Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/connectors_amazon.py +6 -0
- api/odoo_relational.py +37 -1
- api/routes_connectors.py +17 -4
- platform/harness/amazon_store.py +53 -6
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +5 -1
- web/src/connectors/DatabasesTab.tsx +16 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "2c7b866bf",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
2c7b866bf
|
api/connectors_amazon.py
CHANGED
|
@@ -603,6 +603,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 603 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 604 |
"unknownCause": " ".join(unknown),
|
| 605 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 606 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 607 |
answer[table]["limit"] = limit
|
| 608 |
answer["sku_keywords"] = {
|
|
|
|
| 603 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 604 |
"unknownCause": " ".join(unknown),
|
| 605 |
}
|
| 606 |
+
# A source-internal mismatch published the table. Account markers carry a connection prefix.
|
| 607 |
+
warned = az.entity_key("warning_" + table)
|
| 608 |
+
warning = next((str(value.get("cursor") or "") for key, value in entities.items()
|
| 609 |
+
if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "")
|
| 610 |
+
if warning:
|
| 611 |
+
answer[table]["warning"] = warning
|
| 612 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 613 |
answer[table]["limit"] = limit
|
| 614 |
answer["sku_keywords"] = {
|
api/odoo_relational.py
CHANGED
|
@@ -7362,8 +7362,19 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
|
|
| 7362 |
with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
|
| 7363 |
datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
|
| 7364 |
static = datastore.ENTITIES.get(source)
|
|
|
|
| 7365 |
if static:
|
| 7366 |
datastore._ensure_table(con, source, static)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7367 |
else:
|
| 7368 |
types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
|
| 7369 |
columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
|
|
@@ -7468,13 +7479,38 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
|
|
| 7468 |
"snapshotId": publication["snapshot_id"]}
|
| 7469 |
table["readThrough"], table["rows"] = True, {}
|
| 7470 |
table.pop(ut.OVERSIZE_FIELD, None)
|
| 7471 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7472 |
return current
|
| 7473 |
rt.update(ut.STORE_KEY, publish, flush="sync")
|
| 7474 |
ut.register_connected(key)
|
| 7475 |
return {"tables": written, "errors": []}
|
| 7476 |
|
| 7477 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7478 |
def _model_ident(name):
|
| 7479 |
"""Quote a native field identifier without accepting SQL syntax."""
|
| 7480 |
name = str(name)
|
|
|
|
| 7362 |
with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
|
| 7363 |
datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
|
| 7364 |
static = datastore.ENTITIES.get(source)
|
| 7365 |
+
unbound = []
|
| 7366 |
if static:
|
| 7367 |
datastore._ensure_table(con, source, static)
|
| 7368 |
+
# A shared static mirror stores only its declared columns: publish the fields it binds.
|
| 7369 |
+
catalogue_row, unbound = _mirror_bound_catalogue(
|
| 7370 |
+
catalogue_row, datastore.columns_of(source, cur=con) or set())
|
| 7371 |
+
if unbound:
|
| 7372 |
+
fields = _model_field_specs(catalogue_row)
|
| 7373 |
+
metadata = {"model": model, "fields": {
|
| 7374 |
+
name: {prop: meta[prop] for prop in ("type", "relation") if prop in meta}
|
| 7375 |
+
for name, meta in (catalogue_row.get("fields") or {}).items()}}
|
| 7376 |
+
print(f"[odoo] {key}: {len(unbound)} fields have no mirror column and are not "
|
| 7377 |
+
f"published: {', '.join(unbound)}", flush=True)
|
| 7378 |
else:
|
| 7379 |
types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
|
| 7380 |
columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
|
|
|
|
| 7479 |
"snapshotId": publication["snapshot_id"]}
|
| 7480 |
table["readThrough"], table["rows"] = True, {}
|
| 7481 |
table.pop(ut.OVERSIZE_FIELD, None)
|
| 7482 |
+
if unbound:
|
| 7483 |
+
# A machine field with no mirror column would read empty on every row: it leaves.
|
| 7484 |
+
table["fields"] = [f for f in table.get("fields") or ()
|
| 7485 |
+
if f.get("key") not in unbound
|
| 7486 |
+
or not (f.get("automation") or {}).get("preset") or ut.user_edited(f)]
|
| 7487 |
+
written[key] = {**counts, "rows": count, **({"unboundFields": unbound} if unbound else {})}
|
| 7488 |
return current
|
| 7489 |
rt.update(ut.STORE_KEY, publish, flush="sync")
|
| 7490 |
ut.register_connected(key)
|
| 7491 |
return {"tables": written, "errors": []}
|
| 7492 |
|
| 7493 |
|
| 7494 |
+
def _mirror_bound_catalogue(catalogue_row, have):
|
| 7495 |
+
"""The catalogue row cut to the fields a static mirror table binds, and the names it left out."""
|
| 7496 |
+
fields = catalogue_row.get("fields") if isinstance(catalogue_row, dict) else {}
|
| 7497 |
+
fields = fields if isinstance(fields, dict) else {}
|
| 7498 |
+
have = {str(name).lower() for name in have}
|
| 7499 |
+
|
| 7500 |
+
def bound(name, meta):
|
| 7501 |
+
kind = (meta if isinstance(meta, dict) else {}).get("type")
|
| 7502 |
+
if name == "id" or kind == "binary":
|
| 7503 |
+
return True
|
| 7504 |
+
label = name.removesuffix("_id") + "_name"
|
| 7505 |
+
return name.lower() in have and (kind != "many2one" or label.lower() in have)
|
| 7506 |
+
|
| 7507 |
+
unbound = sorted(str(name) for name, meta in fields.items() if not bound(str(name), meta))
|
| 7508 |
+
if not unbound:
|
| 7509 |
+
return catalogue_row, []
|
| 7510 |
+
return {**catalogue_row, "fields": {name: meta for name, meta in fields.items()
|
| 7511 |
+
if str(name) not in unbound}}, unbound
|
| 7512 |
+
|
| 7513 |
+
|
| 7514 |
def _model_ident(name):
|
| 7515 |
"""Quote a native field identifier without accepting SQL syntax."""
|
| 7516 |
name = str(name)
|
api/routes_connectors.py
CHANGED
|
@@ -2199,7 +2199,14 @@ def _odoo_current_jobs(rt, jobs):
|
|
| 2199 |
offered, current = {}, []
|
| 2200 |
for job in jobs:
|
| 2201 |
if job.get("state") != "failed":
|
| 2202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2203 |
continue
|
| 2204 |
cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
|
| 2205 |
if cid not in offered:
|
|
@@ -3187,7 +3194,9 @@ def amazon_hold_cause(row=None, held="", *, table=""):
|
|
| 3187 |
cause = "The last pass started this table and did not finish it. The next pass continues it."
|
| 3188 |
elif state == "refused":
|
| 3189 |
plain = plain_refusal_reason(reason or held, status=status, table=table)
|
| 3190 |
-
|
|
|
|
|
|
|
| 3191 |
"Amazon refused this report: " + plain)
|
| 3192 |
else:
|
| 3193 |
problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
|
|
@@ -3254,7 +3263,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3254 |
if any(not state["updated_at"] for state in states.values()):
|
| 3255 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3256 |
# A row's cause reads the saved refusal and SQP limit markers, never source rows.
|
| 3257 |
-
cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES
|
|
|
|
| 3258 |
+ [prefix + az.SQP_LIMIT_KEY])
|
| 3259 |
saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
|
| 3260 |
[cause_keys]).fetchall())
|
|
@@ -3294,10 +3304,13 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3294 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3295 |
else tick.get("state") or "never_run")
|
| 3296 |
reason = row_reason(table, state)
|
|
|
|
|
|
|
| 3297 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3298 |
"lastSyncAt": (completed.get("updated_at")
|
| 3299 |
if completed.get("state") == "done" else None),
|
| 3300 |
-
"state": state, **({"reason": reason} if reason else {})
|
|
|
|
| 3301 |
return schedule
|
| 3302 |
|
| 3303 |
try:
|
|
|
|
| 2199 |
offered, current = {}, []
|
| 2200 |
for job in jobs:
|
| 2201 |
if job.get("state") != "failed":
|
| 2202 |
+
# A ready receipt carries its database's newest clean pass, which dates the card top.
|
| 2203 |
+
database = ((connections.get(str(job.get("connectionId") or "")) or {}).get("databases") or {}
|
| 2204 |
+
).get(str(job.get("databaseKey") or ""))
|
| 2205 |
+
success = (_odoo_timestamp(_odoo_receipt_success(database))
|
| 2206 |
+
if job.get("state") == "ready" and isinstance(database, dict) else None)
|
| 2207 |
+
stamp = _odoo_timestamp(job.get("at"))
|
| 2208 |
+
current.append({**job, "at": success} if success is not None and (stamp is None or success > stamp)
|
| 2209 |
+
else job)
|
| 2210 |
continue
|
| 2211 |
cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
|
| 2212 |
if cid not in offered:
|
|
|
|
| 3194 |
cause = "The last pass started this table and did not finish it. The next pass continues it."
|
| 3195 |
elif state == "refused":
|
| 3196 |
plain = plain_refusal_reason(reason or held, status=status, table=table)
|
| 3197 |
+
# A reconciliation hold is our own check, never an Amazon refusal.
|
| 3198 |
+
reconciled = "reconciliation refused" in (reason or held)
|
| 3199 |
+
cause = plain if reconciled or plain.startswith(("Amazon refused", "Amazon does not offer")) else (
|
| 3200 |
"Amazon refused this report: " + plain)
|
| 3201 |
else:
|
| 3202 |
problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
|
|
|
|
| 3263 |
if any(not state["updated_at"] for state in states.values()):
|
| 3264 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3265 |
# A row's cause reads the saved refusal and SQP limit markers, never source rows.
|
| 3266 |
+
cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES
|
| 3267 |
+
for family in ("", "refused_", "warning_")]
|
| 3268 |
+ [prefix + az.SQP_LIMIT_KEY])
|
| 3269 |
saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
|
| 3270 |
[cause_keys]).fetchall())
|
|
|
|
| 3304 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3305 |
else tick.get("state") or "never_run")
|
| 3306 |
reason = row_reason(table, state)
|
| 3307 |
+
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3308 |
+
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3309 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3310 |
"lastSyncAt": (completed.get("updated_at")
|
| 3311 |
if completed.get("state") == "done" else None),
|
| 3312 |
+
"state": state, **({"reason": reason} if reason else {}),
|
| 3313 |
+
**({"warning": warning} if warning else {})})
|
| 3314 |
return schedule
|
| 3315 |
|
| 3316 |
try:
|
platform/harness/amazon_store.py
CHANGED
|
@@ -493,6 +493,8 @@ SCHEDULED_FLOOR_SECONDS = 86400
|
|
| 493 |
SCHEDULED_FLOOR_GRACE_SECONDS = 0
|
| 494 |
# The resident cadence checks once a minute.
|
| 495 |
SCHEDULE_PHASE_GRACE_SECONDS = 60
|
|
|
|
|
|
|
| 496 |
_SCHEDULE_OFFSET_READER = None
|
| 497 |
|
| 498 |
|
|
@@ -509,10 +511,15 @@ def _scheduled_phase_due(table, state, now, earliest):
|
|
| 509 |
shift = int(_SCHEDULE_OFFSET_READER())
|
| 510 |
interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
|
| 511 |
phase = ((now.timestamp() - shift) // interval) * interval + shift
|
| 512 |
-
stamp = _utc(state["updated_at"]).timestamp() if state and state.get("
|
| 513 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 514 |
# The second half is left alone: a staging copy of a live tick stays off the live phase.
|
| 515 |
-
if stamp is
|
| 516 |
return earliest
|
| 517 |
if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
|
| 518 |
or (stamp is not None and stamp >= phase)):
|
|
@@ -607,6 +614,20 @@ def scheduled_sync_plan(tenant_key, *, rt, now=None):
|
|
| 607 |
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 608 |
"WHERE starts_with(entity, ?)", [prefix]).fetchall()}
|
| 609 |
plan = sync_plan(ticks, tables, now=now)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
due.update(dict.fromkeys(plan["tables"]))
|
| 611 |
for table, reason in plan["deferred"].items():
|
| 612 |
deferred.setdefault(table, []).append(reason)
|
|
@@ -5114,6 +5135,14 @@ def _refused_reports(con):
|
|
| 5114 |
def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
| 5115 |
"""Project refusals to cause and action copy, retaining the fee API's vendor reason."""
|
| 5116 |
text = str(reason or "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5117 |
feedback = table in ("review_topics", "review_trends")
|
| 5118 |
subject = "review trends" if table == "review_trends" else "review topics"
|
| 5119 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
|
@@ -8606,7 +8635,8 @@ def _validate_orders(con):
|
|
| 8606 |
return {"state": "unproven", "expected": expected, "actual": actual,
|
| 8607 |
"reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
|
| 8608 |
f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
|
| 8609 |
-
|
|
|
|
| 8610 |
"reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
|
| 8611 |
except Exception:
|
| 8612 |
return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
|
|
@@ -9146,7 +9176,20 @@ def _refuse_unreconciled(tenant_key, rt, report, log):
|
|
| 9146 |
"the table keeps its previous rows until it runs"} for table in synced}
|
| 9147 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 9148 |
for table in synced:
|
| 9149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9150 |
continue
|
| 9151 |
reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
|
| 9152 |
report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
|
|
@@ -9391,6 +9434,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9391 |
continue
|
| 9392 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 9393 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
|
|
|
|
|
|
| 9394 |
tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
|
| 9395 |
if tick and tick.get("state") == "throttled":
|
| 9396 |
# The first load owns SQP now; a passed SQP deadline must not make it due every minute.
|
|
@@ -9407,7 +9452,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9407 |
mark = _read_state(con, entity_key(table)) or {}
|
| 9408 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 9409 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 9410 |
-
|
|
|
|
|
|
|
| 9411 |
for table in want:
|
| 9412 |
tick = ticks.get(table)
|
| 9413 |
# A queued resync is a repair: only Amazon's own floor and the stagger hold it.
|
|
|
|
| 493 |
SCHEDULED_FLOOR_GRACE_SECONDS = 0
|
| 494 |
# The resident cadence checks once a minute.
|
| 495 |
SCHEDULE_PHASE_GRACE_SECONDS = 60
|
| 496 |
+
#: `_sync_state` marker: the scheduled first-load guard deferred this table; never a tick, so no card reads it.
|
| 497 |
+
PHASE_HOLD_PREFIX = "phase_hold_"
|
| 498 |
_SCHEDULE_OFFSET_READER = None
|
| 499 |
|
| 500 |
|
|
|
|
| 511 |
shift = int(_SCHEDULE_OFFSET_READER())
|
| 512 |
interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
|
| 513 |
phase = ((now.timestamp() - shift) // interval) * interval + shift
|
| 514 |
+
stamp = _utc(state["updated_at"]).timestamp() if state and state.get("updated_at") else None
|
| 515 |
+
window = interval / 2
|
| 516 |
+
fastest = SCHEDULED_FLOOR_SECONDS // max(cadence["perDay"] for cadence in _client().CADENCE.values())
|
| 517 |
+
if shift % fastest:
|
| 518 |
+
# A shifted deployment's late run ends 30 minutes before the next live slot.
|
| 519 |
+
window = min(window, fastest - phase % fastest - 1800)
|
| 520 |
+
# A tick that missed its phase, in any state or absent, runs late inside the first half of the interval.
|
| 521 |
# The second half is left alone: a staging copy of a live tick stays off the live phase.
|
| 522 |
+
if (stamp is None or stamp < phase) and earliest.timestamp() - phase < window:
|
| 523 |
return earliest
|
| 524 |
if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
|
| 525 |
or (stamp is not None and stamp >= phase)):
|
|
|
|
| 614 |
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 615 |
"WHERE starts_with(entity, ?)", [prefix]).fetchall()}
|
| 616 |
plan = sync_plan(ticks, tables, now=now)
|
| 617 |
+
hold = _account_prefix(entry["connection_id"]) + entity_key(PHASE_HOLD_PREFIX)
|
| 618 |
+
# The planner reads cadence rows through this one query shape only, never source data.
|
| 619 |
+
holds = {entity[len(hold):]: {"state": phase, "rows": 0, "updated_at": at, "detail": None}
|
| 620 |
+
for entity, phase, _rows, at, _detail in con.execute(
|
| 621 |
+
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 622 |
+
"WHERE starts_with(entity, ?)", [hold]).fetchall()}
|
| 623 |
+
for table in list(plan["tables"]):
|
| 624 |
+
held, tick = holds.get(table), ticks.get(table)
|
| 625 |
+
# A first-load deferral newer than the tick waits for the next phase, as the sync defers it.
|
| 626 |
+
if held and (tick is None or _utc(held["updated_at"]) > _utc(tick["updated_at"])):
|
| 627 |
+
reason = _interval_reason(table, held, now, floor_s=_cadence_seconds(table, tick))
|
| 628 |
+
if reason:
|
| 629 |
+
plan["tables"].remove(table)
|
| 630 |
+
plan["deferred"][table] = reason
|
| 631 |
due.update(dict.fromkeys(plan["tables"]))
|
| 632 |
for table, reason in plan["deferred"].items():
|
| 633 |
deferred.setdefault(table, []).append(reason)
|
|
|
|
| 5135 |
def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
| 5136 |
"""Project refusals to cause and action copy, retaining the fee API's vendor reason."""
|
| 5137 |
text = str(reason or "")
|
| 5138 |
+
# A reconciliation hold is our own check: never call it a load failure.
|
| 5139 |
+
reconciliation = re.search(r"\breconciliation refused:\s*([\s\S]*)$", text, re.I)
|
| 5140 |
+
if reconciliation:
|
| 5141 |
+
detail = re.sub(r"[.\s]+$", "", reconciliation[1].strip()).replace("\u2014", ". ").replace("\u2013", " to ")
|
| 5142 |
+
if re.match(r"the check could not run", detail, re.I):
|
| 5143 |
+
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5144 |
+
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5145 |
+
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5146 |
feedback = table in ("review_topics", "review_trends")
|
| 5147 |
subject = "review trends" if table == "review_trends" else "review topics"
|
| 5148 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
|
|
|
| 8635 |
return {"state": "unproven", "expected": expected, "actual": actual,
|
| 8636 |
"reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
|
| 8637 |
f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
|
| 8638 |
+
# Two Amazon reports disagree: a check on the source's own data warns and publishes.
|
| 8639 |
+
return {"state": "refused", "expected": expected, "actual": actual, "revived": revived, "publish": True,
|
| 8640 |
"reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
|
| 8641 |
except Exception:
|
| 8642 |
return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
|
|
|
|
| 9176 |
"the table keeps its previous rows until it runs"} for table in synced}
|
| 9177 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 9178 |
for table in synced:
|
| 9179 |
+
verdict, warned = checked.get(table) or {}, entity_key("warning_" + table)
|
| 9180 |
+
if verdict.get("state") == "refused" and verdict.get("publish") is True:
|
| 9181 |
+
# A check on the source's own data warns and publishes. It never holds the sync.
|
| 9182 |
+
expected, actual = verdict.get("expected"), verdict.get("actual")
|
| 9183 |
+
warning = (f"Amazon Sales counted {expected:,} orders and the orders report holds {actual:,}. "
|
| 9184 |
+
"The table is published." if type(expected) is int and type(actual) is int else
|
| 9185 |
+
f"{str(verdict.get('reason') or 'Amazon reports disagree').rstrip('. ')}. The table is published.")
|
| 9186 |
+
report["tables"][table] = {**report["tables"][table], "warning": warning}
|
| 9187 |
+
_record_state(con, warned, "warning", 0, warning)
|
| 9188 |
+
log(f"{table}: published with a warning: {warning}")
|
| 9189 |
+
continue
|
| 9190 |
+
if verdict.get("state") != "refused":
|
| 9191 |
+
# A clean reconcile clears the last warning. A held table keeps it with its kept rows.
|
| 9192 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [warned])
|
| 9193 |
continue
|
| 9194 |
reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
|
| 9195 |
report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
|
|
|
|
| 9434 |
continue
|
| 9435 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 9436 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
| 9437 |
+
# The planner waits for the next phase on this hold. The card's tick stays as it was.
|
| 9438 |
+
_record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred")
|
| 9439 |
tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
|
| 9440 |
if tick and tick.get("state") == "throttled":
|
| 9441 |
# The first load owns SQP now; a passed SQP deadline must not make it due every minute.
|
|
|
|
| 9452 |
mark = _read_state(con, entity_key(table)) or {}
|
| 9453 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 9454 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 9455 |
+
# The re-asked tick keeps its stamp, so the phase rule decides as the planner does.
|
| 9456 |
+
ticks.update({name: ticks[name] and {**ticks[name], "state": "refused"}
|
| 9457 |
+
for name in (table, pairs.get(table)) if name in want})
|
| 9458 |
for table in want:
|
| 9459 |
tick = ticks.get(table)
|
| 9460 |
# A queued resync is a repair: only Amazon's own floor and the stagger hold it.
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
92d4222f3262a02dae11929a0f9f8215d84e91c80bcd86e5208005f9c7a6577c
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -265,6 +265,8 @@ export interface AmazonTableSync {
|
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
|
|
|
|
|
|
| 268 |
}
|
| 269 |
|
| 270 |
export interface AmazonSyncScheduleReading {
|
|
@@ -298,7 +300,8 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
|
|
| 298 |
if (identities.has(identity)) return unavailable;
|
| 299 |
identities.add(identity);
|
| 300 |
const reason = typeof row.reason === "string" ? row.reason.trim() : "";
|
| 301 |
-
|
|
|
|
| 302 |
}
|
| 303 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 304 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
@@ -344,6 +347,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 344 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 345 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 346 |
{row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
|
|
|
|
| 347 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 348 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 349 |
</td>
|
|
|
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
| 268 |
+
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 269 |
+
warning?: string;
|
| 270 |
}
|
| 271 |
|
| 272 |
export interface AmazonSyncScheduleReading {
|
|
|
|
| 300 |
if (identities.has(identity)) return unavailable;
|
| 301 |
identities.add(identity);
|
| 302 |
const reason = typeof row.reason === "string" ? row.reason.trim() : "";
|
| 303 |
+
const warning = typeof row.warning === "string" ? row.warning.trim() : "";
|
| 304 |
+
tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined, warning: warning || undefined });
|
| 305 |
}
|
| 306 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 307 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
|
|
| 347 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 348 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 349 |
{row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
|
| 350 |
+
: row.warning && row.state === "done" ? <p role="status">{row.warning}</p>
|
| 351 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 352 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 353 |
</td>
|
web/src/connectors/DatabasesTab.tsx
CHANGED
|
@@ -135,6 +135,8 @@ interface AmazonTableSync {
|
|
| 135 |
stuck: boolean | null;
|
| 136 |
stuckCause: string;
|
| 137 |
unknownCause: string;
|
|
|
|
|
|
|
| 138 |
}
|
| 139 |
|
| 140 |
export interface DatabaseSyncHealth {
|
|
@@ -564,6 +566,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
|
|
| 564 |
}
|
| 565 |
const reason = syncText(syncObject(state.deferred).reason);
|
| 566 |
const held = syncObject(state.held);
|
|
|
|
| 567 |
states[database.key] = {
|
| 568 |
rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
|
| 569 |
inProgress: state.inProgress === true,
|
|
@@ -575,6 +578,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
|
|
| 575 |
firstLoad: load,
|
| 576 |
stuck: state.stuck === true ? true : state.stuck === false ? false : null,
|
| 577 |
stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
|
|
|
|
| 578 |
};
|
| 579 |
}
|
| 580 |
return states;
|
|
@@ -686,6 +690,16 @@ export function firstLoadEstimate(seconds: number): string {
|
|
| 686 |
function amazonRefusalReason(reason: string, status = ""): string {
|
| 687 |
const text = reason.trim();
|
| 688 |
const lower = text.toLowerCase();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 689 |
if (/within (?:the past )?two years/.test(lower)) {
|
| 690 |
return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
|
| 691 |
}
|
|
@@ -2323,7 +2337,8 @@ const DatabasesTab = forwardRef<DatabasesTabHandle, {
|
|
| 2323 |
catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
|
| 2324 |
{health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
|
| 2325 |
{health.error.message}</p> : null}
|
| 2326 |
-
{health?.warning ? <p className="conn-db-note">
|
|
|
|
| 2327 |
{job?.state === "failed" && job.retryable ? <button type="button"
|
| 2328 |
className="conn-job-retry" disabled={Boolean(retrying)}
|
| 2329 |
onClick={() => void retryJob(job)}>Retry sync</button> : null}
|
|
|
|
| 135 |
stuck: boolean | null;
|
| 136 |
stuckCause: string;
|
| 137 |
unknownCause: string;
|
| 138 |
+
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 139 |
+
warning?: string;
|
| 140 |
}
|
| 141 |
|
| 142 |
export interface DatabaseSyncHealth {
|
|
|
|
| 566 |
}
|
| 567 |
const reason = syncText(syncObject(state.deferred).reason);
|
| 568 |
const held = syncObject(state.held);
|
| 569 |
+
const warning = syncText(state.warning).replace(/\u2014/g, ". ").replace(/\u2013/g, " to ").trim();
|
| 570 |
states[database.key] = {
|
| 571 |
rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
|
| 572 |
inProgress: state.inProgress === true,
|
|
|
|
| 578 |
firstLoad: load,
|
| 579 |
stuck: state.stuck === true ? true : state.stuck === false ? false : null,
|
| 580 |
stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
|
| 581 |
+
...(warning ? { warning } : {}),
|
| 582 |
};
|
| 583 |
}
|
| 584 |
return states;
|
|
|
|
| 690 |
function amazonRefusalReason(reason: string, status = ""): string {
|
| 691 |
const text = reason.trim();
|
| 692 |
const lower = text.toLowerCase();
|
| 693 |
+
// Mirrors plain_refusal_reason: a reconciliation hold is our own check, never a load failure.
|
| 694 |
+
const reconciliation = /\breconciliation refused:\s*([\s\S]*)$/i.exec(text);
|
| 695 |
+
if (reconciliation) {
|
| 696 |
+
const detail = reconciliation[1].trim().replace(/[.\s]+$/, "").replace(/\u2014/g, ". ").replace(/\u2013/g, " to ");
|
| 697 |
+
if (/^the check could not run/i.test(detail)) {
|
| 698 |
+
return "The reconciliation check could not run. The last good rows are kept until it runs.";
|
| 699 |
+
}
|
| 700 |
+
return detail ? `Reconciliation did not tie: ${detail}. The last good rows are kept.`
|
| 701 |
+
: "Reconciliation did not tie. The last good rows are kept.";
|
| 702 |
+
}
|
| 703 |
if (/within (?:the past )?two years/.test(lower)) {
|
| 704 |
return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
|
| 705 |
}
|
|
|
|
| 2337 |
catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
|
| 2338 |
{health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
|
| 2339 |
{health.error.message}</p> : null}
|
| 2340 |
+
{health?.warning || amazonState?.warning ? <p className="conn-db-note">
|
| 2341 |
+
{health?.warning || amazonState?.warning}</p> : null}
|
| 2342 |
{job?.state === "failed" && job.retryable ? <button type="button"
|
| 2343 |
className="conn-job-retry" disabled={Boolean(retrying)}
|
| 2344 |
onClick={() => void retryJob(job)}>Retry sync</button> : null}
|