Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/connectors_amazon.py +6 -0
- api/main.py +34 -9
- api/odoo_relational.py +37 -1
- api/routes_connectors.py +17 -4
- api/sync_alerts.py +42 -9
- platform/core/connector_sync.py +40 -0
- platform/harness/amazon_store.py +53 -6
- platform/harness/mirror_store.py +63 -12
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +34 -9
- web/src/connectors/DatabasesTab.tsx +16 -1
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "50dae147b (50dae147b20ab)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
50dae147b (50dae147b20ab)
|
api/connectors_amazon.py
CHANGED
|
@@ -603,6 +603,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 603 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 604 |
"unknownCause": " ".join(unknown),
|
| 605 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 606 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 607 |
answer[table]["limit"] = limit
|
| 608 |
answer["sku_keywords"] = {
|
|
|
|
| 603 |
"stuck": stuck, "stuckCause": stuck_cause,
|
| 604 |
"unknownCause": " ".join(unknown),
|
| 605 |
}
|
| 606 |
+
# A source-internal mismatch published the table. Account markers carry a connection prefix.
|
| 607 |
+
warned = az.entity_key("warning_" + table)
|
| 608 |
+
warning = next((str(value.get("cursor") or "") for key, value in entities.items()
|
| 609 |
+
if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "")
|
| 610 |
+
if warning:
|
| 611 |
+
answer[table]["warning"] = warning
|
| 612 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 613 |
answer[table]["limit"] = limit
|
| 614 |
answer["sku_keywords"] = {
|
api/main.py
CHANGED
|
@@ -1188,16 +1188,29 @@ def _pull_meta(why, tenant_key=None):
|
|
| 1188 |
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1189 |
on_error=runtime_failed):
|
| 1190 |
try:
|
| 1191 |
-
|
|
|
|
| 1192 |
continue
|
| 1193 |
from core import connector_sync as _meta_status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1194 |
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1195 |
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1196 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1197 |
log=lambda *_a: None, insights_days=7)
|
| 1198 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1199 |
-
detail =
|
|
|
|
| 1200 |
_status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
|
|
|
|
|
|
|
| 1201 |
for problem in rep.get("problems") or []:
|
| 1202 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
|
| 1203 |
except Exception as exc:
|
|
@@ -1356,17 +1369,29 @@ def _pull_amazon(why, vendor=True):
|
|
| 1356 |
if not vendor:
|
| 1357 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1358 |
return True
|
|
|
|
| 1359 |
from harness import mirror_store
|
| 1360 |
from core import connector_sync as _amazon_status
|
| 1361 |
-
|
| 1362 |
-
|
| 1363 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1364 |
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1365 |
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1366 |
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
|
| 1367 |
-
|
| 1368 |
-
|
| 1369 |
-
|
| 1370 |
_holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
|
| 1371 |
_problems = _holds.get("unsettled", rep.get("problems") or [])
|
| 1372 |
_status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
|
|
@@ -1613,7 +1638,7 @@ def _sync_now_cause(cause):
|
|
| 1613 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 1614 |
("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 1615 |
("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 1616 |
-
"expired", "authentication"), "
|
| 1617 |
("mirror_constraint", ("duplicate key", "primary key", "constraint"),
|
| 1618 |
"The mirror rejected rows because they violate a database constraint."),
|
| 1619 |
("connect_failed", ("connection", "network", "name resolution"),
|
|
|
|
| 1188 |
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1189 |
on_error=runtime_failed):
|
| 1190 |
try:
|
| 1191 |
+
connections = _meta._connection_rows(rt, key, {"problems": []})
|
| 1192 |
+
if not connections:
|
| 1193 |
continue
|
| 1194 |
from core import connector_sync as _meta_status
|
| 1195 |
+
# A refused credential is not retried until it changes. An unreadable fingerprint runs the pass.
|
| 1196 |
+
try:
|
| 1197 |
+
fingerprint = _meta_status.credential_fingerprint(
|
| 1198 |
+
[f"{row[0]}\t{row[2]}" for row in connections])
|
| 1199 |
+
except Exception: # noqa: BLE001
|
| 1200 |
+
fingerprint = None
|
| 1201 |
+
if _meta_status.credential_held(rt, "meta_ads", fingerprint):
|
| 1202 |
+
print("[sync] meta_ads held: the provider refused the credential; waiting for a new one")
|
| 1203 |
+
continue
|
| 1204 |
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1205 |
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1206 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1207 |
log=lambda *_a: None, insights_days=7)
|
| 1208 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1209 |
+
cause, detail = (_sync_now_cause(" ".join(rep.get("problems") or []))
|
| 1210 |
+
if not outcomes[key] else ("", ""))
|
| 1211 |
_status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
|
| 1212 |
+
if cause == "credential_refused":
|
| 1213 |
+
_meta_status.hold_credential(rt, "meta_ads", fingerprint)
|
| 1214 |
for problem in rep.get("problems") or []:
|
| 1215 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
|
| 1216 |
except Exception as exc:
|
|
|
|
| 1369 |
if not vendor:
|
| 1370 |
print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
|
| 1371 |
return True
|
| 1372 |
+
from contextlib import ExitStack
|
| 1373 |
from harness import mirror_store
|
| 1374 |
from core import connector_sync as _amazon_status
|
| 1375 |
+
with ExitStack() as _lease:
|
| 1376 |
+
try:
|
| 1377 |
+
_lease.enter_context(_amazon_role.connector_writer(rt, "amazon"))
|
| 1378 |
+
except Exception as exc:
|
| 1379 |
+
# A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
|
| 1380 |
+
if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
|
| 1381 |
+
print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
|
| 1382 |
+
flush=True)
|
| 1383 |
+
return False
|
| 1384 |
+
_amazon_status.record_pass_start(rt, "amazon_sp")
|
| 1385 |
+
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1386 |
+
raise
|
| 1387 |
+
_amazon_status.record_pass_start(rt, "amazon_sp")
|
| 1388 |
+
try:
|
| 1389 |
rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
|
| 1390 |
slot_pass=bool(RESYNC_STATE.get("slotPass")),
|
| 1391 |
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
|
| 1392 |
+
except Exception:
|
| 1393 |
+
_status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
|
| 1394 |
+
raise
|
| 1395 |
_holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
|
| 1396 |
_problems = _holds.get("unsettled", rep.get("problems") or [])
|
| 1397 |
_status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
|
|
|
|
| 1638 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 1639 |
("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 1640 |
("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 1641 |
+
"expired", "authentication"), "Credential expired. Reconnect to resume."),
|
| 1642 |
("mirror_constraint", ("duplicate key", "primary key", "constraint"),
|
| 1643 |
"The mirror rejected rows because they violate a database constraint."),
|
| 1644 |
("connect_failed", ("connection", "network", "name resolution"),
|
api/odoo_relational.py
CHANGED
|
@@ -7362,8 +7362,19 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
|
|
| 7362 |
with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
|
| 7363 |
datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
|
| 7364 |
static = datastore.ENTITIES.get(source)
|
|
|
|
| 7365 |
if static:
|
| 7366 |
datastore._ensure_table(con, source, static)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7367 |
else:
|
| 7368 |
types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
|
| 7369 |
columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
|
|
@@ -7468,13 +7479,38 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
|
|
| 7468 |
"snapshotId": publication["snapshot_id"]}
|
| 7469 |
table["readThrough"], table["rows"] = True, {}
|
| 7470 |
table.pop(ut.OVERSIZE_FIELD, None)
|
| 7471 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7472 |
return current
|
| 7473 |
rt.update(ut.STORE_KEY, publish, flush="sync")
|
| 7474 |
ut.register_connected(key)
|
| 7475 |
return {"tables": written, "errors": []}
|
| 7476 |
|
| 7477 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7478 |
def _model_ident(name):
|
| 7479 |
"""Quote a native field identifier without accepting SQL syntax."""
|
| 7480 |
name = str(name)
|
|
|
|
| 7362 |
with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
|
| 7363 |
datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
|
| 7364 |
static = datastore.ENTITIES.get(source)
|
| 7365 |
+
unbound = []
|
| 7366 |
if static:
|
| 7367 |
datastore._ensure_table(con, source, static)
|
| 7368 |
+
# A shared static mirror stores only its declared columns: publish the fields it binds.
|
| 7369 |
+
catalogue_row, unbound = _mirror_bound_catalogue(
|
| 7370 |
+
catalogue_row, datastore.columns_of(source, cur=con) or set())
|
| 7371 |
+
if unbound:
|
| 7372 |
+
fields = _model_field_specs(catalogue_row)
|
| 7373 |
+
metadata = {"model": model, "fields": {
|
| 7374 |
+
name: {prop: meta[prop] for prop in ("type", "relation") if prop in meta}
|
| 7375 |
+
for name, meta in (catalogue_row.get("fields") or {}).items()}}
|
| 7376 |
+
print(f"[odoo] {key}: {len(unbound)} fields have no mirror column and are not "
|
| 7377 |
+
f"published: {', '.join(unbound)}", flush=True)
|
| 7378 |
else:
|
| 7379 |
types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
|
| 7380 |
columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
|
|
|
|
| 7479 |
"snapshotId": publication["snapshot_id"]}
|
| 7480 |
table["readThrough"], table["rows"] = True, {}
|
| 7481 |
table.pop(ut.OVERSIZE_FIELD, None)
|
| 7482 |
+
if unbound:
|
| 7483 |
+
# A machine field with no mirror column would read empty on every row: it leaves.
|
| 7484 |
+
table["fields"] = [f for f in table.get("fields") or ()
|
| 7485 |
+
if f.get("key") not in unbound
|
| 7486 |
+
or not (f.get("automation") or {}).get("preset") or ut.user_edited(f)]
|
| 7487 |
+
written[key] = {**counts, "rows": count, **({"unboundFields": unbound} if unbound else {})}
|
| 7488 |
return current
|
| 7489 |
rt.update(ut.STORE_KEY, publish, flush="sync")
|
| 7490 |
ut.register_connected(key)
|
| 7491 |
return {"tables": written, "errors": []}
|
| 7492 |
|
| 7493 |
|
| 7494 |
+
def _mirror_bound_catalogue(catalogue_row, have):
|
| 7495 |
+
"""The catalogue row cut to the fields a static mirror table binds, and the names it left out."""
|
| 7496 |
+
fields = catalogue_row.get("fields") if isinstance(catalogue_row, dict) else {}
|
| 7497 |
+
fields = fields if isinstance(fields, dict) else {}
|
| 7498 |
+
have = {str(name).lower() for name in have}
|
| 7499 |
+
|
| 7500 |
+
def bound(name, meta):
|
| 7501 |
+
kind = (meta if isinstance(meta, dict) else {}).get("type")
|
| 7502 |
+
if name == "id" or kind == "binary":
|
| 7503 |
+
return True
|
| 7504 |
+
label = name.removesuffix("_id") + "_name"
|
| 7505 |
+
return name.lower() in have and (kind != "many2one" or label.lower() in have)
|
| 7506 |
+
|
| 7507 |
+
unbound = sorted(str(name) for name, meta in fields.items() if not bound(str(name), meta))
|
| 7508 |
+
if not unbound:
|
| 7509 |
+
return catalogue_row, []
|
| 7510 |
+
return {**catalogue_row, "fields": {name: meta for name, meta in fields.items()
|
| 7511 |
+
if str(name) not in unbound}}, unbound
|
| 7512 |
+
|
| 7513 |
+
|
| 7514 |
def _model_ident(name):
|
| 7515 |
"""Quote a native field identifier without accepting SQL syntax."""
|
| 7516 |
name = str(name)
|
api/routes_connectors.py
CHANGED
|
@@ -2199,7 +2199,14 @@ def _odoo_current_jobs(rt, jobs):
|
|
| 2199 |
offered, current = {}, []
|
| 2200 |
for job in jobs:
|
| 2201 |
if job.get("state") != "failed":
|
| 2202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2203 |
continue
|
| 2204 |
cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
|
| 2205 |
if cid not in offered:
|
|
@@ -3187,7 +3194,9 @@ def amazon_hold_cause(row=None, held="", *, table=""):
|
|
| 3187 |
cause = "The last pass started this table and did not finish it. The next pass continues it."
|
| 3188 |
elif state == "refused":
|
| 3189 |
plain = plain_refusal_reason(reason or held, status=status, table=table)
|
| 3190 |
-
|
|
|
|
|
|
|
| 3191 |
"Amazon refused this report: " + plain)
|
| 3192 |
else:
|
| 3193 |
problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
|
|
@@ -3254,7 +3263,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3254 |
if any(not state["updated_at"] for state in states.values()):
|
| 3255 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3256 |
# A row's cause reads the saved refusal and SQP limit markers, never source rows.
|
| 3257 |
-
cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES
|
|
|
|
| 3258 |
+ [prefix + az.SQP_LIMIT_KEY])
|
| 3259 |
saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
|
| 3260 |
[cause_keys]).fetchall())
|
|
@@ -3294,10 +3304,13 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3294 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3295 |
else tick.get("state") or "never_run")
|
| 3296 |
reason = row_reason(table, state)
|
|
|
|
|
|
|
| 3297 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3298 |
"lastSyncAt": (completed.get("updated_at")
|
| 3299 |
if completed.get("state") == "done" else None),
|
| 3300 |
-
"state": state, **({"reason": reason} if reason else {})
|
|
|
|
| 3301 |
return schedule
|
| 3302 |
|
| 3303 |
try:
|
|
|
|
| 2199 |
offered, current = {}, []
|
| 2200 |
for job in jobs:
|
| 2201 |
if job.get("state") != "failed":
|
| 2202 |
+
# A ready receipt carries its database's newest clean pass, which dates the card top.
|
| 2203 |
+
database = ((connections.get(str(job.get("connectionId") or "")) or {}).get("databases") or {}
|
| 2204 |
+
).get(str(job.get("databaseKey") or ""))
|
| 2205 |
+
success = (_odoo_timestamp(_odoo_receipt_success(database))
|
| 2206 |
+
if job.get("state") == "ready" and isinstance(database, dict) else None)
|
| 2207 |
+
stamp = _odoo_timestamp(job.get("at"))
|
| 2208 |
+
current.append({**job, "at": success} if success is not None and (stamp is None or success > stamp)
|
| 2209 |
+
else job)
|
| 2210 |
continue
|
| 2211 |
cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
|
| 2212 |
if cid not in offered:
|
|
|
|
| 3194 |
cause = "The last pass started this table and did not finish it. The next pass continues it."
|
| 3195 |
elif state == "refused":
|
| 3196 |
plain = plain_refusal_reason(reason or held, status=status, table=table)
|
| 3197 |
+
# A reconciliation hold is our own check, never an Amazon refusal.
|
| 3198 |
+
reconciled = "reconciliation refused" in (reason or held)
|
| 3199 |
+
cause = plain if reconciled or plain.startswith(("Amazon refused", "Amazon does not offer")) else (
|
| 3200 |
"Amazon refused this report: " + plain)
|
| 3201 |
else:
|
| 3202 |
problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
|
|
|
|
| 3263 |
if any(not state["updated_at"] for state in states.values()):
|
| 3264 |
raise ValueError("Amazon cadence metadata has an invalid timestamp")
|
| 3265 |
# A row's cause reads the saved refusal and SQP limit markers, never source rows.
|
| 3266 |
+
cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES
|
| 3267 |
+
for family in ("", "refused_", "warning_")]
|
| 3268 |
+ [prefix + az.SQP_LIMIT_KEY])
|
| 3269 |
saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
|
| 3270 |
[cause_keys]).fetchall())
|
|
|
|
| 3304 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3305 |
else tick.get("state") or "never_run")
|
| 3306 |
reason = row_reason(table, state)
|
| 3307 |
+
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3308 |
+
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3309 |
schedule.append({**row, "connectionId": cid, "account": entry["label"],
|
| 3310 |
"lastSyncAt": (completed.get("updated_at")
|
| 3311 |
if completed.get("state") == "done" else None),
|
| 3312 |
+
"state": state, **({"reason": reason} if reason else {}),
|
| 3313 |
+
**({"warning": warning} if warning else {})})
|
| 3314 |
return schedule
|
| 3315 |
|
| 3316 |
try:
|
api/sync_alerts.py
CHANGED
|
@@ -27,8 +27,18 @@ SYNC_KEY = "sync"
|
|
| 27 |
# Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
|
| 28 |
AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
-
# A
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
# A throttle or a held table alerts only when it repeats on this many passes in a row.
|
| 33 |
REPEAT_CODES = {"throttled": 2, "held": 2}
|
| 34 |
|
|
@@ -120,9 +130,14 @@ def _clean(failure):
|
|
| 120 |
if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
|
| 121 |
return None
|
| 122 |
key = scrub(failure["key"], 120)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
return {"key": key, "label": scrub(failure.get("label") or key, 120),
|
| 124 |
-
"code":
|
| 125 |
-
"cause": scrub(failure.get("cause") or "no cause given"),
|
| 126 |
"last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
|
| 127 |
|
| 128 |
|
|
@@ -332,6 +347,16 @@ def _item(alert):
|
|
| 332 |
return f"{alert['label']} ({alert['cause']}, {tail})"
|
| 333 |
|
| 334 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
def environment():
|
| 336 |
"""LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
|
| 337 |
space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
|
|
@@ -341,14 +366,22 @@ def environment():
|
|
| 341 |
|
| 342 |
|
| 343 |
def compose(tenant, connector, plan, document):
|
| 344 |
-
"""One line for the whole pass: FAILED, STILL FAILING
|
| 345 |
alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
|
| 346 |
where = f"{environment()} {tenant} {connector}"
|
| 347 |
parts = []
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
|
| 353 |
if names:
|
| 354 |
parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
|
|
|
|
| 27 |
# Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
|
| 28 |
AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
+
# A refused or expired credential is one cause: the owner reconnects or disconnects the connector.
|
| 31 |
+
CREDENTIAL_CAUSE = "credential expired, reconnect or disconnect it"
|
| 32 |
+
CREDENTIAL_CODES = frozenset({"credential", "credential_refused", "source_auth_failed"})
|
| 33 |
+
# A TLS "certificate has expired" names no token or credential, so it never matches.
|
| 34 |
+
CREDENTIAL = re.compile(
|
| 35 |
+
r"\brefused the connector credential\b|\bcredential expired\b|\binvalid_grant\b|\bcredential_refused\b"
|
| 36 |
+
r"|\berror validating access token\b|\bsession has expired\b"
|
| 37 |
+
r"|\b(?:\w+_)?tokens?\b[^.]{0,40}\b(?:expired|revoked|invalid)\b"
|
| 38 |
+
r"|\b(?:expired|revoked|invalid)\b[^.]{0,20}\b(?:\w+_)?tokens?\b"
|
| 39 |
+
r"|\b(?:refused|rejected)\b[^.]{0,40}\bcredential\b", re.I)
|
| 40 |
+
# A missing role or a refused credential is a standing account fact: it alerts once and never reminds.
|
| 41 |
+
ONCE_CODES = frozenset({"role", "credential"})
|
| 42 |
# A throttle or a held table alerts only when it repeats on this many passes in a row.
|
| 43 |
REPEAT_CODES = {"throttled": 2, "held": 2}
|
| 44 |
|
|
|
|
| 130 |
if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
|
| 131 |
return None
|
| 132 |
key = scrub(failure["key"], 120)
|
| 133 |
+
code = scrub(failure.get("code") or "error", 60) or "error"
|
| 134 |
+
cause = scrub(failure.get("cause") or "no cause given")
|
| 135 |
+
raw = f"{failure.get('code') or ''} {failure.get('cause') or ''}"
|
| 136 |
+
# A missing Amazon role stays a role: it is checked before the credential.
|
| 137 |
+
if code != "role" and not AMAZON_ROLE.search(raw) and (code in CREDENTIAL_CODES or CREDENTIAL.search(raw)):
|
| 138 |
+
code, cause = "credential", CREDENTIAL_CAUSE
|
| 139 |
return {"key": key, "label": scrub(failure.get("label") or key, 120),
|
| 140 |
+
"code": code, "cause": cause,
|
|
|
|
| 141 |
"last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
|
| 142 |
|
| 143 |
|
|
|
|
| 347 |
return f"{alert['label']} ({alert['cause']}, {tail})"
|
| 348 |
|
| 349 |
|
| 350 |
+
#: Connectors whose cards the watchdog reads, so their causes also reach the AI inbox.
|
| 351 |
+
INBOX_CONNECTORS = frozenset({"amazon", "odoo", "meta_ads", "amazon_ads"})
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def _names_table(connector, alert):
|
| 355 |
+
"""True when the alert's key is one table: a watchdog `table:` key or an Amazon pass table."""
|
| 356 |
+
key = str(alert.get("key") or "")
|
| 357 |
+
return key.startswith("table:") or (connector == "amazon" and key != SYNC_KEY)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
def environment():
|
| 361 |
"""LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
|
| 362 |
space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
|
|
|
|
| 366 |
|
| 367 |
|
| 368 |
def compose(tenant, connector, plan, document):
|
| 369 |
+
"""One line for the whole pass: FAILED lists each alert, STILL FAILING counts them, RECOVERED names them."""
|
| 370 |
alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
|
| 371 |
where = f"{environment()} {tenant} {connector}"
|
| 372 |
parts = []
|
| 373 |
+
items = [_item(alerts[a]) for a in sorted(plan["failed"]) if a in alerts]
|
| 374 |
+
if items:
|
| 375 |
+
parts.append(f"Loopable sync FAILED: {where}: " + ", ".join(items) + ".")
|
| 376 |
+
# The daily reminder is one short line per connector: the inbox keeps the full list.
|
| 377 |
+
reminded = [alerts[a] for a in sorted(plan["reminder"]) if a in alerts]
|
| 378 |
+
if reminded:
|
| 379 |
+
tables = all(_names_table(connector, alert) for alert in reminded)
|
| 380 |
+
noun = ("table" if tables else "cause") + ("" if len(reminded) == 1 else "s")
|
| 381 |
+
# Only a connector the watchdog reads has its causes in the AI inbox.
|
| 382 |
+
told = connector.startswith("watchdog ") or connector in INBOX_CONNECTORS
|
| 383 |
+
parts.append(f"Loopable sync STILL FAILING: {where}: {len(reminded)} {noun} still failing"
|
| 384 |
+
+ (", see the inbox." if told else "."))
|
| 385 |
names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
|
| 386 |
if names:
|
| 387 |
parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
|
platform/core/connector_sync.py
CHANGED
|
@@ -1110,10 +1110,50 @@ def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync
|
|
| 1110 |
"minutes": round(duration / 60, 3) if duration is not None else None,
|
| 1111 |
"budgetMinutes": 30, "outcome": outcome}
|
| 1112 |
entry["dailyPasses"] = days
|
|
|
|
|
|
|
|
|
|
| 1113 |
_advance_good_sync(entry, last_good_sync_at)
|
| 1114 |
return _status_update(rt, provider, connection_id, change)
|
| 1115 |
|
| 1116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1117 |
def record_resync_state(rt, provider, state, *, connection_id=None):
|
| 1118 |
"""Replace the connector's resync state with a JSON-safe copy of `state`."""
|
| 1119 |
try:
|
|
|
|
| 1110 |
"minutes": round(duration / 60, 3) if duration is not None else None,
|
| 1111 |
"budgetMinutes": 30, "outcome": outcome}
|
| 1112 |
entry["dailyPasses"] = days
|
| 1113 |
+
if outcome != "failed":
|
| 1114 |
+
# Any pass that did not fail proves the credential works again.
|
| 1115 |
+
entry.pop("credentialHold", None)
|
| 1116 |
_advance_good_sync(entry, last_good_sync_at)
|
| 1117 |
return _status_update(rt, provider, connection_id, change)
|
| 1118 |
|
| 1119 |
|
| 1120 |
+
def credential_fingerprint(material):
|
| 1121 |
+
"""The first 16 hex of sha256 over the credential material, or None when there is none.
|
| 1122 |
+
|
| 1123 |
+
A list or tuple is joined line by line. The secret itself is never stored or returned.
|
| 1124 |
+
"""
|
| 1125 |
+
if isinstance(material, (list, tuple)):
|
| 1126 |
+
material = "\n".join(str(part) for part in material)
|
| 1127 |
+
if not isinstance(material, str) or not material:
|
| 1128 |
+
return None
|
| 1129 |
+
return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
def hold_credential(rt, provider, fingerprint, *, connection_id=None):
|
| 1133 |
+
"""Record that the provider refused the credential with this fingerprint."""
|
| 1134 |
+
if not fingerprint:
|
| 1135 |
+
return False
|
| 1136 |
+
at = _utc_stamp()
|
| 1137 |
+
|
| 1138 |
+
def change(entry):
|
| 1139 |
+
entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at}
|
| 1140 |
+
return _status_update(rt, provider, connection_id, change)
|
| 1141 |
+
|
| 1142 |
+
|
| 1143 |
+
def credential_held(rt, provider, fingerprint, *, connection_id=None):
|
| 1144 |
+
"""True when a scheduled pass must skip: the last pass failed on this same refused credential.
|
| 1145 |
+
|
| 1146 |
+
An unknown fingerprint or an unreadable record answers False, so the pass runs.
|
| 1147 |
+
"""
|
| 1148 |
+
if not fingerprint:
|
| 1149 |
+
return False
|
| 1150 |
+
entry = status(rt, provider, connection_id, fresh=True) or {}
|
| 1151 |
+
hold = entry.get("credentialHold")
|
| 1152 |
+
last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
|
| 1153 |
+
return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint
|
| 1154 |
+
and last_pass.get("outcome") == "failed")
|
| 1155 |
+
|
| 1156 |
+
|
| 1157 |
def record_resync_state(rt, provider, state, *, connection_id=None):
|
| 1158 |
"""Replace the connector's resync state with a JSON-safe copy of `state`."""
|
| 1159 |
try:
|
platform/harness/amazon_store.py
CHANGED
|
@@ -493,6 +493,8 @@ SCHEDULED_FLOOR_SECONDS = 86400
|
|
| 493 |
SCHEDULED_FLOOR_GRACE_SECONDS = 0
|
| 494 |
# The resident cadence checks once a minute.
|
| 495 |
SCHEDULE_PHASE_GRACE_SECONDS = 60
|
|
|
|
|
|
|
| 496 |
_SCHEDULE_OFFSET_READER = None
|
| 497 |
|
| 498 |
|
|
@@ -509,10 +511,15 @@ def _scheduled_phase_due(table, state, now, earliest):
|
|
| 509 |
shift = int(_SCHEDULE_OFFSET_READER())
|
| 510 |
interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
|
| 511 |
phase = ((now.timestamp() - shift) // interval) * interval + shift
|
| 512 |
-
stamp = _utc(state["updated_at"]).timestamp() if state and state.get("
|
| 513 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 514 |
# The second half is left alone: a staging copy of a live tick stays off the live phase.
|
| 515 |
-
if stamp is
|
| 516 |
return earliest
|
| 517 |
if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
|
| 518 |
or (stamp is not None and stamp >= phase)):
|
|
@@ -607,6 +614,20 @@ def scheduled_sync_plan(tenant_key, *, rt, now=None):
|
|
| 607 |
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 608 |
"WHERE starts_with(entity, ?)", [prefix]).fetchall()}
|
| 609 |
plan = sync_plan(ticks, tables, now=now)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
due.update(dict.fromkeys(plan["tables"]))
|
| 611 |
for table, reason in plan["deferred"].items():
|
| 612 |
deferred.setdefault(table, []).append(reason)
|
|
@@ -5114,6 +5135,14 @@ def _refused_reports(con):
|
|
| 5114 |
def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
| 5115 |
"""Project refusals to cause and action copy, retaining the fee API's vendor reason."""
|
| 5116 |
text = str(reason or "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5117 |
feedback = table in ("review_topics", "review_trends")
|
| 5118 |
subject = "review trends" if table == "review_trends" else "review topics"
|
| 5119 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
|
@@ -8606,7 +8635,8 @@ def _validate_orders(con):
|
|
| 8606 |
return {"state": "unproven", "expected": expected, "actual": actual,
|
| 8607 |
"reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
|
| 8608 |
f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
|
| 8609 |
-
|
|
|
|
| 8610 |
"reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
|
| 8611 |
except Exception:
|
| 8612 |
return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
|
|
@@ -9146,7 +9176,20 @@ def _refuse_unreconciled(tenant_key, rt, report, log):
|
|
| 9146 |
"the table keeps its previous rows until it runs"} for table in synced}
|
| 9147 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 9148 |
for table in synced:
|
| 9149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9150 |
continue
|
| 9151 |
reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
|
| 9152 |
report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
|
|
@@ -9391,6 +9434,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9391 |
continue
|
| 9392 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 9393 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
|
|
|
|
|
|
| 9394 |
tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
|
| 9395 |
if tick and tick.get("state") == "throttled":
|
| 9396 |
# The first load owns SQP now; a passed SQP deadline must not make it due every minute.
|
|
@@ -9407,7 +9452,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 9407 |
mark = _read_state(con, entity_key(table)) or {}
|
| 9408 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 9409 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 9410 |
-
|
|
|
|
|
|
|
| 9411 |
for table in want:
|
| 9412 |
tick = ticks.get(table)
|
| 9413 |
# A queued resync is a repair: only Amazon's own floor and the stagger hold it.
|
|
|
|
| 493 |
SCHEDULED_FLOOR_GRACE_SECONDS = 0
|
| 494 |
# The resident cadence checks once a minute.
|
| 495 |
SCHEDULE_PHASE_GRACE_SECONDS = 60
|
| 496 |
+
#: `_sync_state` marker: the scheduled first-load guard deferred this table; never a tick, so no card reads it.
|
| 497 |
+
PHASE_HOLD_PREFIX = "phase_hold_"
|
| 498 |
_SCHEDULE_OFFSET_READER = None
|
| 499 |
|
| 500 |
|
|
|
|
| 511 |
shift = int(_SCHEDULE_OFFSET_READER())
|
| 512 |
interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
|
| 513 |
phase = ((now.timestamp() - shift) // interval) * interval + shift
|
| 514 |
+
stamp = _utc(state["updated_at"]).timestamp() if state and state.get("updated_at") else None
|
| 515 |
+
window = interval / 2
|
| 516 |
+
fastest = SCHEDULED_FLOOR_SECONDS // max(cadence["perDay"] for cadence in _client().CADENCE.values())
|
| 517 |
+
if shift % fastest:
|
| 518 |
+
# A shifted deployment's late run ends 30 minutes before the next live slot.
|
| 519 |
+
window = min(window, fastest - phase % fastest - 1800)
|
| 520 |
+
# A tick that missed its phase, in any state or absent, runs late inside the first half of the interval.
|
| 521 |
# The second half is left alone: a staging copy of a live tick stays off the live phase.
|
| 522 |
+
if (stamp is None or stamp < phase) and earliest.timestamp() - phase < window:
|
| 523 |
return earliest
|
| 524 |
if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
|
| 525 |
or (stamp is not None and stamp >= phase)):
|
|
|
|
| 614 |
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 615 |
"WHERE starts_with(entity, ?)", [prefix]).fetchall()}
|
| 616 |
plan = sync_plan(ticks, tables, now=now)
|
| 617 |
+
hold = _account_prefix(entry["connection_id"]) + entity_key(PHASE_HOLD_PREFIX)
|
| 618 |
+
# The planner reads cadence rows through this one query shape only, never source data.
|
| 619 |
+
holds = {entity[len(hold):]: {"state": phase, "rows": 0, "updated_at": at, "detail": None}
|
| 620 |
+
for entity, phase, _rows, at, _detail in con.execute(
|
| 621 |
+
"SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
|
| 622 |
+
"WHERE starts_with(entity, ?)", [hold]).fetchall()}
|
| 623 |
+
for table in list(plan["tables"]):
|
| 624 |
+
held, tick = holds.get(table), ticks.get(table)
|
| 625 |
+
# A first-load deferral newer than the tick waits for the next phase, as the sync defers it.
|
| 626 |
+
if held and (tick is None or _utc(held["updated_at"]) > _utc(tick["updated_at"])):
|
| 627 |
+
reason = _interval_reason(table, held, now, floor_s=_cadence_seconds(table, tick))
|
| 628 |
+
if reason:
|
| 629 |
+
plan["tables"].remove(table)
|
| 630 |
+
plan["deferred"][table] = reason
|
| 631 |
due.update(dict.fromkeys(plan["tables"]))
|
| 632 |
for table, reason in plan["deferred"].items():
|
| 633 |
deferred.setdefault(table, []).append(reason)
|
|
|
|
| 5135 |
def plain_refusal_reason(reason, *, status="", table="", operation=""):
|
| 5136 |
"""Project refusals to cause and action copy, retaining the fee API's vendor reason."""
|
| 5137 |
text = str(reason or "")
|
| 5138 |
+
# A reconciliation hold is our own check: never call it a load failure.
|
| 5139 |
+
reconciliation = re.search(r"\breconciliation refused:\s*([\s\S]*)$", text, re.I)
|
| 5140 |
+
if reconciliation:
|
| 5141 |
+
detail = re.sub(r"[.\s]+$", "", reconciliation[1].strip()).replace("\u2014", ". ").replace("\u2013", " to ")
|
| 5142 |
+
if re.match(r"the check could not run", detail, re.I):
|
| 5143 |
+
return "The reconciliation check could not run. The last good rows are kept until it runs."
|
| 5144 |
+
return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
|
| 5145 |
+
else "Reconciliation did not tie. The last good rows are kept.")
|
| 5146 |
feedback = table in ("review_topics", "review_trends")
|
| 5147 |
subject = "review trends" if table == "review_trends" else "review topics"
|
| 5148 |
if feedback and (str(status) in ("403", "NOT_OFFERED")
|
|
|
|
| 8635 |
return {"state": "unproven", "expected": expected, "actual": actual,
|
| 8636 |
"reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
|
| 8637 |
f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
|
| 8638 |
+
# Two Amazon reports disagree: a check on the source's own data warns and publishes.
|
| 8639 |
+
return {"state": "refused", "expected": expected, "actual": actual, "revived": revived, "publish": True,
|
| 8640 |
"reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
|
| 8641 |
except Exception:
|
| 8642 |
return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
|
|
|
|
| 9176 |
"the table keeps its previous rows until it runs"} for table in synced}
|
| 9177 |
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 9178 |
for table in synced:
|
| 9179 |
+
verdict, warned = checked.get(table) or {}, entity_key("warning_" + table)
|
| 9180 |
+
if verdict.get("state") == "refused" and verdict.get("publish") is True:
|
| 9181 |
+
# A check on the source's own data warns and publishes. It never holds the sync.
|
| 9182 |
+
expected, actual = verdict.get("expected"), verdict.get("actual")
|
| 9183 |
+
warning = (f"Amazon Sales counted {expected:,} orders and the orders report holds {actual:,}. "
|
| 9184 |
+
"The table is published." if type(expected) is int and type(actual) is int else
|
| 9185 |
+
f"{str(verdict.get('reason') or 'Amazon reports disagree').rstrip('. ')}. The table is published.")
|
| 9186 |
+
report["tables"][table] = {**report["tables"][table], "warning": warning}
|
| 9187 |
+
_record_state(con, warned, "warning", 0, warning)
|
| 9188 |
+
log(f"{table}: published with a warning: {warning}")
|
| 9189 |
+
continue
|
| 9190 |
+
if verdict.get("state") != "refused":
|
| 9191 |
+
# A clean reconcile clears the last warning. A held table keeps it with its kept rows.
|
| 9192 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [warned])
|
| 9193 |
continue
|
| 9194 |
reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
|
| 9195 |
report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
|
|
|
|
| 9434 |
continue
|
| 9435 |
report["tables"][table] = {"state": "deferred", "reason":
|
| 9436 |
"The first load is continuing in bounded passes on the resident sync thread."}
|
| 9437 |
+
# The planner waits for the next phase on this hold. The card's tick stays as it was.
|
| 9438 |
+
_record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred")
|
| 9439 |
tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
|
| 9440 |
if tick and tick.get("state") == "throttled":
|
| 9441 |
# The first load owns SQP now; a passed SQP deadline must not make it due every minute.
|
|
|
|
| 9452 |
mark = _read_state(con, entity_key(table)) or {}
|
| 9453 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 9454 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 9455 |
+
# The re-asked tick keeps its stamp, so the phase rule decides as the planner does.
|
| 9456 |
+
ticks.update({name: ticks[name] and {**ticks[name], "state": "refused"}
|
| 9457 |
+
for name in (table, pairs.get(table)) if name in want})
|
| 9458 |
for table in want:
|
| 9459 |
tick = ticks.get(table)
|
| 9460 |
# A queued resync is a repair: only Amazon's own floor and the stagger hold it.
|
platform/harness/mirror_store.py
CHANGED
|
@@ -37,6 +37,8 @@ _KEY_LEDGER = "keys.json"
|
|
| 37 |
_TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
|
| 38 |
_PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
|
| 39 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
|
|
|
|
|
|
|
| 40 |
_PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
|
| 41 |
"shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
|
| 42 |
ORPHAN_GRACE = timedelta(hours=24)
|
|
@@ -726,10 +728,59 @@ class MirrorStore:
|
|
| 726 |
return key
|
| 727 |
|
| 728 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 729 |
def _is_mirror_blob_key(self, key):
|
| 730 |
match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
|
| 731 |
return bool((match and self._valid_table(match.group(1)))
|
| 732 |
-
or _STATE_BLOB_RE.fullmatch(str(key)))
|
| 733 |
|
| 734 |
|
| 735 |
def _manifest_keys(self, manifest, *, strict=True):
|
|
@@ -747,7 +798,8 @@ class MirrorStore:
|
|
| 747 |
state = manifest.get("sync_state")
|
| 748 |
if isinstance(state, dict):
|
| 749 |
try:
|
| 750 |
-
|
|
|
|
| 751 |
except (ValueError, AttributeError, TypeError):
|
| 752 |
pass
|
| 753 |
return out
|
|
@@ -805,8 +857,9 @@ class MirrorStore:
|
|
| 805 |
raise ValueError("Mirror publication identity or times are invalid")
|
| 806 |
allow_partial = False
|
| 807 |
state_meta = manifest["sync_state"]
|
| 808 |
-
|
| 809 |
-
|
|
|
|
| 810 |
if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
|
| 811 |
or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
|
| 812 |
or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
|
|
@@ -1351,18 +1404,16 @@ class MirrorStore:
|
|
| 1351 |
continue
|
| 1352 |
pending.append((key, path))
|
| 1353 |
rows = self._state_rows(con)
|
| 1354 |
-
|
| 1355 |
-
state_meta = {**_blob_metadata(state), "rows": len(rows)}
|
| 1356 |
-
state_meta["key"] = _content_key("sync_state", suffix + ".json", state_meta)
|
| 1357 |
published = None
|
| 1358 |
if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
|
| 1359 |
or manifest["tables"] != old.get("tables", {})
|
| 1360 |
or old.get("version") != MANIFEST_VERSION):
|
| 1361 |
-
|
| 1362 |
-
|
| 1363 |
-
|
| 1364 |
-
|
| 1365 |
-
|
| 1366 |
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1367 |
raise ValueError("mirror part exceeds the byte limit")
|
| 1368 |
if pending:
|
|
|
|
| 37 |
_TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
|
| 38 |
_PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
|
| 39 |
_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
|
| 40 |
+
#: State over one part's byte limit is saved as consecutive row parts, as a table is.
|
| 41 |
+
_STATE_PART_RE = re.compile(r"sync_state\.p([0-9]{1,6})\.([0-9a-f]{16})(?:\.sealed-v1)?\.json")
|
| 42 |
_PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
|
| 43 |
"shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
|
| 44 |
ORPHAN_GRACE = timedelta(hours=24)
|
|
|
|
| 728 |
return key
|
| 729 |
|
| 730 |
|
| 731 |
+
def _state_blob_keys(self, metadata, *, strict=True):
|
| 732 |
+
"""[(key, metadata)] for the manifest's state entry: its row parts, or its one blob."""
|
| 733 |
+
if not (isinstance(metadata, dict) and "parts" in metadata):
|
| 734 |
+
return [(self._state_blob_key(metadata, strict=strict), metadata)]
|
| 735 |
+
parts = metadata["parts"]
|
| 736 |
+
if not isinstance(parts, list) or not parts:
|
| 737 |
+
raise ValueError(f"{self.label} mirror manifest names an empty state part list")
|
| 738 |
+
out = []
|
| 739 |
+
for part in parts:
|
| 740 |
+
key = str(part.get("key") or "") if isinstance(part, dict) else ""
|
| 741 |
+
match = _STATE_PART_RE.fullmatch(key)
|
| 742 |
+
if not match or (strict and not str(part.get("sha256") or "").startswith(match.group(2))):
|
| 743 |
+
raise ValueError(f"{self.label} mirror manifest names an invalid state part key")
|
| 744 |
+
out.append((key, part))
|
| 745 |
+
return out
|
| 746 |
+
|
| 747 |
+
|
| 748 |
+
@staticmethod
|
| 749 |
+
def _state_entry(rows, limit, suffix):
|
| 750 |
+
"""(manifest state entry, [(key, bytes)]): one blob while the rows fit `limit`, else row parts."""
|
| 751 |
+
state = _mirror_json(rows)
|
| 752 |
+
if len(state) <= limit:
|
| 753 |
+
meta = {**_blob_metadata(state), "rows": len(rows)}
|
| 754 |
+
meta["key"] = _content_key("sync_state", suffix + ".json", meta)
|
| 755 |
+
return meta, [(meta["key"], state)]
|
| 756 |
+
parts, blobs, chunk, size = [], [], [], 2
|
| 757 |
+
|
| 758 |
+
def close():
|
| 759 |
+
data = _mirror_json(chunk)
|
| 760 |
+
meta = {**_blob_metadata(data), "rows": len(chunk), "part": len(parts)}
|
| 761 |
+
meta["key"] = _content_key(f"sync_state.p{len(parts)}", suffix + ".json", meta)
|
| 762 |
+
parts.append(meta)
|
| 763 |
+
blobs.append((meta["key"], data))
|
| 764 |
+
|
| 765 |
+
for row in rows:
|
| 766 |
+
width = len(_mirror_json(row)) + 1
|
| 767 |
+
if chunk and size + width > limit:
|
| 768 |
+
close()
|
| 769 |
+
chunk, size = [], 2
|
| 770 |
+
chunk.append(row)
|
| 771 |
+
size += width
|
| 772 |
+
if chunk:
|
| 773 |
+
close()
|
| 774 |
+
# A single row wider than `limit` stays one part, and the pending byte check names it.
|
| 775 |
+
combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
|
| 776 |
+
return ({"parts": parts, "rows": len(rows), "sha256": combined,
|
| 777 |
+
"bytes": sum(p["bytes"] for p in parts)}, blobs)
|
| 778 |
+
|
| 779 |
+
|
| 780 |
def _is_mirror_blob_key(self, key):
|
| 781 |
match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
|
| 782 |
return bool((match and self._valid_table(match.group(1)))
|
| 783 |
+
or _STATE_BLOB_RE.fullmatch(str(key)) or _STATE_PART_RE.fullmatch(str(key)))
|
| 784 |
|
| 785 |
|
| 786 |
def _manifest_keys(self, manifest, *, strict=True):
|
|
|
|
| 798 |
state = manifest.get("sync_state")
|
| 799 |
if isinstance(state, dict):
|
| 800 |
try:
|
| 801 |
+
for key, blob in self._state_blob_keys(state, strict=strict):
|
| 802 |
+
out[key] = blob.get("sha256")
|
| 803 |
except (ValueError, AttributeError, TypeError):
|
| 804 |
pass
|
| 805 |
return out
|
|
|
|
| 857 |
raise ValueError("Mirror publication identity or times are invalid")
|
| 858 |
allow_partial = False
|
| 859 |
state_meta = manifest["sync_state"]
|
| 860 |
+
parts = [json.loads(self._checked_mirror_blob(rt, key, blob))
|
| 861 |
+
for key, blob in self._state_blob_keys(state_meta)]
|
| 862 |
+
rows = [row for part in parts for row in part] if all(isinstance(p, list) for p in parts) else None
|
| 863 |
if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
|
| 864 |
or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
|
| 865 |
or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
|
|
|
|
| 1404 |
continue
|
| 1405 |
pending.append((key, path))
|
| 1406 |
rows = self._state_rows(con)
|
| 1407 |
+
state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
|
|
|
|
|
|
|
| 1408 |
published = None
|
| 1409 |
if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
|
| 1410 |
or manifest["tables"] != old.get("tables", {})
|
| 1411 |
or old.get("version") != MANIFEST_VERSION):
|
| 1412 |
+
for index, (state_key, state) in enumerate(state_blobs):
|
| 1413 |
+
if state_key not in named:
|
| 1414 |
+
state_path = Path(tmp) / f"sync_state.{index}.json"
|
| 1415 |
+
state_path.write_bytes(state)
|
| 1416 |
+
pending.append((state_key, state_path))
|
| 1417 |
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1418 |
raise ValueError("mirror part exceeds the byte limit")
|
| 1419 |
if pending:
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
fc20fb27640c854759ec5061266f0c3a70ad8f7831d67eeb401f0b2125c01625
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -265,6 +265,8 @@ export interface AmazonTableSync {
|
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
|
|
|
|
|
|
| 268 |
}
|
| 269 |
|
| 270 |
export interface AmazonSyncScheduleReading {
|
|
@@ -298,7 +300,8 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
|
|
| 298 |
if (identities.has(identity)) return unavailable;
|
| 299 |
identities.add(identity);
|
| 300 |
const reason = typeof row.reason === "string" ? row.reason.trim() : "";
|
| 301 |
-
|
|
|
|
| 302 |
}
|
| 303 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 304 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
@@ -344,6 +347,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
|
|
| 344 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 345 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 346 |
{row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
|
|
|
|
| 347 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 348 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 349 |
</td>
|
|
@@ -433,14 +437,32 @@ export function staleSyncWarnings(name: string, body: unknown): string[] {
|
|
| 433 |
return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
|
| 434 |
}
|
| 435 |
|
| 436 |
-
/**
|
| 437 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
try {
|
| 439 |
const res = await fetch(`${API_V1}/connectors/${encodeURIComponent(connectorKey)}/sync`, {
|
| 440 |
credentials: CREDENTIALS, signal,
|
| 441 |
});
|
| 442 |
if (!res.ok) return [];
|
| 443 |
const body: unknown = await res.json().catch(() => null);
|
|
|
|
|
|
|
|
|
|
| 444 |
return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
|
| 445 |
} catch {
|
| 446 |
return [];
|
|
@@ -1073,17 +1095,18 @@ export default function ConnectorsPage({
|
|
| 1073 |
return () => controller.abort();
|
| 1074 |
}, [hasOdooCard]);
|
| 1075 |
const dailySourceKeys = load.phase === "ready"
|
| 1076 |
-
? load.rows.filter((row) => row.key in DAILY_SOURCE_NAMES
|
| 1077 |
-
.map((row) => row.key).join(",")
|
| 1078 |
: "";
|
| 1079 |
const [dailyWarnings, setDailyWarnings] = useState<Record<string, string[]>>({});
|
| 1080 |
useEffect(() => {
|
| 1081 |
if (!dailySourceKeys) return;
|
| 1082 |
const controller = new AbortController();
|
| 1083 |
-
const
|
| 1084 |
-
|
|
|
|
| 1085 |
if (!controller.signal.aborted) {
|
| 1086 |
-
setDailyWarnings(Object.fromEntries(
|
| 1087 |
}
|
| 1088 |
});
|
| 1089 |
return () => controller.abort();
|
|
@@ -1571,7 +1594,9 @@ export function ConnectorCard({
|
|
| 1571 |
const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
|
| 1572 |
// D-2679: databases that failed alone are named; a stale connector still reads "Stale".
|
| 1573 |
const databasesStale = failedDatabasesStatus(syncWarnings);
|
| 1574 |
-
const status =
|
|
|
|
|
|
|
| 1575 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1576 |
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
| 1577 |
|| (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
|
|
|
|
| 265 |
perDay: number;
|
| 266 |
state: string;
|
| 267 |
reason?: string;
|
| 268 |
+
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 269 |
+
warning?: string;
|
| 270 |
}
|
| 271 |
|
| 272 |
export interface AmazonSyncScheduleReading {
|
|
|
|
| 300 |
if (identities.has(identity)) return unavailable;
|
| 301 |
identities.add(identity);
|
| 302 |
const reason = typeof row.reason === "string" ? row.reason.trim() : "";
|
| 303 |
+
const warning = typeof row.warning === "string" ? row.warning.trim() : "";
|
| 304 |
+
tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined, warning: warning || undefined });
|
| 305 |
}
|
| 306 |
const state = body.scheduleState as AmazonSyncScheduleReading["state"];
|
| 307 |
if ((state === "ready") !== (tables.length > 0)) return unavailable;
|
|
|
|
| 347 |
{Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
|
| 348 |
: <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
|
| 349 |
{row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
|
| 350 |
+
: row.warning && row.state === "done" ? <p role="status">{row.warning}</p>
|
| 351 |
: row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
|
| 352 |
: ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
|
| 353 |
</td>
|
|
|
|
| 437 |
return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
|
| 438 |
}
|
| 439 |
|
| 440 |
+
/** The server's sentence when the provider refused the connector credential. */
|
| 441 |
+
export const CREDENTIAL_EXPIRED = "Credential expired. Reconnect to resume.";
|
| 442 |
+
/** The sentence a pass stored before this one: the card reads it the same way until the next pass. */
|
| 443 |
+
const CREDENTIAL_REFUSED_BEFORE = "The provider refused the connector credential";
|
| 444 |
+
|
| 445 |
+
/** The card line for a failed pass on a refused credential, PURE: none for any other state. */
|
| 446 |
+
export function credentialSyncWarnings(body: unknown): string[] {
|
| 447 |
+
if (!body || typeof body !== "object") return [];
|
| 448 |
+
const { state, detail } = body as { state?: unknown; detail?: unknown };
|
| 449 |
+
return state === "error" && typeof detail === "string"
|
| 450 |
+
&& (detail.startsWith(CREDENTIAL_EXPIRED) || detail.startsWith(CREDENTIAL_REFUSED_BEFORE))
|
| 451 |
+
? [CREDENTIAL_EXPIRED] : [];
|
| 452 |
+
}
|
| 453 |
+
|
| 454 |
+
/** One read of a daily source's sync route. A failed read is no warning, never a thrown error.
|
| 455 |
+
* A card the directory does not count as connected shows only a refused credential: its pass still runs. */
|
| 456 |
+
async function dailySyncWarnings(connectorKey: string, signal: AbortSignal, connected = true): Promise<string[]> {
|
| 457 |
try {
|
| 458 |
const res = await fetch(`${API_V1}/connectors/${encodeURIComponent(connectorKey)}/sync`, {
|
| 459 |
credentials: CREDENTIALS, signal,
|
| 460 |
});
|
| 461 |
if (!res.ok) return [];
|
| 462 |
const body: unknown = await res.json().catch(() => null);
|
| 463 |
+
const credential = credentialSyncWarnings(body);
|
| 464 |
+
if (credential.length) return credential;
|
| 465 |
+
if (!connected) return [];
|
| 466 |
return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
|
| 467 |
} catch {
|
| 468 |
return [];
|
|
|
|
| 1095 |
return () => controller.abort();
|
| 1096 |
}, [hasOdooCard]);
|
| 1097 |
const dailySourceKeys = load.phase === "ready"
|
| 1098 |
+
? load.rows.filter((row) => row.key in DAILY_SOURCE_NAMES)
|
| 1099 |
+
.map((row) => `${row.key}:${row.state === "connected" ? 1 : 0}`).join(",")
|
| 1100 |
: "";
|
| 1101 |
const [dailyWarnings, setDailyWarnings] = useState<Record<string, string[]>>({});
|
| 1102 |
useEffect(() => {
|
| 1103 |
if (!dailySourceKeys) return;
|
| 1104 |
const controller = new AbortController();
|
| 1105 |
+
const entries = dailySourceKeys.split(",").map((entry) => entry.split(":"));
|
| 1106 |
+
const reads = entries.map(([key, connected]) => dailySyncWarnings(key, controller.signal, connected === "1"));
|
| 1107 |
+
void Promise.all(reads).then((lines) => {
|
| 1108 |
if (!controller.signal.aborted) {
|
| 1109 |
+
setDailyWarnings(Object.fromEntries(entries.map(([key], index) => [key, lines[index]])));
|
| 1110 |
}
|
| 1111 |
});
|
| 1112 |
return () => controller.abort();
|
|
|
|
| 1594 |
const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
|
| 1595 |
// D-2679: databases that failed alone are named; a stale connector still reads "Stale".
|
| 1596 |
const databasesStale = failedDatabasesStatus(syncWarnings);
|
| 1597 |
+
const status = syncWarnings?.includes(CREDENTIAL_EXPIRED) ? (
|
| 1598 |
+
<span className="conn-card-status" role="status">{CREDENTIAL_EXPIRED}</span>
|
| 1599 |
+
) : catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
|
| 1600 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1601 |
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
| 1602 |
|| (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
|
web/src/connectors/DatabasesTab.tsx
CHANGED
|
@@ -135,6 +135,8 @@ interface AmazonTableSync {
|
|
| 135 |
stuck: boolean | null;
|
| 136 |
stuckCause: string;
|
| 137 |
unknownCause: string;
|
|
|
|
|
|
|
| 138 |
}
|
| 139 |
|
| 140 |
export interface DatabaseSyncHealth {
|
|
@@ -564,6 +566,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
|
|
| 564 |
}
|
| 565 |
const reason = syncText(syncObject(state.deferred).reason);
|
| 566 |
const held = syncObject(state.held);
|
|
|
|
| 567 |
states[database.key] = {
|
| 568 |
rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
|
| 569 |
inProgress: state.inProgress === true,
|
|
@@ -575,6 +578,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
|
|
| 575 |
firstLoad: load,
|
| 576 |
stuck: state.stuck === true ? true : state.stuck === false ? false : null,
|
| 577 |
stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
|
|
|
|
| 578 |
};
|
| 579 |
}
|
| 580 |
return states;
|
|
@@ -686,6 +690,16 @@ export function firstLoadEstimate(seconds: number): string {
|
|
| 686 |
function amazonRefusalReason(reason: string, status = ""): string {
|
| 687 |
const text = reason.trim();
|
| 688 |
const lower = text.toLowerCase();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 689 |
if (/within (?:the past )?two years/.test(lower)) {
|
| 690 |
return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
|
| 691 |
}
|
|
@@ -2323,7 +2337,8 @@ const DatabasesTab = forwardRef<DatabasesTabHandle, {
|
|
| 2323 |
catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
|
| 2324 |
{health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
|
| 2325 |
{health.error.message}</p> : null}
|
| 2326 |
-
{health?.warning ? <p className="conn-db-note">
|
|
|
|
| 2327 |
{job?.state === "failed" && job.retryable ? <button type="button"
|
| 2328 |
className="conn-job-retry" disabled={Boolean(retrying)}
|
| 2329 |
onClick={() => void retryJob(job)}>Retry sync</button> : null}
|
|
|
|
| 135 |
stuck: boolean | null;
|
| 136 |
stuckCause: string;
|
| 137 |
unknownCause: string;
|
| 138 |
+
/** A source-internal mismatch: the table published, and both counts are named. */
|
| 139 |
+
warning?: string;
|
| 140 |
}
|
| 141 |
|
| 142 |
export interface DatabaseSyncHealth {
|
|
|
|
| 566 |
}
|
| 567 |
const reason = syncText(syncObject(state.deferred).reason);
|
| 568 |
const held = syncObject(state.held);
|
| 569 |
+
const warning = syncText(state.warning).replace(/\u2014/g, ". ").replace(/\u2013/g, " to ").trim();
|
| 570 |
states[database.key] = {
|
| 571 |
rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
|
| 572 |
inProgress: state.inProgress === true,
|
|
|
|
| 578 |
firstLoad: load,
|
| 579 |
stuck: state.stuck === true ? true : state.stuck === false ? false : null,
|
| 580 |
stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
|
| 581 |
+
...(warning ? { warning } : {}),
|
| 582 |
};
|
| 583 |
}
|
| 584 |
return states;
|
|
|
|
| 690 |
function amazonRefusalReason(reason: string, status = ""): string {
|
| 691 |
const text = reason.trim();
|
| 692 |
const lower = text.toLowerCase();
|
| 693 |
+
// Mirrors plain_refusal_reason: a reconciliation hold is our own check, never a load failure.
|
| 694 |
+
const reconciliation = /\breconciliation refused:\s*([\s\S]*)$/i.exec(text);
|
| 695 |
+
if (reconciliation) {
|
| 696 |
+
const detail = reconciliation[1].trim().replace(/[.\s]+$/, "").replace(/\u2014/g, ". ").replace(/\u2013/g, " to ");
|
| 697 |
+
if (/^the check could not run/i.test(detail)) {
|
| 698 |
+
return "The reconciliation check could not run. The last good rows are kept until it runs.";
|
| 699 |
+
}
|
| 700 |
+
return detail ? `Reconciliation did not tie: ${detail}. The last good rows are kept.`
|
| 701 |
+
: "Reconciliation did not tie. The last good rows are kept.";
|
| 702 |
+
}
|
| 703 |
if (/within (?:the past )?two years/.test(lower)) {
|
| 704 |
return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
|
| 705 |
}
|
|
|
|
| 2337 |
catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
|
| 2338 |
{health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
|
| 2339 |
{health.error.message}</p> : null}
|
| 2340 |
+
{health?.warning || amazonState?.warning ? <p className="conn-db-note">
|
| 2341 |
+
{health?.warning || amazonState?.warning}</p> : null}
|
| 2342 |
{job?.state === "failed" && job.retryable ? <button type="button"
|
| 2343 |
className="conn-job-retry" disabled={Boolean(retrying)}
|
| 2344 |
onClick={() => void retryJob(job)}>Retry sync</button> : null}
|