fsanyoto commited on
Commit
463918b
·
verified ·
1 Parent(s): d27064e

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "42c956192 (42c956192ae7b)",
3
  "releases": [
4
  {
5
  "version": "v86",
 
1
  {
2
+ "current": "50dae147b (50dae147b20ab)",
3
  "releases": [
4
  {
5
  "version": "v86",
VERSION CHANGED
@@ -1 +1 @@
1
- 42c956192 (42c956192ae7b)
 
1
+ 50dae147b (50dae147b20ab)
api/connectors_amazon.py CHANGED
@@ -603,6 +603,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
603
  "stuck": stuck, "stuckCause": stuck_cause,
604
  "unknownCause": " ".join(unknown),
605
  }
 
 
 
 
 
 
606
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
607
  answer[table]["limit"] = limit
608
  answer["sku_keywords"] = {
 
603
  "stuck": stuck, "stuckCause": stuck_cause,
604
  "unknownCause": " ".join(unknown),
605
  }
606
+ # A source-internal mismatch published the table. Account markers carry a connection prefix.
607
+ warned = az.entity_key("warning_" + table)
608
+ warning = next((str(value.get("cursor") or "") for key, value in entities.items()
609
+ if (key == warned or key.endswith("__" + warned)) and value.get("cursor")), "")
610
+ if warning:
611
+ answer[table]["warning"] = warning
612
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
613
  answer[table]["limit"] = limit
614
  answer["sku_keywords"] = {
api/main.py CHANGED
@@ -1188,16 +1188,29 @@ def _pull_meta(why, tenant_key=None):
1188
  for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
1189
  on_error=runtime_failed):
1190
  try:
1191
- if not _meta._connection_rows(rt, key, {"problems": []}):
 
1192
  continue
1193
  from core import connector_sync as _meta_status
 
 
 
 
 
 
 
 
 
1194
  _meta_status.record_pass_start(rt, "meta_ads")
1195
  _status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
1196
  rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
1197
  log=lambda *_a: None, insights_days=7)
1198
  outcomes[key] = mirror_store._sync_result("meta", rep)[1]
1199
- detail = _sync_now_failure(" ".join(rep.get("problems") or [])) if not outcomes[key] else ""
 
1200
  _status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
 
 
1201
  for problem in rep.get("problems") or []:
1202
  print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
1203
  except Exception as exc:
@@ -1356,17 +1369,29 @@ def _pull_amazon(why, vendor=True):
1356
  if not vendor:
1357
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
1358
  return True
 
1359
  from harness import mirror_store
1360
  from core import connector_sync as _amazon_status
1361
- _amazon_status.record_pass_start(rt, "amazon_sp")
1362
- try:
1363
- with _amazon_role.connector_writer(rt, "amazon"):
 
 
 
 
 
 
 
 
 
 
 
1364
  rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1365
  slot_pass=bool(RESYNC_STATE.get("slotPass")),
1366
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
1367
- except Exception:
1368
- _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1369
- raise
1370
  _holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
1371
  _problems = _holds.get("unsettled", rep.get("problems") or [])
1372
  _status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
@@ -1613,7 +1638,7 @@ def _sync_now_cause(cause):
1613
  "The provider rate limit stopped the sync. Retry after the provider delay."),
1614
  ("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
1615
  ("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
1616
- "expired", "authentication"), "The provider refused the connector credential or its permissions."),
1617
  ("mirror_constraint", ("duplicate key", "primary key", "constraint"),
1618
  "The mirror rejected rows because they violate a database constraint."),
1619
  ("connect_failed", ("connection", "network", "name resolution"),
 
1188
  for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
1189
  on_error=runtime_failed):
1190
  try:
1191
+ connections = _meta._connection_rows(rt, key, {"problems": []})
1192
+ if not connections:
1193
  continue
1194
  from core import connector_sync as _meta_status
1195
+ # A refused credential is not retried until it changes. An unreadable fingerprint runs the pass.
1196
+ try:
1197
+ fingerprint = _meta_status.credential_fingerprint(
1198
+ [f"{row[0]}\t{row[2]}" for row in connections])
1199
+ except Exception: # noqa: BLE001
1200
+ fingerprint = None
1201
+ if _meta_status.credential_held(rt, "meta_ads", fingerprint):
1202
+ print("[sync] meta_ads held: the provider refused the credential; waiting for a new one")
1203
+ continue
1204
  _meta_status.record_pass_start(rt, "meta_ads")
1205
  _status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
1206
  rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
1207
  log=lambda *_a: None, insights_days=7)
1208
  outcomes[key] = mirror_store._sync_result("meta", rep)[1]
1209
+ cause, detail = (_sync_now_cause(" ".join(rep.get("problems") or []))
1210
+ if not outcomes[key] else ("", ""))
1211
  _status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
1212
+ if cause == "credential_refused":
1213
+ _meta_status.hold_credential(rt, "meta_ads", fingerprint)
1214
  for problem in rep.get("problems") or []:
1215
  print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
1216
  except Exception as exc:
 
1369
  if not vendor:
1370
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
1371
  return True
1372
+ from contextlib import ExitStack
1373
  from harness import mirror_store
1374
  from core import connector_sync as _amazon_status
1375
+ with ExitStack() as _lease:
1376
+ try:
1377
+ _lease.enter_context(_amazon_role.connector_writer(rt, "amazon"))
1378
+ except Exception as exc:
1379
+ # A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
1380
+ if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
1381
+ print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
1382
+ flush=True)
1383
+ return False
1384
+ _amazon_status.record_pass_start(rt, "amazon_sp")
1385
+ _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1386
+ raise
1387
+ _amazon_status.record_pass_start(rt, "amazon_sp")
1388
+ try:
1389
  rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1390
  slot_pass=bool(RESYNC_STATE.get("slotPass")),
1391
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
1392
+ except Exception:
1393
+ _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1394
+ raise
1395
  _holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
1396
  _problems = _holds.get("unsettled", rep.get("problems") or [])
1397
  _status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
 
1638
  "The provider rate limit stopped the sync. Retry after the provider delay."),
1639
  ("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
1640
  ("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
1641
+ "expired", "authentication"), "Credential expired. Reconnect to resume."),
1642
  ("mirror_constraint", ("duplicate key", "primary key", "constraint"),
1643
  "The mirror rejected rows because they violate a database constraint."),
1644
  ("connect_failed", ("connection", "network", "name resolution"),
api/odoo_relational.py CHANGED
@@ -7362,8 +7362,19 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
7362
  with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
7363
  datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
7364
  static = datastore.ENTITIES.get(source)
 
7365
  if static:
7366
  datastore._ensure_table(con, source, static)
 
 
 
 
 
 
 
 
 
 
7367
  else:
7368
  types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
7369
  columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
@@ -7468,13 +7479,38 @@ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, to
7468
  "snapshotId": publication["snapshot_id"]}
7469
  table["readThrough"], table["rows"] = True, {}
7470
  table.pop(ut.OVERSIZE_FIELD, None)
7471
- written[key] = {**counts, "rows": count}
 
 
 
 
 
7472
  return current
7473
  rt.update(ut.STORE_KEY, publish, flush="sync")
7474
  ut.register_connected(key)
7475
  return {"tables": written, "errors": []}
7476
 
7477
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7478
  def _model_ident(name):
7479
  """Quote a native field identifier without accepting SQL syntax."""
7480
  name = str(name)
 
7362
  with mirror_store.sync_mirror("odoo", rt.key, rt, expected_tables=[]) as con:
7363
  datastore.register_odoo_dynamic_model(rt, con, connection_id=cid, model=model, table_key=key)
7364
  static = datastore.ENTITIES.get(source)
7365
+ unbound = []
7366
  if static:
7367
  datastore._ensure_table(con, source, static)
7368
+ # A shared static mirror stores only its declared columns: publish the fields it binds.
7369
+ catalogue_row, unbound = _mirror_bound_catalogue(
7370
+ catalogue_row, datastore.columns_of(source, cur=con) or set())
7371
+ if unbound:
7372
+ fields = _model_field_specs(catalogue_row)
7373
+ metadata = {"model": model, "fields": {
7374
+ name: {prop: meta[prop] for prop in ("type", "relation") if prop in meta}
7375
+ for name, meta in (catalogue_row.get("fields") or {}).items()}}
7376
+ print(f"[odoo] {key}: {len(unbound)} fields have no mirror column and are not "
7377
+ f"published: {', '.join(unbound)}", flush=True)
7378
  else:
7379
  types = {"boolean": "BOOLEAN", "integer": "BIGINT", "float": "DOUBLE", "monetary": "DOUBLE"}
7380
  columns = [f'{_model_ident(f["key"])} {types.get((metadata["fields"].get(f["key"]) or {}).get("type"), "VARCHAR")}'
 
7479
  "snapshotId": publication["snapshot_id"]}
7480
  table["readThrough"], table["rows"] = True, {}
7481
  table.pop(ut.OVERSIZE_FIELD, None)
7482
+ if unbound:
7483
+ # A machine field with no mirror column would read empty on every row: it leaves.
7484
+ table["fields"] = [f for f in table.get("fields") or ()
7485
+ if f.get("key") not in unbound
7486
+ or not (f.get("automation") or {}).get("preset") or ut.user_edited(f)]
7487
+ written[key] = {**counts, "rows": count, **({"unboundFields": unbound} if unbound else {})}
7488
  return current
7489
  rt.update(ut.STORE_KEY, publish, flush="sync")
7490
  ut.register_connected(key)
7491
  return {"tables": written, "errors": []}
7492
 
7493
 
7494
+ def _mirror_bound_catalogue(catalogue_row, have):
7495
+ """The catalogue row cut to the fields a static mirror table binds, and the names it left out."""
7496
+ fields = catalogue_row.get("fields") if isinstance(catalogue_row, dict) else {}
7497
+ fields = fields if isinstance(fields, dict) else {}
7498
+ have = {str(name).lower() for name in have}
7499
+
7500
+ def bound(name, meta):
7501
+ kind = (meta if isinstance(meta, dict) else {}).get("type")
7502
+ if name == "id" or kind == "binary":
7503
+ return True
7504
+ label = name.removesuffix("_id") + "_name"
7505
+ return name.lower() in have and (kind != "many2one" or label.lower() in have)
7506
+
7507
+ unbound = sorted(str(name) for name, meta in fields.items() if not bound(str(name), meta))
7508
+ if not unbound:
7509
+ return catalogue_row, []
7510
+ return {**catalogue_row, "fields": {name: meta for name, meta in fields.items()
7511
+ if str(name) not in unbound}}, unbound
7512
+
7513
+
7514
  def _model_ident(name):
7515
  """Quote a native field identifier without accepting SQL syntax."""
7516
  name = str(name)
api/routes_connectors.py CHANGED
@@ -2199,7 +2199,14 @@ def _odoo_current_jobs(rt, jobs):
2199
  offered, current = {}, []
2200
  for job in jobs:
2201
  if job.get("state") != "failed":
2202
- current.append(job)
 
 
 
 
 
 
 
2203
  continue
2204
  cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
2205
  if cid not in offered:
@@ -3187,7 +3194,9 @@ def amazon_hold_cause(row=None, held="", *, table=""):
3187
  cause = "The last pass started this table and did not finish it. The next pass continues it."
3188
  elif state == "refused":
3189
  plain = plain_refusal_reason(reason or held, status=status, table=table)
3190
- cause = plain if plain.startswith(("Amazon refused", "Amazon does not offer")) else (
 
 
3191
  "Amazon refused this report: " + plain)
3192
  else:
3193
  problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
@@ -3254,7 +3263,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3254
  if any(not state["updated_at"] for state in states.values()):
3255
  raise ValueError("Amazon cadence metadata has an invalid timestamp")
3256
  # A row's cause reads the saved refusal and SQP limit markers, never source rows.
3257
- cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES for family in ("", "refused_")]
 
3258
  + [prefix + az.SQP_LIMIT_KEY])
3259
  saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
3260
  [cause_keys]).fetchall())
@@ -3294,10 +3304,13 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3294
  state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
3295
  else tick.get("state") or "never_run")
3296
  reason = row_reason(table, state)
 
 
3297
  schedule.append({**row, "connectionId": cid, "account": entry["label"],
3298
  "lastSyncAt": (completed.get("updated_at")
3299
  if completed.get("state") == "done" else None),
3300
- "state": state, **({"reason": reason} if reason else {})})
 
3301
  return schedule
3302
 
3303
  try:
 
2199
  offered, current = {}, []
2200
  for job in jobs:
2201
  if job.get("state") != "failed":
2202
+ # A ready receipt carries its database's newest clean pass, which dates the card top.
2203
+ database = ((connections.get(str(job.get("connectionId") or "")) or {}).get("databases") or {}
2204
+ ).get(str(job.get("databaseKey") or ""))
2205
+ success = (_odoo_timestamp(_odoo_receipt_success(database))
2206
+ if job.get("state") == "ready" and isinstance(database, dict) else None)
2207
+ stamp = _odoo_timestamp(job.get("at"))
2208
+ current.append({**job, "at": success} if success is not None and (stamp is None or success > stamp)
2209
+ else job)
2210
  continue
2211
  cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
2212
  if cid not in offered:
 
3194
  cause = "The last pass started this table and did not finish it. The next pass continues it."
3195
  elif state == "refused":
3196
  plain = plain_refusal_reason(reason or held, status=status, table=table)
3197
+ # A reconciliation hold is our own check, never an Amazon refusal.
3198
+ reconciled = "reconciliation refused" in (reason or held)
3199
+ cause = plain if reconciled or plain.startswith(("Amazon refused", "Amazon does not offer")) else (
3200
  "Amazon refused this report: " + plain)
3201
  else:
3202
  problem = held[len("a problem names this table: "):] if held.startswith("a problem names this table: ") else ""
 
3263
  if any(not state["updated_at"] for state in states.values()):
3264
  raise ValueError("Amazon cadence metadata has an invalid timestamp")
3265
  # A row's cause reads the saved refusal and SQP limit markers, never source rows.
3266
+ cause_keys = ([prefix + az.entity_key(family + table) for table in az.TABLES
3267
+ for family in ("", "refused_", "warning_")]
3268
  + [prefix + az.SQP_LIMIT_KEY])
3269
  saved = dict(cursor.execute("SELECT entity, cursor_wd FROM _sync_state WHERE entity IN (SELECT unnest(?))",
3270
  [cause_keys]).fetchall())
 
3304
  state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
3305
  else tick.get("state") or "never_run")
3306
  reason = row_reason(table, state)
3307
+ # A source-internal mismatch published the table. Its warning stays until a clean reconcile.
3308
+ warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
3309
  schedule.append({**row, "connectionId": cid, "account": entry["label"],
3310
  "lastSyncAt": (completed.get("updated_at")
3311
  if completed.get("state") == "done" else None),
3312
+ "state": state, **({"reason": reason} if reason else {}),
3313
+ **({"warning": warning} if warning else {})})
3314
  return schedule
3315
 
3316
  try:
api/sync_alerts.py CHANGED
@@ -27,8 +27,18 @@ SYNC_KEY = "sync"
27
  # Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
28
  AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
29
  AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
30
- # A missing role is a standing account fact: it alerts once and never reminds.
31
- ONCE_CODES = frozenset({"role"})
 
 
 
 
 
 
 
 
 
 
32
  # A throttle or a held table alerts only when it repeats on this many passes in a row.
33
  REPEAT_CODES = {"throttled": 2, "held": 2}
34
 
@@ -120,9 +130,14 @@ def _clean(failure):
120
  if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
121
  return None
122
  key = scrub(failure["key"], 120)
 
 
 
 
 
 
123
  return {"key": key, "label": scrub(failure.get("label") or key, 120),
124
- "code": scrub(failure.get("code") or "error", 60) or "error",
125
- "cause": scrub(failure.get("cause") or "no cause given"),
126
  "last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
127
 
128
 
@@ -332,6 +347,16 @@ def _item(alert):
332
  return f"{alert['label']} ({alert['cause']}, {tail})"
333
 
334
 
 
 
 
 
 
 
 
 
 
 
335
  def environment():
336
  """LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
337
  space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
@@ -341,14 +366,22 @@ def environment():
341
 
342
 
343
  def compose(tenant, connector, plan, document):
344
- """One line for the whole pass: FAILED, STILL FAILING and RECOVERED, each listed once."""
345
  alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
346
  where = f"{environment()} {tenant} {connector}"
347
  parts = []
348
- for kind, title in (("failed", "FAILED"), ("reminder", "STILL FAILING")):
349
- items = [_item(alerts[a]) for a in sorted(plan[kind]) if a in alerts]
350
- if items:
351
- parts.append(f"Loopable sync {title}: {where}: " + ", ".join(items) + ".")
 
 
 
 
 
 
 
 
352
  names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
353
  if names:
354
  parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
 
27
  # Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
28
  AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
29
  AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
30
+ # A refused or expired credential is one cause: the owner reconnects or disconnects the connector.
31
+ CREDENTIAL_CAUSE = "credential expired, reconnect or disconnect it"
32
+ CREDENTIAL_CODES = frozenset({"credential", "credential_refused", "source_auth_failed"})
33
+ # A TLS "certificate has expired" names no token or credential, so it never matches.
34
+ CREDENTIAL = re.compile(
35
+ r"\brefused the connector credential\b|\bcredential expired\b|\binvalid_grant\b|\bcredential_refused\b"
36
+ r"|\berror validating access token\b|\bsession has expired\b"
37
+ r"|\b(?:\w+_)?tokens?\b[^.]{0,40}\b(?:expired|revoked|invalid)\b"
38
+ r"|\b(?:expired|revoked|invalid)\b[^.]{0,20}\b(?:\w+_)?tokens?\b"
39
+ r"|\b(?:refused|rejected)\b[^.]{0,40}\bcredential\b", re.I)
40
+ # A missing role or a refused credential is a standing account fact: it alerts once and never reminds.
41
+ ONCE_CODES = frozenset({"role", "credential"})
42
  # A throttle or a held table alerts only when it repeats on this many passes in a row.
43
  REPEAT_CODES = {"throttled": 2, "held": 2}
44
 
 
130
  if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
131
  return None
132
  key = scrub(failure["key"], 120)
133
+ code = scrub(failure.get("code") or "error", 60) or "error"
134
+ cause = scrub(failure.get("cause") or "no cause given")
135
+ raw = f"{failure.get('code') or ''} {failure.get('cause') or ''}"
136
+ # A missing Amazon role stays a role: it is checked before the credential.
137
+ if code != "role" and not AMAZON_ROLE.search(raw) and (code in CREDENTIAL_CODES or CREDENTIAL.search(raw)):
138
+ code, cause = "credential", CREDENTIAL_CAUSE
139
  return {"key": key, "label": scrub(failure.get("label") or key, 120),
140
+ "code": code, "cause": cause,
 
141
  "last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
142
 
143
 
 
347
  return f"{alert['label']} ({alert['cause']}, {tail})"
348
 
349
 
350
+ #: Connectors whose cards the watchdog reads, so their causes also reach the AI inbox.
351
+ INBOX_CONNECTORS = frozenset({"amazon", "odoo", "meta_ads", "amazon_ads"})
352
+
353
+
354
+ def _names_table(connector, alert):
355
+ """True when the alert's key is one table: a watchdog `table:` key or an Amazon pass table."""
356
+ key = str(alert.get("key") or "")
357
+ return key.startswith("table:") or (connector == "amazon" and key != SYNC_KEY)
358
+
359
+
360
  def environment():
361
  """LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
362
  space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
 
366
 
367
 
368
  def compose(tenant, connector, plan, document):
369
+ """One line for the whole pass: FAILED lists each alert, STILL FAILING counts them, RECOVERED names them."""
370
  alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
371
  where = f"{environment()} {tenant} {connector}"
372
  parts = []
373
+ items = [_item(alerts[a]) for a in sorted(plan["failed"]) if a in alerts]
374
+ if items:
375
+ parts.append(f"Loopable sync FAILED: {where}: " + ", ".join(items) + ".")
376
+ # The daily reminder is one short line per connector: the inbox keeps the full list.
377
+ reminded = [alerts[a] for a in sorted(plan["reminder"]) if a in alerts]
378
+ if reminded:
379
+ tables = all(_names_table(connector, alert) for alert in reminded)
380
+ noun = ("table" if tables else "cause") + ("" if len(reminded) == 1 else "s")
381
+ # Only a connector the watchdog reads has its causes in the AI inbox.
382
+ told = connector.startswith("watchdog ") or connector in INBOX_CONNECTORS
383
+ parts.append(f"Loopable sync STILL FAILING: {where}: {len(reminded)} {noun} still failing"
384
+ + (", see the inbox." if told else "."))
385
  names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
386
  if names:
387
  parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
platform/core/connector_sync.py CHANGED
@@ -1110,10 +1110,50 @@ def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync
1110
  "minutes": round(duration / 60, 3) if duration is not None else None,
1111
  "budgetMinutes": 30, "outcome": outcome}
1112
  entry["dailyPasses"] = days
 
 
 
1113
  _advance_good_sync(entry, last_good_sync_at)
1114
  return _status_update(rt, provider, connection_id, change)
1115
 
1116
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1117
  def record_resync_state(rt, provider, state, *, connection_id=None):
1118
  """Replace the connector's resync state with a JSON-safe copy of `state`."""
1119
  try:
 
1110
  "minutes": round(duration / 60, 3) if duration is not None else None,
1111
  "budgetMinutes": 30, "outcome": outcome}
1112
  entry["dailyPasses"] = days
1113
+ if outcome != "failed":
1114
+ # Any pass that did not fail proves the credential works again.
1115
+ entry.pop("credentialHold", None)
1116
  _advance_good_sync(entry, last_good_sync_at)
1117
  return _status_update(rt, provider, connection_id, change)
1118
 
1119
 
1120
+ def credential_fingerprint(material):
1121
+ """The first 16 hex of sha256 over the credential material, or None when there is none.
1122
+
1123
+ A list or tuple is joined line by line. The secret itself is never stored or returned.
1124
+ """
1125
+ if isinstance(material, (list, tuple)):
1126
+ material = "\n".join(str(part) for part in material)
1127
+ if not isinstance(material, str) or not material:
1128
+ return None
1129
+ return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
1130
+
1131
+
1132
+ def hold_credential(rt, provider, fingerprint, *, connection_id=None):
1133
+ """Record that the provider refused the credential with this fingerprint."""
1134
+ if not fingerprint:
1135
+ return False
1136
+ at = _utc_stamp()
1137
+
1138
+ def change(entry):
1139
+ entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at}
1140
+ return _status_update(rt, provider, connection_id, change)
1141
+
1142
+
1143
+ def credential_held(rt, provider, fingerprint, *, connection_id=None):
1144
+ """True when a scheduled pass must skip: the last pass failed on this same refused credential.
1145
+
1146
+ An unknown fingerprint or an unreadable record answers False, so the pass runs.
1147
+ """
1148
+ if not fingerprint:
1149
+ return False
1150
+ entry = status(rt, provider, connection_id, fresh=True) or {}
1151
+ hold = entry.get("credentialHold")
1152
+ last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
1153
+ return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint
1154
+ and last_pass.get("outcome") == "failed")
1155
+
1156
+
1157
  def record_resync_state(rt, provider, state, *, connection_id=None):
1158
  """Replace the connector's resync state with a JSON-safe copy of `state`."""
1159
  try:
platform/harness/amazon_store.py CHANGED
@@ -493,6 +493,8 @@ SCHEDULED_FLOOR_SECONDS = 86400
493
  SCHEDULED_FLOOR_GRACE_SECONDS = 0
494
  # The resident cadence checks once a minute.
495
  SCHEDULE_PHASE_GRACE_SECONDS = 60
 
 
496
  _SCHEDULE_OFFSET_READER = None
497
 
498
 
@@ -509,10 +511,15 @@ def _scheduled_phase_due(table, state, now, earliest):
509
  shift = int(_SCHEDULE_OFFSET_READER())
510
  interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
511
  phase = ((now.timestamp() - shift) // interval) * interval + shift
512
- stamp = _utc(state["updated_at"]).timestamp() if state and state.get("state") == "done" else None
513
- # A done tick that missed its phase runs late inside the first half of the interval.
 
 
 
 
 
514
  # The second half is left alone: a staging copy of a live tick stays off the live phase.
515
- if stamp is not None and stamp < phase and earliest.timestamp() - phase < interval / 2:
516
  return earliest
517
  if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
518
  or (stamp is not None and stamp >= phase)):
@@ -607,6 +614,20 @@ def scheduled_sync_plan(tenant_key, *, rt, now=None):
607
  "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
608
  "WHERE starts_with(entity, ?)", [prefix]).fetchall()}
609
  plan = sync_plan(ticks, tables, now=now)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
610
  due.update(dict.fromkeys(plan["tables"]))
611
  for table, reason in plan["deferred"].items():
612
  deferred.setdefault(table, []).append(reason)
@@ -5114,6 +5135,14 @@ def _refused_reports(con):
5114
  def plain_refusal_reason(reason, *, status="", table="", operation=""):
5115
  """Project refusals to cause and action copy, retaining the fee API's vendor reason."""
5116
  text = str(reason or "")
 
 
 
 
 
 
 
 
5117
  feedback = table in ("review_topics", "review_trends")
5118
  subject = "review trends" if table == "review_trends" else "review topics"
5119
  if feedback and (str(status) in ("403", "NOT_OFFERED")
@@ -8606,7 +8635,8 @@ def _validate_orders(con):
8606
  return {"state": "unproven", "expected": expected, "actual": actual,
8607
  "reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
8608
  f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
8609
- return {"state": "refused", "expected": expected, "actual": actual, "revived": revived,
 
8610
  "reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
8611
  except Exception:
8612
  return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
@@ -9146,7 +9176,20 @@ def _refuse_unreconciled(tenant_key, rt, report, log):
9146
  "the table keeps its previous rows until it runs"} for table in synced}
9147
  con, _path = _connect_tenant_mirror(tenant_key, rt)
9148
  for table in synced:
9149
- if (checked.get(table) or {}).get("state") != "refused":
 
 
 
 
 
 
 
 
 
 
 
 
 
9150
  continue
9151
  reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
9152
  report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
@@ -9391,6 +9434,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9391
  continue
9392
  report["tables"][table] = {"state": "deferred", "reason":
9393
  "The first load is continuing in bounded passes on the resident sync thread."}
 
 
9394
  tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
9395
  if tick and tick.get("state") == "throttled":
9396
  # The first load owns SQP now; a passed SQP deadline must not make it due every minute.
@@ -9407,7 +9452,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9407
  mark = _read_state(con, entity_key(table)) or {}
9408
  if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
9409
  and not _interval_reason(table, ticks.get(table), end)):
9410
- ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
 
 
9411
  for table in want:
9412
  tick = ticks.get(table)
9413
  # A queued resync is a repair: only Amazon's own floor and the stagger hold it.
 
493
  SCHEDULED_FLOOR_GRACE_SECONDS = 0
494
  # The resident cadence checks once a minute.
495
  SCHEDULE_PHASE_GRACE_SECONDS = 60
496
+ #: `_sync_state` marker: the scheduled first-load guard deferred this table; never a tick, so no card reads it.
497
+ PHASE_HOLD_PREFIX = "phase_hold_"
498
  _SCHEDULE_OFFSET_READER = None
499
 
500
 
 
511
  shift = int(_SCHEDULE_OFFSET_READER())
512
  interval = SCHEDULED_FLOOR_SECONDS // _client().CADENCE[table]["perDay"]
513
  phase = ((now.timestamp() - shift) // interval) * interval + shift
514
+ stamp = _utc(state["updated_at"]).timestamp() if state and state.get("updated_at") else None
515
+ window = interval / 2
516
+ fastest = SCHEDULED_FLOOR_SECONDS // max(cadence["perDay"] for cadence in _client().CADENCE.values())
517
+ if shift % fastest:
518
+ # A shifted deployment's late run ends 30 minutes before the next live slot.
519
+ window = min(window, fastest - phase % fastest - 1800)
520
+ # A tick that missed its phase, in any state or absent, runs late inside the first half of the interval.
521
  # The second half is left alone: a staging copy of a live tick stays off the live phase.
522
+ if (stamp is None or stamp < phase) and earliest.timestamp() - phase < window:
523
  return earliest
524
  if (now.timestamp() - phase > SCHEDULE_PHASE_GRACE_SECONDS
525
  or (stamp is not None and stamp >= phase)):
 
614
  "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
615
  "WHERE starts_with(entity, ?)", [prefix]).fetchall()}
616
  plan = sync_plan(ticks, tables, now=now)
617
+ hold = _account_prefix(entry["connection_id"]) + entity_key(PHASE_HOLD_PREFIX)
618
+ # The planner reads cadence rows through this one query shape only, never source data.
619
+ holds = {entity[len(hold):]: {"state": phase, "rows": 0, "updated_at": at, "detail": None}
620
+ for entity, phase, _rows, at, _detail in con.execute(
621
+ "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
622
+ "WHERE starts_with(entity, ?)", [hold]).fetchall()}
623
+ for table in list(plan["tables"]):
624
+ held, tick = holds.get(table), ticks.get(table)
625
+ # A first-load deferral newer than the tick waits for the next phase, as the sync defers it.
626
+ if held and (tick is None or _utc(held["updated_at"]) > _utc(tick["updated_at"])):
627
+ reason = _interval_reason(table, held, now, floor_s=_cadence_seconds(table, tick))
628
+ if reason:
629
+ plan["tables"].remove(table)
630
+ plan["deferred"][table] = reason
631
  due.update(dict.fromkeys(plan["tables"]))
632
  for table, reason in plan["deferred"].items():
633
  deferred.setdefault(table, []).append(reason)
 
5135
  def plain_refusal_reason(reason, *, status="", table="", operation=""):
5136
  """Project refusals to cause and action copy, retaining the fee API's vendor reason."""
5137
  text = str(reason or "")
5138
+ # A reconciliation hold is our own check: never call it a load failure.
5139
+ reconciliation = re.search(r"\breconciliation refused:\s*([\s\S]*)$", text, re.I)
5140
+ if reconciliation:
5141
+ detail = re.sub(r"[.\s]+$", "", reconciliation[1].strip()).replace("\u2014", ". ").replace("\u2013", " to ")
5142
+ if re.match(r"the check could not run", detail, re.I):
5143
+ return "The reconciliation check could not run. The last good rows are kept until it runs."
5144
+ return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
5145
+ else "Reconciliation did not tie. The last good rows are kept.")
5146
  feedback = table in ("review_topics", "review_trends")
5147
  subject = "review trends" if table == "review_trends" else "review topics"
5148
  if feedback and (str(status) in ("403", "NOT_OFFERED")
 
8635
  return {"state": "unproven", "expected": expected, "actual": actual,
8636
  "reason": (f"Amazon Sales counted {expected} orders after the stored orders were read; "
8637
  f"stored orders contain {actual} {span}. The next orders sync reconciles them.")}
8638
+ # Two Amazon reports disagree: a check on the source's own data warns and publishes.
8639
+ return {"state": "refused", "expected": expected, "actual": actual, "revived": revived, "publish": True,
8640
  "reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} {span}"}
8641
  except Exception:
8642
  return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
 
9176
  "the table keeps its previous rows until it runs"} for table in synced}
9177
  con, _path = _connect_tenant_mirror(tenant_key, rt)
9178
  for table in synced:
9179
+ verdict, warned = checked.get(table) or {}, entity_key("warning_" + table)
9180
+ if verdict.get("state") == "refused" and verdict.get("publish") is True:
9181
+ # A check on the source's own data warns and publishes. It never holds the sync.
9182
+ expected, actual = verdict.get("expected"), verdict.get("actual")
9183
+ warning = (f"Amazon Sales counted {expected:,} orders and the orders report holds {actual:,}. "
9184
+ "The table is published." if type(expected) is int and type(actual) is int else
9185
+ f"{str(verdict.get('reason') or 'Amazon reports disagree').rstrip('. ')}. The table is published.")
9186
+ report["tables"][table] = {**report["tables"][table], "warning": warning}
9187
+ _record_state(con, warned, "warning", 0, warning)
9188
+ log(f"{table}: published with a warning: {warning}")
9189
+ continue
9190
+ if verdict.get("state") != "refused":
9191
+ # A clean reconcile clears the last warning. A held table keeps it with its kept rows.
9192
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [warned])
9193
  continue
9194
  reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
9195
  report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
 
9434
  continue
9435
  report["tables"][table] = {"state": "deferred", "reason":
9436
  "The first load is continuing in bounded passes on the resident sync thread."}
9437
+ # The planner waits for the next phase on this hold. The card's tick stays as it was.
9438
+ _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred")
9439
  tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
9440
  if tick and tick.get("state") == "throttled":
9441
  # The first load owns SQP now; a passed SQP deadline must not make it due every minute.
 
9452
  mark = _read_state(con, entity_key(table)) or {}
9453
  if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
9454
  and not _interval_reason(table, ticks.get(table), end)):
9455
+ # The re-asked tick keeps its stamp, so the phase rule decides as the planner does.
9456
+ ticks.update({name: ticks[name] and {**ticks[name], "state": "refused"}
9457
+ for name in (table, pairs.get(table)) if name in want})
9458
  for table in want:
9459
  tick = ticks.get(table)
9460
  # A queued resync is a repair: only Amazon's own floor and the stagger hold it.
platform/harness/mirror_store.py CHANGED
@@ -37,6 +37,8 @@ _KEY_LEDGER = "keys.json"
37
  _TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
38
  _PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
39
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
 
 
40
  _PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
41
  "shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
42
  ORPHAN_GRACE = timedelta(hours=24)
@@ -726,10 +728,59 @@ class MirrorStore:
726
  return key
727
 
728
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
729
  def _is_mirror_blob_key(self, key):
730
  match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
731
  return bool((match and self._valid_table(match.group(1)))
732
- or _STATE_BLOB_RE.fullmatch(str(key)))
733
 
734
 
735
  def _manifest_keys(self, manifest, *, strict=True):
@@ -747,7 +798,8 @@ class MirrorStore:
747
  state = manifest.get("sync_state")
748
  if isinstance(state, dict):
749
  try:
750
- out[self._state_blob_key(state, strict=strict)] = state.get("sha256")
 
751
  except (ValueError, AttributeError, TypeError):
752
  pass
753
  return out
@@ -805,8 +857,9 @@ class MirrorStore:
805
  raise ValueError("Mirror publication identity or times are invalid")
806
  allow_partial = False
807
  state_meta = manifest["sync_state"]
808
- state_bytes = self._checked_mirror_blob(rt, self._state_blob_key(state_meta), state_meta)
809
- rows = json.loads(state_bytes)
 
810
  if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
811
  or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
812
  or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
@@ -1351,18 +1404,16 @@ class MirrorStore:
1351
  continue
1352
  pending.append((key, path))
1353
  rows = self._state_rows(con)
1354
- state = _mirror_json(rows)
1355
- state_meta = {**_blob_metadata(state), "rows": len(rows)}
1356
- state_meta["key"] = _content_key("sync_state", suffix + ".json", state_meta)
1357
  published = None
1358
  if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1359
  or manifest["tables"] != old.get("tables", {})
1360
  or old.get("version") != MANIFEST_VERSION):
1361
- state_path = None
1362
- if state_meta["key"] not in named:
1363
- state_path = Path(tmp) / "sync_state.json"
1364
- state_path.write_bytes(state)
1365
- pending.append((state_meta["key"], state_path))
1366
  if any(path.stat().st_size > part_limit for _key, path in pending):
1367
  raise ValueError("mirror part exceeds the byte limit")
1368
  if pending:
 
37
  _TABLE_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.parquet")
38
  _PART_BLOB_RE = re.compile(r"([a-z][a-z0-9_]*)\.p([0-9a-z-]{1,48})\.([0-9a-f]{16})(?:\.sealed-v1)?\.parquet")
39
  _STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?(?:\.sealed-v1)?\.json")
40
+ #: State over one part's byte limit is saved as consecutive row parts, as a table is.
41
+ _STATE_PART_RE = re.compile(r"sync_state\.p([0-9]{1,6})\.([0-9a-f]{16})(?:\.sealed-v1)?\.json")
42
  _PART_COLUMNS = ("_period_start", "month", "_snapshot_date", "posted_date", "purchase_date",
43
  "shipment_date", "date", "snapshot_date", "last_updated_date", "settlement_start_date")
44
  ORPHAN_GRACE = timedelta(hours=24)
 
728
  return key
729
 
730
 
731
+ def _state_blob_keys(self, metadata, *, strict=True):
732
+ """[(key, metadata)] for the manifest's state entry: its row parts, or its one blob."""
733
+ if not (isinstance(metadata, dict) and "parts" in metadata):
734
+ return [(self._state_blob_key(metadata, strict=strict), metadata)]
735
+ parts = metadata["parts"]
736
+ if not isinstance(parts, list) or not parts:
737
+ raise ValueError(f"{self.label} mirror manifest names an empty state part list")
738
+ out = []
739
+ for part in parts:
740
+ key = str(part.get("key") or "") if isinstance(part, dict) else ""
741
+ match = _STATE_PART_RE.fullmatch(key)
742
+ if not match or (strict and not str(part.get("sha256") or "").startswith(match.group(2))):
743
+ raise ValueError(f"{self.label} mirror manifest names an invalid state part key")
744
+ out.append((key, part))
745
+ return out
746
+
747
+
748
+ @staticmethod
749
+ def _state_entry(rows, limit, suffix):
750
+ """(manifest state entry, [(key, bytes)]): one blob while the rows fit `limit`, else row parts."""
751
+ state = _mirror_json(rows)
752
+ if len(state) <= limit:
753
+ meta = {**_blob_metadata(state), "rows": len(rows)}
754
+ meta["key"] = _content_key("sync_state", suffix + ".json", meta)
755
+ return meta, [(meta["key"], state)]
756
+ parts, blobs, chunk, size = [], [], [], 2
757
+
758
+ def close():
759
+ data = _mirror_json(chunk)
760
+ meta = {**_blob_metadata(data), "rows": len(chunk), "part": len(parts)}
761
+ meta["key"] = _content_key(f"sync_state.p{len(parts)}", suffix + ".json", meta)
762
+ parts.append(meta)
763
+ blobs.append((meta["key"], data))
764
+
765
+ for row in rows:
766
+ width = len(_mirror_json(row)) + 1
767
+ if chunk and size + width > limit:
768
+ close()
769
+ chunk, size = [], 2
770
+ chunk.append(row)
771
+ size += width
772
+ if chunk:
773
+ close()
774
+ # A single row wider than `limit` stays one part, and the pending byte check names it.
775
+ combined = hashlib.sha256("".join(p["sha256"] for p in parts).encode()).hexdigest()
776
+ return ({"parts": parts, "rows": len(rows), "sha256": combined,
777
+ "bytes": sum(p["bytes"] for p in parts)}, blobs)
778
+
779
+
780
  def _is_mirror_blob_key(self, key):
781
  match = _TABLE_BLOB_RE.fullmatch(str(key)) or _PART_BLOB_RE.fullmatch(str(key))
782
  return bool((match and self._valid_table(match.group(1)))
783
+ or _STATE_BLOB_RE.fullmatch(str(key)) or _STATE_PART_RE.fullmatch(str(key)))
784
 
785
 
786
  def _manifest_keys(self, manifest, *, strict=True):
 
798
  state = manifest.get("sync_state")
799
  if isinstance(state, dict):
800
  try:
801
+ for key, blob in self._state_blob_keys(state, strict=strict):
802
+ out[key] = blob.get("sha256")
803
  except (ValueError, AttributeError, TypeError):
804
  pass
805
  return out
 
857
  raise ValueError("Mirror publication identity or times are invalid")
858
  allow_partial = False
859
  state_meta = manifest["sync_state"]
860
+ parts = [json.loads(self._checked_mirror_blob(rt, key, blob))
861
+ for key, blob in self._state_blob_keys(state_meta)]
862
+ rows = [row for part in parts for row in part] if all(isinstance(p, list) for p in parts) else None
863
  if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
864
  or not self._state_complete(rows) or any(not isinstance(row, list) or len(row) != 6
865
  or not isinstance(row[0], str) or not self._owns_state(row[0]) for row in rows)
 
1404
  continue
1405
  pending.append((key, path))
1406
  rows = self._state_rows(con)
1407
+ state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
 
 
1408
  published = None
1409
  if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1410
  or manifest["tables"] != old.get("tables", {})
1411
  or old.get("version") != MANIFEST_VERSION):
1412
+ for index, (state_key, state) in enumerate(state_blobs):
1413
+ if state_key not in named:
1414
+ state_path = Path(tmp) / f"sync_state.{index}.json"
1415
+ state_path.write_bytes(state)
1416
+ pending.append((state_key, state_path))
1417
  if any(path.stat().st_size > part_limit for _key, path in pending):
1418
  raise ValueError("mirror part exceeds the byte limit")
1419
  if pending:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 34a79558f6e41fb1f5af8f8070e906e3d4e64a9ecf8ad4c49834cc5f546da272
 
1
+ fc20fb27640c854759ec5061266f0c3a70ad8f7831d67eeb401f0b2125c01625
web/src/connectors/ConnectorsPage.tsx CHANGED
@@ -265,6 +265,8 @@ export interface AmazonTableSync {
265
  perDay: number;
266
  state: string;
267
  reason?: string;
 
 
268
  }
269
 
270
  export interface AmazonSyncScheduleReading {
@@ -298,7 +300,8 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
298
  if (identities.has(identity)) return unavailable;
299
  identities.add(identity);
300
  const reason = typeof row.reason === "string" ? row.reason.trim() : "";
301
- tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined });
 
302
  }
303
  const state = body.scheduleState as AmazonSyncScheduleReading["state"];
304
  if ((state === "ready") !== (tables.length > 0)) return unavailable;
@@ -344,6 +347,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
344
  {Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
345
  : <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
346
  {row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
 
347
  : row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
348
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
349
  </td>
@@ -433,14 +437,32 @@ export function staleSyncWarnings(name: string, body: unknown): string[] {
433
  return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
434
  }
435
 
436
- /** One read of a daily source's sync route. A failed read is no warning, never a thrown error. */
437
- async function dailySyncWarnings(connectorKey: string, signal: AbortSignal): Promise<string[]> {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  try {
439
  const res = await fetch(`${API_V1}/connectors/${encodeURIComponent(connectorKey)}/sync`, {
440
  credentials: CREDENTIALS, signal,
441
  });
442
  if (!res.ok) return [];
443
  const body: unknown = await res.json().catch(() => null);
 
 
 
444
  return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
445
  } catch {
446
  return [];
@@ -1073,17 +1095,18 @@ export default function ConnectorsPage({
1073
  return () => controller.abort();
1074
  }, [hasOdooCard]);
1075
  const dailySourceKeys = load.phase === "ready"
1076
- ? load.rows.filter((row) => row.key in DAILY_SOURCE_NAMES && row.state === "connected")
1077
- .map((row) => row.key).join(",")
1078
  : "";
1079
  const [dailyWarnings, setDailyWarnings] = useState<Record<string, string[]>>({});
1080
  useEffect(() => {
1081
  if (!dailySourceKeys) return;
1082
  const controller = new AbortController();
1083
- const keys = dailySourceKeys.split(",");
1084
- void Promise.all(keys.map((key) => dailySyncWarnings(key, controller.signal))).then((lines) => {
 
1085
  if (!controller.signal.aborted) {
1086
- setDailyWarnings(Object.fromEntries(keys.map((key, index) => [key, lines[index]])));
1087
  }
1088
  });
1089
  return () => controller.abort();
@@ -1571,7 +1594,9 @@ export function ConnectorCard({
1571
  const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
1572
  // D-2679: databases that failed alone are named; a stale connector still reads "Stale".
1573
  const databasesStale = failedDatabasesStatus(syncWarnings);
1574
- const status = catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
 
 
1575
  <span className="conn-card-status conn-catch-up" role="status">
1576
  {catchUp?.active ? catchUpLabel(catchUp) : catchUpError
1577
  || (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
 
265
  perDay: number;
266
  state: string;
267
  reason?: string;
268
+ /** A source-internal mismatch: the table published, and both counts are named. */
269
+ warning?: string;
270
  }
271
 
272
  export interface AmazonSyncScheduleReading {
 
300
  if (identities.has(identity)) return unavailable;
301
  identities.add(identity);
302
  const reason = typeof row.reason === "string" ? row.reason.trim() : "";
303
+ const warning = typeof row.warning === "string" ? row.warning.trim() : "";
304
+ tables.push({ ...(row as unknown as AmazonTableSync), reason: reason || undefined, warning: warning || undefined });
305
  }
306
  const state = body.scheduleState as AmazonSyncScheduleReading["state"];
307
  if ((state === "ready") !== (tables.length > 0)) return unavailable;
 
347
  {Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now"
348
  : <time dateTime={row.nextSyncAt}>{amazonSyncStamp(row.nextSyncAt, reading.timezone)}</time>}
349
  {row.reason && row.state !== "done" ? <p role="status">{row.reason}</p>
350
+ : row.warning && row.state === "done" ? <p role="status">{row.warning}</p>
351
  : row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
352
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
353
  </td>
 
437
  return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
438
  }
439
 
440
+ /** The server's sentence when the provider refused the connector credential. */
441
+ export const CREDENTIAL_EXPIRED = "Credential expired. Reconnect to resume.";
442
+ /** The sentence a pass stored before this one: the card reads it the same way until the next pass. */
443
+ const CREDENTIAL_REFUSED_BEFORE = "The provider refused the connector credential";
444
+
445
+ /** The card line for a failed pass on a refused credential, PURE: none for any other state. */
446
+ export function credentialSyncWarnings(body: unknown): string[] {
447
+ if (!body || typeof body !== "object") return [];
448
+ const { state, detail } = body as { state?: unknown; detail?: unknown };
449
+ return state === "error" && typeof detail === "string"
450
+ && (detail.startsWith(CREDENTIAL_EXPIRED) || detail.startsWith(CREDENTIAL_REFUSED_BEFORE))
451
+ ? [CREDENTIAL_EXPIRED] : [];
452
+ }
453
+
454
+ /** One read of a daily source's sync route. A failed read is no warning, never a thrown error.
455
+ * A card the directory does not count as connected shows only a refused credential: its pass still runs. */
456
+ async function dailySyncWarnings(connectorKey: string, signal: AbortSignal, connected = true): Promise<string[]> {
457
  try {
458
  const res = await fetch(`${API_V1}/connectors/${encodeURIComponent(connectorKey)}/sync`, {
459
  credentials: CREDENTIALS, signal,
460
  });
461
  if (!res.ok) return [];
462
  const body: unknown = await res.json().catch(() => null);
463
+ const credential = credentialSyncWarnings(body);
464
+ if (credential.length) return credential;
465
+ if (!connected) return [];
466
  return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
467
  } catch {
468
  return [];
 
1095
  return () => controller.abort();
1096
  }, [hasOdooCard]);
1097
  const dailySourceKeys = load.phase === "ready"
1098
+ ? load.rows.filter((row) => row.key in DAILY_SOURCE_NAMES)
1099
+ .map((row) => `${row.key}:${row.state === "connected" ? 1 : 0}`).join(",")
1100
  : "";
1101
  const [dailyWarnings, setDailyWarnings] = useState<Record<string, string[]>>({});
1102
  useEffect(() => {
1103
  if (!dailySourceKeys) return;
1104
  const controller = new AbortController();
1105
+ const entries = dailySourceKeys.split(",").map((entry) => entry.split(":"));
1106
+ const reads = entries.map(([key, connected]) => dailySyncWarnings(key, controller.signal, connected === "1"));
1107
+ void Promise.all(reads).then((lines) => {
1108
  if (!controller.signal.aborted) {
1109
+ setDailyWarnings(Object.fromEntries(entries.map(([key], index) => [key, lines[index]])));
1110
  }
1111
  });
1112
  return () => controller.abort();
 
1594
  const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
1595
  // D-2679: databases that failed alone are named; a stale connector still reads "Stale".
1596
  const databasesStale = failedDatabasesStatus(syncWarnings);
1597
+ const status = syncWarnings?.includes(CREDENTIAL_EXPIRED) ? (
1598
+ <span className="conn-card-status" role="status">{CREDENTIAL_EXPIRED}</span>
1599
+ ) : catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
1600
  <span className="conn-card-status conn-catch-up" role="status">
1601
  {catchUp?.active ? catchUpLabel(catchUp) : catchUpError
1602
  || (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
web/src/connectors/DatabasesTab.tsx CHANGED
@@ -135,6 +135,8 @@ interface AmazonTableSync {
135
  stuck: boolean | null;
136
  stuckCause: string;
137
  unknownCause: string;
 
 
138
  }
139
 
140
  export interface DatabaseSyncHealth {
@@ -564,6 +566,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
564
  }
565
  const reason = syncText(syncObject(state.deferred).reason);
566
  const held = syncObject(state.held);
 
567
  states[database.key] = {
568
  rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
569
  inProgress: state.inProgress === true,
@@ -575,6 +578,7 @@ export function parseAmazonSyncStates(value: unknown): Record<string, AmazonTabl
575
  firstLoad: load,
576
  stuck: state.stuck === true ? true : state.stuck === false ? false : null,
577
  stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
 
578
  };
579
  }
580
  return states;
@@ -686,6 +690,16 @@ export function firstLoadEstimate(seconds: number): string {
686
  function amazonRefusalReason(reason: string, status = ""): string {
687
  const text = reason.trim();
688
  const lower = text.toLowerCase();
 
 
 
 
 
 
 
 
 
 
689
  if (/within (?:the past )?two years/.test(lower)) {
690
  return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
691
  }
@@ -2323,7 +2337,8 @@ const DatabasesTab = forwardRef<DatabasesTabHandle, {
2323
  catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
2324
  {health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
2325
  {health.error.message}</p> : null}
2326
- {health?.warning ? <p className="conn-db-note">{health.warning}</p> : null}
 
2327
  {job?.state === "failed" && job.retryable ? <button type="button"
2328
  className="conn-job-retry" disabled={Boolean(retrying)}
2329
  onClick={() => void retryJob(job)}>Retry sync</button> : null}
 
135
  stuck: boolean | null;
136
  stuckCause: string;
137
  unknownCause: string;
138
+ /** A source-internal mismatch: the table published, and both counts are named. */
139
+ warning?: string;
140
  }
141
 
142
  export interface DatabaseSyncHealth {
 
566
  }
567
  const reason = syncText(syncObject(state.deferred).reason);
568
  const held = syncObject(state.held);
569
+ const warning = syncText(state.warning).replace(/\u2014/g, ". ").replace(/\u2013/g, " to ").trim();
570
  states[database.key] = {
571
  rowsLoaded: syncCount(state.rowsLoaded), lastSuccess: syncStamp(state.lastSuccess),
572
  inProgress: state.inProgress === true,
 
578
  firstLoad: load,
579
  stuck: state.stuck === true ? true : state.stuck === false ? false : null,
580
  stuckCause: syncText(state.stuckCause), unknownCause: syncText(state.unknownCause),
581
+ ...(warning ? { warning } : {}),
582
  };
583
  }
584
  return states;
 
690
  function amazonRefusalReason(reason: string, status = ""): string {
691
  const text = reason.trim();
692
  const lower = text.toLowerCase();
693
+ // Mirrors plain_refusal_reason: a reconciliation hold is our own check, never a load failure.
694
+ const reconciliation = /\breconciliation refused:\s*([\s\S]*)$/i.exec(text);
695
+ if (reconciliation) {
696
+ const detail = reconciliation[1].trim().replace(/[.\s]+$/, "").replace(/\u2014/g, ". ").replace(/\u2013/g, " to ");
697
+ if (/^the check could not run/i.test(detail)) {
698
+ return "The reconciliation check could not run. The last good rows are kept until it runs.";
699
+ }
700
+ return detail ? `Reconciliation did not tie: ${detail}. The last good rows are kept.`
701
+ : "Reconciliation did not tie. The last good rows are kept.";
702
+ }
703
  if (/within (?:the past )?two years/.test(lower)) {
704
  return "Amazon refused this date range. Choose a start date within the past two years and retry sync.";
705
  }
 
2337
  catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />}
2338
  {health?.error && !amazonState?.refused && !amazonState?.held ? <p className="conn-db-error" role="alert">
2339
  {health.error.message}</p> : null}
2340
+ {health?.warning || amazonState?.warning ? <p className="conn-db-note">
2341
+ {health?.warning || amazonState?.warning}</p> : null}
2342
  {job?.state === "failed" && job.retryable ? <button type="button"
2343
  className="conn-job-retry" disabled={Boolean(retrying)}
2344
  onClick={() => void retryJob(job)}>Retry sync</button> : null}