fsanyoto commited on
Commit
aa9ac4a
·
verified ·
1 Parent(s): e198d0b

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "30fc33afffe (30fc33afffe)",
3
  "releases": [
4
  {
5
  "version": "v80",
 
1
  {
2
+ "current": "982dca9899c (982dca9899c)",
3
  "releases": [
4
  {
5
  "version": "v80",
VERSION CHANGED
@@ -1 +1 @@
1
- 30fc33afffe (30fc33afffe)
 
1
+ 982dca9899c (982dca9899c)
api/main.py CHANGED
@@ -736,6 +736,8 @@ def _seed_and_sync_store():
736
  _ds.reconcile_deletes(entities=selected_entities, log=lambda *a, **k: None)
737
  except Exception:
738
  pass
 
 
739
  print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
740
  f"(ready={_ds.ready()})")
741
  # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
@@ -999,6 +1001,8 @@ class _DeskAwareOdooClient:
999
  self._factory = factory
1000
  self._client = None
1001
  self._next_desk = 0.0
 
 
1002
 
1003
  def __getattr__(self, name):
1004
  if name not in {"search_read", "search_count", "read_group", "fields_get"}:
@@ -1010,7 +1014,7 @@ class _DeskAwareOdooClient:
1010
  result, finished = {}, threading.Event()
1011
 
1012
  def serve_desk():
1013
- if time.monotonic() >= self._next_desk:
1014
  outcome = {}
1015
  _run_sync_requests(outcome, during_odoo=True)
1016
  self._next_desk = time.monotonic() + (30 if "continuing" in outcome.values() else 0.25)
@@ -1651,7 +1655,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1651
  _health(cid, "freshness", "failed",
1652
  failure={**failure, "stage": "freshness"})
1653
  else:
1654
- _health(cid, "source", "ready")
 
 
1655
  _health(cid, "materialization", "ready")
1656
  # A receipt is the authoritative per-database result. ``tables`` alone omits
1657
  # read-through grains and the two compiled canonical databases, which previously
@@ -2146,6 +2152,11 @@ def _resync_mirror_pass(manual, passes, entities=None):
2146
  _resync_log(f"failed after {_pass_clock.monotonic() - pass_started:.1f}s ({type(e).__name__})")
2147
  # A partial report must never be mistaken for a confirmed zero-change pass.
2148
  return None, None, _classify_resync_error(e)
 
 
 
 
 
2149
 
2150
 
2151
  def _absent_mirror_entities(ds, entities=None):
@@ -2270,14 +2281,14 @@ def _resync_memory_read():
2270
  return {}
2271
 
2272
 
2273
- def _resync_memory_write(pass_at, full_rebuild_at, fingerprints):
2274
  """Persist the loop's restart memory. Best effort: a failed write only costs a rebuild."""
2275
  try:
2276
  import odoo_relational as _rel_memory
2277
  from harness import runtime as _runtime_memory
2278
  _rel_memory.record_resync_memory(
2279
  _runtime_memory.get_runtime(_resync_tenant()[0]), pass_at=pass_at,
2280
- full_rebuild_at=full_rebuild_at, fingerprints=fingerprints)
2281
  except Exception as e: # noqa: BLE001
2282
  print(f"[aios-api] store resync memory not saved ({type(e).__name__}); "
2283
  "the next restart rebuilds in full")
@@ -2325,6 +2336,8 @@ def _store_resync_loop():
2325
  last_relational_success_at = _memory.get("fullRebuildAt")
2326
  last_source_fingerprints = _memory.get("fingerprints")
2327
  last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
 
 
2328
  catch_up_retries = 0
2329
  last_meta_fingerprint = None
2330
  last_meta_applied_at = None
@@ -2461,6 +2474,15 @@ def _store_resync_loop():
2461
  RESYNC_STATE["consecutiveFailures"] = 0
2462
  if not _manual:
2463
  RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
 
 
 
 
 
 
 
 
 
2464
  else:
2465
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
2466
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
@@ -2517,8 +2539,36 @@ def _store_resync_loop():
2517
  # correctly. A connector that can only ever be established at boot is one bad boot away
2518
  # from being permanently absent.
2519
  # ⚠ Cheap when there is nothing to do: no token => one line and return.
2520
- _meta_pull_confirmed = _pull_meta("resync")
2521
- _pull_amazon("resync")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2522
  _meta_now = _t.time()
2523
  _meta_reason = _meta_rebuild_force_reason(
2524
  _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
@@ -2610,10 +2660,14 @@ def _store_resync_loop():
2610
  last_meta_fingerprint = None
2611
  last_meta_applied_at = None
2612
  # HOTFIX 2026-09-29: remember this pass for the next container (see the top of the loop).
2613
- if _err is None and _rebuild_ok:
2614
- last_good_pass_at = _meta_now
 
 
 
 
2615
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
2616
- last_source_fingerprints)
2617
 
2618
 
2619
  def _boot_app_migrations(why):
 
736
  _ds.reconcile_deletes(entities=selected_entities, log=lambda *a, **k: None)
737
  except Exception:
738
  pass
739
+ if isinstance(getattr(_odoo._tlocal, "client", None), _DeskAwareOdooClient):
740
+ _odoo._tlocal.client.desk = False # the rebuild below must not serve the desk
741
  print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
742
  f"(ready={_ds.ready()})")
743
  # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
 
1001
  self._factory = factory
1002
  self._client = None
1003
  self._next_desk = 0.0
1004
+ # Only the Odoo pull serves the desk; the relational rebuild after it on this thread must not.
1005
+ self.desk = True
1006
 
1007
  def __getattr__(self, name):
1008
  if name not in {"search_read", "search_count", "read_group", "fields_get"}:
 
1014
  result, finished = {}, threading.Event()
1015
 
1016
  def serve_desk():
1017
+ if self.desk and time.monotonic() >= self._next_desk:
1018
  outcome = {}
1019
  _run_sync_requests(outcome, during_odoo=True)
1020
  self._next_desk = time.monotonic() + (30 if "continuing" in outcome.values() else 0.25)
 
1655
  _health(cid, "freshness", "failed",
1656
  failure={**failure, "stage": "freshness"})
1657
  else:
1658
+ if str(cid) != str(default_cid):
1659
+ # A named connection's rebuild is its pull. The default's source stamp is the published mirror pass.
1660
+ _health(cid, "source", "ready")
1661
  _health(cid, "materialization", "ready")
1662
  # A receipt is the authoritative per-database result. ``tables`` alone omits
1663
  # read-through grains and the two compiled canonical databases, which previously
 
2152
  _resync_log(f"failed after {_pass_clock.monotonic() - pass_started:.1f}s ({type(e).__name__})")
2153
  # A partial report must never be mistaken for a confirmed zero-change pass.
2154
  return None, None, _classify_resync_error(e)
2155
+ finally:
2156
+ # The rebuild reuses this thread's client: a desk served there ran the Amazon first load inside it.
2157
+ _client = getattr(getattr(sys.modules.get("core.odoo"), "_tlocal", None), "client", None)
2158
+ if isinstance(_client, _DeskAwareOdooClient):
2159
+ _client.desk = False
2160
 
2161
 
2162
  def _absent_mirror_entities(ds, entities=None):
 
2281
  return {}
2282
 
2283
 
2284
+ def _resync_memory_write(pass_at, full_rebuild_at, fingerprints, pulls=None):
2285
  """Persist the loop's restart memory. Best effort: a failed write only costs a rebuild."""
2286
  try:
2287
  import odoo_relational as _rel_memory
2288
  from harness import runtime as _runtime_memory
2289
  _rel_memory.record_resync_memory(
2290
  _runtime_memory.get_runtime(_resync_tenant()[0]), pass_at=pass_at,
2291
+ full_rebuild_at=full_rebuild_at, fingerprints=fingerprints, pulls=pulls)
2292
  except Exception as e: # noqa: BLE001
2293
  print(f"[aios-api] store resync memory not saved ({type(e).__name__}); "
2294
  "the next restart rebuilds in full")
 
2336
  last_relational_success_at = _memory.get("fullRebuildAt")
2337
  last_source_fingerprints = _memory.get("fingerprints")
2338
  last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
2339
+ # The pulls an unfinished pass already made; its first pass after this boot resumes it.
2340
+ pending_pulls = _memory.get("pulls")
2341
  catch_up_retries = 0
2342
  last_meta_fingerprint = None
2343
  last_meta_applied_at = None
 
2474
  RESYNC_STATE["consecutiveFailures"] = 0
2475
  if not _manual:
2476
  RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
2477
+ # A published pull is the Odoo source stage's success, so the receipt's source stamp follows it.
2478
+ try:
2479
+ from harness import runtime as _source_runtime
2480
+ import routes_connectors as _source_receipts
2481
+ _source_receipts.record_odoo_stage(
2482
+ _source_runtime.get_runtime(_cad["tenant"]), RESYNC_STATE["connectionId"] or "default",
2483
+ "source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
2484
+ except Exception: # noqa: BLE001
2485
+ pass # The resident mirror state above still reports this pass.
2486
  else:
2487
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
2488
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
 
2539
  # correctly. A connector that can only ever be established at boot is one bad boot away
2540
  # from being permanently absent.
2541
  # ⚠ Cheap when there is nothing to do: no token => one line and return.
2542
+ # ⛔⛔ HOTFIX sdlc74 #3: a restart RESUMES an unfinished pass and never repeats its pulls.
2543
+ # Each pull checkpoints when it returns. The first pass after a boot skips a pull that an
2544
+ # unfinished pass younger than one cadence already made, and runs only the rest.
2545
+ _pending = (pending_pulls if passes == 1 and not _odoo_requested
2546
+ and isinstance(pending_pulls, dict) else {})
2547
+ _resumed = ({stage: _pending[stage] for stage in ("meta", "amazon") if _pending.get(stage)}
2548
+ if _resync_boot_recent(_pending.get("startedAt"), _t.time(), _every) else {})
2549
+ _pulls = {"startedAt": last_full_pass_at}
2550
+ if _resumed:
2551
+ _pulls = dict(_resumed, startedAt=_pending["startedAt"])
2552
+ last_full_pass_at = _pulls["startedAt"]
2553
+ _age = max(0, int(_t.time() - last_full_pass_at))
2554
+ print(f"[aios-api] store resync: restarted {_age}s into a pass that never finished; "
2555
+ f"its {' and '.join(sorted(_resumed))} pull already reached the store, so that "
2556
+ f"vendor is not called again and only the rest of the pass runs now; the next "
2557
+ f"scheduled pass is due in {max(0, _every - _age)}s")
2558
+ if _resumed.get("meta"):
2559
+ _meta_pull_confirmed = False
2560
+ else:
2561
+ _meta_pull_confirmed = _pull_meta("resync")
2562
+ _pulls["meta"] = _t.time()
2563
+ _resync_memory_write(last_good_pass_at, last_relational_success_at,
2564
+ last_source_fingerprints, dict(_pulls))
2565
+ if _resumed.get("amazon"):
2566
+ _pull_amazon("boot", False)
2567
+ else:
2568
+ _pull_amazon("resync")
2569
+ _pulls["amazon"] = _t.time()
2570
+ _resync_memory_write(last_good_pass_at, last_relational_success_at,
2571
+ last_source_fingerprints, dict(_pulls))
2572
  _meta_now = _t.time()
2573
  _meta_reason = _meta_rebuild_force_reason(
2574
  _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
 
2660
  last_meta_fingerprint = None
2661
  last_meta_applied_at = None
2662
  # HOTFIX 2026-09-29: remember this pass for the next container (see the top of the loop).
2663
+ # A pass that did not end good keeps its pulls checkpoint, so a restart resumes it.
2664
+ _pass_good = _err is None and _rebuild_ok
2665
+ if _pass_good:
2666
+ # A resumed pass ended its pulls when they ran, so a restart never shifts the cadence.
2667
+ last_good_pass_at = (max(_pulls.get("meta") or 0, _pulls.get("amazon") or 0)
2668
+ if _resumed else _meta_now)
2669
  _resync_memory_write(last_good_pass_at, last_relational_success_at,
2670
+ last_source_fingerprints, None if _pass_good else dict(_pulls))
2671
 
2672
 
2673
  def _boot_app_migrations(why):
api/odoo_relational.py CHANGED
@@ -5216,6 +5216,8 @@ def record_sync_outcome(rt, ok, at=None):
5216
  #: * `passAt` the last scheduled pass that completed without a mirror error (ISO);
5217
  #: * `fullRebuildAt` the last FULL relational rebuild that succeeded (ISO), for the weekly recovery;
5218
  #: * `fingerprints` `{entity: [count, hash]}` as the last successful apply saw the mirror.
 
 
5219
  #: ⚠ The two numbers are STRINGS: `bit_xor(hash())` is an unsigned 64-bit value, and a JSON reader
5220
  #: that is not Python may round it. `read_resync_memory` turns them back into the int TUPLES the
5221
  #: loop compares, because a list never equals a tuple and every pass would read as changed.
@@ -5264,7 +5266,8 @@ def decode_fingerprints(stored):
5264
 
5265
  def read_resync_memory(rt):
5266
  """What the last container's resync loop knew, or empty values. Never raises."""
5267
- empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None}
 
5268
  try:
5269
  cur = rt.get(CONFIG_KEY) if rt is not None else None
5270
  except Exception: # noqa: BLE001
@@ -5272,14 +5275,17 @@ def read_resync_memory(rt):
5272
  cur = cur if isinstance(cur, dict) else {}
5273
  memory = cur.get(RESYNC_MEMORY_FIELD)
5274
  memory = memory if isinstance(memory, dict) else {}
 
5275
  return {"passAt": _memory_epoch(memory.get("passAt")),
5276
  "fullRebuildAt": _memory_epoch(memory.get("fullRebuildAt")),
5277
  "fingerprints": decode_fingerprints(memory.get("fingerprints")),
5278
- "lastGoodSyncAt": _memory_epoch(cur.get("lastGoodSyncAt"))}
 
 
5279
 
5280
 
5281
- def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints):
5282
- """Persist the loop's three facts exactly as it now holds them (None clears one)."""
5283
  def _iso(epoch):
5284
  if epoch is None:
5285
  return None
@@ -5287,6 +5293,8 @@ def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints):
5287
 
5288
  memory = {"passAt": _iso(pass_at), "fullRebuildAt": _iso(full_rebuild_at),
5289
  "fingerprints": encode_fingerprints(fingerprints)}
 
 
5290
 
5291
  def _up(cur):
5292
  cur = dict(cur) if isinstance(cur, dict) else {}
 
5216
  #: * `passAt` the last scheduled pass that completed without a mirror error (ISO);
5217
  #: * `fullRebuildAt` the last FULL relational rebuild that succeeded (ISO), for the weekly recovery;
5218
  #: * `fingerprints` `{entity: [count, hash]}` as the last successful apply saw the mirror.
5219
+ #: * `pulls` `{startedAt, meta, amazon}` (ISO) of a pass whose vendor pulls finished but
5220
+ #: whose end was never written; a restart inside the cadence resumes it.
5221
  #: ⚠ The two numbers are STRINGS: `bit_xor(hash())` is an unsigned 64-bit value, and a JSON reader
5222
  #: that is not Python may round it. `read_resync_memory` turns them back into the int TUPLES the
5223
  #: loop compares, because a list never equals a tuple and every pass would read as changed.
 
5266
 
5267
  def read_resync_memory(rt):
5268
  """What the last container's resync loop knew, or empty values. Never raises."""
5269
+ empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
5270
+ "pulls": None}
5271
  try:
5272
  cur = rt.get(CONFIG_KEY) if rt is not None else None
5273
  except Exception: # noqa: BLE001
 
5275
  cur = cur if isinstance(cur, dict) else {}
5276
  memory = cur.get(RESYNC_MEMORY_FIELD)
5277
  memory = memory if isinstance(memory, dict) else {}
5278
+ pulls = memory.get("pulls")
5279
  return {"passAt": _memory_epoch(memory.get("passAt")),
5280
  "fullRebuildAt": _memory_epoch(memory.get("fullRebuildAt")),
5281
  "fingerprints": decode_fingerprints(memory.get("fingerprints")),
5282
+ "lastGoodSyncAt": _memory_epoch(cur.get("lastGoodSyncAt")),
5283
+ "pulls": ({str(k): _memory_epoch(v) for k, v in pulls.items()}
5284
+ if isinstance(pulls, dict) and pulls else None)}
5285
 
5286
 
5287
+ def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
5288
+ """Persist the loop's facts exactly as it now holds them (None clears one)."""
5289
  def _iso(epoch):
5290
  if epoch is None:
5291
  return None
 
5293
 
5294
  memory = {"passAt": _iso(pass_at), "fullRebuildAt": _iso(full_rebuild_at),
5295
  "fingerprints": encode_fingerprints(fingerprints)}
5296
+ if isinstance(pulls, dict) and pulls.get("startedAt") is not None:
5297
+ memory["pulls"] = {str(k): _iso(v) for k, v in pulls.items() if v is not None}
5298
 
5299
  def _up(cur):
5300
  cur = dict(cur) if isinstance(cur, dict) else {}
api/routes_odoo_tables.py CHANGED
@@ -399,7 +399,7 @@ def _catch_up_status(session):
399
  try:
400
  status = mirror_store.catch_up_status(session.runtime)
401
  except Exception:
402
- return {"active": True, "done": 0, "total": 0, "newest": {}}
403
  status["newest"] = {key: value for key, value in status["newest"].items()
404
  if perm_scope.may_read(session.user, key, st=session.runtime)}
405
  return status
 
399
  try:
400
  status = mirror_store.catch_up_status(session.runtime)
401
  except Exception:
402
+ return {"active": True, "done": 0, "total": 0, "newest": {}, "sources": {}}
403
  status["newest"] = {key: value for key, value in status["newest"].items()
404
  if perm_scope.may_read(session.user, key, st=session.runtime)}
405
  return status
api/routes_sheet.py CHANGED
@@ -138,10 +138,36 @@ def _sheet_catch_up(session):
138
 
139
 
140
  def _database_catching_up(session, database):
141
- """Connected mirrors wait for C1; native authored tables remain readable."""
142
- if _sheet_catch_up(session).get("active") is not True:
 
 
 
143
  return False
144
- return _database_connected(session, database)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
 
146
 
147
  def _database_connected(session, database):
@@ -172,9 +198,14 @@ def _binding_catching_up(session, binding):
172
  if _sheet_catch_up(session).get("active") is not True:
173
  return False
174
  from modules import sheet as sheet_mod
175
- nodes, _ = _binding_filter_nodes(session, database, binding)
176
- return any(_database_catching_up(session, _binding_related_entry(session, database, via)["target"])
177
- for via in sheet_mod.related_vias(nodes))
 
 
 
 
 
178
 
179
 
180
  def _mirror(session: Session):
@@ -2626,27 +2657,58 @@ def read_sheet_activity(workbook_id: str, session: Session = Depends(require_ses
2626
  def read_sheet_catch_up(workbook_id: str, session: Session = Depends(require_session)):
2627
  """Scope C1 to the workbook, including tabs the browser has not loaded."""
2628
  from core import sheets
 
2629
  try:
2630
  book = sheets.get_workbook(_store(session), workbook_id)
2631
  _require_reader(session, book)
2632
  except sheets.SheetRefusal as exc:
2633
  raise _refuse(exc)
2634
  session._sheet_scope_context = {}
2635
- status = _sheet_catch_up(session)
2636
- if status.get("active") is True:
2637
- for tab in book.get("sheets") or []:
2638
- for cell in (tab.get("cells") or {}).values():
2639
- binding = cell.get("binding") or {}
2640
- if not binding.get("database"):
2641
- continue
2642
- try:
2643
- _require_database(session, binding["database"])
2644
- if _binding_catching_up(session, binding):
2645
- # C1's newest map can name unrelated databases. Return only progress.
2646
- return {"catchUp": {key: status.get(key) for key in ("active", "done", "total")}}
2647
- except HTTPException:
2648
- continue
2649
- return {"catchUp": {"active": False}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2650
 
2651
 
2652
  @router.get("/sheets/{workbook_id}/refresh-history")
 
138
 
139
 
140
  def _database_catching_up(session, database):
141
+ """Connected mirrors wait for their own source's C1; native authored tables remain readable."""
142
+ status = _sheet_catch_up(session)
143
+ if status.get("active") is not True:
144
+ return False
145
+ if not _database_connected(session, database):
146
  return False
147
+ # An unmapped database, or a status without per-source progress, waits for the tenant.
148
+ own = (status.get("sources") or {}).get(_database_source(session, database))
149
+ return not isinstance(own, dict) or own.get("active") is True
150
+
151
+
152
+ #: The connector name each catch-up source shows a person.
153
+ _SOURCE_LABELS = {"odoo": "Odoo", "amazon": "Amazon", "meta": "Meta Ads"}
154
+
155
+
156
+ def _database_source(session, database):
157
+ """Name the catch-up source a connected database waits on, or None when it is unknown."""
158
+ import odoo_relational as rel
159
+ from core import connector_spec
160
+ database = str(database or "")
161
+ if (database in rel._REGISTRY_GRID_FIELDS or database == rel.SHIPMENTS_KEY
162
+ or any(database == key for _, key, _, _ in rel.TABLES)):
163
+ return "odoo"
164
+ connector = connector_spec.connector_of(database) or next((name for prefix, name in (
165
+ ("ut_amzn_", "amazon"), ("ut_meta_", "meta"), ("ut_odoo_", "odoo"),
166
+ ) if database.startswith(prefix)), None)
167
+ if connector is None and database in {"customer_data", "product_data"}:
168
+ connector = "odoo"
169
+ connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
170
+ return connector if connector in _SOURCE_LABELS else None
171
 
172
 
173
  def _database_connected(session, database):
 
198
  if _sheet_catch_up(session).get("active") is not True:
199
  return False
200
  from modules import sheet as sheet_mod
201
+ try:
202
+ nodes, _ = _binding_filter_nodes(session, database, binding)
203
+ targets = [_binding_related_entry(session, database, via)["target"]
204
+ for via in sheet_mod.related_vias(nodes)]
205
+ except HTTPException:
206
+ # The cell's own read refuses an unreadable relation; this check never refuses the workbook.
207
+ return False
208
+ return any(_database_catching_up(session, target) for target in targets)
209
 
210
 
211
  def _mirror(session: Session):
 
2657
  def read_sheet_catch_up(workbook_id: str, session: Session = Depends(require_session)):
2658
  """Scope C1 to the workbook, including tabs the browser has not loaded."""
2659
  from core import sheets
2660
+ from modules import sheet as sheet_mod
2661
  try:
2662
  book = sheets.get_workbook(_store(session), workbook_id)
2663
  _require_reader(session, book)
2664
  except sheets.SheetRefusal as exc:
2665
  raise _refuse(exc)
2666
  session._sheet_scope_context = {}
2667
+ # One memo per read: without it every bound cell re-read the whole catalogue (64.9 s on a template).
2668
+ with _sheet_request_scope(session):
2669
+ status = _sheet_catch_up(session)
2670
+ sources = status.get("sources") if isinstance(status.get("sources"), dict) else {}
2671
+ behind = {name for name, entry in sources.items() if isinstance(entry, dict) and entry.get("behind")}
2672
+ held, bound, readable = set(), set(), {}
2673
+ if status.get("active") is True or behind:
2674
+ for tab in book.get("sheets") or []:
2675
+ for cell in (tab.get("cells") or {}).values():
2676
+ binding = cell.get("binding") or {}
2677
+ if not binding.get("database"):
2678
+ continue
2679
+ # The page polls every 2 s, so each distinct database is checked once per read.
2680
+ if binding["database"] not in readable:
2681
+ try:
2682
+ _require_database(session, binding["database"])
2683
+ readable[binding["database"]] = _database_source(session, binding["database"])
2684
+ except HTTPException:
2685
+ readable[binding["database"]] = HTTPException
2686
+ if readable[binding["database"]] is HTTPException:
2687
+ continue
2688
+ try:
2689
+ bound.add(readable[binding["database"]])
2690
+ if status.get("active") is True and _binding_catching_up(session, binding):
2691
+ database = str(binding["database"])
2692
+ nodes, _ = _binding_filter_nodes(session, database, binding)
2693
+ targets = [database] + [_binding_related_entry(session, database, via)["target"]
2694
+ for via in sheet_mod.related_vias(nodes)]
2695
+ held.update(_database_source(session, target) for target in targets
2696
+ if _database_catching_up(session, target))
2697
+ except HTTPException:
2698
+ continue
2699
+ if held:
2700
+ break # The first held cell answers, as before: one held template took 53 s per poll.
2701
+ if held:
2702
+ break
2703
+ if held:
2704
+ # C1's newest map can name unrelated databases. Return only the held sources' progress.
2705
+ if None in held or not held <= set(sources):
2706
+ return {"catchUp": {key: status.get(key) for key in ("active", "done", "total")}}
2707
+ return {"catchUp": {"active": True, "done": sum(sources[name].get("done", 0) for name in held),
2708
+ "total": sum(sources[name].get("total", 0) for name in held)}}
2709
+ lines = [{"source": _SOURCE_LABELS[name], "reason": sources[name]["behind"]}
2710
+ for name in _SOURCE_LABELS if name in behind and name in bound]
2711
+ return {"catchUp": {"active": False, "behind": lines} if lines else {"active": False}}
2712
 
2713
 
2714
  @router.get("/sheets/{workbook_id}/refresh-history")
api/test_odoo_rank_refresh.py CHANGED
@@ -484,7 +484,7 @@ class RankRefreshTests(unittest.TestCase):
484
 
485
  # ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
486
  def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
487
- errors=(), max_sleeps=1, frozen=False):
488
  """Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
489
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
490
  functions = {'_store_resync_loop', '_resync_rebuild_reason',
@@ -516,7 +516,7 @@ class RankRefreshTests(unittest.TestCase):
516
 
517
  def rebuild(why, previous, meta_state, **scope):
518
  events.append(('rebuild', scope.get('table_keys')))
519
- return True
520
 
521
  clock = SimpleNamespace(sleep=sleep, time=lambda: now,
522
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
@@ -583,9 +583,64 @@ class RankRefreshTests(unittest.TestCase):
583
  events, _ = self.run_daily_loop(
584
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
585
  'fingerprints': fps}, fps)
586
- self.assertEqual([e[0] for e in events], ['sync', 'meta', 'amazon', 'fresh', 'memory'])
587
- self.assertEqual(events[2], ('amazon', 'resync'))
588
- self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
589
 
590
  def test_persisted_fingerprints_skip_the_rebuild_after_a_restart(self):
591
  fps = {'res_partner': (3, 7)}
@@ -674,6 +729,14 @@ class RankRefreshTests(unittest.TestCase):
674
  memory = rel.read_resync_memory(rt)
675
  self.assertEqual((memory['passAt'], memory['fullRebuildAt'], memory['fingerprints']),
676
  (1_000.0, None, fps))
 
 
 
 
 
 
 
 
677
  self.assertEqual(rt.docs[rel.CONFIG_KEY]['syncEvery'], 'daily')
678
  self.assertIsNotNone(memory['lastGoodSyncAt'])
679
 
 
484
 
485
  # ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
486
  def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
487
+ errors=(), max_sleeps=1, frozen=False, rebuild_ok=True):
488
  """Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
489
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
490
  functions = {'_store_resync_loop', '_resync_rebuild_reason',
 
516
 
517
  def rebuild(why, previous, meta_state, **scope):
518
  events.append(('rebuild', scope.get('table_keys')))
519
+ return rebuild_ok
520
 
521
  clock = SimpleNamespace(sleep=sleep, time=lambda: now,
522
  strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
 
583
  events, _ = self.run_daily_loop(
584
  {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
585
  'fingerprints': fps}, fps)
586
+ self.assertEqual([e[0] for e in events],
587
+ ['sync', 'meta', 'memory', 'amazon', 'memory', 'fresh', 'memory'])
588
+ self.assertEqual(events[3], ('amazon', 'resync'))
589
+ self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
590
+
591
+ # ── HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" ─────────────────────
592
+ def test_each_pull_checkpoints_before_the_rebuild_and_a_bad_end_keeps_it(self):
593
+ old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
594
+ events, _ = self.run_daily_loop(
595
+ {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
596
+ new, rebuild_ok=False)
597
+ kinds = [e[0] for e in events]
598
+ self.assertEqual(kinds, ['sync', 'meta', 'memory', 'amazon', 'memory', 'rebuild', 'memory'])
599
+ # The checkpoints change none of the three facts: a death in the rebuild leaves them as
600
+ # they were, plus the pulls the pass already made.
601
+ self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
602
+ {'startedAt': 10_000.0, 'meta': 10_000.0}))
603
+ both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
604
+ self.assertEqual(events[4], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old, both))
605
+ self.assertEqual(events[-1][-1], both)
606
+
607
+ def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
608
+ old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
609
+ events, sleeps = self.run_daily_loop(
610
+ {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
611
+ 'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
612
+ 'amazon': 10_000.0 - 600}}, new)
613
+ # No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
614
+ # catches up, and the rebuild that never reached the store runs.
615
+ self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'rebuild', 'memory'])
616
+ self.assertEqual(events[1], ('amazon', 'boot', False))
617
+ self.assertIn('ut_odoo_orders', events[2][1])
618
+ # The pass ends good at the time its pulls ended, and its checkpoint is cleared.
619
+ self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
620
+ # The next scheduled pass is one cadence after the unfinished pass STARTED.
621
+ self.assertEqual(sleeps, [86400 - 3600])
622
+ self.assertIn('restarted 3600s into a pass that never finished', self.loop_output)
623
+
624
+ def test_restart_during_the_amazon_pull_repeats_only_amazon(self):
625
+ old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
626
+ events, _ = self.run_daily_loop(
627
+ {'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
628
+ 'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
629
+ self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'memory', 'rebuild', 'memory'])
630
+ self.assertEqual(events[1], ('amazon', 'resync'))
631
+ self.assertEqual(events[2][-1], {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
632
+ 'amazon': 10_000.0})
633
+
634
+ def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
635
+ """NC: the same checkpoint one cadence old runs the whole scheduled pass again"""
636
+ old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
637
+ events, _ = self.run_daily_loop(
638
+ {'passAt': 10_000.0 - 190_000, 'fullRebuildAt': 10_000.0 - 190_000, 'fingerprints': old,
639
+ 'pulls': {'startedAt': 10_000.0 - 86_400, 'meta': 10_000.0 - 86_000,
640
+ 'amazon': 10_000.0 - 84_000}}, new)
641
+ self.assertIn(('meta', 'resync'), events)
642
+ self.assertIn(('amazon', 'resync'), events)
643
+ self.assertNotIn(('amazon', 'boot', False), events)
644
 
645
  def test_persisted_fingerprints_skip_the_rebuild_after_a_restart(self):
646
  fps = {'res_partner': (3, 7)}
 
729
  memory = rel.read_resync_memory(rt)
730
  self.assertEqual((memory['passAt'], memory['fullRebuildAt'], memory['fingerprints']),
731
  (1_000.0, None, fps))
732
+ self.assertIsNone(memory['pulls'])
733
+ # HOTFIX sdlc74 #3: the pulls checkpoint round-trips, and the next write without it clears it.
734
+ pulls = {'startedAt': 2_000.0, 'meta': 2_100.0, 'amazon': 4_900.0}
735
+ rel.record_resync_memory(rt, pass_at=1_000.0, full_rebuild_at=None, fingerprints=fps,
736
+ pulls=pulls)
737
+ self.assertEqual(rel.read_resync_memory(rt)['pulls'], pulls)
738
+ rel.record_resync_memory(rt, pass_at=5_000.0, full_rebuild_at=None, fingerprints=fps)
739
+ self.assertIsNone(rel.read_resync_memory(rt)['pulls'])
740
  self.assertEqual(rt.docs[rel.CONFIG_KEY]['syncEvery'], 'daily')
741
  self.assertIsNotNone(memory['lastGoodSyncAt'])
742
 
platform/harness/amazon_client.py CHANGED
@@ -45,9 +45,13 @@ Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-con
45
  from __future__ import annotations
46
 
47
  import contextvars
 
48
  import json
 
49
  import random
50
  import re
 
 
51
  import threading
52
  import time
53
  from datetime import date, datetime, timezone, timedelta
@@ -1994,7 +1998,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
1994
  url = str(meta.get("url") or "")
1995
  if not url:
1996
  raise AmazonError(f"the {report_type} report document carried no download URL")
1997
- text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1998
  say(f" {report_type}: {len(text):,} chars")
1999
  return text, {"reportId": rid, "reportDocumentId": doc_id,
2000
  "dataStartTime": got.get("dataStartTime"),
@@ -2090,8 +2094,106 @@ def _workbook_as_tsv(raw):
2090
  return "\n".join(lines)
2091
 
2092
 
2093
- def _download_document(url, compression="", timeout=120):
2094
- """Fetch one presigned report document and decode it. -> text.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2095
 
2096
  ⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
2097
  string, which for an S3 presigned URL is the entire credential.
@@ -2103,11 +2205,20 @@ def _download_document(url, compression="", timeout=120):
2103
  # apply OPERATIONS here: its host and signature belong to Amazon's document download service.
2104
  req = urllib.request.Request(url, method="GET")
2105
  req.add_header("User-Agent", _UA)
 
2106
  try:
2107
  with urllib.request.urlopen(req, timeout=timeout) as r:
2108
- raw = r.read()
 
 
 
 
2109
  except Exception as e: # noqa: BLE001
 
 
2110
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
 
 
2111
  # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
2112
  # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
2113
  # algorithm the bytes were decoded as text and every economics window failed to parse. The
 
45
  from __future__ import annotations
46
 
47
  import contextvars
48
+ import gzip
49
  import json
50
+ import os
51
  import random
52
  import re
53
+ import shutil
54
+ import tempfile
55
  import threading
56
  import time
57
  from datetime import date, datetime, timezone, timedelta
 
1998
  url = str(meta.get("url") or "")
1999
  if not url:
2000
  raise AmazonError(f"the {report_type} report document carried no download URL")
2001
+ text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
2002
  say(f" {report_type}: {len(text):,} chars")
2003
  return text, {"reportId": rid, "reportDocumentId": doc_id,
2004
  "dataStartTime": got.get("dataStartTime"),
 
2094
  return "\n".join(lines)
2095
 
2096
 
2097
+ class ReportDocument:
2098
+ """One downloaded report document spooled to a temporary file, read back as text in pieces.
2099
+
2100
+ ⛔ A report has no row cap (standing rule 1), so its text never sits in memory whole: the Brand
2101
+ Analytics month held the body, its decompressed copy and its decoded text at once (D-7081).
2102
+ The file is removed when the object is closed or collected.
2103
+ """
2104
+
2105
+ def __init__(self, path, encoding, errors, chars, size, compressed):
2106
+ self.path, self.encoding, self.errors = path, encoding, errors
2107
+ self.chars, self.size, self.compressed = chars, size, compressed
2108
+
2109
+ def open_text(self):
2110
+ """The decoded text as a stream; `newline=""` keeps line ends exactly as `bytes.decode` did."""
2111
+ return open(self.path, "r", encoding=self.encoding, errors=self.errors, newline="")
2112
+
2113
+ def read_text(self):
2114
+ """The whole text, for a consumer whose document is small."""
2115
+ with self.open_text() as handle:
2116
+ return handle.read()
2117
+
2118
+ def __len__(self):
2119
+ return self.chars
2120
+
2121
+ def close(self):
2122
+ path, self.path = getattr(self, "path", None), None
2123
+ if path:
2124
+ try:
2125
+ os.unlink(path)
2126
+ except Exception: # noqa: BLE001
2127
+ pass
2128
+
2129
+ def __del__(self):
2130
+ self.close()
2131
+
2132
+
2133
+ _SPOOL_CHUNK = 1 << 20
2134
+
2135
+
2136
+ def _spool_file(prefix):
2137
+ handle, path = tempfile.mkstemp(prefix=prefix, suffix=".part")
2138
+ os.close(handle)
2139
+ return path
2140
+
2141
+
2142
+ def _unlink(path):
2143
+ try:
2144
+ os.unlink(path)
2145
+ except OSError:
2146
+ pass
2147
+
2148
+
2149
+ def _decoded_chars(path, encoding, errors="strict"):
2150
+ """Characters in the file under one encoding; raises UnicodeDecodeError like `bytes.decode`."""
2151
+ chars = 0
2152
+ with open(path, "r", encoding=encoding, errors=errors, newline="") as handle:
2153
+ while True:
2154
+ piece = handle.read(_SPOOL_CHUNK)
2155
+ if not piece:
2156
+ return chars
2157
+ chars += len(piece)
2158
+
2159
+
2160
+ def _spooled_document(body, compression):
2161
+ """The spooled body as a `ReportDocument`: gunzip and decode stream through disk, never memory."""
2162
+ compressed = os.path.getsize(body)
2163
+ with open(body, "rb") as handle:
2164
+ head = handle.read(4)
2165
+ if not compression and head[:2] == b"\x1f\x8b":
2166
+ compression = "GZIP"
2167
+ if str(compression).upper() == "GZIP":
2168
+ plain = _spool_file("amazon-document-")
2169
+ try:
2170
+ with gzip.open(body, "rb") as source, open(plain, "wb") as out:
2171
+ shutil.copyfileobj(source, out, _SPOOL_CHUNK)
2172
+ except Exception as e: # noqa: BLE001
2173
+ _unlink(plain)
2174
+ raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
2175
+ finally:
2176
+ _unlink(body)
2177
+ body = plain
2178
+ with open(body, "rb") as handle:
2179
+ head = handle.read(4)
2180
+ if head[:4] == b"PK\x03\x04":
2181
+ try:
2182
+ with open(body, "rb") as handle:
2183
+ return _workbook_as_tsv(handle.read())
2184
+ finally:
2185
+ _unlink(body)
2186
+ size = os.path.getsize(body)
2187
+ for enc in ("utf-8", "cp1252"):
2188
+ try:
2189
+ return ReportDocument(body, enc, "strict", _decoded_chars(body, enc), size, compressed)
2190
+ except UnicodeDecodeError:
2191
+ continue
2192
+ return ReportDocument(body, "utf-8", "replace", _decoded_chars(body, "utf-8", "replace"), size, compressed)
2193
+
2194
+
2195
+ def _download_document(url, compression="", timeout=120, *, spool=False):
2196
+ """Fetch one presigned report document and decode it. -> text, or a `ReportDocument` when spooled.
2197
 
2198
  ⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
2199
  string, which for an S3 presigned URL is the entire credential.
 
2205
  # apply OPERATIONS here: its host and signature belong to Amazon's document download service.
2206
  req = urllib.request.Request(url, method="GET")
2207
  req.add_header("User-Agent", _UA)
2208
+ body = _spool_file("amazon-document-") if spool else None
2209
  try:
2210
  with urllib.request.urlopen(req, timeout=timeout) as r:
2211
+ if body:
2212
+ with open(body, "wb") as out:
2213
+ shutil.copyfileobj(r, out, _SPOOL_CHUNK)
2214
+ else:
2215
+ raw = r.read()
2216
  except Exception as e: # noqa: BLE001
2217
+ if body:
2218
+ _unlink(body)
2219
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
2220
+ if body:
2221
+ return _spooled_document(body, compression)
2222
  # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
2223
  # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
2224
  # algorithm the bytes were decoded as text and every economics window failed to parse. The
platform/harness/amazon_store.py CHANGED
@@ -228,6 +228,10 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
228
  #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
229
  #: thread for hours and the Odoo resync behind it would never run.
230
  FIRST_LOAD_REPORTS_PER_PASS = 15
 
 
 
 
231
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
232
  FIRST_LOAD_END_KEY = "sync_first_load_end"
233
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
@@ -327,6 +331,234 @@ def _without_buyer_data(value):
327
  return value
328
 
329
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330
  def parse_flat_file(text, *, document_id=None, report_type=None):
331
  """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
332
 
@@ -339,29 +571,15 @@ def parse_flat_file(text, *, document_id=None, report_type=None):
339
  swallow the rest of the row and silently merge it with the next one, so a catalogue with one
340
  inch-mark in a title would lose rows with nothing raising.
341
  """
 
 
342
  if not str(text or "").strip():
343
  return [], []
344
  buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
345
- reader = csv.reader(buf, delimiter="\t", quoting=csv.QUOTE_NONE)
346
- try:
347
- header = next(reader)
348
- except StopIteration:
349
  return [], []
350
- cols = _dedupe([_snake(h) for h in header])
351
- width = len(cols)
352
- rows = []
353
- # The read-only connectors_amazon report probe measures headers without capture metadata.
354
- # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
355
- # BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
356
- attributed = document_id is not None or report_type is not None
357
- for row_number, raw in enumerate(reader, 1):
358
- if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
359
- continue
360
- vals = list(raw[:width]) + [None] * max(0, width - len(raw))
361
- row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
362
- if not buyer_column(c)}
363
- rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
364
- return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows
365
 
366
 
367
  def parse_json_report(text, root, *, document_id=None, report_type=None):
@@ -372,6 +590,8 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
372
  with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
373
  artifact is the authority; a field the payload omits is NULL.
374
  """
 
 
375
  try:
376
  doc = json.loads(str(text or "") or "{}")
377
  except (ValueError, TypeError) as exc:
@@ -387,12 +607,151 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
387
  body = [body]
388
  if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
389
  raise ValueError(f"Amazon JSON report has an invalid {root} row list")
390
- rows = []
 
 
 
 
 
391
  attributed = document_id is not None or report_type is not None
392
  for row_number, item in enumerate(body, 1):
393
  row = _flatten_row(_without_buyer_data(item))
394
- rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
395
- return sorted({k for r in rows for k in r}), rows
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
396
 
397
 
398
  def _flatten_row(obj, prefix=""):
@@ -601,6 +960,8 @@ def _awd_rows(inventory, operation_id=None):
601
 
602
  def _tag_fba_inventory(rows):
603
  """Mark report rows so one combined inventory table never conflates FBA with AWD."""
 
 
604
  for row in rows or ():
605
  row["inventory_source"] = "FBA"
606
  return rows
@@ -618,7 +979,15 @@ def _pricing_asins(rows):
618
 
619
 
620
  def _apply_competitive(con, table, responses):
621
- """Add every competitive response field to all matching Sales and Traffic ASIN rows."""
 
 
 
 
 
 
 
 
622
  rows = _competitive_rows(responses)
623
  if not rows:
624
  return 0, 0
@@ -628,13 +997,15 @@ def _apply_competitive(con, table, responses):
628
  raise ValueError("Pricing mirror has no ASIN column; sync Sales and Traffic before price enrichment")
629
  types = _ensure(con, table, cols, rows)
630
  assignments = ", ".join(f'"{c}"=?' for c in cols)
 
 
 
631
  updated = 0
632
  for row in rows:
633
  asin = row["competitive_body_asin"]
634
  values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
635
- result = con.execute(
636
- f'UPDATE {table} SET {assignments} WHERE ' + " OR ".join(f'"{c}"=?' for c in match_columns),
637
- values + [asin] * len(match_columns))
638
  try:
639
  updated += int(result.fetchone()[0])
640
  except Exception: # noqa: BLE001
@@ -680,36 +1051,18 @@ def _coltype(name, rows=None):
680
  return "VARCHAR"
681
  if not rows:
682
  return "VARCHAR"
 
 
683
  seen, kinds = 0, set()
684
  for r in rows:
685
  v = r.get(name)
686
  if v is None or v == "":
687
  continue
688
- if isinstance(v, bool):
689
- kinds.add("str")
690
- elif isinstance(v, int):
691
- kinds.add("int")
692
- elif isinstance(v, float):
693
- kinds.add("float")
694
- elif isinstance(v, (list, dict)):
695
- kinds.add("str")
696
- else:
697
- s = str(v).strip()
698
- try:
699
- int(s)
700
- kinds.add("int")
701
- except ValueError:
702
- try:
703
- float(s)
704
- kinds.add("float")
705
- except ValueError:
706
- kinds.add("str")
707
  seen += 1
708
  if seen >= 500:
709
  break
710
- if not kinds or "str" in kinds:
711
- return "VARCHAR"
712
- return "DOUBLE" if "float" in kinds else "BIGINT"
713
 
714
 
715
  def _cell(value, kind):
@@ -788,11 +1141,27 @@ def _replace(con, table, cols, types, rows):
788
  or type(r.get("_report_row_number")) is not int
789
  or r["_report_row_number"] < 1 for r in rows):
790
  raise ValueError("refusing Amazon rows without complete source attribution")
791
- ph = ", ".join("?" for _ in cols)
792
- con.executemany(
793
- f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES ({ph})',
794
- [[_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols] for r in rows])
795
- return len(rows)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
796
 
797
 
798
  def replace_keys_for(table):
@@ -812,28 +1181,26 @@ def replace_window(con, table, cols, types, rows, keys):
812
  return _replace(con, table, cols, types, rows), None
813
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
814
  target = quote(table)
815
- temporary = quote("amazon_window_keys_" + uuid.uuid4().hex)
816
- columns = ", ".join(quote(c) for c in keys)
817
- # Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree.
818
- incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR"))
819
- for c in keys) for row in rows))
820
  con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
821
  try:
822
- con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
823
- incoming)
824
  predicate = " AND ".join(
825
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
826
  con.execute("BEGIN TRANSACTION")
827
  try:
 
828
  con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
829
- count = _replace(con, table, cols, types, rows)
830
  con.execute("COMMIT")
831
  except Exception:
832
  con.execute("ROLLBACK")
833
  raise
834
  return count, None
835
  finally:
836
- con.execute(f"DROP TABLE {temporary}")
837
 
838
 
839
  def _windows(start, end, size=MAX_WINDOW_DAYS):
@@ -1179,6 +1546,7 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
1179
  return 0
1180
 
1181
  # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
 
1182
  doc = json.loads(text)
1183
  raw = doc.get(spec["json_root"])
1184
  if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
@@ -1741,6 +2109,58 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
1741
  def _connect(self, tenant_key, rt):
1742
  return _connect_tenant_mirror(tenant_key, rt)
1743
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1744
  def _before_restore(self, tenant_key, rt):
1745
  obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
1746
  if obligation["earliestTriggerAt"]:
@@ -2685,6 +3105,16 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
2685
 
2686
  def document_row_count(text, roots=()):
2687
  """Count source records before flattening, privacy filtering or mirror transformations."""
 
 
 
 
 
 
 
 
 
 
2688
  if roots:
2689
  doc = json.loads(text)
2690
  return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
@@ -2729,23 +3159,31 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
2729
  stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
2730
  if not stamp and (table == "listings" or not rows):
2731
  raise ValueError("Amazon snapshot returned no data date or report creation date")
2732
- counts, identities = {}, set()
2733
- for row in rows:
2734
  period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
2735
  row["_snapshot_date"] = period
2736
  if table == "inventory":
2737
  row["date"] = period
2738
  row["inventory_source"] = "FBA"
 
 
 
 
 
 
 
2739
  identity = _snapshot_identity(table, row)
2740
  if identity in identities:
2741
  raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
2742
  identities.add(identity)
2743
  counts[period] = counts.get(period, 0) + 1
2744
- if len(rows) != source_count:
 
2745
  raise ValueError(f"Amazon {table} snapshot row count differs from its document")
2746
  if not counts:
2747
  counts[_utc(stamp).date().isoformat()] = 0
2748
- columns = sorted(set(columns) | {"_snapshot_date"} | {k for row in rows for k in row})
 
2749
  if table == "inventory":
2750
  columns = sorted(set(columns) | {"inventory_source"})
2751
  mirror = table_name(table)
@@ -2892,8 +3330,9 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2892
  if options:
2893
  kwargs["options"] = options
2894
  text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
 
2895
  parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
2896
- if spec.get("json_root") and meta.get("cancelled") and not str(text or "").strip():
2897
  # ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
2898
  # returns an empty document for it, S66-T42), but the JSON parser read "" as a document
2899
  # with no row root and REFUSED it. So the two B2B opportunity reports never earned a
@@ -2903,14 +3342,14 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2903
  elif spec.get("json_root"):
2904
  columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
2905
  if spec.get("json_extra_roots"):
2906
- for row in rows:
2907
- row["_json_root"] = spec["json_root"]
2908
  for root in spec["json_extra_roots"]:
2909
  extra_columns, extra = parse_json_report(text, root, **parser_args)
2910
- for row in extra:
2911
- row["_json_root"] = root
2912
  columns = merge_columns(columns, extra_columns)
2913
- rows.extend(extra)
 
2914
  else:
2915
  columns, rows = parse_flat_file(text, **parser_args)
2916
  if table in LEGACY_SNAPSHOT_TABLES:
@@ -2924,7 +3363,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2924
  period, column = _utc(w0).strftime("%Y-%m"), "month"
2925
  else:
2926
  period, column = _iso(w0), "_period_start"
2927
- for row in rows:
2928
  row[column] = period
2929
  if w1 is not None:
2930
  row["_period_end"] = _iso(w1)
@@ -2932,19 +3371,31 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
2932
  row["fulfillment_center"] = (row.get("fulfillment_center") or
2933
  row.get("fulfillment_center_id") or
2934
  row.get("fulfilment_center") or "all")
2935
- columns = sorted(set(columns) | {k for row in rows for k in row})
 
 
 
 
2936
  roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
2937
  # HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
2938
- empty = bool(roots) and ((meta.get("cancelled") and not str(text or "").strip()) or
2939
  (table in ("b2b_recommended", "b2b_not_yet_on_amazon")
2940
- and isinstance(json.loads(text or "{}"), dict)
2941
- and spec["json_root"] not in json.loads(text or "{}")))
2942
  marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
2943
  "column": column, "count": 0 if empty else document_row_count(text, roots)}
2944
  return _replace_period(con, table, rows, columns, column, period, marker=marker,
2945
  snapshot=bool(spec.get("snapshot")))
2946
 
2947
 
 
 
 
 
 
 
 
 
2948
  def _order_metric_totals(records, *, flattened=False):
2949
  totals = {"orders": 0, "units": 0, "sales": {}}
2950
  for row in records:
@@ -2971,11 +3422,15 @@ def _validate_orders(con):
2971
  if not marker.get("count"):
2972
  return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"}
2973
  expected = marker["metrics"]["orders"]
 
 
 
 
2974
  actual = con.execute(
2975
  "SELECT COUNT(DISTINCT amazon_order_id) FROM amzn_orders "
2976
  "WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) "
2977
  "AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ) "
2978
- "AND lower(COALESCE(order_status, '')) NOT IN ('cancelled', 'canceled')",
2979
  [marker["start"], marker["end"]]).fetchone()[0]
2980
  return {"state": "ok" if expected == actual else "refused",
2981
  "expected": expected, "actual": actual,
@@ -3098,9 +3553,34 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
3098
  expected.append("settlements")
3099
  with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
3100
  report.update(_sync_candidate(tenant_key, **options))
 
3101
  return report
3102
 
3103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3104
  def _sync_credential(tenant_key, rt):
3105
  """Refuse unusable credentials before storage or vendor work at either sync entry."""
3106
  fields = credential(tenant_key, rt)
@@ -3222,8 +3702,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3222
  or table in completed
3223
  or (table == "settlements" and not refresh_loaded)]
3224
  if scheduled and not first_load:
3225
- plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
3226
- for table in TABLES}, want, now=end)
 
 
 
 
 
 
 
3227
  want = plan["tables"]
3228
  for table, reason in plan["deferred"].items():
3229
  report["tables"][table] = {"state": "deferred", "reason": reason,
@@ -3261,7 +3748,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3261
  return count
3262
 
3263
  work, plans, paired, contexts = [], {}, {}, {}
3264
- documents, settlement_remaining = 0, 0
3265
  failed, landed = set(), set()
3266
  pending_reports = set()
3267
  sqp_asins, sqp_limit = [], None
@@ -3281,6 +3768,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3281
  "refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []),
3282
  "partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."}
3283
 
 
 
 
 
 
 
 
3284
  def sqp_problem(message):
3285
  report["problems"].append(message)
3286
  log(message)
@@ -3444,11 +3938,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3444
  settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
3445
  for item in discovered[issued_index:])
3446
  break
 
3447
  operation = "getReportDocument"
3448
  documents += 1
3449
  _sync_event("amazon.sync", tenant_key, table,
3450
  {"state": "access", "operation": operation}, rt, report, log)
3451
  text = cx.settlement_document(fields, document_id)
 
3452
  cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
3453
  count = 0
3454
  if cols and rows:
@@ -3457,6 +3953,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3457
  # A newly issued document still replaces its own settlement's lines.
3458
  count = write_window(table, cols, types, rows)
3459
  written += count
 
3460
  # Record completion after the write, including header-only and blank files.
3461
  _record_state(con, entity_key("settlement_doc_" + document_id), "done",
3462
  count, issued.get("dataEndTime"))
@@ -3520,11 +4017,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3520
  _record_state(con, entity_key("limit_" + table), "done", detail="{}")
3521
  for w0, w1 in margin_windows:
3522
  # One detail window is one budget task, including its reconciliation query.
 
3523
  documents += 1
3524
  _sync_event("amazon.sync", tenant_key, table,
3525
  {"state": "access", "operation": operation}, rt, report, log)
3526
- written += _sync_margin_window(con, table, fields, w0, w1,
3527
- max_wait=max_wait, log=log)
 
 
 
3528
  landed_window = True
3529
  if table in LEGACY_SNAPSHOT_TABLES:
3530
  # The dated replacement already landed; AWD must not clear that
@@ -3544,6 +4045,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3544
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
3545
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
3546
  documents += 1
 
3547
  with _report_resume(
3548
  con, table, rtype, start=_iso(w0),
3549
  end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
@@ -3554,6 +4056,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3554
  _sync_event("amazon.sync", tenant_key, table,
3555
  {"state": "access", "operation": "report_document"}, rt, report, log)
3556
  text, meta = cx.report_document(fields, rtype, **resume_kwargs)
 
3557
  landed_window = True
3558
  if root:
3559
  parsed_cols, rows = parse_json_report(
@@ -3629,6 +4132,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3629
  if cx.REPORT_TYPES[table].get("monthly"):
3630
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
3631
  "count": len(rows)})
 
 
3632
  if first_load:
3633
  for reached in (table, sibling):
3634
  if reached:
@@ -3649,10 +4154,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
3649
  if table == "pricing":
3650
  if first_load:
3651
  # Completion can be in another process/pass than the recent ASIN report.
 
 
3652
  available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
3653
- pricing_asins = list(dict.fromkeys(pricing_asins + [str(row[0])
3654
- for col in ("child_asin", "parent_asin") if col in available
3655
- for row in con.execute(f'SELECT DISTINCT "{col}" FROM {mirror} WHERE "{col}" IS NOT NULL').fetchall()]))
 
 
3656
  state_key = ENTITY_STATE_KEYS[table]
3657
  if not pricing_asins:
3658
  _record_state(con, state_key, "unproven", 0,
@@ -3859,15 +4368,20 @@ def _dedupe_orders(rows):
3859
  lines by construction, so this drops line detail rather than losing an order. `order_items`
3860
  keeps every row, which is why both tables read the same document instead of costing two.
3861
  """
3862
- seen, out = set(), []
 
 
 
 
 
 
3863
  for r in rows:
3864
  oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
3865
  if oid and oid in seen:
3866
  continue
3867
  if oid:
3868
  seen.add(oid)
3869
- out.append(r)
3870
- return out
3871
 
3872
 
3873
  def validate(tenant_key="royal-imports", rt=None, log=print):
@@ -5541,6 +6055,12 @@ def group_a_selftest(log=print, check=None):
5541
  "('O3','2026-07-04T00:00:00Z','Cancelled')")
5542
  ck("orders independently reconcile Sales API counts and exclude cancelled orders",
5543
  red and _validate_orders(con)["state"] == "ok")
 
 
 
 
 
 
5544
  spec = cx.REPORT_TYPES["reimbursements"]
5545
  with patch.object(cx, "report_document", return_value=(
5546
  "sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})):
@@ -6038,6 +6558,28 @@ def selftest(log=print):
6038
  ck("NC-AZ19 competitive summary enriches every matching report row without appending",
6039
  response_count == 1 and updated == 2 and enriched == 2,
6040
  (response_count, updated, enriched))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6041
 
6042
  mem.execute("""CREATE TABLE _sync_state (
6043
  entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
@@ -6173,10 +6715,113 @@ def selftest(log=print):
6173
  ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
6174
  ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
6175
  ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
6176
- log(f"\n{'GREEN' if not fails else 'RED'}: {31 - len(fails)}/31")
 
 
 
 
6177
  return 0 if not fails else 1
6178
 
6179
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6180
  def main(argv=None):
6181
  ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
6182
  ap.add_argument("--sync", action="store_true")
 
228
  #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
229
  #: thread for hours and the Odoo resync behind it would never run.
230
  FIRST_LOAD_REPORTS_PER_PASS = 15
231
+ #: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
232
+ #: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
233
+ #: minutes each) never hold the sync desk for hours (D-7081).
234
+ FIRST_LOAD_ROWS_PER_PASS = 3_000_000
235
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
236
  FIRST_LOAD_END_KEY = "sync_first_load_end"
237
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
 
331
  return value
332
 
333
 
334
+ #: Rows per INSERT statement. A batch, not a cap: every row of a document is written (standing rule 1).
335
+ INSERT_BATCH_ROWS = 500
336
+ #: Bound parameters per INSERT statement, so a wide table takes fewer rows per statement.
337
+ INSERT_BATCH_VALUES = 30_000
338
+ #: Characters read per step of a spooled document.
339
+ DOCUMENT_READ_CHARS = 1 << 20
340
+
341
+
342
+ def _spooled(document):
343
+ """True for a `ReportDocument` the client spooled to disk; False for in-memory text."""
344
+ return hasattr(document, "open_text")
345
+
346
+
347
+ def _document_text(document):
348
+ """The whole text of a document, for a consumer whose documents are small."""
349
+ return document.read_text() if _spooled(document) else document
350
+
351
+
352
+ def _blank(document):
353
+ """`not str(text or "").strip()`, read in pieces when the document is spooled."""
354
+ if not _spooled(document):
355
+ return not str(document or "").strip()
356
+ with document.open_text() as handle:
357
+ while True:
358
+ piece = handle.read(DOCUMENT_READ_CHARS)
359
+ if not piece:
360
+ return True
361
+ if piece.strip():
362
+ return False
363
+
364
+
365
+ def _value_kind(v):
366
+ """One non-empty value's sniffed kind: `int`, `float` or `str`."""
367
+ if isinstance(v, bool):
368
+ return "str"
369
+ if isinstance(v, int):
370
+ return "int"
371
+ if isinstance(v, float):
372
+ return "float"
373
+ if isinstance(v, (list, dict)):
374
+ return "str"
375
+ s = str(v).strip()
376
+ try:
377
+ int(s)
378
+ return "int"
379
+ except ValueError:
380
+ try:
381
+ float(s)
382
+ return "float"
383
+ except ValueError:
384
+ return "str"
385
+
386
+
387
+ def _kinds_type(kinds):
388
+ if not kinds or "str" in kinds:
389
+ return "VARCHAR"
390
+ return "DOUBLE" if "float" in kinds else "BIGINT"
391
+
392
+
393
+ class _DocumentRows:
394
+ """Rows parsed again from a spooled report document on every pass, so no pass holds them all.
395
+
396
+ ⛔ D-7081: one list of every row of the Brand Analytics month took the Space to 16 GB. A caller
397
+ iterates, takes `len()` or tests truth; `map` and `pipe` replay a row change on every pass.
398
+ """
399
+
400
+ def __init__(self, produce, steps=()):
401
+ self._produce, self._steps = produce, tuple(steps)
402
+ self._count, self._sniff = None, None
403
+
404
+ def __iter__(self):
405
+ rows = self._produce()
406
+ for step in self._steps:
407
+ rows = step(rows)
408
+ return iter(rows)
409
+
410
+ def pipe(self, step):
411
+ return _DocumentRows(self._produce, self._steps + (step,))
412
+
413
+ def map(self, change):
414
+ return self.pipe(lambda rows: (change(row) for row in rows))
415
+
416
+ @staticmethod
417
+ def chain(*parts):
418
+ return _DocumentRows(lambda: (row for part in parts for row in part))
419
+
420
+ def _profile(self):
421
+ """One pass: the row count, every key, and each key's first 500 non-empty kinds."""
422
+ count, sniff = 0, {}
423
+ for row in self:
424
+ count += 1
425
+ for key, value in row.items():
426
+ entry = sniff.get(key)
427
+ if entry is None:
428
+ entry = sniff[key] = [0, set()]
429
+ if entry[0] >= 500 or value is None or value == "":
430
+ continue
431
+ entry[1].add(_value_kind(value))
432
+ entry[0] += 1
433
+ self._count, self._sniff = count, sniff
434
+
435
+ def keys(self):
436
+ if self._sniff is None:
437
+ self._profile()
438
+ return set(self._sniff)
439
+
440
+ def sniffed(self, name):
441
+ """`_coltype(name, rows)` for these rows, from the one profile pass."""
442
+ if self._sniff is None:
443
+ self._profile()
444
+ return _kinds_type(self._sniff.get(name, (0, ()))[1])
445
+
446
+ def __len__(self):
447
+ if self._count is None:
448
+ self._profile()
449
+ return self._count
450
+
451
+ def __bool__(self):
452
+ if self._count is not None:
453
+ return self._count > 0
454
+ rows = iter(self)
455
+ try:
456
+ return next(rows, None) is not None
457
+ finally:
458
+ getattr(rows, "close", lambda: None)()
459
+
460
+
461
+ def _rss_gb():
462
+ """This process's resident memory in GB, or None where /proc is absent."""
463
+ try:
464
+ with open("/proc/self/status", encoding="ascii") as handle:
465
+ for line in handle:
466
+ if line.startswith("VmRSS:"):
467
+ return int(line.split()[1]) / 1048576
468
+ except Exception: # noqa: BLE001
469
+ return None
470
+ return None
471
+
472
+
473
+ class _DocumentProbe:
474
+ """One server-log line per report document: table, bytes, rows, seconds and the memory peak.
475
+
476
+ ⚠ `print`, not `log`: the sync desk passes a silent log, so its first load logged nothing until a
477
+ pass ended and the document in flight at an out-of-memory kill was unknown (D-7081).
478
+ """
479
+
480
+ _lock = threading.Lock()
481
+ _active = None
482
+
483
+ def __init__(self, table, report_type):
484
+ self.table, self.report_type = table, report_type
485
+ self.started, self.size, self.peak = time.monotonic(), None, _rss_gb()
486
+ self._halt = threading.Event()
487
+ with _DocumentProbe._lock:
488
+ previous, _DocumentProbe._active = _DocumentProbe._active, self
489
+ if previous is not None:
490
+ previous._halt.set()
491
+ threading.Thread(target=self._sample, name="amazon-document-probe", daemon=True).start()
492
+
493
+ def _sample(self):
494
+ deadline = time.monotonic() + 6 * 3600
495
+ while not self._halt.wait(0.5) and time.monotonic() < deadline:
496
+ rss = _rss_gb()
497
+ if rss is not None and (self.peak is None or rss > self.peak):
498
+ self.peak = rss
499
+
500
+ @classmethod
501
+ def downloaded(cls, document):
502
+ probe = cls._active
503
+ if probe is not None and not probe._halt.is_set():
504
+ probe.size = (document.size if _spooled(document)
505
+ else len(str(document or "").encode("utf-8", "replace")))
506
+ probe._say("downloaded", None)
507
+
508
+ def finish(self, rows):
509
+ self._halt.set()
510
+ self._say("written", rows)
511
+
512
+ def _say(self, phase, rows):
513
+ rss = _rss_gb()
514
+ peak = max(x for x in (self.peak, rss, 0.0) if x is not None)
515
+ size = "unknown" if self.size is None else f"{self.size:,}"
516
+ print(f"[aios-api] amazon document {phase}: table {self.table}, {self.report_type}, "
517
+ f"{size} bytes, {'-' if rows is None else f'{rows:,}'} rows, "
518
+ f"{time.monotonic() - self.started:.0f}s, rss {rss or 0:.2f} GB, peak {peak:.2f} GB",
519
+ flush=True)
520
+
521
+
522
+ def _flat_rows(lines, document_id, report_type):
523
+ """`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line."""
524
+ reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)
525
+ try:
526
+ header = next(reader)
527
+ except StopIteration:
528
+ return None, ()
529
+ cols = _dedupe([_snake(h) for h in header])
530
+ width = len(cols)
531
+ # The read-only connectors_amazon report probe measures headers without capture metadata.
532
+ # Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
533
+ # BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
534
+ attributed = document_id is not None or report_type is not None
535
+
536
+ def rows():
537
+ for row_number, raw in enumerate(reader, 1):
538
+ if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
539
+ continue
540
+ vals = list(raw[:width]) + [None] * max(0, width - len(raw))
541
+ row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
542
+ if not buyer_column(c)}
543
+ yield _origin(row, document_id, row_number, report_type) if attributed else row
544
+ return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows()
545
+
546
+
547
+ def _parse_flat_document(document, *, document_id=None, report_type=None):
548
+ """`parse_flat_file` over a spooled document: the header now, the rows again on every pass."""
549
+ if _blank(document):
550
+ return [], []
551
+ with document.open_text() as handle:
552
+ cols, _rows = _flat_rows(handle, document_id, report_type)
553
+ if cols is None:
554
+ return [], []
555
+
556
+ def produce():
557
+ with document.open_text() as handle:
558
+ yield from _flat_rows(handle, document_id, report_type)[1]
559
+ return cols, _DocumentRows(produce)
560
+
561
+
562
  def parse_flat_file(text, *, document_id=None, report_type=None):
563
  """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
564
 
 
571
  swallow the rest of the row and silently merge it with the next one, so a catalogue with one
572
  inch-mark in a title would lose rows with nothing raising.
573
  """
574
+ if _spooled(text):
575
+ return _parse_flat_document(text, document_id=document_id, report_type=report_type)
576
  if not str(text or "").strip():
577
  return [], []
578
  buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
579
+ cols, rows = _flat_rows(buf, document_id, report_type)
580
+ if cols is None:
 
 
581
  return [], []
582
+ return cols, list(rows)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
583
 
584
 
585
  def parse_json_report(text, root, *, document_id=None, report_type=None):
 
590
  with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
591
  artifact is the authority; a field the payload omits is NULL.
592
  """
593
+ if _spooled(text):
594
+ return _parse_json_document(text, root, document_id=document_id, report_type=report_type)
595
  try:
596
  doc = json.loads(str(text or "") or "{}")
597
  except (ValueError, TypeError) as exc:
 
607
  body = [body]
608
  if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
609
  raise ValueError(f"Amazon JSON report has an invalid {root} row list")
610
+ rows = list(_json_rows(body, document_id, report_type))
611
+ return sorted({k for r in rows for k in r}), rows
612
+
613
+
614
+ def _json_rows(body, document_id, report_type):
615
+ """Flatten and attribute each row object, numbering from one."""
616
  attributed = document_id is not None or report_type is not None
617
  for row_number, item in enumerate(body, 1):
618
  row = _flatten_row(_without_buyer_data(item))
619
+ yield _origin(row, document_id, row_number, report_type) if attributed else row
620
+
621
+
622
+ class _JsonFallback(Exception):
623
+ """A shape only the whole-text parser answers exactly: malformed, not an object, a repeated root."""
624
+
625
+
626
+ _JSON_WS = re.compile(r"[ \t\n\r]*")
627
+ _JSON_DECODER = json.JSONDecoder()
628
+ #: A single value may span this many characters before the stream hands it to `json.loads`.
629
+ _JSON_VALUE_CHARS = 256 << 20
630
+
631
+
632
+ class _JsonStream:
633
+ """JSON read from a text stream one value at a time, with `json.loads` value semantics."""
634
+
635
+ def __init__(self, handle):
636
+ self.handle, self.buf, self.pos = handle, "", 0
637
+
638
+ def _fill(self):
639
+ piece = self.handle.read(DOCUMENT_READ_CHARS)
640
+ if not piece:
641
+ return False
642
+ self.buf = self.buf[self.pos:] + piece
643
+ self.pos = 0
644
+ return True
645
+
646
+ def peek(self):
647
+ while True:
648
+ self.pos = _JSON_WS.match(self.buf, self.pos).end()
649
+ if self.pos < len(self.buf) or not self._fill():
650
+ return self.buf[self.pos:self.pos + 1]
651
+
652
+ def take(self, char):
653
+ if self.peek() != char:
654
+ raise _JsonFallback()
655
+ self.pos += 1
656
+
657
+ def value(self):
658
+ self.peek()
659
+ while True:
660
+ try:
661
+ obj, end = _JSON_DECODER.raw_decode(self.buf, self.pos)
662
+ except ValueError:
663
+ if len(self.buf) - self.pos < _JSON_VALUE_CHARS and self._fill():
664
+ continue
665
+ raise _JsonFallback() from None
666
+ # A number ending at the buffer's edge may continue in the next piece.
667
+ if end == len(self.buf) and self._fill():
668
+ continue
669
+ self.pos = end
670
+ return obj
671
+
672
+ def skip(self):
673
+ if self.peek() != "[":
674
+ self.value()
675
+ return
676
+ self.pos += 1
677
+ if self.peek() == "]":
678
+ self.pos += 1
679
+ return
680
+ while True:
681
+ self.skip()
682
+ char = self.peek()
683
+ self.pos += 1
684
+ if char == "]":
685
+ return
686
+ if char != ",":
687
+ raise _JsonFallback()
688
+
689
+ def items(self):
690
+ if self.peek() == "{":
691
+ yield self.value()
692
+ return
693
+ self.take("[")
694
+ if self.peek() == "]":
695
+ self.pos += 1
696
+ return
697
+ while True:
698
+ item = self.value()
699
+ if not isinstance(item, dict):
700
+ raise _JsonFallback()
701
+ yield item
702
+ char = self.peek()
703
+ self.pos += 1
704
+ if char == "]":
705
+ return
706
+ if char != ",":
707
+ raise _JsonFallback()
708
+
709
+
710
+ def _json_root_items(document, root):
711
+ """Each row object under the top-level `root` key of a spooled document, decoded one at a time.
712
+
713
+ Raises `_JsonFallback` for any shape `json.loads` must judge: malformed text, a document that is
714
+ not an object, a missing or repeated root, or a root that is not a list of objects.
715
+ """
716
+ with document.open_text() as handle:
717
+ stream, found = _JsonStream(handle), False
718
+ stream.take("{")
719
+ if stream.peek() == "}":
720
+ stream.pos += 1
721
+ else:
722
+ while True:
723
+ key = stream.value()
724
+ if not isinstance(key, str):
725
+ raise _JsonFallback()
726
+ stream.take(":")
727
+ if key == root:
728
+ if found:
729
+ raise _JsonFallback()
730
+ found = True
731
+ yield from stream.items()
732
+ else:
733
+ stream.skip()
734
+ char = stream.peek()
735
+ stream.pos += 1
736
+ if char == "}":
737
+ break
738
+ if char != ",":
739
+ raise _JsonFallback()
740
+ if stream.peek() or not found:
741
+ raise _JsonFallback()
742
+
743
+
744
+ def _parse_json_document(document, root, *, document_id=None, report_type=None):
745
+ """`parse_json_report` over a spooled document; any shape it cannot stream reads the whole text."""
746
+ rows = _DocumentRows(lambda: _json_rows(_json_root_items(document, root), document_id, report_type))
747
+ try:
748
+ keys = rows.keys()
749
+ except _JsonFallback:
750
+ return parse_json_report(document.read_text(), root, document_id=document_id,
751
+ report_type=report_type)
752
+ if not len(rows):
753
+ return [], []
754
+ return sorted(keys), rows
755
 
756
 
757
  def _flatten_row(obj, prefix=""):
 
960
 
961
  def _tag_fba_inventory(rows):
962
  """Mark report rows so one combined inventory table never conflates FBA with AWD."""
963
+ if isinstance(rows, _DocumentRows):
964
+ return _stamp_rows(rows, {"inventory_source": "FBA"})
965
  for row in rows or ():
966
  row["inventory_source"] = "FBA"
967
  return rows
 
979
 
980
 
981
  def _apply_competitive(con, table, responses):
982
+ """Replace the mirror's competitive fields with this pass's responses on matching ASIN rows.
983
+
984
+ ⛔ A row whose ASIN did not answer this pass is cleared, never left holding an older price
985
+ shown as current; `validate()` counts competitive ASINs against this pass's responses.
986
+ """
987
+ have = _columns_of(con, table)
988
+ stale = sorted(c for c in have if c.startswith("competitive_"))
989
+ if stale:
990
+ con.execute(f"UPDATE {table} SET " + ", ".join(f'"{c}"=NULL' for c in stale))
991
  rows = _competitive_rows(responses)
992
  if not rows:
993
  return 0, 0
 
997
  raise ValueError("Pricing mirror has no ASIN column; sync Sales and Traffic before price enrichment")
998
  types = _ensure(con, table, cols, rows)
999
  assignments = ", ".join(f'"{c}"=?' for c in cols)
1000
+ # A parent ASIN's summary prices only a row with no child ASIN, never that parent's SKU rows.
1001
+ where = ('"child_asin"=? OR ("child_asin" IS NULL AND "parent_asin"=?)' if len(match_columns) == 2
1002
+ else f'"{match_columns[0]}"=?')
1003
  updated = 0
1004
  for row in rows:
1005
  asin = row["competitive_body_asin"]
1006
  values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
1007
+ result = con.execute(f'UPDATE {table} SET {assignments} WHERE {where}',
1008
+ values + [asin] * len(match_columns))
 
1009
  try:
1010
  updated += int(result.fetchone()[0])
1011
  except Exception: # noqa: BLE001
 
1051
  return "VARCHAR"
1052
  if not rows:
1053
  return "VARCHAR"
1054
+ if isinstance(rows, _DocumentRows):
1055
+ return rows.sniffed(name)
1056
  seen, kinds = 0, set()
1057
  for r in rows:
1058
  v = r.get(name)
1059
  if v is None or v == "":
1060
  continue
1061
+ kinds.add(_value_kind(v))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1062
  seen += 1
1063
  if seen >= 500:
1064
  break
1065
+ return _kinds_type(kinds)
 
 
1066
 
1067
 
1068
  def _cell(value, kind):
 
1141
  or type(r.get("_report_row_number")) is not int
1142
  or r["_report_row_number"] < 1 for r in rows):
1143
  raise ValueError("refusing Amazon rows without complete source attribution")
1144
+ ph = "(" + ", ".join("?" for _ in cols) + ")"
1145
+ insert = f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES '
1146
+ per = max(1, min(INSERT_BATCH_ROWS, INSERT_BATCH_VALUES // len(cols)))
1147
+ full = insert + ", ".join([ph] * per)
1148
+
1149
+ def write(batch):
1150
+ con.execute(full if len(batch) == per else insert + ", ".join([ph] * len(batch)),
1151
+ [value for row in batch for value in row])
1152
+
1153
+ # Multi-row statements in batches: no copy of every row exists at once, the caller's transaction
1154
+ # keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did.
1155
+ count, batch = 0, []
1156
+ for r in rows:
1157
+ batch.append([_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols])
1158
+ if len(batch) == per:
1159
+ write(batch)
1160
+ count, batch = count + len(batch), []
1161
+ if batch:
1162
+ write(batch)
1163
+ count += len(batch)
1164
+ return count
1165
 
1166
 
1167
  def replace_keys_for(table):
 
1181
  return _replace(con, table, cols, types, rows), None
1182
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
1183
  target = quote(table)
1184
+ # The window's rows are staged in batches inside the transaction, typed as the mirror and
1185
+ # converted by the insert's own cells, so NULL and numeric keys agree and no key list exists.
1186
+ temporary = quote("amazon_window_rows_" + uuid.uuid4().hex)
1187
+ columns = ", ".join(quote(c) for c in cols)
 
1188
  con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
1189
  try:
 
 
1190
  predicate = " AND ".join(
1191
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
1192
  con.execute("BEGIN TRANSACTION")
1193
  try:
1194
+ count = _replace(con, temporary, cols, types, rows)
1195
  con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
1196
+ con.execute(f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}")
1197
  con.execute("COMMIT")
1198
  except Exception:
1199
  con.execute("ROLLBACK")
1200
  raise
1201
  return count, None
1202
  finally:
1203
+ con.execute(f"DROP TABLE IF EXISTS {temporary}")
1204
 
1205
 
1206
  def _windows(start, end, size=MAX_WINDOW_DAYS):
 
1546
  return 0
1547
 
1548
  # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
1549
+ text = _document_text(text)
1550
  doc = json.loads(text)
1551
  raw = doc.get(spec["json_root"])
1552
  if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
 
2109
  def _connect(self, tenant_key, rt):
2110
  return _connect_tenant_mirror(tenant_key, rt)
2111
 
2112
+ def _publication_holds(self, report, waiting):
2113
+ """Publish what a refresh pass completed; hold each unfinished table at its previous rows.
2114
+
2115
+ A first-load pass (it reports a state) keeps SDLC 73's whole-source publication. A problem no table owns
2116
+ holds the pass.
2117
+ """
2118
+ if report.get("state") in {"partial", "failed", "done"}:
2119
+ return None
2120
+ rows = report.get("tables") or {}
2121
+ unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
2122
+ unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
2123
+ and not _mirrors._table_sync_complete("amazon", row)}
2124
+ # Head and lines come from one document, so neither publishes without the other.
2125
+ pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}}
2126
+ partner = {pairs[name]: name for name in unfinished if name in pairs and pairs[name] not in unfinished}
2127
+ unfinished |= set(partner)
2128
+ owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))}
2129
+ blocked = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []
2130
+ if str(problem).split(":", 1)[0].strip() not in owners]
2131
+ held = {}
2132
+ for name in sorted(unfinished):
2133
+ if name in partner:
2134
+ held[name] = f"held with {partner[name]}, which reads the same report document"
2135
+ continue
2136
+ row = rows.get(name) if isinstance(rows.get(name), dict) else {}
2137
+ state = " ".join(str(part) for part in (row.get("state") or "not reached in this pass",
2138
+ row.get("amazonStatus")) if part)
2139
+ held[name] = state + (f": {row['reason']}" if row.get("reason") else "")
2140
+ return {"held": held, "tables": {table_name(name) for name in held}, "blocked": blocked}
2141
+
2142
+ def _held_state_rows(self, rows, tables):
2143
+ """A held table's data markers: watermark, floor, proofs, limits, role results and documents.
2144
+
2145
+ Its status, tick, refusal and pending-report markers describe the pass, so they publish with it.
2146
+ """
2147
+ tables = set(tables)
2148
+ names = sorted({table_name(t) for t in TABLES} | tables, key=lambda name: (-len(name), name))
2149
+
2150
+ def owner(entity):
2151
+ if entity.startswith(entity_key("settlement_doc_")):
2152
+ return table_name("settlements")
2153
+ for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_"):
2154
+ for name in names:
2155
+ marker = entity_key(family + name[len(PREFIX):])
2156
+ if entity == marker or entity.startswith(marker + "_"):
2157
+ return name
2158
+ if entity in names:
2159
+ return None
2160
+ return next((name for name in names if entity.startswith(name + "_")), None)
2161
+
2162
+ return [row for row in rows if owner(str(row[0])) in tables]
2163
+
2164
  def _before_restore(self, tenant_key, rt):
2165
  obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
2166
  if obligation["earliestTriggerAt"]:
 
3105
 
3106
  def document_row_count(text, roots=()):
3107
  """Count source records before flattening, privacy filtering or mirror transformations."""
3108
+ if _spooled(text):
3109
+ try:
3110
+ if roots:
3111
+ return sum(sum(1 for _item in _json_root_items(text, root)) for root in roots)
3112
+ with text.open_text() as handle:
3113
+ reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
3114
+ next(reader, None)
3115
+ return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader)
3116
+ except _JsonFallback:
3117
+ return document_row_count(text.read_text(), roots)
3118
  if roots:
3119
  doc = json.loads(text)
3120
  return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
 
3159
  stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
3160
  if not stamp and (table == "listings" or not rows):
3161
  raise ValueError("Amazon snapshot returned no data date or report creation date")
3162
+ def stamped(row):
 
3163
  period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
3164
  row["_snapshot_date"] = period
3165
  if table == "inventory":
3166
  row["date"] = period
3167
  row["inventory_source"] = "FBA"
3168
+ return row
3169
+
3170
+ lazy = isinstance(rows, _DocumentRows)
3171
+ rows = rows.map(stamped) if lazy else rows
3172
+ counts, identities, total = {}, set(), 0
3173
+ for row in rows:
3174
+ period = (row if lazy else stamped(row))["_snapshot_date"]
3175
  identity = _snapshot_identity(table, row)
3176
  if identity in identities:
3177
  raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
3178
  identities.add(identity)
3179
  counts[period] = counts.get(period, 0) + 1
3180
+ total += 1
3181
+ if total != source_count:
3182
  raise ValueError(f"Amazon {table} snapshot row count differs from its document")
3183
  if not counts:
3184
  counts[_utc(stamp).date().isoformat()] = 0
3185
+ columns = sorted(set(columns) | {"_snapshot_date"}
3186
+ | (rows.keys() if lazy else {k for row in rows for k in row}))
3187
  if table == "inventory":
3188
  columns = sorted(set(columns) | {"inventory_source"})
3189
  mirror = table_name(table)
 
3330
  if options:
3331
  kwargs["options"] = options
3332
  text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
3333
+ _DocumentProbe.downloaded(text)
3334
  parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
3335
+ if spec.get("json_root") and meta.get("cancelled") and _blank(text):
3336
  # ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
3337
  # returns an empty document for it, S66-T42), but the JSON parser read "" as a document
3338
  # with no row root and REFUSED it. So the two B2B opportunity reports never earned a
 
3342
  elif spec.get("json_root"):
3343
  columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
3344
  if spec.get("json_extra_roots"):
3345
+ rows = _stamp_rows(rows, {"_json_root": spec["json_root"]})
3346
+ parts = [rows]
3347
  for root in spec["json_extra_roots"]:
3348
  extra_columns, extra = parse_json_report(text, root, **parser_args)
3349
+ parts.append(_stamp_rows(extra, {"_json_root": root}))
 
3350
  columns = merge_columns(columns, extra_columns)
3351
+ rows = (_DocumentRows.chain(*parts) if any(isinstance(part, _DocumentRows) for part in parts)
3352
+ else [row for part in parts for row in part])
3353
  else:
3354
  columns, rows = parse_flat_file(text, **parser_args)
3355
  if table in LEGACY_SNAPSHOT_TABLES:
 
3363
  period, column = _utc(w0).strftime("%Y-%m"), "month"
3364
  else:
3365
  period, column = _iso(w0), "_period_start"
3366
+ def stamped(row):
3367
  row[column] = period
3368
  if w1 is not None:
3369
  row["_period_end"] = _iso(w1)
 
3371
  row["fulfillment_center"] = (row.get("fulfillment_center") or
3372
  row.get("fulfillment_center_id") or
3373
  row.get("fulfilment_center") or "all")
3374
+ return row
3375
+
3376
+ rows = rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
3377
+ columns = sorted(set(columns) | (rows.keys() if isinstance(rows, _DocumentRows)
3378
+ else {k for row in rows for k in row}))
3379
  roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
3380
  # HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
3381
+ empty = bool(roots) and ((meta.get("cancelled") and _blank(text)) or
3382
  (table in ("b2b_recommended", "b2b_not_yet_on_amazon")
3383
+ and isinstance(json.loads(_document_text(text) or "{}"), dict)
3384
+ and spec["json_root"] not in json.loads(_document_text(text) or "{}")))
3385
  marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
3386
  "column": column, "count": 0 if empty else document_row_count(text, roots)}
3387
  return _replace_period(con, table, rows, columns, column, period, marker=marker,
3388
  snapshot=bool(spec.get("snapshot")))
3389
 
3390
 
3391
+ def _stamp_rows(rows, values):
3392
+ """Set `values` on every row, on each pass when the rows are lazy."""
3393
+ def stamped(row):
3394
+ row.update(values)
3395
+ return row
3396
+ return rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
3397
+
3398
+
3399
  def _order_metric_totals(records, *, flattened=False):
3400
  totals = {"orders": 0, "units": 0, "sales": {}}
3401
  for row in records:
 
3422
  if not marker.get("count"):
3423
  return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"}
3424
  expected = marker["metrics"]["orders"]
3425
+ # getOrderMetrics counts the US marketplace only; the order report also carries CA, MX and MCF orders.
3426
+ columns = {row[0] for row in con.execute(
3427
+ "SELECT column_name FROM information_schema.columns WHERE table_name='amzn_orders'").fetchall()}
3428
+ channel = " AND sales_channel = 'Amazon.com'" if "sales_channel" in columns else ""
3429
  actual = con.execute(
3430
  "SELECT COUNT(DISTINCT amazon_order_id) FROM amzn_orders "
3431
  "WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) "
3432
  "AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ) "
3433
+ "AND lower(COALESCE(order_status, '')) NOT IN ('cancelled', 'canceled')" + channel,
3434
  [marker["start"], marker["end"]]).fetchone()[0]
3435
  return {"state": "ok" if expected == actual else "refused",
3436
  "expected": expected, "actual": actual,
 
3553
  expected.append("settlements")
3554
  with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
3555
  report.update(_sync_candidate(tenant_key, **options))
3556
+ _refuse_unreconciled(tenant_key, rt, report, log)
3557
  return report
3558
 
3559
 
3560
+ def _refuse_unreconciled(tenant_key, rt, report, log):
3561
+ """Standing rule 8: a table synced in this pass whose reconciliation refuses keeps its previous rows."""
3562
+ synced = [table for table, row in (report.get("tables") or {}).items()
3563
+ if isinstance(row, dict) and row.get("state", "done") == "done" and "rows" in row]
3564
+ if not synced or report.get("state") in {"partial", "failed", "done"}:
3565
+ return
3566
+ try:
3567
+ checked = validate(tenant_key, rt=rt, log=lambda *_: None)
3568
+ except Exception as exc: # noqa: BLE001
3569
+ report["problems"].append(f"Amazon reconciliation unavailable ({type(exc).__name__}); "
3570
+ "this pass publishes nothing new until it runs.")
3571
+ log(report["problems"][-1])
3572
+ return
3573
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
3574
+ for table in synced:
3575
+ if (checked.get(table) or {}).get("state") != "refused":
3576
+ continue
3577
+ reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
3578
+ report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
3579
+ report["problems"].append(f"{table}: {reason}")
3580
+ _record_state(con, entity_key(table), "refused", 0, reason)
3581
+ log(f"{table}: {reason}")
3582
+
3583
+
3584
  def _sync_credential(tenant_key, rt):
3585
  """Refuse unusable credentials before storage or vendor work at either sync entry."""
3586
  fields = credential(tenant_key, rt)
 
3702
  or table in completed
3703
  or (table == "settlements" and not refresh_loaded)]
3704
  if scheduled and not first_load:
3705
+ ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
3706
+ # A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
3707
+ pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}}
3708
+ for table in list(want):
3709
+ mark = _read_state(con, entity_key(table)) or {}
3710
+ if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
3711
+ and not _interval_reason(table, ticks.get(table), end)):
3712
+ ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
3713
+ plan = sync_plan(ticks, want, now=end)
3714
  want = plan["tables"]
3715
  for table, reason in plan["deferred"].items():
3716
  report["tables"][table] = {"state": "deferred", "reason": reason,
 
3748
  return count
3749
 
3750
  work, plans, paired, contexts = [], {}, {}, {}
3751
+ documents, settlement_remaining, pass_rows = 0, 0, 0
3752
  failed, landed = set(), set()
3753
  pending_reports = set()
3754
  sqp_asins, sqp_limit = [], None
 
3768
  "refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []),
3769
  "partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."}
3770
 
3771
+ def spend_rows(rows):
3772
+ """A first-load pass past its row bound spends its document budget; the next pass continues."""
3773
+ nonlocal documents, pass_rows
3774
+ pass_rows += rows
3775
+ if first_load and isinstance(report_budget, int) and pass_rows >= FIRST_LOAD_ROWS_PER_PASS:
3776
+ documents = max(documents, report_budget)
3777
+
3778
  def sqp_problem(message):
3779
  report["problems"].append(message)
3780
  log(message)
 
3938
  settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
3939
  for item in discovered[issued_index:])
3940
  break
3941
+ probe = _DocumentProbe(table, rtype)
3942
  operation = "getReportDocument"
3943
  documents += 1
3944
  _sync_event("amazon.sync", tenant_key, table,
3945
  {"state": "access", "operation": operation}, rt, report, log)
3946
  text = cx.settlement_document(fields, document_id)
3947
+ _DocumentProbe.downloaded(text)
3948
  cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
3949
  count = 0
3950
  if cols and rows:
 
3953
  # A newly issued document still replaces its own settlement's lines.
3954
  count = write_window(table, cols, types, rows)
3955
  written += count
3956
+ probe.finish(count)
3957
  # Record completion after the write, including header-only and blank files.
3958
  _record_state(con, entity_key("settlement_doc_" + document_id), "done",
3959
  count, issued.get("dataEndTime"))
 
4017
  _record_state(con, entity_key("limit_" + table), "done", detail="{}")
4018
  for w0, w1 in margin_windows:
4019
  # One detail window is one budget task, including its reconciliation query.
4020
+ probe = _DocumentProbe(table, spec.get("report") or operation)
4021
  documents += 1
4022
  _sync_event("amazon.sync", tenant_key, table,
4023
  {"state": "access", "operation": operation}, rt, report, log)
4024
+ count = _sync_margin_window(con, table, fields, w0, w1,
4025
+ max_wait=max_wait, log=log)
4026
+ probe.finish(count)
4027
+ spend_rows(count)
4028
+ written += count
4029
  landed_window = True
4030
  if table in LEGACY_SNAPSHOT_TABLES:
4031
  # The dated replacement already landed; AWD must not clear that
 
4045
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
4046
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
4047
  documents += 1
4048
+ probe = _DocumentProbe(table, rtype)
4049
  with _report_resume(
4050
  con, table, rtype, start=_iso(w0),
4051
  end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
 
4056
  _sync_event("amazon.sync", tenant_key, table,
4057
  {"state": "access", "operation": "report_document"}, rt, report, log)
4058
  text, meta = cx.report_document(fields, rtype, **resume_kwargs)
4059
+ _DocumentProbe.downloaded(text)
4060
  landed_window = True
4061
  if root:
4062
  parsed_cols, rows = parse_json_report(
 
4132
  if cx.REPORT_TYPES[table].get("monthly"):
4133
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
4134
  "count": len(rows)})
4135
+ probe.finish(len(rows) if cols else 0)
4136
+ spend_rows(len(rows) if cols else 0)
4137
  if first_load:
4138
  for reached in (table, sibling):
4139
  if reached:
 
4154
  if table == "pricing":
4155
  if first_load:
4156
  # Completion can be in another process/pass than the recent ASIN report.
4157
+ # One ASIN per row, as `_pricing_asins` picks it: a parent beside its children
4158
+ # would answer for no row and `validate()` would refuse the pass.
4159
  available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
4160
+ picked = [c for c in ("child_asin", "parent_asin") if c in available]
4161
+ select = ", ".join(f'"{c}"' for c in picked)
4162
+ mirror_rows = [dict(zip(picked, row)) for row in con.execute(
4163
+ f"SELECT DISTINCT {select} FROM {mirror}").fetchall()] if picked else []
4164
+ pricing_asins = list(dict.fromkeys(pricing_asins + _pricing_asins(mirror_rows)))
4165
  state_key = ENTITY_STATE_KEYS[table]
4166
  if not pricing_asins:
4167
  _record_state(con, state_key, "unproven", 0,
 
4368
  lines by construction, so this drops line detail rather than losing an order. `order_items`
4369
  keeps every row, which is why both tables read the same document instead of costing two.
4370
  """
4371
+ if isinstance(rows, _DocumentRows):
4372
+ return rows.pipe(_first_row_per_order)
4373
+ return list(_first_row_per_order(rows))
4374
+
4375
+
4376
+ def _first_row_per_order(rows):
4377
+ seen = set()
4378
  for r in rows:
4379
  oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
4380
  if oid and oid in seen:
4381
  continue
4382
  if oid:
4383
  seen.add(oid)
4384
+ yield r
 
4385
 
4386
 
4387
  def validate(tenant_key="royal-imports", rt=None, log=print):
 
6055
  "('O3','2026-07-04T00:00:00Z','Cancelled')")
6056
  ck("orders independently reconcile Sales API counts and exclude cancelled orders",
6057
  red and _validate_orders(con)["state"] == "ok")
6058
+ con.execute("ALTER TABLE amzn_orders ADD COLUMN sales_channel VARCHAR")
6059
+ con.execute("UPDATE amzn_orders SET sales_channel='Amazon.com'")
6060
+ con.execute("INSERT INTO amzn_orders VALUES ('O4','2026-07-03T00:00:00Z','Shipped','Amazon.ca'), "
6061
+ "('O5','2026-07-03T00:00:00Z','Shipped','Non-Amazon')")
6062
+ ck("orders reconcile only the US marketplace that Amazon Sales counts",
6063
+ _validate_orders(con)["state"] == "ok")
6064
  spec = cx.REPORT_TYPES["reimbursements"]
6065
  with patch.object(cx, "report_document", return_value=(
6066
  "sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})):
 
6558
  ck("NC-AZ19 competitive summary enriches every matching report row without appending",
6559
  response_count == 1 and updated == 2 and enriched == 2,
6560
  (response_count, updated, enriched))
6561
+ held = duckdb.connect(":memory:")
6562
+ held.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, "
6563
+ "competitive_body_asin VARCHAR, competitive_status_status_code BIGINT)")
6564
+ held.execute("INSERT INTO amzn_pricing VALUES ('B01','P01',NULL,NULL), ('B02','P02','B02',200)")
6565
+ response_count, _ = _apply_competitive(held, "amzn_pricing", competitive)
6566
+ left = held.execute("SELECT competitive_body_asin, competitive_status_status_code FROM amzn_pricing "
6567
+ "WHERE child_asin='B02'").fetchone()
6568
+ carried = held.execute("SELECT COUNT(DISTINCT competitive_body_asin) FROM amzn_pricing "
6569
+ "WHERE competitive_body_asin IS NOT NULL").fetchone()[0]
6570
+ held.close()
6571
+ ck("NC-AZ32 an ASIN absent from this pass keeps no earlier competitive price",
6572
+ left == (None, None) and carried == response_count == 1, (left, carried, response_count))
6573
+ sku = duckdb.connect(":memory:")
6574
+ sku.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
6575
+ sku.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'), ('B03','P01'), (NULL,'P04')")
6576
+ family = [{"body": {"asin": a, "price": p}} for a, p in (("B01", 1), ("B03", 3), ("P01", 9), ("P04", 4))]
6577
+ response_count, _ = _apply_competitive(sku, "amzn_pricing", family)
6578
+ priced = sku.execute("SELECT COALESCE(child_asin, parent_asin), competitive_body_asin, competitive_body_price "
6579
+ "FROM amzn_pricing ORDER BY 1").fetchall()
6580
+ sku.close()
6581
+ ck("NC-AZ33 a parent ASIN's summary never overwrites its SKU rows",
6582
+ priced == [("B01", "B01", 1), ("B03", "B03", 3), ("P04", "P04", 4)] and response_count == 4, priced)
6583
 
6584
  mem.execute("""CREATE TABLE _sync_state (
6585
  entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
 
6715
  ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
6716
  ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
6717
  ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
6718
+ same, detail = _spooled_parse_selftest()
6719
+ ck("NC-AZ34 a spooled document parses and counts exactly as its text does", same, detail)
6720
+ bounded, detail = _bounded_document_selftest()
6721
+ ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
6722
+ log(f"\n{'GREEN' if not fails else 'RED'}: {35 - len(fails)}/35")
6723
  return 0 if not fails else 1
6724
 
6725
 
6726
+ def _spool_fixture(text, *, gz=False, encoding="utf-8"):
6727
+ """A `ReportDocument` made by the client's own spool path from local bytes, no network."""
6728
+ import gzip
6729
+ raw = text.encode(encoding) if isinstance(text, str) else text
6730
+ handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
6731
+ with os.fdopen(handle, "wb") as out:
6732
+ out.write(gzip.compress(raw) if gz else raw)
6733
+ return _client()._spooled_document(path, "GZIP" if gz else "")
6734
+
6735
+
6736
+ def _spooled_parse_selftest():
6737
+ """D-7081: the streaming readers answer every fixture exactly as the whole-text parsers do."""
6738
+ origin = {"document_id": "selftest-document", "report_type": "selftest-report"}
6739
+
6740
+ def outcome(parse):
6741
+ try:
6742
+ cols, rows = parse()
6743
+ return "ok", list(cols), [dict(row) for row in rows]
6744
+ except Exception as exc: # noqa: BLE001
6745
+ return "error", type(exc).__name__, str(exc)
6746
+
6747
+ def count(text, roots):
6748
+ try:
6749
+ return document_row_count(text, roots)
6750
+ except Exception as exc: # noqa: BLE001
6751
+ return type(exc).__name__
6752
+
6753
+ flat = ["", " \n\t", "a\tb\r\n1\t2\r\n\r\n3\t4", "\ufeffsku\tsku\tbuyer-email\nA\tB\tx\n\"q\t1\n",
6754
+ "x\ty\r1\r2\t3\r\r", "sku\tname\nA\tsmart \u201cquote\u201d\n"]
6755
+ blobs = [(t, {}) for t in flat] + [(t, {"gz": True}) for t in flat] + [(flat[-1], {"encoding": "cp1252"})]
6756
+ misses = []
6757
+ for index, (text, how) in enumerate(blobs):
6758
+ expected = text if how.get("encoding") != "cp1252" else text.encode("cp1252").decode("cp1252")
6759
+ got = outcome(lambda: parse_flat_file(_spool_fixture(text, **how), **origin))
6760
+ if got != outcome(lambda: parse_flat_file(expected, **origin)):
6761
+ misses.append(f"flat {index}")
6762
+ if expected.strip() and count(_spool_fixture(text, **how), ()) != count(expected, ()):
6763
+ misses.append(f"flat count {index}")
6764
+ documents = ['', '[]', '{"rows": []}', '{"rows": {"a": 1}}', '{"rows": [1]}', '{"rows": [{"a": 1},]}',
6765
+ '{"rows": [{"a": 1}]} x', '{"rows": [{"a": 1}], "rows": [{"b": 2}]}', '{"other": 1}',
6766
+ ' {"spec": {"q": [1, [2]]}, "rows": [{"a": 1, "b": null, "n": {"c": "\u201c"}}, '
6767
+ '{"a": 2.5, "l": [1, 2]}], "tail": [[1], []]} ',
6768
+ '{"rows": [{"_report_document_id": "vendor"}]}', '{"rows": [{"buyerEmail": "e", "a": 123456789012345678901}]}',
6769
+ '{"x": NaN, "rows": [{"v": Infinity, "w": -0.0, "s": "\\ud83d"}]}']
6770
+ for index, text in enumerate(documents):
6771
+ if outcome(lambda: parse_json_report(_spool_fixture(text), "rows", **origin)) != outcome(
6772
+ lambda: parse_json_report(text, "rows", **origin)):
6773
+ misses.append(f"json {index}")
6774
+ if count(_spool_fixture(text), ("rows",)) != count(text, ("rows",)):
6775
+ misses.append(f"json count {index}")
6776
+ return not misses, misses
6777
+
6778
+
6779
+ def _bounded_document_selftest(rows=24_000):
6780
+ """D-7081: a report larger than the bound parses, sniffs and writes with a bounded Python peak.
6781
+
6782
+ The whole-text path holds the decoded text and every row at once, so it fails this bound; the
6783
+ spooled path holds one read piece and one insert batch.
6784
+ """
6785
+ import gzip
6786
+ import tracemalloc
6787
+ bound = 10 << 20
6788
+ handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
6789
+ os.close(handle)
6790
+ with gzip.open(path, "wt", encoding="utf-8") as out:
6791
+ out.write('{"reportSpecification": {"reportType": "selftest"}, "dataByDepartmentAndSearchTerm": [')
6792
+ for i in range(rows):
6793
+ out.write(("," if i else "") + json.dumps({
6794
+ "departmentName": "Amazon.com", "searchTerm": f"term {i} \u2122 caf\u00e9",
6795
+ "searchFrequencyRank": i + 1, "clickedAsin": f"B0{i:08d}",
6796
+ "clickedItemName": f"Product title number {i} with several words", "clickShareRank": i % 3 + 1,
6797
+ "clickShare": round(i / 7 % 1, 4), "conversionShare": round(i / 11 % 1, 4)}))
6798
+ out.write("]}")
6799
+ document = _client()._spooled_document(path, "GZIP")
6800
+ text_bytes = 2 * len(document)
6801
+ con = datastore.duckdb.connect(":memory:")
6802
+ table = "amzn_brand_analytics"
6803
+ # A first bound statement imports DuckDB's lazy modules; that one-time cost is not a row's.
6804
+ con.execute("SELECT ?", [1]).fetchall()
6805
+ tracemalloc.start()
6806
+ try:
6807
+ cols, parsed = parse_json_report(document, "dataByDepartmentAndSearchTerm",
6808
+ document_id="selftest-document", report_type="selftest")
6809
+ cols = merge_columns([c for c, _t in amazon_schema.columns_for("brand_analytics")], cols)
6810
+ types = _ensure(con, table, cols, parsed, declared=dict(amazon_schema.columns_for("brand_analytics")))
6811
+ first, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
6812
+ peak = tracemalloc.get_traced_memory()[1]
6813
+ finally:
6814
+ tracemalloc.stop()
6815
+ again, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
6816
+ stored = con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _report_row_number) FROM {table}").fetchone()
6817
+ con.close()
6818
+ document.close()
6819
+ detail = {"rows": len(parsed), "first": first, "again": again, "stored": stored,
6820
+ "text_bytes": text_bytes, "peak": peak, "bound": bound}
6821
+ return (len(parsed) == first == again == rows and stored == (rows, rows)
6822
+ and text_bytes > bound and peak < bound), detail
6823
+
6824
+
6825
  def main(argv=None):
6826
  ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
6827
  ap.add_argument("--sync", action="store_true")
platform/harness/datastore.py CHANGED
@@ -546,7 +546,9 @@ def check_odoo_snapshot(rt, candidate, published, row_counts):
546
  checks = []
547
 
548
  def record(name, diffs=(), **details):
549
- checks.append({"check": name, "state": "failed" if diffs else "passed",
 
 
550
  "diffs": list(diffs), **details})
551
 
552
  def columns(con, table):
@@ -592,6 +594,19 @@ def check_odoo_snapshot(rt, candidate, published, row_counts):
592
  balance_diffs.append(item)
593
  elif not balanced and company in exceptions:
594
  accepted.append({**item, "reason": exceptions[company]["reason"]})
 
 
 
 
 
 
 
 
 
 
 
 
 
595
  if invalid:
596
  balance_diffs.append({"invalid_ledger_lines": invalid})
597
  record("ledger_balance", balance_diffs, exceptions=accepted)
 
546
  checks = []
547
 
548
  def record(name, diffs=(), **details):
549
+ # A check of Odoo's own ledger reports a warning and never holds a sync. Only our copy's row drop fails.
550
+ found = "failed" if name == "row_count_drop" else "warning"
551
+ checks.append({"check": name, "state": found if diffs else "passed",
552
  "diffs": list(diffs), **details})
553
 
554
  def columns(con, table):
 
594
  balance_diffs.append(item)
595
  elif not balanced and company in exceptions:
596
  accepted.append({**item, "reason": exceptions[company]["reason"]})
597
+ move_cols = ledger & {"move_id", "move_name"}
598
+ if "move_id" in move_cols:
599
+ # Name the posted moves whose own lines do not sum to zero: that is the cause in Odoo.
600
+ named = ", any_value(move_name)" if "move_name" in move_cols else ""
601
+ for item in balance_diffs:
602
+ moves = candidate.execute(
603
+ f"SELECT move_id, sum(CAST(balance AS DECIMAL(38,6))) AS total{named} FROM account_move_line "
604
+ f"WHERE {where} AND isfinite(balance) AND company_id=? AND move_id IS NOT NULL "
605
+ "GROUP BY move_id HAVING abs(sum(CAST(balance AS DECIMAL(38,6)))) > ? ORDER BY abs(total) DESC, move_id",
606
+ [item["company_id"], Decimal(str(policy["balance_tolerance"]))]).fetchall()
607
+ item["unbalanced_moves"] = len(moves)
608
+ item["moves"] = [{"move_id": row[0], **({"move_name": row[2]} if named else {}),
609
+ "balance": float(row[1])} for row in moves[:10]]
610
  if invalid:
611
  balance_diffs.append({"invalid_ledger_lines": invalid})
612
  record("ledger_balance", balance_diffs, exceptions=accepted)
platform/harness/mirror_store.py CHANGED
@@ -66,7 +66,8 @@ def _catch_up_source(rt, connector):
66
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
67
  return _CATCH_UP.setdefault(identity, {}).setdefault(connector, {
68
  "current": False, "parts": set(), "part_total": 0,
69
- "passes": 0, "pass_pending": False, "newest": {}, "refused": False})
 
70
 
71
 
72
  def connected_sources(rt):
@@ -90,17 +91,68 @@ def catch_up_status(rt, sources=None):
90
  if sources is None:
91
  sources = connected_sources(rt)
92
  with _CATCH_UP_LOCK:
93
- states = [_catch_up_source(rt, name) for name in dict.fromkeys(sources)]
94
- done = sum(len(s["parts"]) + s["passes"] for s in states)
95
- total = sum(s["part_total"] + s["passes"] + int(s["pass_pending"]) for s in states)
96
- return {"active": any(not s["current"] for s in states), "done": done,
97
- "total": total, "newest": {k: v for s in states for k, v in s["newest"].items()}}
 
 
 
 
 
 
 
 
98
 
99
 
100
  def source_catching_up(rt, connector):
101
  return catch_up_status(rt, [connector])["active"]
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _record_newest(con, connector):
105
  """Measure record dates, never checkpoint time or the sync cursor."""
106
  preferred = ("invoice_date", "date_order", "date", "purchase_date", "posted_date",
@@ -139,7 +191,12 @@ def _sync_result(connector, report):
139
  for row in report.values())
140
  return completed, completed and all(row["phase"] == "live" for row in report.values())
141
  if report.get("problems") or report.get("audit_problems"):
142
- return False, False
 
 
 
 
 
143
  rows = report.get("tables") or {}
144
  if connector == "meta":
145
  completed = bool(rows) and not any(row.get("capped") for row in rows.values())
@@ -160,12 +217,21 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
160
  with _CATCH_UP_LOCK:
161
  state = _catch_up_source(rt, connector)
162
  if not state["current"]:
163
- state["pass_pending"] = True
164
- report = pull(*args, **kwargs)
 
 
 
 
 
 
 
 
165
  completed, current = _sync_result(connector, report)
166
  current = current and _catch_up_complete and Path(rt.datastore_path).exists()
167
  newest = None
168
- if current:
 
169
  try:
170
  con, _ = datastore._mirror_connection(rt.key, rt)
171
  try:
@@ -179,8 +245,18 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
179
  state = _catch_up_source(rt, connector)
180
  if completed:
181
  state["passes"] += 1
 
 
182
  if current and not state["refused"]:
183
- state.update(current=True, pass_pending=False, newest=newest or {})
 
 
 
 
 
 
 
 
184
  return report
185
 
186
 
@@ -300,6 +376,15 @@ class MirrorStore:
300
  def _log_restored(self, paths, total, rows, log):
301
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
302
 
 
 
 
 
 
 
 
 
 
303
  def _mirror_cache_key(self, tenant_key, rt):
304
  key = _tenant_key(tenant_key)
305
  if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
@@ -570,7 +655,8 @@ class MirrorStore:
570
  newest = _record_newest(con, self.connector)
571
  with _CATCH_UP_LOCK:
572
  _catch_up_source(rt, self.connector).update(
573
- current=current, pass_pending=not current, newest=newest)
 
574
  except Exception as exc:
575
  result["state"] = "refused"
576
  message = (f"{self.label} mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
@@ -770,8 +856,6 @@ class MirrorStore:
770
  return # The enclosing source pass owns publication after every table completes.
771
  uploaded = 0
772
  with self.lock:
773
- if _held_manifest(self, tenant_key, rt):
774
- raise PublicationHeld("Release the held snapshot before syncing this source.")
775
  cache = self.restore_mirror(tenant_key, rt, log=log)
776
  cache["dirty"].update(changed_tables)
777
  try:
@@ -855,13 +939,22 @@ class MirrorStore:
855
  **(_publication or {}))
856
  if _publication is not None:
857
  manifest["completed_at"] = stamp
858
- published = _mirror_json(manifest)
859
  if _hold is not None:
860
  manifest["hold"] = _hold
861
  rt.upload_bytes(self.prefix + _HELD_KEY, _mirror_json(manifest))
862
  return manifest
 
 
 
 
863
  rt.upload_bytes(self.prefix + _MANIFEST_KEY, published)
864
  uploaded += len(published)
 
 
 
 
 
 
865
  if _publication is None:
866
  datastore._write_publication(con, self.connector, manifest)
867
  cache["manifest"] = manifest
@@ -930,9 +1023,31 @@ def _held_manifest(store, tenant_key, rt):
930
  if (published.get("tenant") == tenant_key and published.get("snapshot_id") == held["snapshot_id"]
931
  and published.get("release")):
932
  return None
 
 
 
 
933
  return held
934
 
935
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
936
  def _publication_summary(manifest):
937
  return {**{key: manifest.get(key) or "unknown" for key in
938
  ("snapshot_id", "started_at", "completed_at", "state")},
@@ -1044,6 +1159,42 @@ def _open_candidate(store, tenant_key, rt, published):
1044
  raise
1045
 
1046
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1047
  @contextmanager
1048
  def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
1049
  expected_tables=None):
@@ -1072,14 +1223,13 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1072
  with store.lock:
1073
  if getattr(datastore._SYNC_TLS, "candidate", None) is not None:
1074
  raise RuntimeError("A source publication is already active on this thread")
1075
- if _held_manifest(store, tenant_key, rt):
1076
- raise PublicationHeld("Release the held snapshot before syncing this source.")
1077
  restored = store.restore_mirror(tenant_key, rt, log=log)
1078
  if restored["state"] == "refused":
1079
  raise RuntimeError("The saved mirror could not be restored. Repair store access and restart.")
1080
  published, _path = store._connect(tenant_key, rt)
1081
  candidate = None
1082
  installed = False
 
1083
  save_started = None
1084
  try:
1085
  opened = time.monotonic()
@@ -1107,20 +1257,31 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1107
  or (row.get("state") == "disabled" and name not in pending))}
1108
  if done:
1109
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
1110
- if candidate.execute("SELECT 1 FROM _candidate_pending LIMIT 1").fetchone():
 
 
 
1111
  if odoo:
1112
  note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
1113
  "SELECT name FROM _candidate_pending").fetchall()])
1114
  return
1115
  if report is not None:
1116
  completed, current = _sync_result(store.connector, report)
1117
- if not (completed if store.connector == "meta" else current):
1118
  if odoo:
1119
  note(f"held, the pass is not current (completed={completed}), entities not live",
1120
  names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
1121
  for name, row in report.items()
1122
  if not (isinstance(row, dict) and row.get("phase") == "live")])
 
 
 
 
1123
  return
 
 
 
 
1124
  if store.connector == "odoo" and any(row[1] not in {"live", "done"}
1125
  for row in store._state_rows(candidate)):
1126
  note("held, sync state not live or done", names=lambda: [
@@ -1139,8 +1300,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1139
  validating = time.monotonic()
1140
  metadata["validation"] = datastore.check_odoo_snapshot(rt, candidate, published, metadata["rows"])
1141
  failed = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "failed"]
1142
- note("validation " + ("failed" if failed else "passed"), validating,
1143
- names=(lambda: failed) if failed else None)
 
1144
  if failed:
1145
  prior = _publication_summary(datastore._publication(published, store.connector))
1146
  hold = {"previous_snapshot_id": prior["snapshot_id"], "previous_publication": prior,
@@ -1152,7 +1314,8 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1152
  if key != "rows"}, _hold=hold)
1153
  if not held:
1154
  raise RuntimeError("The failed candidate could not be saved. " + "; ".join(result["problems"]))
1155
- raise PublicationHeld("Snapshot held: " + ", ".join(failed) + ". Release held snapshot requires an Administrator.")
 
1156
  metadata.pop("rows")
1157
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
1158
  result = {"problems": []}
@@ -1182,6 +1345,13 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1182
  raise
1183
  if odoo:
1184
  note("install", installing)
 
 
 
 
 
 
 
1185
  finally:
1186
  if save_started is not None and store.connector == "odoo":
1187
  status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
@@ -1189,7 +1359,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1189
  print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
1190
  if candidate is not None:
1191
  candidate.close()
1192
- if installed:
1193
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1194
  path.unlink(missing_ok=True)
1195
  published.close()
 
66
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
67
  return _CATCH_UP.setdefault(identity, {}).setdefault(connector, {
68
  "current": False, "parts": set(), "part_total": 0,
69
+ "passes": 0, "pass_pending": False, "newest": {}, "refused": False, "misses": 0, "behind": "",
70
+ "attempt_at": 0.0, "missed_at": 0.0, "miss_reason": "", "since": time.time()})
71
 
72
 
73
  def connected_sources(rt):
 
91
  if sources is None:
92
  sources = connected_sources(rt)
93
  with _CATCH_UP_LOCK:
94
+ states = {connector_spec.CONNECTOR_SPEC_KEYS.get(name, name): _catch_up_source(rt, name)
95
+ for name in dict.fromkeys(sources)}
96
+ # A behind source has settled: its pending pass no longer holds the tenant or its count.
97
+ behind = {name: _behind(s) for name, s in states.items()}
98
+ per = {name: {"active": not s["current"] and not behind[name],
99
+ "done": len(s["parts"]) + s["passes"],
100
+ "total": s["part_total"] + s["passes"] + int(s["pass_pending"] and not behind[name]),
101
+ "behind": behind[name]} for name, s in states.items()}
102
+ return {"active": any(p["active"] for p in per.values()),
103
+ "done": sum(p["done"] for p in per.values()),
104
+ "total": sum(p["total"] for p in per.values()),
105
+ "newest": {k: v for s in states.values() for k, v in s["newest"].items()},
106
+ "sources": per}
107
 
108
 
109
  def source_catching_up(rt, connector):
110
  return catch_up_status(rt, [connector])["active"]
111
 
112
 
113
+ #: Consecutive unsettled passes before a source reads behind. Odoo retries within minutes;
114
+ #: Amazon and Meta pass about daily, so one unsettled pass names them behind.
115
+ BEHIND_AFTER_PASSES = {"odoo": 3}
116
+ #: A miss that no new pass followed within this many seconds also reads behind.
117
+ BEHIND_AFTER_SECONDS = 900
118
+ #: A restored source that no pass has started on within this many seconds reads behind.
119
+ IDLE_BEHIND_SECONDS = 1800
120
+
121
+
122
+ def _behind(state):
123
+ """The settled reason, or a plain reason once no pass followed a miss or a restore in time."""
124
+ if state["behind"] or state["current"]:
125
+ return state["behind"]
126
+ now = time.time()
127
+ if state["misses"]:
128
+ waited = state["missed_at"] >= state["attempt_at"] and now - state["missed_at"] >= BEHIND_AFTER_SECONDS
129
+ return state["miss_reason"] if waited else ""
130
+ # A restored source whose pass was skipped (a resumed pass already pulled it) waits a whole cadence.
131
+ idle = (state["pass_pending"] and not state.get("attempt_at") and len(state["parts"]) >= state["part_total"]
132
+ and now - state.get("since", now) >= IDLE_BEHIND_SECONDS)
133
+ return "Its sync has not run since the restart." if idle else ""
134
+
135
+
136
+ def _record_miss(connector, state, reason):
137
+ """Count one unsettled pass on a source that is not current, and name it behind at the limit."""
138
+ if state["current"]:
139
+ return
140
+ state["misses"] += 1
141
+ if state["refused"]:
142
+ reason = "Its saved copy could not be restored."
143
+ state.update(missed_at=time.time(), miss_reason=reason)
144
+ if state["misses"] >= BEHIND_AFTER_PASSES.get(connector, 1):
145
+ state["behind"] = reason
146
+
147
+
148
+ def _skipped_pass(report):
149
+ """An empty report ran no pass, so it neither settles nor misses."""
150
+ if not isinstance(report, dict) or not report:
151
+ return True
152
+ return ("tables" in report and not report.get("tables") and not report.get("problems")
153
+ and not report.get("audit_problems"))
154
+
155
+
156
  def _record_newest(con, connector):
157
  """Measure record dates, never checkpoint time or the sync cursor."""
158
  preferred = ("invoice_date", "date_order", "date", "purchase_date", "posted_date",
 
191
  for row in report.values())
192
  return completed, completed and all(row["phase"] == "live" for row in report.values())
193
  if report.get("problems") or report.get("audit_problems"):
194
+ # A problem owned by one held table publishes the rest; it never makes the pass current.
195
+ holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
196
+ if holds is None or holds["blocked"]:
197
+ return False, False
198
+ rows = report.get("tables") or {}
199
+ return bool(rows) and any(_table_sync_complete(connector, row) for row in rows.values()), False
200
  rows = report.get("tables") or {}
201
  if connector == "meta":
202
  completed = bool(rows) and not any(row.get("capped") for row in rows.values())
 
217
  with _CATCH_UP_LOCK:
218
  state = _catch_up_source(rt, connector)
219
  if not state["current"]:
220
+ state.update(pass_pending=True, attempt_at=time.time())
221
+ try:
222
+ report = pull(*args, **kwargs)
223
+ except Exception as exc:
224
+ if _catch_up_complete:
225
+ with _CATCH_UP_LOCK:
226
+ _record_miss(connector, _catch_up_source(rt, connector),
227
+ "Its last update is held for review." if isinstance(exc, PublicationHeld)
228
+ else "Its last sync failed.")
229
+ raise
230
  completed, current = _sync_result(connector, report)
231
  current = current and _catch_up_complete and Path(rt.datastore_path).exists()
232
  newest = None
233
+ # An Amazon pass that published part of the source still moves the measured record dates.
234
+ if current or (completed and connector == "amazon" and Path(rt.datastore_path).exists()):
235
  try:
236
  con, _ = datastore._mirror_connection(rt.key, rt)
237
  try:
 
245
  state = _catch_up_source(rt, connector)
246
  if completed:
247
  state["passes"] += 1
248
+ if newest and not state["refused"]:
249
+ state["newest"] = newest
250
  if current and not state["refused"]:
251
+ state.update(current=True, pass_pending=False, newest=newest or {}, misses=0, behind="")
252
+ elif not _catch_up_complete or _skipped_pass(report):
253
+ pass
254
+ elif completed and connector == "odoo" and not state["refused"]:
255
+ # A bounded Odoo backfill pass is progress: the next pass resumes it.
256
+ state["misses"] = 0
257
+ else:
258
+ _record_miss(connector, state, "Its last sync left some tables unavailable." if completed
259
+ else "Its last sync did not finish.")
260
  return report
261
 
262
 
 
376
  def _log_restored(self, paths, total, rows, log):
377
  log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
378
 
379
+ def _publication_holds(self, report, waiting):
380
+ """None publishes the whole source or nothing. A connector may return
381
+ {"held": {name: reason}, "tables": held mirror tables, "blocked": [problems that hold the pass]}."""
382
+ return None
383
+
384
+ def _held_state_rows(self, rows, tables):
385
+ """The state rows that describe a held table's data, so they ride with its previous rows."""
386
+ return [row for row in rows if row[0] in tables or row[0].split(".", 1)[0] in tables]
387
+
388
  def _mirror_cache_key(self, tenant_key, rt):
389
  key = _tenant_key(tenant_key)
390
  if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
 
655
  newest = _record_newest(con, self.connector)
656
  with _CATCH_UP_LOCK:
657
  _catch_up_source(rt, self.connector).update(
658
+ current=current, pass_pending=not current, newest=newest,
659
+ **({"misses": 0, "behind": ""} if current else {}))
660
  except Exception as exc:
661
  result["state"] = "refused"
662
  message = (f"{self.label} mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
 
856
  return # The enclosing source pass owns publication after every table completes.
857
  uploaded = 0
858
  with self.lock:
 
 
859
  cache = self.restore_mirror(tenant_key, rt, log=log)
860
  cache["dirty"].update(changed_tables)
861
  try:
 
939
  **(_publication or {}))
940
  if _publication is not None:
941
  manifest["completed_at"] = stamp
 
942
  if _hold is not None:
943
  manifest["hold"] = _hold
944
  rt.upload_bytes(self.prefix + _HELD_KEY, _mirror_json(manifest))
945
  return manifest
946
+ superseded = _superseded_hold(self, tenant_key, rt)
947
+ if superseded:
948
+ manifest["superseded_hold"] = superseded
949
+ published = _mirror_json(manifest)
950
  rt.upload_bytes(self.prefix + _MANIFEST_KEY, published)
951
  uploaded += len(published)
952
+ if superseded is not None:
953
+ # A newer publication replaces a stored hold. Nobody has to release it.
954
+ try:
955
+ rt.delete_path(self.prefix + _HELD_KEY)
956
+ except Exception: # noqa: BLE001
957
+ log(f"{self.label} superseded hold pointer stays until the next publication")
958
  if _publication is None:
959
  datastore._write_publication(con, self.connector, manifest)
960
  cache["manifest"] = manifest
 
1023
  if (published.get("tenant") == tenant_key and published.get("snapshot_id") == held["snapshot_id"]
1024
  and published.get("release")):
1025
  return None
1026
+ # A publication that already superseded this hold leaves only a stale pointer.
1027
+ if (published.get("tenant") == _tenant_key(tenant_key) and isinstance(published.get("superseded_hold"), dict)
1028
+ and published["superseded_hold"].get("snapshot_id") == held["snapshot_id"]):
1029
+ return None
1030
  return held
1031
 
1032
 
1033
+ def _superseded_hold(store, tenant_key, rt):
1034
+ """Describe the stored hold a new publication replaces: None if there is none, {} for a stale pointer."""
1035
+ try:
1036
+ if rt.download_bytes(store.prefix + _HELD_KEY) is None:
1037
+ return None
1038
+ except Exception: # noqa: BLE001
1039
+ return None
1040
+ try:
1041
+ held = _held_manifest(store, tenant_key, rt)
1042
+ except Exception: # noqa: BLE001
1043
+ # An unreadable hold is deleted with the publication, never a crash.
1044
+ return {"snapshot_id": "unknown", "held_at": "unknown", "failed_checks": [], "unreadable": True}
1045
+ if held is None:
1046
+ return {}
1047
+ return {"snapshot_id": held["snapshot_id"], "held_at": held["hold"].get("held_at") or "unknown",
1048
+ "failed_checks": list(held["hold"].get("failed_checks") or [])}
1049
+
1050
+
1051
  def _publication_summary(manifest):
1052
  return {**{key: manifest.get(key) or "unknown" for key in
1053
  ("snapshot_id", "started_at", "completed_at", "state")},
 
1159
  raise
1160
 
1161
 
1162
+ def _hold_previous(store, candidate, published, tables):
1163
+ """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1164
+ prior = store._tables(published)
1165
+ with tempfile.TemporaryDirectory(prefix="mirror-held-") as tmp:
1166
+ for name in sorted(tables):
1167
+ if name not in prior:
1168
+ candidate.execute(f"DROP TABLE IF EXISTS {_quoted_identifier(name)}")
1169
+ continue
1170
+ path = Path(tmp) / (name + ".parquet")
1171
+ published.execute(f"COPY {_quoted_identifier(name)} TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1172
+ candidate.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
1173
+ f"SELECT * FROM {_parquet_source([path])}")
1174
+ private = store._held_state_rows(store._state_rows(candidate), tables)
1175
+ if private:
1176
+ candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in private])
1177
+ previous = store._held_state_rows(store._state_rows(published), tables)
1178
+ if previous:
1179
+ candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1180
+
1181
+
1182
+ def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1183
+ """One log line: what a pass published and what it held, each with its reason."""
1184
+ rows = report.get("tables") or {}
1185
+ synced = sorted(name for name in done if name not in holds["held"] and "rows" in (rows.get(name) or {}))
1186
+ synced_text = ", ".join(f"{name} {rows[name]['rows']:,} rows" if isinstance(rows[name]["rows"], int)
1187
+ else name for name in synced) or "none"
1188
+ held = "; ".join(f"{name} ({str(reason)[:160]})" for name, reason in sorted(holds["held"].items())) or "none"
1189
+ if snapshot_id:
1190
+ text = (f"{store.label} publication {snapshot_id}: published {len(synced)} synced ({synced_text}) and "
1191
+ f"{len(set(done) - set(synced) - set(holds['held']))} current; "
1192
+ f"held {len(holds['held'])} at their previous rows: {held}")
1193
+ else:
1194
+ text = f"{store.label} publication held, the previous snapshot stays: {why}; unfinished: {held}"
1195
+ return text.replace("—", "-").replace("–", "-").replace("\n", " ").replace("\r", " ")
1196
+
1197
+
1198
  @contextmanager
1199
  def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
1200
  expected_tables=None):
 
1223
  with store.lock:
1224
  if getattr(datastore._SYNC_TLS, "candidate", None) is not None:
1225
  raise RuntimeError("A source publication is already active on this thread")
 
 
1226
  restored = store.restore_mirror(tenant_key, rt, log=log)
1227
  if restored["state"] == "refused":
1228
  raise RuntimeError("The saved mirror could not be restored. Repair store access and restart.")
1229
  published, _path = store._connect(tenant_key, rt)
1230
  candidate = None
1231
  installed = False
1232
+ kept = False
1233
  save_started = None
1234
  try:
1235
  opened = time.monotonic()
 
1257
  or (row.get("state") == "disabled" and name not in pending))}
1258
  if done:
1259
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
1260
+ # A connector policy may publish what completed and hold each unfinished table at its previous rows.
1261
+ holds = None if report is None or odoo else store._publication_holds(report, {
1262
+ row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()})
1263
+ if holds is None and candidate.execute("SELECT 1 FROM _candidate_pending LIMIT 1").fetchone():
1264
  if odoo:
1265
  note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
1266
  "SELECT name FROM _candidate_pending").fetchall()])
1267
  return
1268
  if report is not None:
1269
  completed, current = _sync_result(store.connector, report)
1270
+ if not (completed if store.connector == "meta" or holds is not None else current):
1271
  if odoo:
1272
  note(f"held, the pass is not current (completed={completed}), entities not live",
1273
  names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
1274
  for name, row in report.items()
1275
  if not (isinstance(row, dict) and row.get("phase") == "live")])
1276
+ if holds is not None:
1277
+ log(_publication_line(store, report, done, holds, why=(
1278
+ "a problem no table owns: " + "; ".join(str(p)[:200] for p in holds["blocked"][:3])
1279
+ if holds["blocked"] else "no table completed in this pass")))
1280
  return
1281
+ if holds is not None and holds["tables"]:
1282
+ # Rolled back after publication, so the candidate keeps each held table's private progress.
1283
+ candidate.execute("BEGIN TRANSACTION")
1284
+ _hold_previous(store, candidate, published, holds["tables"])
1285
  if store.connector == "odoo" and any(row[1] not in {"live", "done"}
1286
  for row in store._state_rows(candidate)):
1287
  note("held, sync state not live or done", names=lambda: [
 
1300
  validating = time.monotonic()
1301
  metadata["validation"] = datastore.check_odoo_snapshot(rt, candidate, published, metadata["rows"])
1302
  failed = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "failed"]
1303
+ warned = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "warning"]
1304
+ note("validation " + ("failed" if failed else "passed with warnings" if warned else "passed"),
1305
+ validating, names=(lambda: failed or warned) if failed or warned else None)
1306
  if failed:
1307
  prior = _publication_summary(datastore._publication(published, store.connector))
1308
  hold = {"previous_snapshot_id": prior["snapshot_id"], "previous_publication": prior,
 
1314
  if key != "rows"}, _hold=hold)
1315
  if not held:
1316
  raise RuntimeError("The failed candidate could not be saved. " + "; ".join(result["problems"]))
1317
+ raise PublicationHeld("Snapshot held: " + ", ".join(failed) + ". The previous snapshot is kept "
1318
+ "and the next pass checks again.")
1319
  metadata.pop("rows")
1320
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
1321
  result = {"problems": []}
 
1345
  raise
1346
  if odoo:
1347
  note("install", installing)
1348
+ if holds is not None and holds["tables"]:
1349
+ candidate.execute("ROLLBACK")
1350
+ candidate.execute("DELETE FROM _candidate_base")
1351
+ candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
1352
+ kept = True
1353
+ if holds is not None:
1354
+ log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
1355
  finally:
1356
  if save_started is not None and store.connector == "odoo":
1357
  status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
 
1359
  print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
1360
  if candidate is not None:
1361
  candidate.close()
1362
+ if installed and not kept:
1363
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1364
  path.unlink(missing_ok=True)
1365
  published.close()
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 93a9a6294661907a5994900cda306707b200a807ea1472ffbeba3c0d1455613d
 
1
+ a278ac86384656722f918a76c838415bdcb9e46155b910f9335eb24f7b292c8d
web/src/connectors/ConnectorsPage.tsx CHANGED
@@ -54,8 +54,11 @@ import type { ConnectorTab } from "./ConnectorModal";
54
  import MonidRunDialog from "./MonidRunDialog";
55
  import {
56
  DatabaseSyncSummary,
 
57
  catchUpLabel,
 
58
  parseDatabaseSyncHealth,
 
59
  useConnectorCatchUp,
60
  type ConnectorCatchUp,
61
  type DatabaseSyncHealth,
@@ -801,10 +804,13 @@ export default function ConnectorsPage({
801
  const hasOdooCard =
802
  load.phase === "ready" &&
803
  load.rows.some((row) => row.key === "odoo" && row.state === "connected" && !!row.manage);
804
- const hasConnectedOdooCard =
805
  load.phase === "ready" &&
806
- load.rows.some((row) => row.key === "odoo" && row.state === "connected");
807
- const catchUpStatus = useConnectorCatchUp(hasConnectedOdooCard);
 
 
 
808
  const [odooWarnings, setOdooWarnings] = useState<string[]>([]);
809
  useEffect(() => {
810
  if (!hasOdooCard) return;
@@ -933,7 +939,7 @@ export default function ConnectorsPage({
933
  onOpen={() => openConnector(row)}
934
  onPullReviews={pullHandler(row)}
935
  syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
936
- catchUp={row.key === "odoo" ? catchUpStatus.catchUp : undefined}
937
  catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
938
  catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
939
  />
@@ -956,7 +962,7 @@ export default function ConnectorsPage({
956
  onOpen={() => openConnector(row)}
957
  onPullReviews={pullHandler(row)}
958
  syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
959
- catchUp={row.key === "odoo" ? catchUpStatus.catchUp : undefined}
960
  catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
961
  catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
962
  />
@@ -1138,8 +1144,9 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
1138
  });
1139
  return () => controller.abort();
1140
  }, []);
 
1141
  const shownDatabaseHealth = databaseHealth.filter((database) => database.health
1142
- || catchUpStatus.catchUp?.active || catchUpStatus.catchUp?.newest[database.key] || catchUpStatus.error);
1143
  if (problem) return <section className="w45-contract-card"><h4>Sync</h4><p className="set-error">{problem}</p></section>;
1144
  if (!status || !schema) return <section className="w45-contract-card"><h4>Sync</h4><p>Loading connector status…</p></section>;
1145
  return (
@@ -1148,9 +1155,10 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
1148
  <OdooFreshnessNote lines={freshnessWarnings(freshness)} />
1149
  <dl className="voice-agent-details">
1150
  <div><dt>Schedule</dt><dd>{status.schedule || "Not scheduled"}</dd></div>
1151
- <div><dt>Health</dt><dd>{catchUpStatus.catchUp?.active ? catchUpLabel(catchUpStatus.catchUp)
1152
- : catchUpStatus.error || (catchUpStatus.loading ? "Checking sync" : status.health)}
1153
- {catchUpStatus.catchUp?.active && catchUpStatus.error ? `, ${catchUpStatus.error}` : ""}</dd></div>
 
1154
  <div><dt>Last run</dt><dd>{status.lastRun || "No run reported"}</dd></div>
1155
  {freshness ? (
1156
  <div>
@@ -1165,8 +1173,8 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
1165
  <div key={database.key}>
1166
  <strong>{database.label}</strong>
1167
  <DatabaseSyncSummary health={database.health} databaseKey={database.key}
1168
- catchUp={catchUpStatus.catchUp} catchUpError={catchUpStatus.error}
1169
- catchUpLoading={catchUpStatus.loading} />
1170
  </div>
1171
  )) : <p>No database sync state has been published.</p>}
1172
  <h4>Field catalog</h4>
@@ -1371,9 +1379,10 @@ export function ConnectorCard({
1371
  return (
1372
  <div className="conn-card" data-connector={row.key}>
1373
  <ConnectorCardHead row={row} />
1374
- {catchUp?.active || catchUpError || catchUpLoading ? (
1375
  <span className="conn-card-status conn-catch-up" role="status">
1376
- {catchUp?.active ? catchUpLabel(catchUp) : catchUpError || "Checking sync"}
 
1377
  {catchUp?.active && catchUpError ? `, ${catchUpError}` : ""}
1378
  </span>
1379
  ) : syncWarnings?.length ? (
 
54
  import MonidRunDialog from "./MonidRunDialog";
55
  import {
56
  DatabaseSyncSummary,
57
+ behindLabel,
58
  catchUpLabel,
59
+ catchUpSourceKey,
60
  parseDatabaseSyncHealth,
61
+ sourceCatchUp,
62
  useConnectorCatchUp,
63
  type ConnectorCatchUp,
64
  type DatabaseSyncHealth,
 
804
  const hasOdooCard =
805
  load.phase === "ready" &&
806
  load.rows.some((row) => row.key === "odoo" && row.state === "connected" && !!row.manage);
807
+ const hasConnectedSourceCard =
808
  load.phase === "ready" &&
809
+ load.rows.some((row) => ["odoo", "amazon_sp", "meta_ads"].includes(row.key) && row.state === "connected");
810
+ const catchUpStatus = useConnectorCatchUp(hasConnectedSourceCard);
811
+ // Each connected source card shows its own catch-up; the Odoo card alone reports a status read error.
812
+ const sourceCard = (row: ConnectorRow) => ["odoo", "amazon_sp", "meta_ads"].includes(row.key)
813
+ && row.state === "connected" ? sourceCatchUp(catchUpStatus.catchUp, catchUpSourceKey(row.key)) : undefined;
814
  const [odooWarnings, setOdooWarnings] = useState<string[]>([]);
815
  useEffect(() => {
816
  if (!hasOdooCard) return;
 
939
  onOpen={() => openConnector(row)}
940
  onPullReviews={pullHandler(row)}
941
  syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
942
+ catchUp={sourceCard(row)}
943
  catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
944
  catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
945
  />
 
962
  onOpen={() => openConnector(row)}
963
  onPullReviews={pullHandler(row)}
964
  syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
965
+ catchUp={sourceCard(row)}
966
  catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
967
  catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
968
  />
 
1144
  });
1145
  return () => controller.abort();
1146
  }, []);
1147
+ const odooCatchUp = sourceCatchUp(catchUpStatus.catchUp, "odoo");
1148
  const shownDatabaseHealth = databaseHealth.filter((database) => database.health
1149
+ || odooCatchUp?.active || odooCatchUp?.newest[database.key] || catchUpStatus.error);
1150
  if (problem) return <section className="w45-contract-card"><h4>Sync</h4><p className="set-error">{problem}</p></section>;
1151
  if (!status || !schema) return <section className="w45-contract-card"><h4>Sync</h4><p>Loading connector status…</p></section>;
1152
  return (
 
1155
  <OdooFreshnessNote lines={freshnessWarnings(freshness)} />
1156
  <dl className="voice-agent-details">
1157
  <div><dt>Schedule</dt><dd>{status.schedule || "Not scheduled"}</dd></div>
1158
+ <div><dt>Health</dt><dd>{odooCatchUp?.active ? catchUpLabel(odooCatchUp)
1159
+ : catchUpStatus.error || (catchUpStatus.loading ? "Checking sync"
1160
+ : odooCatchUp?.behind ? behindLabel(odooCatchUp.behind) : status.health)}
1161
+ {odooCatchUp?.active && catchUpStatus.error ? `, ${catchUpStatus.error}` : ""}</dd></div>
1162
  <div><dt>Last run</dt><dd>{status.lastRun || "No run reported"}</dd></div>
1163
  {freshness ? (
1164
  <div>
 
1173
  <div key={database.key}>
1174
  <strong>{database.label}</strong>
1175
  <DatabaseSyncSummary health={database.health} databaseKey={database.key}
1176
+ catchUp={sourceCatchUp(catchUpStatus.catchUp, catchUpSourceKey(database.key))}
1177
+ catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />
1178
  </div>
1179
  )) : <p>No database sync state has been published.</p>}
1180
  <h4>Field catalog</h4>
 
1379
  return (
1380
  <div className="conn-card" data-connector={row.key}>
1381
  <ConnectorCardHead row={row} />
1382
+ {catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
1383
  <span className="conn-card-status conn-catch-up" role="status">
1384
+ {catchUp?.active ? catchUpLabel(catchUp) : catchUpError
1385
+ || (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
1386
  {catchUp?.active && catchUpError ? `, ${catchUpError}` : ""}
1387
  </span>
1388
  ) : syncWarnings?.length ? (
web/src/connectors/DatabasesTab.tsx CHANGED
@@ -124,11 +124,22 @@ function syncStamp(value: unknown): string | null {
124
  return text && Number.isFinite(Date.parse(text)) ? text : null;
125
  }
126
 
 
 
 
 
 
 
 
127
  export interface ConnectorCatchUp {
128
  active: boolean;
129
  done: number;
130
  total: number;
131
  newest: Record<string, string>;
 
 
 
 
132
  }
133
 
134
  /** Missing catchUp is compatible; a malformed status is not a settled sync. */
@@ -149,7 +160,40 @@ export function parseConnectorCatchUp(value: unknown): ConnectorCatchUp | null {
149
  if (!stamp) throw new Error("Sync status unavailable");
150
  newest[key] = stamp;
151
  }
152
- return { active: raw.active, done, total, newest };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  }
154
 
155
  export async function readConnectorCatchUp(signal: AbortSignal): Promise<ConnectorCatchUp | null> {
@@ -162,6 +206,10 @@ export function catchUpLabel(catchUp: ConnectorCatchUp): string {
162
  return `Updating ${catchUp.done.toLocaleString()} of ${catchUp.total.toLocaleString()}`;
163
  }
164
 
 
 
 
 
165
  export function useConnectorCatchUp(enabled: boolean) {
166
  const [state, setState] = useState<{
167
  catchUp: ConnectorCatchUp | null; error: string; loading: boolean;
 
124
  return text && Number.isFinite(Date.parse(text)) ? text : null;
125
  }
126
 
127
+ interface ConnectorSourceCatchUp {
128
+ active: boolean;
129
+ done: number;
130
+ total: number;
131
+ behind: string;
132
+ }
133
+
134
  export interface ConnectorCatchUp {
135
  active: boolean;
136
  done: number;
137
  total: number;
138
  newest: Record<string, string>;
139
+ /** Per-source progress; null when the server reports only the tenant total. */
140
+ sources: Record<string, ConnectorSourceCatchUp> | null;
141
+ /** Set when this status is scoped to one source that settled behind. */
142
+ behind?: string;
143
  }
144
 
145
  /** Missing catchUp is compatible; a malformed status is not a settled sync. */
 
160
  if (!stamp) throw new Error("Sync status unavailable");
161
  newest[key] = stamp;
162
  }
163
+ let sources: Record<string, ConnectorSourceCatchUp> | null = null;
164
+ if (Object.prototype.hasOwnProperty.call(raw, "sources")) {
165
+ const rawSources = syncObject(raw.sources);
166
+ sources = {};
167
+ for (const [name, value] of Object.entries(rawSources)) {
168
+ const entry = syncObject(value);
169
+ const sourceDone = syncCount(entry.done);
170
+ const sourceTotal = syncCount(entry.total);
171
+ if (typeof entry.active !== "boolean" || sourceDone === null || sourceTotal === null
172
+ || sourceDone > sourceTotal || typeof entry.behind !== "string") {
173
+ throw new Error("Sync status unavailable");
174
+ }
175
+ sources[name] = { active: entry.active, done: sourceDone, total: sourceTotal,
176
+ behind: syncText(entry.behind) };
177
+ }
178
+ }
179
+ return { active: raw.active, done, total, newest, sources };
180
+ }
181
+
182
+ /** The catch-up source a connector card or database waits on. */
183
+ export function catchUpSourceKey(key: string): string {
184
+ if (key === "amazon_sp" || key === "amazon" || key.startsWith("ut_amzn_")) return "amazon";
185
+ if (key === "meta_ads" || key === "meta" || key.startsWith("ut_meta_")) return "meta";
186
+ return "odoo";
187
+ }
188
+
189
+ /** Scope the tenant status to one source; an older server keeps the tenant total on Odoo only. */
190
+ export function sourceCatchUp(catchUp: ConnectorCatchUp | null, source: string): ConnectorCatchUp | null {
191
+ if (!catchUp) return null;
192
+ if (catchUp.sources === null) return source === "odoo" ? catchUp : null;
193
+ const own = catchUp.sources[source];
194
+ if (!own) return null;
195
+ return { ...catchUp, active: catchUp.active && own.active, done: own.done, total: own.total,
196
+ behind: own.behind || undefined };
197
  }
198
 
199
  export async function readConnectorCatchUp(signal: AbortSignal): Promise<ConnectorCatchUp | null> {
 
206
  return `Updating ${catchUp.done.toLocaleString()} of ${catchUp.total.toLocaleString()}`;
207
  }
208
 
209
+ export function behindLabel(reason: string): string {
210
+ return `Behind. ${reason}`;
211
+ }
212
+
213
  export function useConnectorCatchUp(enabled: boolean) {
214
  const [state, setState] = useState<{
215
  catchUp: ConnectorCatchUp | null; error: string; loading: boolean;
web/src/customer-grid/GridActivity.tsx CHANGED
@@ -135,11 +135,16 @@ export function GridActivity({ database }: { database: string }) {
135
  if (!response.ok) throw new Error("status unavailable");
136
  const body = await response.json() as {
137
  tables?: Record<string, unknown>;
138
- catchUp?: { active: boolean; done: number; total: number; newest: Record<string, string> };
 
139
  };
140
  connected = Object.hasOwn(body.tables ?? {}, database);
141
- // C1 owns catch-up; ordinary row requests never suppress activity.
142
- if (live) setCatchUp({ database, active: connected && body.catchUp?.active === true });
 
 
 
 
143
  } catch {
144
  if (live) setCatchUp({ database, active: false });
145
  } finally {
 
135
  if (!response.ok) throw new Error("status unavailable");
136
  const body = await response.json() as {
137
  tables?: Record<string, unknown>;
138
+ catchUp?: { active: boolean; done: number; total: number; newest: Record<string, string>;
139
+ sources?: Record<string, { active?: boolean }> };
140
  };
141
  connected = Object.hasOwn(body.tables ?? {}, database);
142
+ // C1 owns catch-up for this database's own source; an unscoped status holds every source.
143
+ const source = database.startsWith("ut_amzn_") ? "amazon"
144
+ : database.startsWith("ut_meta_") ? "meta" : "odoo";
145
+ const own = body.catchUp?.sources?.[source];
146
+ if (live) setCatchUp({ database, active: connected && body.catchUp?.active === true
147
+ && (own ? own.active === true : true) });
148
  } catch {
149
  if (live) setCatchUp({ database, active: false });
150
  } finally {
web/src/sheet/SheetCatchUp.tsx CHANGED
@@ -35,7 +35,15 @@ export default function SheetCatchUp({ workbookId, revision, children }: {
35
  };
36
  }, [workbookId, revision]);
37
 
38
- if (!status.active) return children;
 
 
 
 
 
 
 
 
39
  const { done, total } = status;
40
  const measured = typeof done === "number" && Number.isSafeInteger(done) && done >= 0
41
  && typeof total === "number" && Number.isSafeInteger(total) && total > 0 && done <= total;
 
35
  };
36
  }, [workbookId, revision]);
37
 
38
+ if (!status.active) {
39
+ if (!status.behind?.length) return children;
40
+ return <>
41
+ <span className="sheet-catch-up-behind" role="status">
42
+ {status.behind.map((entry) => `${entry.source} is behind. ${entry.reason}`).join(" ")}
43
+ </span>
44
+ {children}
45
+ </>;
46
+ }
47
  const { done, total } = status;
48
  const measured = typeof done === "number" && Number.isSafeInteger(done) && done >= 0
49
  && typeof total === "number" && Number.isSafeInteger(total) && total > 0 && done <= total;
web/src/sheet/sheetApi.ts CHANGED
@@ -63,12 +63,18 @@ export interface SheetCatchUpStatus {
63
  active: boolean;
64
  done?: number;
65
  total?: number;
 
 
66
  }
67
 
68
  export const readSheetCatchUp = (workbookId: string, signal?: AbortSignal): Promise<Result<SheetCatchUpStatus>> =>
69
  call(`/sheets/${encodeURIComponent(workbookId)}/catch-up`, { method: "GET", signal }, body => {
70
  const status = (body as { catchUp?: SheetCatchUpStatus } | null)?.catchUp;
71
- return status?.active === true ? status : { active: false };
 
 
 
 
72
  });
73
 
74
  const errorOf = (body: unknown): { code: string; message: string; rev_current?: number } => {
 
63
  active: boolean;
64
  done?: number;
65
  total?: number;
66
+ /** Sources this workbook binds that settled behind, each with a plain reason. */
67
+ behind?: { source: string; reason: string }[];
68
  }
69
 
70
  export const readSheetCatchUp = (workbookId: string, signal?: AbortSignal): Promise<Result<SheetCatchUpStatus>> =>
71
  call(`/sheets/${encodeURIComponent(workbookId)}/catch-up`, { method: "GET", signal }, body => {
72
  const status = (body as { catchUp?: SheetCatchUpStatus } | null)?.catchUp;
73
+ if (status?.active === true) return status;
74
+ const listed = status?.behind;
75
+ const behind = (Array.isArray(listed) ? listed : []).filter((entry) =>
76
+ typeof entry?.source === "string" && entry.source && typeof entry.reason === "string" && entry.reason);
77
+ return behind.length ? { active: false, behind } : { active: false };
78
  });
79
 
80
  const errorOf = (body: unknown): { code: string; message: string; rev_current?: number } => {
web/src/styles/sdlc73/sheet.css CHANGED
@@ -160,6 +160,13 @@
160
  font-variant-numeric: tabular-nums;
161
  }
162
 
 
 
 
 
 
 
 
163
  .sheet-last-edit {
164
  appearance: none;
165
  border: 0;
 
160
  font-variant-numeric: tabular-nums;
161
  }
162
 
163
+ .sheet-catch-up-behind {
164
+ padding: 8px 0;
165
+ color: var(--lp-muted);
166
+ font-size: var(--lp-fs-2xs);
167
+ white-space: nowrap;
168
+ }
169
+
170
  .sheet-last-edit {
171
  appearance: none;
172
  border: 0;