Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +63 -9
- api/odoo_relational.py +12 -4
- api/routes_odoo_tables.py +1 -1
- api/routes_sheet.py +83 -21
- api/test_odoo_rank_refresh.py +68 -5
- platform/harness/amazon_client.py +115 -4
- platform/harness/amazon_store.py +735 -90
- platform/harness/datastore.py +16 -1
- platform/harness/mirror_store.py +193 -23
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +22 -13
- web/src/connectors/DatabasesTab.tsx +49 -1
- web/src/customer-grid/GridActivity.tsx +8 -3
- web/src/sheet/SheetCatchUp.tsx +9 -1
- web/src/sheet/sheetApi.ts +7 -1
- web/src/styles/sdlc73/sheet.css +7 -0
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v80",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "982dca9899c (982dca9899c)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v80",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
982dca9899c (982dca9899c)
|
api/main.py
CHANGED
|
@@ -736,6 +736,8 @@ def _seed_and_sync_store():
|
|
| 736 |
_ds.reconcile_deletes(entities=selected_entities, log=lambda *a, **k: None)
|
| 737 |
except Exception:
|
| 738 |
pass
|
|
|
|
|
|
|
| 739 |
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
| 740 |
f"(ready={_ds.ready()})")
|
| 741 |
# ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
|
|
@@ -999,6 +1001,8 @@ class _DeskAwareOdooClient:
|
|
| 999 |
self._factory = factory
|
| 1000 |
self._client = None
|
| 1001 |
self._next_desk = 0.0
|
|
|
|
|
|
|
| 1002 |
|
| 1003 |
def __getattr__(self, name):
|
| 1004 |
if name not in {"search_read", "search_count", "read_group", "fields_get"}:
|
|
@@ -1010,7 +1014,7 @@ class _DeskAwareOdooClient:
|
|
| 1010 |
result, finished = {}, threading.Event()
|
| 1011 |
|
| 1012 |
def serve_desk():
|
| 1013 |
-
if time.monotonic() >= self._next_desk:
|
| 1014 |
outcome = {}
|
| 1015 |
_run_sync_requests(outcome, during_odoo=True)
|
| 1016 |
self._next_desk = time.monotonic() + (30 if "continuing" in outcome.values() else 0.25)
|
|
@@ -1651,7 +1655,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 1651 |
_health(cid, "freshness", "failed",
|
| 1652 |
failure={**failure, "stage": "freshness"})
|
| 1653 |
else:
|
| 1654 |
-
|
|
|
|
|
|
|
| 1655 |
_health(cid, "materialization", "ready")
|
| 1656 |
# A receipt is the authoritative per-database result. ``tables`` alone omits
|
| 1657 |
# read-through grains and the two compiled canonical databases, which previously
|
|
@@ -2146,6 +2152,11 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 2146 |
_resync_log(f"failed after {_pass_clock.monotonic() - pass_started:.1f}s ({type(e).__name__})")
|
| 2147 |
# A partial report must never be mistaken for a confirmed zero-change pass.
|
| 2148 |
return None, None, _classify_resync_error(e)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2149 |
|
| 2150 |
|
| 2151 |
def _absent_mirror_entities(ds, entities=None):
|
|
@@ -2270,14 +2281,14 @@ def _resync_memory_read():
|
|
| 2270 |
return {}
|
| 2271 |
|
| 2272 |
|
| 2273 |
-
def _resync_memory_write(pass_at, full_rebuild_at, fingerprints):
|
| 2274 |
"""Persist the loop's restart memory. Best effort: a failed write only costs a rebuild."""
|
| 2275 |
try:
|
| 2276 |
import odoo_relational as _rel_memory
|
| 2277 |
from harness import runtime as _runtime_memory
|
| 2278 |
_rel_memory.record_resync_memory(
|
| 2279 |
_runtime_memory.get_runtime(_resync_tenant()[0]), pass_at=pass_at,
|
| 2280 |
-
full_rebuild_at=full_rebuild_at, fingerprints=fingerprints)
|
| 2281 |
except Exception as e: # noqa: BLE001
|
| 2282 |
print(f"[aios-api] store resync memory not saved ({type(e).__name__}); "
|
| 2283 |
"the next restart rebuilds in full")
|
|
@@ -2325,6 +2336,8 @@ def _store_resync_loop():
|
|
| 2325 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2326 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2327 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
|
|
|
|
|
|
| 2328 |
catch_up_retries = 0
|
| 2329 |
last_meta_fingerprint = None
|
| 2330 |
last_meta_applied_at = None
|
|
@@ -2461,6 +2474,15 @@ def _store_resync_loop():
|
|
| 2461 |
RESYNC_STATE["consecutiveFailures"] = 0
|
| 2462 |
if not _manual:
|
| 2463 |
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2464 |
else:
|
| 2465 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 2466 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
|
@@ -2517,8 +2539,36 @@ def _store_resync_loop():
|
|
| 2517 |
# correctly. A connector that can only ever be established at boot is one bad boot away
|
| 2518 |
# from being permanently absent.
|
| 2519 |
# ⚠ Cheap when there is nothing to do: no token => one line and return.
|
| 2520 |
-
|
| 2521 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2522 |
_meta_now = _t.time()
|
| 2523 |
_meta_reason = _meta_rebuild_force_reason(
|
| 2524 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
|
@@ -2610,10 +2660,14 @@ def _store_resync_loop():
|
|
| 2610 |
last_meta_fingerprint = None
|
| 2611 |
last_meta_applied_at = None
|
| 2612 |
# HOTFIX 2026-09-29: remember this pass for the next container (see the top of the loop).
|
| 2613 |
-
|
| 2614 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2615 |
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 2616 |
-
last_source_fingerprints)
|
| 2617 |
|
| 2618 |
|
| 2619 |
def _boot_app_migrations(why):
|
|
|
|
| 736 |
_ds.reconcile_deletes(entities=selected_entities, log=lambda *a, **k: None)
|
| 737 |
except Exception:
|
| 738 |
pass
|
| 739 |
+
if isinstance(getattr(_odoo._tlocal, "client", None), _DeskAwareOdooClient):
|
| 740 |
+
_odoo._tlocal.client.desk = False # the rebuild below must not serve the desk
|
| 741 |
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
| 742 |
f"(ready={_ds.ready()})")
|
| 743 |
# ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
|
|
|
|
| 1001 |
self._factory = factory
|
| 1002 |
self._client = None
|
| 1003 |
self._next_desk = 0.0
|
| 1004 |
+
# Only the Odoo pull serves the desk; the relational rebuild after it on this thread must not.
|
| 1005 |
+
self.desk = True
|
| 1006 |
|
| 1007 |
def __getattr__(self, name):
|
| 1008 |
if name not in {"search_read", "search_count", "read_group", "fields_get"}:
|
|
|
|
| 1014 |
result, finished = {}, threading.Event()
|
| 1015 |
|
| 1016 |
def serve_desk():
|
| 1017 |
+
if self.desk and time.monotonic() >= self._next_desk:
|
| 1018 |
outcome = {}
|
| 1019 |
_run_sync_requests(outcome, during_odoo=True)
|
| 1020 |
self._next_desk = time.monotonic() + (30 if "continuing" in outcome.values() else 0.25)
|
|
|
|
| 1655 |
_health(cid, "freshness", "failed",
|
| 1656 |
failure={**failure, "stage": "freshness"})
|
| 1657 |
else:
|
| 1658 |
+
if str(cid) != str(default_cid):
|
| 1659 |
+
# A named connection's rebuild is its pull. The default's source stamp is the published mirror pass.
|
| 1660 |
+
_health(cid, "source", "ready")
|
| 1661 |
_health(cid, "materialization", "ready")
|
| 1662 |
# A receipt is the authoritative per-database result. ``tables`` alone omits
|
| 1663 |
# read-through grains and the two compiled canonical databases, which previously
|
|
|
|
| 2152 |
_resync_log(f"failed after {_pass_clock.monotonic() - pass_started:.1f}s ({type(e).__name__})")
|
| 2153 |
# A partial report must never be mistaken for a confirmed zero-change pass.
|
| 2154 |
return None, None, _classify_resync_error(e)
|
| 2155 |
+
finally:
|
| 2156 |
+
# The rebuild reuses this thread's client: a desk served there ran the Amazon first load inside it.
|
| 2157 |
+
_client = getattr(getattr(sys.modules.get("core.odoo"), "_tlocal", None), "client", None)
|
| 2158 |
+
if isinstance(_client, _DeskAwareOdooClient):
|
| 2159 |
+
_client.desk = False
|
| 2160 |
|
| 2161 |
|
| 2162 |
def _absent_mirror_entities(ds, entities=None):
|
|
|
|
| 2281 |
return {}
|
| 2282 |
|
| 2283 |
|
| 2284 |
+
def _resync_memory_write(pass_at, full_rebuild_at, fingerprints, pulls=None):
|
| 2285 |
"""Persist the loop's restart memory. Best effort: a failed write only costs a rebuild."""
|
| 2286 |
try:
|
| 2287 |
import odoo_relational as _rel_memory
|
| 2288 |
from harness import runtime as _runtime_memory
|
| 2289 |
_rel_memory.record_resync_memory(
|
| 2290 |
_runtime_memory.get_runtime(_resync_tenant()[0]), pass_at=pass_at,
|
| 2291 |
+
full_rebuild_at=full_rebuild_at, fingerprints=fingerprints, pulls=pulls)
|
| 2292 |
except Exception as e: # noqa: BLE001
|
| 2293 |
print(f"[aios-api] store resync memory not saved ({type(e).__name__}); "
|
| 2294 |
"the next restart rebuilds in full")
|
|
|
|
| 2336 |
last_relational_success_at = _memory.get("fullRebuildAt")
|
| 2337 |
last_source_fingerprints = _memory.get("fingerprints")
|
| 2338 |
last_good_pass_at = _memory.get("passAt") or _memory.get("lastGoodSyncAt")
|
| 2339 |
+
# The pulls an unfinished pass already made; its first pass after this boot resumes it.
|
| 2340 |
+
pending_pulls = _memory.get("pulls")
|
| 2341 |
catch_up_retries = 0
|
| 2342 |
last_meta_fingerprint = None
|
| 2343 |
last_meta_applied_at = None
|
|
|
|
| 2474 |
RESYNC_STATE["consecutiveFailures"] = 0
|
| 2475 |
if not _manual:
|
| 2476 |
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
| 2477 |
+
# A published pull is the Odoo source stage's success, so the receipt's source stamp follows it.
|
| 2478 |
+
try:
|
| 2479 |
+
from harness import runtime as _source_runtime
|
| 2480 |
+
import routes_connectors as _source_receipts
|
| 2481 |
+
_source_receipts.record_odoo_stage(
|
| 2482 |
+
_source_runtime.get_runtime(_cad["tenant"]), RESYNC_STATE["connectionId"] or "default",
|
| 2483 |
+
"source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
|
| 2484 |
+
except Exception: # noqa: BLE001
|
| 2485 |
+
pass # The resident mirror state above still reports this pass.
|
| 2486 |
else:
|
| 2487 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 2488 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
|
|
|
| 2539 |
# correctly. A connector that can only ever be established at boot is one bad boot away
|
| 2540 |
# from being permanently absent.
|
| 2541 |
# ⚠ Cheap when there is nothing to do: no token => one line and return.
|
| 2542 |
+
# ⛔⛔ HOTFIX sdlc74 #3: a restart RESUMES an unfinished pass and never repeats its pulls.
|
| 2543 |
+
# Each pull checkpoints when it returns. The first pass after a boot skips a pull that an
|
| 2544 |
+
# unfinished pass younger than one cadence already made, and runs only the rest.
|
| 2545 |
+
_pending = (pending_pulls if passes == 1 and not _odoo_requested
|
| 2546 |
+
and isinstance(pending_pulls, dict) else {})
|
| 2547 |
+
_resumed = ({stage: _pending[stage] for stage in ("meta", "amazon") if _pending.get(stage)}
|
| 2548 |
+
if _resync_boot_recent(_pending.get("startedAt"), _t.time(), _every) else {})
|
| 2549 |
+
_pulls = {"startedAt": last_full_pass_at}
|
| 2550 |
+
if _resumed:
|
| 2551 |
+
_pulls = dict(_resumed, startedAt=_pending["startedAt"])
|
| 2552 |
+
last_full_pass_at = _pulls["startedAt"]
|
| 2553 |
+
_age = max(0, int(_t.time() - last_full_pass_at))
|
| 2554 |
+
print(f"[aios-api] store resync: restarted {_age}s into a pass that never finished; "
|
| 2555 |
+
f"its {' and '.join(sorted(_resumed))} pull already reached the store, so that "
|
| 2556 |
+
f"vendor is not called again and only the rest of the pass runs now; the next "
|
| 2557 |
+
f"scheduled pass is due in {max(0, _every - _age)}s")
|
| 2558 |
+
if _resumed.get("meta"):
|
| 2559 |
+
_meta_pull_confirmed = False
|
| 2560 |
+
else:
|
| 2561 |
+
_meta_pull_confirmed = _pull_meta("resync")
|
| 2562 |
+
_pulls["meta"] = _t.time()
|
| 2563 |
+
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 2564 |
+
last_source_fingerprints, dict(_pulls))
|
| 2565 |
+
if _resumed.get("amazon"):
|
| 2566 |
+
_pull_amazon("boot", False)
|
| 2567 |
+
else:
|
| 2568 |
+
_pull_amazon("resync")
|
| 2569 |
+
_pulls["amazon"] = _t.time()
|
| 2570 |
+
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 2571 |
+
last_source_fingerprints, dict(_pulls))
|
| 2572 |
_meta_now = _t.time()
|
| 2573 |
_meta_reason = _meta_rebuild_force_reason(
|
| 2574 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
|
|
|
| 2660 |
last_meta_fingerprint = None
|
| 2661 |
last_meta_applied_at = None
|
| 2662 |
# HOTFIX 2026-09-29: remember this pass for the next container (see the top of the loop).
|
| 2663 |
+
# A pass that did not end good keeps its pulls checkpoint, so a restart resumes it.
|
| 2664 |
+
_pass_good = _err is None and _rebuild_ok
|
| 2665 |
+
if _pass_good:
|
| 2666 |
+
# A resumed pass ended its pulls when they ran, so a restart never shifts the cadence.
|
| 2667 |
+
last_good_pass_at = (max(_pulls.get("meta") or 0, _pulls.get("amazon") or 0)
|
| 2668 |
+
if _resumed else _meta_now)
|
| 2669 |
_resync_memory_write(last_good_pass_at, last_relational_success_at,
|
| 2670 |
+
last_source_fingerprints, None if _pass_good else dict(_pulls))
|
| 2671 |
|
| 2672 |
|
| 2673 |
def _boot_app_migrations(why):
|
api/odoo_relational.py
CHANGED
|
@@ -5216,6 +5216,8 @@ def record_sync_outcome(rt, ok, at=None):
|
|
| 5216 |
#: * `passAt` the last scheduled pass that completed without a mirror error (ISO);
|
| 5217 |
#: * `fullRebuildAt` the last FULL relational rebuild that succeeded (ISO), for the weekly recovery;
|
| 5218 |
#: * `fingerprints` `{entity: [count, hash]}` as the last successful apply saw the mirror.
|
|
|
|
|
|
|
| 5219 |
#: ⚠ The two numbers are STRINGS: `bit_xor(hash())` is an unsigned 64-bit value, and a JSON reader
|
| 5220 |
#: that is not Python may round it. `read_resync_memory` turns them back into the int TUPLES the
|
| 5221 |
#: loop compares, because a list never equals a tuple and every pass would read as changed.
|
|
@@ -5264,7 +5266,8 @@ def decode_fingerprints(stored):
|
|
| 5264 |
|
| 5265 |
def read_resync_memory(rt):
|
| 5266 |
"""What the last container's resync loop knew, or empty values. Never raises."""
|
| 5267 |
-
empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None
|
|
|
|
| 5268 |
try:
|
| 5269 |
cur = rt.get(CONFIG_KEY) if rt is not None else None
|
| 5270 |
except Exception: # noqa: BLE001
|
|
@@ -5272,14 +5275,17 @@ def read_resync_memory(rt):
|
|
| 5272 |
cur = cur if isinstance(cur, dict) else {}
|
| 5273 |
memory = cur.get(RESYNC_MEMORY_FIELD)
|
| 5274 |
memory = memory if isinstance(memory, dict) else {}
|
|
|
|
| 5275 |
return {"passAt": _memory_epoch(memory.get("passAt")),
|
| 5276 |
"fullRebuildAt": _memory_epoch(memory.get("fullRebuildAt")),
|
| 5277 |
"fingerprints": decode_fingerprints(memory.get("fingerprints")),
|
| 5278 |
-
"lastGoodSyncAt": _memory_epoch(cur.get("lastGoodSyncAt"))
|
|
|
|
|
|
|
| 5279 |
|
| 5280 |
|
| 5281 |
-
def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints):
|
| 5282 |
-
"""Persist the loop's
|
| 5283 |
def _iso(epoch):
|
| 5284 |
if epoch is None:
|
| 5285 |
return None
|
|
@@ -5287,6 +5293,8 @@ def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints):
|
|
| 5287 |
|
| 5288 |
memory = {"passAt": _iso(pass_at), "fullRebuildAt": _iso(full_rebuild_at),
|
| 5289 |
"fingerprints": encode_fingerprints(fingerprints)}
|
|
|
|
|
|
|
| 5290 |
|
| 5291 |
def _up(cur):
|
| 5292 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
|
|
|
| 5216 |
#: * `passAt` the last scheduled pass that completed without a mirror error (ISO);
|
| 5217 |
#: * `fullRebuildAt` the last FULL relational rebuild that succeeded (ISO), for the weekly recovery;
|
| 5218 |
#: * `fingerprints` `{entity: [count, hash]}` as the last successful apply saw the mirror.
|
| 5219 |
+
#: * `pulls` `{startedAt, meta, amazon}` (ISO) of a pass whose vendor pulls finished but
|
| 5220 |
+
#: whose end was never written; a restart inside the cadence resumes it.
|
| 5221 |
#: ⚠ The two numbers are STRINGS: `bit_xor(hash())` is an unsigned 64-bit value, and a JSON reader
|
| 5222 |
#: that is not Python may round it. `read_resync_memory` turns them back into the int TUPLES the
|
| 5223 |
#: loop compares, because a list never equals a tuple and every pass would read as changed.
|
|
|
|
| 5266 |
|
| 5267 |
def read_resync_memory(rt):
|
| 5268 |
"""What the last container's resync loop knew, or empty values. Never raises."""
|
| 5269 |
+
empty = {"passAt": None, "fullRebuildAt": None, "fingerprints": None, "lastGoodSyncAt": None,
|
| 5270 |
+
"pulls": None}
|
| 5271 |
try:
|
| 5272 |
cur = rt.get(CONFIG_KEY) if rt is not None else None
|
| 5273 |
except Exception: # noqa: BLE001
|
|
|
|
| 5275 |
cur = cur if isinstance(cur, dict) else {}
|
| 5276 |
memory = cur.get(RESYNC_MEMORY_FIELD)
|
| 5277 |
memory = memory if isinstance(memory, dict) else {}
|
| 5278 |
+
pulls = memory.get("pulls")
|
| 5279 |
return {"passAt": _memory_epoch(memory.get("passAt")),
|
| 5280 |
"fullRebuildAt": _memory_epoch(memory.get("fullRebuildAt")),
|
| 5281 |
"fingerprints": decode_fingerprints(memory.get("fingerprints")),
|
| 5282 |
+
"lastGoodSyncAt": _memory_epoch(cur.get("lastGoodSyncAt")),
|
| 5283 |
+
"pulls": ({str(k): _memory_epoch(v) for k, v in pulls.items()}
|
| 5284 |
+
if isinstance(pulls, dict) and pulls else None)}
|
| 5285 |
|
| 5286 |
|
| 5287 |
+
def record_resync_memory(rt, *, pass_at, full_rebuild_at, fingerprints, pulls=None):
|
| 5288 |
+
"""Persist the loop's facts exactly as it now holds them (None clears one)."""
|
| 5289 |
def _iso(epoch):
|
| 5290 |
if epoch is None:
|
| 5291 |
return None
|
|
|
|
| 5293 |
|
| 5294 |
memory = {"passAt": _iso(pass_at), "fullRebuildAt": _iso(full_rebuild_at),
|
| 5295 |
"fingerprints": encode_fingerprints(fingerprints)}
|
| 5296 |
+
if isinstance(pulls, dict) and pulls.get("startedAt") is not None:
|
| 5297 |
+
memory["pulls"] = {str(k): _iso(v) for k, v in pulls.items() if v is not None}
|
| 5298 |
|
| 5299 |
def _up(cur):
|
| 5300 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
api/routes_odoo_tables.py
CHANGED
|
@@ -399,7 +399,7 @@ def _catch_up_status(session):
|
|
| 399 |
try:
|
| 400 |
status = mirror_store.catch_up_status(session.runtime)
|
| 401 |
except Exception:
|
| 402 |
-
return {"active": True, "done": 0, "total": 0, "newest": {}}
|
| 403 |
status["newest"] = {key: value for key, value in status["newest"].items()
|
| 404 |
if perm_scope.may_read(session.user, key, st=session.runtime)}
|
| 405 |
return status
|
|
|
|
| 399 |
try:
|
| 400 |
status = mirror_store.catch_up_status(session.runtime)
|
| 401 |
except Exception:
|
| 402 |
+
return {"active": True, "done": 0, "total": 0, "newest": {}, "sources": {}}
|
| 403 |
status["newest"] = {key: value for key, value in status["newest"].items()
|
| 404 |
if perm_scope.may_read(session.user, key, st=session.runtime)}
|
| 405 |
return status
|
api/routes_sheet.py
CHANGED
|
@@ -138,10 +138,36 @@ def _sheet_catch_up(session):
|
|
| 138 |
|
| 139 |
|
| 140 |
def _database_catching_up(session, database):
|
| 141 |
-
"""Connected mirrors wait for C1; native authored tables remain readable."""
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
| 143 |
return False
|
| 144 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
|
| 147 |
def _database_connected(session, database):
|
|
@@ -172,9 +198,14 @@ def _binding_catching_up(session, binding):
|
|
| 172 |
if _sheet_catch_up(session).get("active") is not True:
|
| 173 |
return False
|
| 174 |
from modules import sheet as sheet_mod
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
|
| 179 |
|
| 180 |
def _mirror(session: Session):
|
|
@@ -2626,27 +2657,58 @@ def read_sheet_activity(workbook_id: str, session: Session = Depends(require_ses
|
|
| 2626 |
def read_sheet_catch_up(workbook_id: str, session: Session = Depends(require_session)):
|
| 2627 |
"""Scope C1 to the workbook, including tabs the browser has not loaded."""
|
| 2628 |
from core import sheets
|
|
|
|
| 2629 |
try:
|
| 2630 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2631 |
_require_reader(session, book)
|
| 2632 |
except sheets.SheetRefusal as exc:
|
| 2633 |
raise _refuse(exc)
|
| 2634 |
session._sheet_scope_context = {}
|
| 2635 |
-
|
| 2636 |
-
|
| 2637 |
-
|
| 2638 |
-
|
| 2639 |
-
|
| 2640 |
-
|
| 2641 |
-
|
| 2642 |
-
|
| 2643 |
-
|
| 2644 |
-
|
| 2645 |
-
|
| 2646 |
-
|
| 2647 |
-
|
| 2648 |
-
|
| 2649 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2650 |
|
| 2651 |
|
| 2652 |
@router.get("/sheets/{workbook_id}/refresh-history")
|
|
|
|
| 138 |
|
| 139 |
|
| 140 |
def _database_catching_up(session, database):
|
| 141 |
+
"""Connected mirrors wait for their own source's C1; native authored tables remain readable."""
|
| 142 |
+
status = _sheet_catch_up(session)
|
| 143 |
+
if status.get("active") is not True:
|
| 144 |
+
return False
|
| 145 |
+
if not _database_connected(session, database):
|
| 146 |
return False
|
| 147 |
+
# An unmapped database, or a status without per-source progress, waits for the tenant.
|
| 148 |
+
own = (status.get("sources") or {}).get(_database_source(session, database))
|
| 149 |
+
return not isinstance(own, dict) or own.get("active") is True
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
#: The connector name each catch-up source shows a person.
|
| 153 |
+
_SOURCE_LABELS = {"odoo": "Odoo", "amazon": "Amazon", "meta": "Meta Ads"}
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _database_source(session, database):
|
| 157 |
+
"""Name the catch-up source a connected database waits on, or None when it is unknown."""
|
| 158 |
+
import odoo_relational as rel
|
| 159 |
+
from core import connector_spec
|
| 160 |
+
database = str(database or "")
|
| 161 |
+
if (database in rel._REGISTRY_GRID_FIELDS or database == rel.SHIPMENTS_KEY
|
| 162 |
+
or any(database == key for _, key, _, _ in rel.TABLES)):
|
| 163 |
+
return "odoo"
|
| 164 |
+
connector = connector_spec.connector_of(database) or next((name for prefix, name in (
|
| 165 |
+
("ut_amzn_", "amazon"), ("ut_meta_", "meta"), ("ut_odoo_", "odoo"),
|
| 166 |
+
) if database.startswith(prefix)), None)
|
| 167 |
+
if connector is None and database in {"customer_data", "product_data"}:
|
| 168 |
+
connector = "odoo"
|
| 169 |
+
connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
|
| 170 |
+
return connector if connector in _SOURCE_LABELS else None
|
| 171 |
|
| 172 |
|
| 173 |
def _database_connected(session, database):
|
|
|
|
| 198 |
if _sheet_catch_up(session).get("active") is not True:
|
| 199 |
return False
|
| 200 |
from modules import sheet as sheet_mod
|
| 201 |
+
try:
|
| 202 |
+
nodes, _ = _binding_filter_nodes(session, database, binding)
|
| 203 |
+
targets = [_binding_related_entry(session, database, via)["target"]
|
| 204 |
+
for via in sheet_mod.related_vias(nodes)]
|
| 205 |
+
except HTTPException:
|
| 206 |
+
# The cell's own read refuses an unreadable relation; this check never refuses the workbook.
|
| 207 |
+
return False
|
| 208 |
+
return any(_database_catching_up(session, target) for target in targets)
|
| 209 |
|
| 210 |
|
| 211 |
def _mirror(session: Session):
|
|
|
|
| 2657 |
def read_sheet_catch_up(workbook_id: str, session: Session = Depends(require_session)):
|
| 2658 |
"""Scope C1 to the workbook, including tabs the browser has not loaded."""
|
| 2659 |
from core import sheets
|
| 2660 |
+
from modules import sheet as sheet_mod
|
| 2661 |
try:
|
| 2662 |
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2663 |
_require_reader(session, book)
|
| 2664 |
except sheets.SheetRefusal as exc:
|
| 2665 |
raise _refuse(exc)
|
| 2666 |
session._sheet_scope_context = {}
|
| 2667 |
+
# One memo per read: without it every bound cell re-read the whole catalogue (64.9 s on a template).
|
| 2668 |
+
with _sheet_request_scope(session):
|
| 2669 |
+
status = _sheet_catch_up(session)
|
| 2670 |
+
sources = status.get("sources") if isinstance(status.get("sources"), dict) else {}
|
| 2671 |
+
behind = {name for name, entry in sources.items() if isinstance(entry, dict) and entry.get("behind")}
|
| 2672 |
+
held, bound, readable = set(), set(), {}
|
| 2673 |
+
if status.get("active") is True or behind:
|
| 2674 |
+
for tab in book.get("sheets") or []:
|
| 2675 |
+
for cell in (tab.get("cells") or {}).values():
|
| 2676 |
+
binding = cell.get("binding") or {}
|
| 2677 |
+
if not binding.get("database"):
|
| 2678 |
+
continue
|
| 2679 |
+
# The page polls every 2 s, so each distinct database is checked once per read.
|
| 2680 |
+
if binding["database"] not in readable:
|
| 2681 |
+
try:
|
| 2682 |
+
_require_database(session, binding["database"])
|
| 2683 |
+
readable[binding["database"]] = _database_source(session, binding["database"])
|
| 2684 |
+
except HTTPException:
|
| 2685 |
+
readable[binding["database"]] = HTTPException
|
| 2686 |
+
if readable[binding["database"]] is HTTPException:
|
| 2687 |
+
continue
|
| 2688 |
+
try:
|
| 2689 |
+
bound.add(readable[binding["database"]])
|
| 2690 |
+
if status.get("active") is True and _binding_catching_up(session, binding):
|
| 2691 |
+
database = str(binding["database"])
|
| 2692 |
+
nodes, _ = _binding_filter_nodes(session, database, binding)
|
| 2693 |
+
targets = [database] + [_binding_related_entry(session, database, via)["target"]
|
| 2694 |
+
for via in sheet_mod.related_vias(nodes)]
|
| 2695 |
+
held.update(_database_source(session, target) for target in targets
|
| 2696 |
+
if _database_catching_up(session, target))
|
| 2697 |
+
except HTTPException:
|
| 2698 |
+
continue
|
| 2699 |
+
if held:
|
| 2700 |
+
break # The first held cell answers, as before: one held template took 53 s per poll.
|
| 2701 |
+
if held:
|
| 2702 |
+
break
|
| 2703 |
+
if held:
|
| 2704 |
+
# C1's newest map can name unrelated databases. Return only the held sources' progress.
|
| 2705 |
+
if None in held or not held <= set(sources):
|
| 2706 |
+
return {"catchUp": {key: status.get(key) for key in ("active", "done", "total")}}
|
| 2707 |
+
return {"catchUp": {"active": True, "done": sum(sources[name].get("done", 0) for name in held),
|
| 2708 |
+
"total": sum(sources[name].get("total", 0) for name in held)}}
|
| 2709 |
+
lines = [{"source": _SOURCE_LABELS[name], "reason": sources[name]["behind"]}
|
| 2710 |
+
for name in _SOURCE_LABELS if name in behind and name in bound]
|
| 2711 |
+
return {"catchUp": {"active": False, "behind": lines} if lines else {"active": False}}
|
| 2712 |
|
| 2713 |
|
| 2714 |
@router.get("/sheets/{workbook_id}/refresh-history")
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -484,7 +484,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 484 |
|
| 485 |
# ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
|
| 486 |
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
|
| 487 |
-
errors=(), max_sleeps=1, frozen=False):
|
| 488 |
"""Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
|
| 489 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 490 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
|
@@ -516,7 +516,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 516 |
|
| 517 |
def rebuild(why, previous, meta_state, **scope):
|
| 518 |
events.append(('rebuild', scope.get('table_keys')))
|
| 519 |
-
return
|
| 520 |
|
| 521 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 522 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
|
@@ -583,9 +583,64 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 583 |
events, _ = self.run_daily_loop(
|
| 584 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
|
| 585 |
'fingerprints': fps}, fps)
|
| 586 |
-
self.assertEqual([e[0] for e in events],
|
| 587 |
-
|
| 588 |
-
self.assertEqual(events[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 589 |
|
| 590 |
def test_persisted_fingerprints_skip_the_rebuild_after_a_restart(self):
|
| 591 |
fps = {'res_partner': (3, 7)}
|
|
@@ -674,6 +729,14 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 674 |
memory = rel.read_resync_memory(rt)
|
| 675 |
self.assertEqual((memory['passAt'], memory['fullRebuildAt'], memory['fingerprints']),
|
| 676 |
(1_000.0, None, fps))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
self.assertEqual(rt.docs[rel.CONFIG_KEY]['syncEvery'], 'daily')
|
| 678 |
self.assertIsNotNone(memory['lastGoodSyncAt'])
|
| 679 |
|
|
|
|
| 484 |
|
| 485 |
# ── HOTFIX 2026-09-29 (owner): "only sync daily at the fastest", and only what changed ──────
|
| 486 |
def run_daily_loop(self, memory, current, *, now=10_000.0, cadence=86400, manual=False,
|
| 487 |
+
errors=(), max_sleeps=1, frozen=False, rebuild_ok=True):
|
| 488 |
"""Drive the real loop with a persisted memory; stop at its `max_sleeps`-th wait."""
|
| 489 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 490 |
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
|
|
|
| 516 |
|
| 517 |
def rebuild(why, previous, meta_state, **scope):
|
| 518 |
events.append(('rebuild', scope.get('table_keys')))
|
| 519 |
+
return rebuild_ok
|
| 520 |
|
| 521 |
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
|
| 522 |
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
|
|
|
| 583 |
events, _ = self.run_daily_loop(
|
| 584 |
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000,
|
| 585 |
'fingerprints': fps}, fps)
|
| 586 |
+
self.assertEqual([e[0] for e in events],
|
| 587 |
+
['sync', 'meta', 'memory', 'amazon', 'memory', 'fresh', 'memory'])
|
| 588 |
+
self.assertEqual(events[3], ('amazon', 'resync'))
|
| 589 |
+
self.assertEqual(events[-1], ('memory', 10_000.0, 10_000.0 - 90_000, fps, None))
|
| 590 |
+
|
| 591 |
+
# ── HOTFIX sdlc74 #3 (owner): "we retired rebooting sync at deployment" ─────────────────────
|
| 592 |
+
def test_each_pull_checkpoints_before_the_rebuild_and_a_bad_end_keeps_it(self):
|
| 593 |
+
old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
|
| 594 |
+
events, _ = self.run_daily_loop(
|
| 595 |
+
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old},
|
| 596 |
+
new, rebuild_ok=False)
|
| 597 |
+
kinds = [e[0] for e in events]
|
| 598 |
+
self.assertEqual(kinds, ['sync', 'meta', 'memory', 'amazon', 'memory', 'rebuild', 'memory'])
|
| 599 |
+
# The checkpoints change none of the three facts: a death in the rebuild leaves them as
|
| 600 |
+
# they were, plus the pulls the pass already made.
|
| 601 |
+
self.assertEqual(events[2], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old,
|
| 602 |
+
{'startedAt': 10_000.0, 'meta': 10_000.0}))
|
| 603 |
+
both = {'startedAt': 10_000.0, 'meta': 10_000.0, 'amazon': 10_000.0}
|
| 604 |
+
self.assertEqual(events[4], ('memory', 10_000.0 - 90_000, 10_000.0 - 90_000, old, both))
|
| 605 |
+
self.assertEqual(events[-1][-1], both)
|
| 606 |
+
|
| 607 |
+
def test_restart_after_an_unfinished_pass_calls_no_vendor_and_rebuilds(self):
|
| 608 |
+
old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
|
| 609 |
+
events, sleeps = self.run_daily_loop(
|
| 610 |
+
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
|
| 611 |
+
'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
|
| 612 |
+
'amazon': 10_000.0 - 600}}, new)
|
| 613 |
+
# No Meta pull and no Amazon vendor pull: Amazon's saved mirror is restored, the mirror
|
| 614 |
+
# catches up, and the rebuild that never reached the store runs.
|
| 615 |
+
self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'rebuild', 'memory'])
|
| 616 |
+
self.assertEqual(events[1], ('amazon', 'boot', False))
|
| 617 |
+
self.assertIn('ut_odoo_orders', events[2][1])
|
| 618 |
+
# The pass ends good at the time its pulls ended, and its checkpoint is cleared.
|
| 619 |
+
self.assertEqual(events[-1], ('memory', 10_000.0 - 600, 10_000.0 - 90_000, new, None))
|
| 620 |
+
# The next scheduled pass is one cadence after the unfinished pass STARTED.
|
| 621 |
+
self.assertEqual(sleeps, [86400 - 3600])
|
| 622 |
+
self.assertIn('restarted 3600s into a pass that never finished', self.loop_output)
|
| 623 |
+
|
| 624 |
+
def test_restart_during_the_amazon_pull_repeats_only_amazon(self):
|
| 625 |
+
old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
|
| 626 |
+
events, _ = self.run_daily_loop(
|
| 627 |
+
{'passAt': 10_000.0 - 90_000, 'fullRebuildAt': 10_000.0 - 90_000, 'fingerprints': old,
|
| 628 |
+
'pulls': {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400}}, new)
|
| 629 |
+
self.assertEqual([e[0] for e in events], ['sync', 'amazon', 'memory', 'rebuild', 'memory'])
|
| 630 |
+
self.assertEqual(events[1], ('amazon', 'resync'))
|
| 631 |
+
self.assertEqual(events[2][-1], {'startedAt': 10_000.0 - 3600, 'meta': 10_000.0 - 3400,
|
| 632 |
+
'amazon': 10_000.0})
|
| 633 |
+
|
| 634 |
+
def test_an_unfinished_pass_older_than_the_cadence_is_not_resumed(self):
|
| 635 |
+
"""NC: the same checkpoint one cadence old runs the whole scheduled pass again"""
|
| 636 |
+
old, new = {'res_partner': (3, 7)}, {'res_partner': (4, 9)}
|
| 637 |
+
events, _ = self.run_daily_loop(
|
| 638 |
+
{'passAt': 10_000.0 - 190_000, 'fullRebuildAt': 10_000.0 - 190_000, 'fingerprints': old,
|
| 639 |
+
'pulls': {'startedAt': 10_000.0 - 86_400, 'meta': 10_000.0 - 86_000,
|
| 640 |
+
'amazon': 10_000.0 - 84_000}}, new)
|
| 641 |
+
self.assertIn(('meta', 'resync'), events)
|
| 642 |
+
self.assertIn(('amazon', 'resync'), events)
|
| 643 |
+
self.assertNotIn(('amazon', 'boot', False), events)
|
| 644 |
|
| 645 |
def test_persisted_fingerprints_skip_the_rebuild_after_a_restart(self):
|
| 646 |
fps = {'res_partner': (3, 7)}
|
|
|
|
| 729 |
memory = rel.read_resync_memory(rt)
|
| 730 |
self.assertEqual((memory['passAt'], memory['fullRebuildAt'], memory['fingerprints']),
|
| 731 |
(1_000.0, None, fps))
|
| 732 |
+
self.assertIsNone(memory['pulls'])
|
| 733 |
+
# HOTFIX sdlc74 #3: the pulls checkpoint round-trips, and the next write without it clears it.
|
| 734 |
+
pulls = {'startedAt': 2_000.0, 'meta': 2_100.0, 'amazon': 4_900.0}
|
| 735 |
+
rel.record_resync_memory(rt, pass_at=1_000.0, full_rebuild_at=None, fingerprints=fps,
|
| 736 |
+
pulls=pulls)
|
| 737 |
+
self.assertEqual(rel.read_resync_memory(rt)['pulls'], pulls)
|
| 738 |
+
rel.record_resync_memory(rt, pass_at=5_000.0, full_rebuild_at=None, fingerprints=fps)
|
| 739 |
+
self.assertIsNone(rel.read_resync_memory(rt)['pulls'])
|
| 740 |
self.assertEqual(rt.docs[rel.CONFIG_KEY]['syncEvery'], 'daily')
|
| 741 |
self.assertIsNotNone(memory['lastGoodSyncAt'])
|
| 742 |
|
platform/harness/amazon_client.py
CHANGED
|
@@ -45,9 +45,13 @@ Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-con
|
|
| 45 |
from __future__ import annotations
|
| 46 |
|
| 47 |
import contextvars
|
|
|
|
| 48 |
import json
|
|
|
|
| 49 |
import random
|
| 50 |
import re
|
|
|
|
|
|
|
| 51 |
import threading
|
| 52 |
import time
|
| 53 |
from datetime import date, datetime, timezone, timedelta
|
|
@@ -1994,7 +1998,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
|
|
| 1994 |
url = str(meta.get("url") or "")
|
| 1995 |
if not url:
|
| 1996 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 1997 |
-
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
|
| 1998 |
say(f" {report_type}: {len(text):,} chars")
|
| 1999 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 2000 |
"dataStartTime": got.get("dataStartTime"),
|
|
@@ -2090,8 +2094,106 @@ def _workbook_as_tsv(raw):
|
|
| 2090 |
return "\n".join(lines)
|
| 2091 |
|
| 2092 |
|
| 2093 |
-
|
| 2094 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2095 |
|
| 2096 |
⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
|
| 2097 |
string, which for an S3 presigned URL is the entire credential.
|
|
@@ -2103,11 +2205,20 @@ def _download_document(url, compression="", timeout=120):
|
|
| 2103 |
# apply OPERATIONS here: its host and signature belong to Amazon's document download service.
|
| 2104 |
req = urllib.request.Request(url, method="GET")
|
| 2105 |
req.add_header("User-Agent", _UA)
|
|
|
|
| 2106 |
try:
|
| 2107 |
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 2108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2109 |
except Exception as e: # noqa: BLE001
|
|
|
|
|
|
|
| 2110 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
|
|
|
|
|
|
| 2111 |
# S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
|
| 2112 |
# `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
|
| 2113 |
# algorithm the bytes were decoded as text and every economics window failed to parse. The
|
|
|
|
| 45 |
from __future__ import annotations
|
| 46 |
|
| 47 |
import contextvars
|
| 48 |
+
import gzip
|
| 49 |
import json
|
| 50 |
+
import os
|
| 51 |
import random
|
| 52 |
import re
|
| 53 |
+
import shutil
|
| 54 |
+
import tempfile
|
| 55 |
import threading
|
| 56 |
import time
|
| 57 |
from datetime import date, datetime, timezone, timedelta
|
|
|
|
| 1998 |
url = str(meta.get("url") or "")
|
| 1999 |
if not url:
|
| 2000 |
raise AmazonError(f"the {report_type} report document carried no download URL")
|
| 2001 |
+
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
|
| 2002 |
say(f" {report_type}: {len(text):,} chars")
|
| 2003 |
return text, {"reportId": rid, "reportDocumentId": doc_id,
|
| 2004 |
"dataStartTime": got.get("dataStartTime"),
|
|
|
|
| 2094 |
return "\n".join(lines)
|
| 2095 |
|
| 2096 |
|
| 2097 |
+
class ReportDocument:
|
| 2098 |
+
"""One downloaded report document spooled to a temporary file, read back as text in pieces.
|
| 2099 |
+
|
| 2100 |
+
⛔ A report has no row cap (standing rule 1), so its text never sits in memory whole: the Brand
|
| 2101 |
+
Analytics month held the body, its decompressed copy and its decoded text at once (D-7081).
|
| 2102 |
+
The file is removed when the object is closed or collected.
|
| 2103 |
+
"""
|
| 2104 |
+
|
| 2105 |
+
def __init__(self, path, encoding, errors, chars, size, compressed):
|
| 2106 |
+
self.path, self.encoding, self.errors = path, encoding, errors
|
| 2107 |
+
self.chars, self.size, self.compressed = chars, size, compressed
|
| 2108 |
+
|
| 2109 |
+
def open_text(self):
|
| 2110 |
+
"""The decoded text as a stream; `newline=""` keeps line ends exactly as `bytes.decode` did."""
|
| 2111 |
+
return open(self.path, "r", encoding=self.encoding, errors=self.errors, newline="")
|
| 2112 |
+
|
| 2113 |
+
def read_text(self):
|
| 2114 |
+
"""The whole text, for a consumer whose document is small."""
|
| 2115 |
+
with self.open_text() as handle:
|
| 2116 |
+
return handle.read()
|
| 2117 |
+
|
| 2118 |
+
def __len__(self):
|
| 2119 |
+
return self.chars
|
| 2120 |
+
|
| 2121 |
+
def close(self):
|
| 2122 |
+
path, self.path = getattr(self, "path", None), None
|
| 2123 |
+
if path:
|
| 2124 |
+
try:
|
| 2125 |
+
os.unlink(path)
|
| 2126 |
+
except Exception: # noqa: BLE001
|
| 2127 |
+
pass
|
| 2128 |
+
|
| 2129 |
+
def __del__(self):
|
| 2130 |
+
self.close()
|
| 2131 |
+
|
| 2132 |
+
|
| 2133 |
+
_SPOOL_CHUNK = 1 << 20
|
| 2134 |
+
|
| 2135 |
+
|
| 2136 |
+
def _spool_file(prefix):
|
| 2137 |
+
handle, path = tempfile.mkstemp(prefix=prefix, suffix=".part")
|
| 2138 |
+
os.close(handle)
|
| 2139 |
+
return path
|
| 2140 |
+
|
| 2141 |
+
|
| 2142 |
+
def _unlink(path):
|
| 2143 |
+
try:
|
| 2144 |
+
os.unlink(path)
|
| 2145 |
+
except OSError:
|
| 2146 |
+
pass
|
| 2147 |
+
|
| 2148 |
+
|
| 2149 |
+
def _decoded_chars(path, encoding, errors="strict"):
|
| 2150 |
+
"""Characters in the file under one encoding; raises UnicodeDecodeError like `bytes.decode`."""
|
| 2151 |
+
chars = 0
|
| 2152 |
+
with open(path, "r", encoding=encoding, errors=errors, newline="") as handle:
|
| 2153 |
+
while True:
|
| 2154 |
+
piece = handle.read(_SPOOL_CHUNK)
|
| 2155 |
+
if not piece:
|
| 2156 |
+
return chars
|
| 2157 |
+
chars += len(piece)
|
| 2158 |
+
|
| 2159 |
+
|
| 2160 |
+
def _spooled_document(body, compression):
|
| 2161 |
+
"""The spooled body as a `ReportDocument`: gunzip and decode stream through disk, never memory."""
|
| 2162 |
+
compressed = os.path.getsize(body)
|
| 2163 |
+
with open(body, "rb") as handle:
|
| 2164 |
+
head = handle.read(4)
|
| 2165 |
+
if not compression and head[:2] == b"\x1f\x8b":
|
| 2166 |
+
compression = "GZIP"
|
| 2167 |
+
if str(compression).upper() == "GZIP":
|
| 2168 |
+
plain = _spool_file("amazon-document-")
|
| 2169 |
+
try:
|
| 2170 |
+
with gzip.open(body, "rb") as source, open(plain, "wb") as out:
|
| 2171 |
+
shutil.copyfileobj(source, out, _SPOOL_CHUNK)
|
| 2172 |
+
except Exception as e: # noqa: BLE001
|
| 2173 |
+
_unlink(plain)
|
| 2174 |
+
raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
|
| 2175 |
+
finally:
|
| 2176 |
+
_unlink(body)
|
| 2177 |
+
body = plain
|
| 2178 |
+
with open(body, "rb") as handle:
|
| 2179 |
+
head = handle.read(4)
|
| 2180 |
+
if head[:4] == b"PK\x03\x04":
|
| 2181 |
+
try:
|
| 2182 |
+
with open(body, "rb") as handle:
|
| 2183 |
+
return _workbook_as_tsv(handle.read())
|
| 2184 |
+
finally:
|
| 2185 |
+
_unlink(body)
|
| 2186 |
+
size = os.path.getsize(body)
|
| 2187 |
+
for enc in ("utf-8", "cp1252"):
|
| 2188 |
+
try:
|
| 2189 |
+
return ReportDocument(body, enc, "strict", _decoded_chars(body, enc), size, compressed)
|
| 2190 |
+
except UnicodeDecodeError:
|
| 2191 |
+
continue
|
| 2192 |
+
return ReportDocument(body, "utf-8", "replace", _decoded_chars(body, "utf-8", "replace"), size, compressed)
|
| 2193 |
+
|
| 2194 |
+
|
| 2195 |
+
def _download_document(url, compression="", timeout=120, *, spool=False):
|
| 2196 |
+
"""Fetch one presigned report document and decode it. -> text, or a `ReportDocument` when spooled.
|
| 2197 |
|
| 2198 |
⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query
|
| 2199 |
string, which for an S3 presigned URL is the entire credential.
|
|
|
|
| 2205 |
# apply OPERATIONS here: its host and signature belong to Amazon's document download service.
|
| 2206 |
req = urllib.request.Request(url, method="GET")
|
| 2207 |
req.add_header("User-Agent", _UA)
|
| 2208 |
+
body = _spool_file("amazon-document-") if spool else None
|
| 2209 |
try:
|
| 2210 |
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 2211 |
+
if body:
|
| 2212 |
+
with open(body, "wb") as out:
|
| 2213 |
+
shutil.copyfileobj(r, out, _SPOOL_CHUNK)
|
| 2214 |
+
else:
|
| 2215 |
+
raw = r.read()
|
| 2216 |
except Exception as e: # noqa: BLE001
|
| 2217 |
+
if body:
|
| 2218 |
+
_unlink(body)
|
| 2219 |
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 2220 |
+
if body:
|
| 2221 |
+
return _spooled_document(body, compression)
|
| 2222 |
# S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
|
| 2223 |
# `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
|
| 2224 |
# algorithm the bytes were decoded as text and every economics window failed to parse. The
|
platform/harness/amazon_store.py
CHANGED
|
@@ -228,6 +228,10 @@ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
|
|
| 228 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 229 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 230 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 232 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 233 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
|
@@ -327,6 +331,234 @@ def _without_buyer_data(value):
|
|
| 327 |
return value
|
| 328 |
|
| 329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
def parse_flat_file(text, *, document_id=None, report_type=None):
|
| 331 |
"""One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
|
| 332 |
|
|
@@ -339,29 +571,15 @@ def parse_flat_file(text, *, document_id=None, report_type=None):
|
|
| 339 |
swallow the rest of the row and silently merge it with the next one, so a catalogue with one
|
| 340 |
inch-mark in a title would lose rows with nothing raising.
|
| 341 |
"""
|
|
|
|
|
|
|
| 342 |
if not str(text or "").strip():
|
| 343 |
return [], []
|
| 344 |
buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
header = next(reader)
|
| 348 |
-
except StopIteration:
|
| 349 |
return [], []
|
| 350 |
-
cols
|
| 351 |
-
width = len(cols)
|
| 352 |
-
rows = []
|
| 353 |
-
# The read-only connectors_amazon report probe measures headers without capture metadata.
|
| 354 |
-
# Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
|
| 355 |
-
# BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
|
| 356 |
-
attributed = document_id is not None or report_type is not None
|
| 357 |
-
for row_number, raw in enumerate(reader, 1):
|
| 358 |
-
if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
|
| 359 |
-
continue
|
| 360 |
-
vals = list(raw[:width]) + [None] * max(0, width - len(raw))
|
| 361 |
-
row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
|
| 362 |
-
if not buyer_column(c)}
|
| 363 |
-
rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
|
| 364 |
-
return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows
|
| 365 |
|
| 366 |
|
| 367 |
def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
@@ -372,6 +590,8 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
| 372 |
with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
|
| 373 |
artifact is the authority; a field the payload omits is NULL.
|
| 374 |
"""
|
|
|
|
|
|
|
| 375 |
try:
|
| 376 |
doc = json.loads(str(text or "") or "{}")
|
| 377 |
except (ValueError, TypeError) as exc:
|
|
@@ -387,12 +607,151 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
| 387 |
body = [body]
|
| 388 |
if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
|
| 389 |
raise ValueError(f"Amazon JSON report has an invalid {root} row list")
|
| 390 |
-
rows =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
attributed = document_id is not None or report_type is not None
|
| 392 |
for row_number, item in enumerate(body, 1):
|
| 393 |
row = _flatten_row(_without_buyer_data(item))
|
| 394 |
-
|
| 395 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 396 |
|
| 397 |
|
| 398 |
def _flatten_row(obj, prefix=""):
|
|
@@ -601,6 +960,8 @@ def _awd_rows(inventory, operation_id=None):
|
|
| 601 |
|
| 602 |
def _tag_fba_inventory(rows):
|
| 603 |
"""Mark report rows so one combined inventory table never conflates FBA with AWD."""
|
|
|
|
|
|
|
| 604 |
for row in rows or ():
|
| 605 |
row["inventory_source"] = "FBA"
|
| 606 |
return rows
|
|
@@ -618,7 +979,15 @@ def _pricing_asins(rows):
|
|
| 618 |
|
| 619 |
|
| 620 |
def _apply_competitive(con, table, responses):
|
| 621 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 622 |
rows = _competitive_rows(responses)
|
| 623 |
if not rows:
|
| 624 |
return 0, 0
|
|
@@ -628,13 +997,15 @@ def _apply_competitive(con, table, responses):
|
|
| 628 |
raise ValueError("Pricing mirror has no ASIN column; sync Sales and Traffic before price enrichment")
|
| 629 |
types = _ensure(con, table, cols, rows)
|
| 630 |
assignments = ", ".join(f'"{c}"=?' for c in cols)
|
|
|
|
|
|
|
|
|
|
| 631 |
updated = 0
|
| 632 |
for row in rows:
|
| 633 |
asin = row["competitive_body_asin"]
|
| 634 |
values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
|
| 635 |
-
result = con.execute(
|
| 636 |
-
|
| 637 |
-
values + [asin] * len(match_columns))
|
| 638 |
try:
|
| 639 |
updated += int(result.fetchone()[0])
|
| 640 |
except Exception: # noqa: BLE001
|
|
@@ -680,36 +1051,18 @@ def _coltype(name, rows=None):
|
|
| 680 |
return "VARCHAR"
|
| 681 |
if not rows:
|
| 682 |
return "VARCHAR"
|
|
|
|
|
|
|
| 683 |
seen, kinds = 0, set()
|
| 684 |
for r in rows:
|
| 685 |
v = r.get(name)
|
| 686 |
if v is None or v == "":
|
| 687 |
continue
|
| 688 |
-
|
| 689 |
-
kinds.add("str")
|
| 690 |
-
elif isinstance(v, int):
|
| 691 |
-
kinds.add("int")
|
| 692 |
-
elif isinstance(v, float):
|
| 693 |
-
kinds.add("float")
|
| 694 |
-
elif isinstance(v, (list, dict)):
|
| 695 |
-
kinds.add("str")
|
| 696 |
-
else:
|
| 697 |
-
s = str(v).strip()
|
| 698 |
-
try:
|
| 699 |
-
int(s)
|
| 700 |
-
kinds.add("int")
|
| 701 |
-
except ValueError:
|
| 702 |
-
try:
|
| 703 |
-
float(s)
|
| 704 |
-
kinds.add("float")
|
| 705 |
-
except ValueError:
|
| 706 |
-
kinds.add("str")
|
| 707 |
seen += 1
|
| 708 |
if seen >= 500:
|
| 709 |
break
|
| 710 |
-
|
| 711 |
-
return "VARCHAR"
|
| 712 |
-
return "DOUBLE" if "float" in kinds else "BIGINT"
|
| 713 |
|
| 714 |
|
| 715 |
def _cell(value, kind):
|
|
@@ -788,11 +1141,27 @@ def _replace(con, table, cols, types, rows):
|
|
| 788 |
or type(r.get("_report_row_number")) is not int
|
| 789 |
or r["_report_row_number"] < 1 for r in rows):
|
| 790 |
raise ValueError("refusing Amazon rows without complete source attribution")
|
| 791 |
-
ph = ", ".join("?" for _ in cols)
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 796 |
|
| 797 |
|
| 798 |
def replace_keys_for(table):
|
|
@@ -812,28 +1181,26 @@ def replace_window(con, table, cols, types, rows, keys):
|
|
| 812 |
return _replace(con, table, cols, types, rows), None
|
| 813 |
quote = lambda name: '"' + str(name).replace('"', '""') + '"'
|
| 814 |
target = quote(table)
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
for c in keys) for row in rows))
|
| 820 |
con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
|
| 821 |
try:
|
| 822 |
-
con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
|
| 823 |
-
incoming)
|
| 824 |
predicate = " AND ".join(
|
| 825 |
f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
|
| 826 |
con.execute("BEGIN TRANSACTION")
|
| 827 |
try:
|
|
|
|
| 828 |
con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
|
| 829 |
-
|
| 830 |
con.execute("COMMIT")
|
| 831 |
except Exception:
|
| 832 |
con.execute("ROLLBACK")
|
| 833 |
raise
|
| 834 |
return count, None
|
| 835 |
finally:
|
| 836 |
-
con.execute(f"DROP TABLE {temporary}")
|
| 837 |
|
| 838 |
|
| 839 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
|
@@ -1179,6 +1546,7 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
|
|
| 1179 |
return 0
|
| 1180 |
|
| 1181 |
# Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
|
|
|
|
| 1182 |
doc = json.loads(text)
|
| 1183 |
raw = doc.get(spec["json_root"])
|
| 1184 |
if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
|
|
@@ -1741,6 +2109,58 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 1741 |
def _connect(self, tenant_key, rt):
|
| 1742 |
return _connect_tenant_mirror(tenant_key, rt)
|
| 1743 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1744 |
def _before_restore(self, tenant_key, rt):
|
| 1745 |
obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
|
| 1746 |
if obligation["earliestTriggerAt"]:
|
|
@@ -2685,6 +3105,16 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
|
|
| 2685 |
|
| 2686 |
def document_row_count(text, roots=()):
|
| 2687 |
"""Count source records before flattening, privacy filtering or mirror transformations."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2688 |
if roots:
|
| 2689 |
doc = json.loads(text)
|
| 2690 |
return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
|
|
@@ -2729,23 +3159,31 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
|
|
| 2729 |
stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
|
| 2730 |
if not stamp and (table == "listings" or not rows):
|
| 2731 |
raise ValueError("Amazon snapshot returned no data date or report creation date")
|
| 2732 |
-
|
| 2733 |
-
for row in rows:
|
| 2734 |
period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
|
| 2735 |
row["_snapshot_date"] = period
|
| 2736 |
if table == "inventory":
|
| 2737 |
row["date"] = period
|
| 2738 |
row["inventory_source"] = "FBA"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2739 |
identity = _snapshot_identity(table, row)
|
| 2740 |
if identity in identities:
|
| 2741 |
raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
|
| 2742 |
identities.add(identity)
|
| 2743 |
counts[period] = counts.get(period, 0) + 1
|
| 2744 |
-
|
|
|
|
| 2745 |
raise ValueError(f"Amazon {table} snapshot row count differs from its document")
|
| 2746 |
if not counts:
|
| 2747 |
counts[_utc(stamp).date().isoformat()] = 0
|
| 2748 |
-
columns = sorted(set(columns) | {"_snapshot_date"}
|
|
|
|
| 2749 |
if table == "inventory":
|
| 2750 |
columns = sorted(set(columns) | {"inventory_source"})
|
| 2751 |
mirror = table_name(table)
|
|
@@ -2892,8 +3330,9 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2892 |
if options:
|
| 2893 |
kwargs["options"] = options
|
| 2894 |
text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
|
|
|
|
| 2895 |
parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
|
| 2896 |
-
if spec.get("json_root") and meta.get("cancelled") and
|
| 2897 |
# ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
|
| 2898 |
# returns an empty document for it, S66-T42), but the JSON parser read "" as a document
|
| 2899 |
# with no row root and REFUSED it. So the two B2B opportunity reports never earned a
|
|
@@ -2903,14 +3342,14 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2903 |
elif spec.get("json_root"):
|
| 2904 |
columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
|
| 2905 |
if spec.get("json_extra_roots"):
|
| 2906 |
-
|
| 2907 |
-
|
| 2908 |
for root in spec["json_extra_roots"]:
|
| 2909 |
extra_columns, extra = parse_json_report(text, root, **parser_args)
|
| 2910 |
-
|
| 2911 |
-
row["_json_root"] = root
|
| 2912 |
columns = merge_columns(columns, extra_columns)
|
| 2913 |
-
|
|
|
|
| 2914 |
else:
|
| 2915 |
columns, rows = parse_flat_file(text, **parser_args)
|
| 2916 |
if table in LEGACY_SNAPSHOT_TABLES:
|
|
@@ -2924,7 +3363,7 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2924 |
period, column = _utc(w0).strftime("%Y-%m"), "month"
|
| 2925 |
else:
|
| 2926 |
period, column = _iso(w0), "_period_start"
|
| 2927 |
-
|
| 2928 |
row[column] = period
|
| 2929 |
if w1 is not None:
|
| 2930 |
row["_period_end"] = _iso(w1)
|
|
@@ -2932,19 +3371,31 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
|
|
| 2932 |
row["fulfillment_center"] = (row.get("fulfillment_center") or
|
| 2933 |
row.get("fulfillment_center_id") or
|
| 2934 |
row.get("fulfilment_center") or "all")
|
| 2935 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2936 |
roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
|
| 2937 |
# HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
|
| 2938 |
-
empty = bool(roots) and ((meta.get("cancelled") and
|
| 2939 |
(table in ("b2b_recommended", "b2b_not_yet_on_amazon")
|
| 2940 |
-
and isinstance(json.loads(text or "{}"), dict)
|
| 2941 |
-
and spec["json_root"] not in json.loads(text or "{}")))
|
| 2942 |
marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
|
| 2943 |
"column": column, "count": 0 if empty else document_row_count(text, roots)}
|
| 2944 |
return _replace_period(con, table, rows, columns, column, period, marker=marker,
|
| 2945 |
snapshot=bool(spec.get("snapshot")))
|
| 2946 |
|
| 2947 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2948 |
def _order_metric_totals(records, *, flattened=False):
|
| 2949 |
totals = {"orders": 0, "units": 0, "sales": {}}
|
| 2950 |
for row in records:
|
|
@@ -2971,11 +3422,15 @@ def _validate_orders(con):
|
|
| 2971 |
if not marker.get("count"):
|
| 2972 |
return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"}
|
| 2973 |
expected = marker["metrics"]["orders"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2974 |
actual = con.execute(
|
| 2975 |
"SELECT COUNT(DISTINCT amazon_order_id) FROM amzn_orders "
|
| 2976 |
"WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) "
|
| 2977 |
"AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ) "
|
| 2978 |
-
"AND lower(COALESCE(order_status, '')) NOT IN ('cancelled', 'canceled')",
|
| 2979 |
[marker["start"], marker["end"]]).fetchone()[0]
|
| 2980 |
return {"state": "ok" if expected == actual else "refused",
|
| 2981 |
"expected": expected, "actual": actual,
|
|
@@ -3098,9 +3553,34 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 3098 |
expected.append("settlements")
|
| 3099 |
with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
|
| 3100 |
report.update(_sync_candidate(tenant_key, **options))
|
|
|
|
| 3101 |
return report
|
| 3102 |
|
| 3103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3104 |
def _sync_credential(tenant_key, rt):
|
| 3105 |
"""Refuse unusable credentials before storage or vendor work at either sync entry."""
|
| 3106 |
fields = credential(tenant_key, rt)
|
|
@@ -3222,8 +3702,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3222 |
or table in completed
|
| 3223 |
or (table == "settlements" and not refresh_loaded)]
|
| 3224 |
if scheduled and not first_load:
|
| 3225 |
-
|
| 3226 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3227 |
want = plan["tables"]
|
| 3228 |
for table, reason in plan["deferred"].items():
|
| 3229 |
report["tables"][table] = {"state": "deferred", "reason": reason,
|
|
@@ -3261,7 +3748,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3261 |
return count
|
| 3262 |
|
| 3263 |
work, plans, paired, contexts = [], {}, {}, {}
|
| 3264 |
-
documents, settlement_remaining = 0, 0
|
| 3265 |
failed, landed = set(), set()
|
| 3266 |
pending_reports = set()
|
| 3267 |
sqp_asins, sqp_limit = [], None
|
|
@@ -3281,6 +3768,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3281 |
"refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []),
|
| 3282 |
"partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."}
|
| 3283 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3284 |
def sqp_problem(message):
|
| 3285 |
report["problems"].append(message)
|
| 3286 |
log(message)
|
|
@@ -3444,11 +3938,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3444 |
settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
|
| 3445 |
for item in discovered[issued_index:])
|
| 3446 |
break
|
|
|
|
| 3447 |
operation = "getReportDocument"
|
| 3448 |
documents += 1
|
| 3449 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3450 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3451 |
text = cx.settlement_document(fields, document_id)
|
|
|
|
| 3452 |
cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
|
| 3453 |
count = 0
|
| 3454 |
if cols and rows:
|
|
@@ -3457,6 +3953,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3457 |
# A newly issued document still replaces its own settlement's lines.
|
| 3458 |
count = write_window(table, cols, types, rows)
|
| 3459 |
written += count
|
|
|
|
| 3460 |
# Record completion after the write, including header-only and blank files.
|
| 3461 |
_record_state(con, entity_key("settlement_doc_" + document_id), "done",
|
| 3462 |
count, issued.get("dataEndTime"))
|
|
@@ -3520,11 +4017,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3520 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 3521 |
for w0, w1 in margin_windows:
|
| 3522 |
# One detail window is one budget task, including its reconciliation query.
|
|
|
|
| 3523 |
documents += 1
|
| 3524 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3525 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3526 |
-
|
| 3527 |
-
|
|
|
|
|
|
|
|
|
|
| 3528 |
landed_window = True
|
| 3529 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 3530 |
# The dated replacement already landed; AWD must not clear that
|
|
@@ -3544,6 +4045,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3544 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 3545 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
| 3546 |
documents += 1
|
|
|
|
| 3547 |
with _report_resume(
|
| 3548 |
con, table, rtype, start=_iso(w0),
|
| 3549 |
end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
|
|
@@ -3554,6 +4056,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3554 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3555 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 3556 |
text, meta = cx.report_document(fields, rtype, **resume_kwargs)
|
|
|
|
| 3557 |
landed_window = True
|
| 3558 |
if root:
|
| 3559 |
parsed_cols, rows = parse_json_report(
|
|
@@ -3629,6 +4132,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3629 |
if cx.REPORT_TYPES[table].get("monthly"):
|
| 3630 |
_record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
|
| 3631 |
"count": len(rows)})
|
|
|
|
|
|
|
| 3632 |
if first_load:
|
| 3633 |
for reached in (table, sibling):
|
| 3634 |
if reached:
|
|
@@ -3649,10 +4154,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 3649 |
if table == "pricing":
|
| 3650 |
if first_load:
|
| 3651 |
# Completion can be in another process/pass than the recent ASIN report.
|
|
|
|
|
|
|
| 3652 |
available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
|
| 3653 |
-
|
| 3654 |
-
|
| 3655 |
-
|
|
|
|
|
|
|
| 3656 |
state_key = ENTITY_STATE_KEYS[table]
|
| 3657 |
if not pricing_asins:
|
| 3658 |
_record_state(con, state_key, "unproven", 0,
|
|
@@ -3859,15 +4368,20 @@ def _dedupe_orders(rows):
|
|
| 3859 |
lines by construction, so this drops line detail rather than losing an order. `order_items`
|
| 3860 |
keeps every row, which is why both tables read the same document instead of costing two.
|
| 3861 |
"""
|
| 3862 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3863 |
for r in rows:
|
| 3864 |
oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
|
| 3865 |
if oid and oid in seen:
|
| 3866 |
continue
|
| 3867 |
if oid:
|
| 3868 |
seen.add(oid)
|
| 3869 |
-
|
| 3870 |
-
return out
|
| 3871 |
|
| 3872 |
|
| 3873 |
def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
@@ -5541,6 +6055,12 @@ def group_a_selftest(log=print, check=None):
|
|
| 5541 |
"('O3','2026-07-04T00:00:00Z','Cancelled')")
|
| 5542 |
ck("orders independently reconcile Sales API counts and exclude cancelled orders",
|
| 5543 |
red and _validate_orders(con)["state"] == "ok")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5544 |
spec = cx.REPORT_TYPES["reimbursements"]
|
| 5545 |
with patch.object(cx, "report_document", return_value=(
|
| 5546 |
"sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})):
|
|
@@ -6038,6 +6558,28 @@ def selftest(log=print):
|
|
| 6038 |
ck("NC-AZ19 competitive summary enriches every matching report row without appending",
|
| 6039 |
response_count == 1 and updated == 2 and enriched == 2,
|
| 6040 |
(response_count, updated, enriched))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6041 |
|
| 6042 |
mem.execute("""CREATE TABLE _sync_state (
|
| 6043 |
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
|
@@ -6173,10 +6715,113 @@ def selftest(log=print):
|
|
| 6173 |
ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
|
| 6174 |
ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
|
| 6175 |
ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
|
| 6176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6177 |
return 0 if not fails else 1
|
| 6178 |
|
| 6179 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6180 |
def main(argv=None):
|
| 6181 |
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 6182 |
ap.add_argument("--sync", action="store_true")
|
|
|
|
| 228 |
#: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
|
| 229 |
#: thread for hours and the Odoo resync behind it would never run.
|
| 230 |
FIRST_LOAD_REPORTS_PER_PASS = 15
|
| 231 |
+
#: Rows one first-load pass writes before it takes no further document. A pacing bound, never a cap:
|
| 232 |
+
#: the pass publishes and the next continues, so Brand Analytics months (3.7M rows and about 25
|
| 233 |
+
#: minutes each) never hold the sync desk for hours (D-7081).
|
| 234 |
+
FIRST_LOAD_ROWS_PER_PASS = 3_000_000
|
| 235 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 236 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 237 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
|
|
|
| 331 |
return value
|
| 332 |
|
| 333 |
|
| 334 |
+
#: Rows per INSERT statement. A batch, not a cap: every row of a document is written (standing rule 1).
|
| 335 |
+
INSERT_BATCH_ROWS = 500
|
| 336 |
+
#: Bound parameters per INSERT statement, so a wide table takes fewer rows per statement.
|
| 337 |
+
INSERT_BATCH_VALUES = 30_000
|
| 338 |
+
#: Characters read per step of a spooled document.
|
| 339 |
+
DOCUMENT_READ_CHARS = 1 << 20
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _spooled(document):
|
| 343 |
+
"""True for a `ReportDocument` the client spooled to disk; False for in-memory text."""
|
| 344 |
+
return hasattr(document, "open_text")
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def _document_text(document):
|
| 348 |
+
"""The whole text of a document, for a consumer whose documents are small."""
|
| 349 |
+
return document.read_text() if _spooled(document) else document
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def _blank(document):
|
| 353 |
+
"""`not str(text or "").strip()`, read in pieces when the document is spooled."""
|
| 354 |
+
if not _spooled(document):
|
| 355 |
+
return not str(document or "").strip()
|
| 356 |
+
with document.open_text() as handle:
|
| 357 |
+
while True:
|
| 358 |
+
piece = handle.read(DOCUMENT_READ_CHARS)
|
| 359 |
+
if not piece:
|
| 360 |
+
return True
|
| 361 |
+
if piece.strip():
|
| 362 |
+
return False
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def _value_kind(v):
|
| 366 |
+
"""One non-empty value's sniffed kind: `int`, `float` or `str`."""
|
| 367 |
+
if isinstance(v, bool):
|
| 368 |
+
return "str"
|
| 369 |
+
if isinstance(v, int):
|
| 370 |
+
return "int"
|
| 371 |
+
if isinstance(v, float):
|
| 372 |
+
return "float"
|
| 373 |
+
if isinstance(v, (list, dict)):
|
| 374 |
+
return "str"
|
| 375 |
+
s = str(v).strip()
|
| 376 |
+
try:
|
| 377 |
+
int(s)
|
| 378 |
+
return "int"
|
| 379 |
+
except ValueError:
|
| 380 |
+
try:
|
| 381 |
+
float(s)
|
| 382 |
+
return "float"
|
| 383 |
+
except ValueError:
|
| 384 |
+
return "str"
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def _kinds_type(kinds):
|
| 388 |
+
if not kinds or "str" in kinds:
|
| 389 |
+
return "VARCHAR"
|
| 390 |
+
return "DOUBLE" if "float" in kinds else "BIGINT"
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
class _DocumentRows:
|
| 394 |
+
"""Rows parsed again from a spooled report document on every pass, so no pass holds them all.
|
| 395 |
+
|
| 396 |
+
⛔ D-7081: one list of every row of the Brand Analytics month took the Space to 16 GB. A caller
|
| 397 |
+
iterates, takes `len()` or tests truth; `map` and `pipe` replay a row change on every pass.
|
| 398 |
+
"""
|
| 399 |
+
|
| 400 |
+
def __init__(self, produce, steps=()):
|
| 401 |
+
self._produce, self._steps = produce, tuple(steps)
|
| 402 |
+
self._count, self._sniff = None, None
|
| 403 |
+
|
| 404 |
+
def __iter__(self):
|
| 405 |
+
rows = self._produce()
|
| 406 |
+
for step in self._steps:
|
| 407 |
+
rows = step(rows)
|
| 408 |
+
return iter(rows)
|
| 409 |
+
|
| 410 |
+
def pipe(self, step):
|
| 411 |
+
return _DocumentRows(self._produce, self._steps + (step,))
|
| 412 |
+
|
| 413 |
+
def map(self, change):
|
| 414 |
+
return self.pipe(lambda rows: (change(row) for row in rows))
|
| 415 |
+
|
| 416 |
+
@staticmethod
|
| 417 |
+
def chain(*parts):
|
| 418 |
+
return _DocumentRows(lambda: (row for part in parts for row in part))
|
| 419 |
+
|
| 420 |
+
def _profile(self):
|
| 421 |
+
"""One pass: the row count, every key, and each key's first 500 non-empty kinds."""
|
| 422 |
+
count, sniff = 0, {}
|
| 423 |
+
for row in self:
|
| 424 |
+
count += 1
|
| 425 |
+
for key, value in row.items():
|
| 426 |
+
entry = sniff.get(key)
|
| 427 |
+
if entry is None:
|
| 428 |
+
entry = sniff[key] = [0, set()]
|
| 429 |
+
if entry[0] >= 500 or value is None or value == "":
|
| 430 |
+
continue
|
| 431 |
+
entry[1].add(_value_kind(value))
|
| 432 |
+
entry[0] += 1
|
| 433 |
+
self._count, self._sniff = count, sniff
|
| 434 |
+
|
| 435 |
+
def keys(self):
|
| 436 |
+
if self._sniff is None:
|
| 437 |
+
self._profile()
|
| 438 |
+
return set(self._sniff)
|
| 439 |
+
|
| 440 |
+
def sniffed(self, name):
|
| 441 |
+
"""`_coltype(name, rows)` for these rows, from the one profile pass."""
|
| 442 |
+
if self._sniff is None:
|
| 443 |
+
self._profile()
|
| 444 |
+
return _kinds_type(self._sniff.get(name, (0, ()))[1])
|
| 445 |
+
|
| 446 |
+
def __len__(self):
|
| 447 |
+
if self._count is None:
|
| 448 |
+
self._profile()
|
| 449 |
+
return self._count
|
| 450 |
+
|
| 451 |
+
def __bool__(self):
|
| 452 |
+
if self._count is not None:
|
| 453 |
+
return self._count > 0
|
| 454 |
+
rows = iter(self)
|
| 455 |
+
try:
|
| 456 |
+
return next(rows, None) is not None
|
| 457 |
+
finally:
|
| 458 |
+
getattr(rows, "close", lambda: None)()
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def _rss_gb():
|
| 462 |
+
"""This process's resident memory in GB, or None where /proc is absent."""
|
| 463 |
+
try:
|
| 464 |
+
with open("/proc/self/status", encoding="ascii") as handle:
|
| 465 |
+
for line in handle:
|
| 466 |
+
if line.startswith("VmRSS:"):
|
| 467 |
+
return int(line.split()[1]) / 1048576
|
| 468 |
+
except Exception: # noqa: BLE001
|
| 469 |
+
return None
|
| 470 |
+
return None
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
class _DocumentProbe:
|
| 474 |
+
"""One server-log line per report document: table, bytes, rows, seconds and the memory peak.
|
| 475 |
+
|
| 476 |
+
⚠ `print`, not `log`: the sync desk passes a silent log, so its first load logged nothing until a
|
| 477 |
+
pass ended and the document in flight at an out-of-memory kill was unknown (D-7081).
|
| 478 |
+
"""
|
| 479 |
+
|
| 480 |
+
_lock = threading.Lock()
|
| 481 |
+
_active = None
|
| 482 |
+
|
| 483 |
+
def __init__(self, table, report_type):
|
| 484 |
+
self.table, self.report_type = table, report_type
|
| 485 |
+
self.started, self.size, self.peak = time.monotonic(), None, _rss_gb()
|
| 486 |
+
self._halt = threading.Event()
|
| 487 |
+
with _DocumentProbe._lock:
|
| 488 |
+
previous, _DocumentProbe._active = _DocumentProbe._active, self
|
| 489 |
+
if previous is not None:
|
| 490 |
+
previous._halt.set()
|
| 491 |
+
threading.Thread(target=self._sample, name="amazon-document-probe", daemon=True).start()
|
| 492 |
+
|
| 493 |
+
def _sample(self):
|
| 494 |
+
deadline = time.monotonic() + 6 * 3600
|
| 495 |
+
while not self._halt.wait(0.5) and time.monotonic() < deadline:
|
| 496 |
+
rss = _rss_gb()
|
| 497 |
+
if rss is not None and (self.peak is None or rss > self.peak):
|
| 498 |
+
self.peak = rss
|
| 499 |
+
|
| 500 |
+
@classmethod
|
| 501 |
+
def downloaded(cls, document):
|
| 502 |
+
probe = cls._active
|
| 503 |
+
if probe is not None and not probe._halt.is_set():
|
| 504 |
+
probe.size = (document.size if _spooled(document)
|
| 505 |
+
else len(str(document or "").encode("utf-8", "replace")))
|
| 506 |
+
probe._say("downloaded", None)
|
| 507 |
+
|
| 508 |
+
def finish(self, rows):
|
| 509 |
+
self._halt.set()
|
| 510 |
+
self._say("written", rows)
|
| 511 |
+
|
| 512 |
+
def _say(self, phase, rows):
|
| 513 |
+
rss = _rss_gb()
|
| 514 |
+
peak = max(x for x in (self.peak, rss, 0.0) if x is not None)
|
| 515 |
+
size = "unknown" if self.size is None else f"{self.size:,}"
|
| 516 |
+
print(f"[aios-api] amazon document {phase}: table {self.table}, {self.report_type}, "
|
| 517 |
+
f"{size} bytes, {'-' if rows is None else f'{rows:,}'} rows, "
|
| 518 |
+
f"{time.monotonic() - self.started:.0f}s, rss {rss or 0:.2f} GB, peak {peak:.2f} GB",
|
| 519 |
+
flush=True)
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def _flat_rows(lines, document_id, report_type):
|
| 523 |
+
"""`(columns, row generator)` over one flat file's lines, or `(None, ())` with no header line."""
|
| 524 |
+
reader = csv.reader(lines, delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 525 |
+
try:
|
| 526 |
+
header = next(reader)
|
| 527 |
+
except StopIteration:
|
| 528 |
+
return None, ()
|
| 529 |
+
cols = _dedupe([_snake(h) for h in header])
|
| 530 |
+
width = len(cols)
|
| 531 |
+
# The read-only connectors_amazon report probe measures headers without capture metadata.
|
| 532 |
+
# Preserve that parse-only API; _replace refuses its unattributed rows. Sync always supplies
|
| 533 |
+
# BOTH keywords, and a missing document ID with a report type is an error, never a fallback.
|
| 534 |
+
attributed = document_id is not None or report_type is not None
|
| 535 |
+
|
| 536 |
+
def rows():
|
| 537 |
+
for row_number, raw in enumerate(reader, 1):
|
| 538 |
+
if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
|
| 539 |
+
continue
|
| 540 |
+
vals = list(raw[:width]) + [None] * max(0, width - len(raw))
|
| 541 |
+
row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
|
| 542 |
+
if not buyer_column(c)}
|
| 543 |
+
yield _origin(row, document_id, row_number, report_type) if attributed else row
|
| 544 |
+
return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows()
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def _parse_flat_document(document, *, document_id=None, report_type=None):
|
| 548 |
+
"""`parse_flat_file` over a spooled document: the header now, the rows again on every pass."""
|
| 549 |
+
if _blank(document):
|
| 550 |
+
return [], []
|
| 551 |
+
with document.open_text() as handle:
|
| 552 |
+
cols, _rows = _flat_rows(handle, document_id, report_type)
|
| 553 |
+
if cols is None:
|
| 554 |
+
return [], []
|
| 555 |
+
|
| 556 |
+
def produce():
|
| 557 |
+
with document.open_text() as handle:
|
| 558 |
+
yield from _flat_rows(handle, document_id, report_type)[1]
|
| 559 |
+
return cols, _DocumentRows(produce)
|
| 560 |
+
|
| 561 |
+
|
| 562 |
def parse_flat_file(text, *, document_id=None, report_type=None):
|
| 563 |
"""One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
|
| 564 |
|
|
|
|
| 571 |
swallow the rest of the row and silently merge it with the next one, so a catalogue with one
|
| 572 |
inch-mark in a title would lose rows with nothing raising.
|
| 573 |
"""
|
| 574 |
+
if _spooled(text):
|
| 575 |
+
return _parse_flat_document(text, document_id=document_id, report_type=report_type)
|
| 576 |
if not str(text or "").strip():
|
| 577 |
return [], []
|
| 578 |
buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n"))
|
| 579 |
+
cols, rows = _flat_rows(buf, document_id, report_type)
|
| 580 |
+
if cols is None:
|
|
|
|
|
|
|
| 581 |
return [], []
|
| 582 |
+
return cols, list(rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
|
| 584 |
|
| 585 |
def parse_json_report(text, root, *, document_id=None, report_type=None):
|
|
|
|
| 590 |
with nothing going red — the defect `meta_store.py`'s header describes in as many words. The
|
| 591 |
artifact is the authority; a field the payload omits is NULL.
|
| 592 |
"""
|
| 593 |
+
if _spooled(text):
|
| 594 |
+
return _parse_json_document(text, root, document_id=document_id, report_type=report_type)
|
| 595 |
try:
|
| 596 |
doc = json.loads(str(text or "") or "{}")
|
| 597 |
except (ValueError, TypeError) as exc:
|
|
|
|
| 607 |
body = [body]
|
| 608 |
if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
|
| 609 |
raise ValueError(f"Amazon JSON report has an invalid {root} row list")
|
| 610 |
+
rows = list(_json_rows(body, document_id, report_type))
|
| 611 |
+
return sorted({k for r in rows for k in r}), rows
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def _json_rows(body, document_id, report_type):
|
| 615 |
+
"""Flatten and attribute each row object, numbering from one."""
|
| 616 |
attributed = document_id is not None or report_type is not None
|
| 617 |
for row_number, item in enumerate(body, 1):
|
| 618 |
row = _flatten_row(_without_buyer_data(item))
|
| 619 |
+
yield _origin(row, document_id, row_number, report_type) if attributed else row
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
class _JsonFallback(Exception):
|
| 623 |
+
"""A shape only the whole-text parser answers exactly: malformed, not an object, a repeated root."""
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
_JSON_WS = re.compile(r"[ \t\n\r]*")
|
| 627 |
+
_JSON_DECODER = json.JSONDecoder()
|
| 628 |
+
#: A single value may span this many characters before the stream hands it to `json.loads`.
|
| 629 |
+
_JSON_VALUE_CHARS = 256 << 20
|
| 630 |
+
|
| 631 |
+
|
| 632 |
+
class _JsonStream:
|
| 633 |
+
"""JSON read from a text stream one value at a time, with `json.loads` value semantics."""
|
| 634 |
+
|
| 635 |
+
def __init__(self, handle):
|
| 636 |
+
self.handle, self.buf, self.pos = handle, "", 0
|
| 637 |
+
|
| 638 |
+
def _fill(self):
|
| 639 |
+
piece = self.handle.read(DOCUMENT_READ_CHARS)
|
| 640 |
+
if not piece:
|
| 641 |
+
return False
|
| 642 |
+
self.buf = self.buf[self.pos:] + piece
|
| 643 |
+
self.pos = 0
|
| 644 |
+
return True
|
| 645 |
+
|
| 646 |
+
def peek(self):
|
| 647 |
+
while True:
|
| 648 |
+
self.pos = _JSON_WS.match(self.buf, self.pos).end()
|
| 649 |
+
if self.pos < len(self.buf) or not self._fill():
|
| 650 |
+
return self.buf[self.pos:self.pos + 1]
|
| 651 |
+
|
| 652 |
+
def take(self, char):
|
| 653 |
+
if self.peek() != char:
|
| 654 |
+
raise _JsonFallback()
|
| 655 |
+
self.pos += 1
|
| 656 |
+
|
| 657 |
+
def value(self):
|
| 658 |
+
self.peek()
|
| 659 |
+
while True:
|
| 660 |
+
try:
|
| 661 |
+
obj, end = _JSON_DECODER.raw_decode(self.buf, self.pos)
|
| 662 |
+
except ValueError:
|
| 663 |
+
if len(self.buf) - self.pos < _JSON_VALUE_CHARS and self._fill():
|
| 664 |
+
continue
|
| 665 |
+
raise _JsonFallback() from None
|
| 666 |
+
# A number ending at the buffer's edge may continue in the next piece.
|
| 667 |
+
if end == len(self.buf) and self._fill():
|
| 668 |
+
continue
|
| 669 |
+
self.pos = end
|
| 670 |
+
return obj
|
| 671 |
+
|
| 672 |
+
def skip(self):
|
| 673 |
+
if self.peek() != "[":
|
| 674 |
+
self.value()
|
| 675 |
+
return
|
| 676 |
+
self.pos += 1
|
| 677 |
+
if self.peek() == "]":
|
| 678 |
+
self.pos += 1
|
| 679 |
+
return
|
| 680 |
+
while True:
|
| 681 |
+
self.skip()
|
| 682 |
+
char = self.peek()
|
| 683 |
+
self.pos += 1
|
| 684 |
+
if char == "]":
|
| 685 |
+
return
|
| 686 |
+
if char != ",":
|
| 687 |
+
raise _JsonFallback()
|
| 688 |
+
|
| 689 |
+
def items(self):
|
| 690 |
+
if self.peek() == "{":
|
| 691 |
+
yield self.value()
|
| 692 |
+
return
|
| 693 |
+
self.take("[")
|
| 694 |
+
if self.peek() == "]":
|
| 695 |
+
self.pos += 1
|
| 696 |
+
return
|
| 697 |
+
while True:
|
| 698 |
+
item = self.value()
|
| 699 |
+
if not isinstance(item, dict):
|
| 700 |
+
raise _JsonFallback()
|
| 701 |
+
yield item
|
| 702 |
+
char = self.peek()
|
| 703 |
+
self.pos += 1
|
| 704 |
+
if char == "]":
|
| 705 |
+
return
|
| 706 |
+
if char != ",":
|
| 707 |
+
raise _JsonFallback()
|
| 708 |
+
|
| 709 |
+
|
| 710 |
+
def _json_root_items(document, root):
|
| 711 |
+
"""Each row object under the top-level `root` key of a spooled document, decoded one at a time.
|
| 712 |
+
|
| 713 |
+
Raises `_JsonFallback` for any shape `json.loads` must judge: malformed text, a document that is
|
| 714 |
+
not an object, a missing or repeated root, or a root that is not a list of objects.
|
| 715 |
+
"""
|
| 716 |
+
with document.open_text() as handle:
|
| 717 |
+
stream, found = _JsonStream(handle), False
|
| 718 |
+
stream.take("{")
|
| 719 |
+
if stream.peek() == "}":
|
| 720 |
+
stream.pos += 1
|
| 721 |
+
else:
|
| 722 |
+
while True:
|
| 723 |
+
key = stream.value()
|
| 724 |
+
if not isinstance(key, str):
|
| 725 |
+
raise _JsonFallback()
|
| 726 |
+
stream.take(":")
|
| 727 |
+
if key == root:
|
| 728 |
+
if found:
|
| 729 |
+
raise _JsonFallback()
|
| 730 |
+
found = True
|
| 731 |
+
yield from stream.items()
|
| 732 |
+
else:
|
| 733 |
+
stream.skip()
|
| 734 |
+
char = stream.peek()
|
| 735 |
+
stream.pos += 1
|
| 736 |
+
if char == "}":
|
| 737 |
+
break
|
| 738 |
+
if char != ",":
|
| 739 |
+
raise _JsonFallback()
|
| 740 |
+
if stream.peek() or not found:
|
| 741 |
+
raise _JsonFallback()
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def _parse_json_document(document, root, *, document_id=None, report_type=None):
|
| 745 |
+
"""`parse_json_report` over a spooled document; any shape it cannot stream reads the whole text."""
|
| 746 |
+
rows = _DocumentRows(lambda: _json_rows(_json_root_items(document, root), document_id, report_type))
|
| 747 |
+
try:
|
| 748 |
+
keys = rows.keys()
|
| 749 |
+
except _JsonFallback:
|
| 750 |
+
return parse_json_report(document.read_text(), root, document_id=document_id,
|
| 751 |
+
report_type=report_type)
|
| 752 |
+
if not len(rows):
|
| 753 |
+
return [], []
|
| 754 |
+
return sorted(keys), rows
|
| 755 |
|
| 756 |
|
| 757 |
def _flatten_row(obj, prefix=""):
|
|
|
|
| 960 |
|
| 961 |
def _tag_fba_inventory(rows):
|
| 962 |
"""Mark report rows so one combined inventory table never conflates FBA with AWD."""
|
| 963 |
+
if isinstance(rows, _DocumentRows):
|
| 964 |
+
return _stamp_rows(rows, {"inventory_source": "FBA"})
|
| 965 |
for row in rows or ():
|
| 966 |
row["inventory_source"] = "FBA"
|
| 967 |
return rows
|
|
|
|
| 979 |
|
| 980 |
|
| 981 |
def _apply_competitive(con, table, responses):
|
| 982 |
+
"""Replace the mirror's competitive fields with this pass's responses on matching ASIN rows.
|
| 983 |
+
|
| 984 |
+
⛔ A row whose ASIN did not answer this pass is cleared, never left holding an older price
|
| 985 |
+
shown as current; `validate()` counts competitive ASINs against this pass's responses.
|
| 986 |
+
"""
|
| 987 |
+
have = _columns_of(con, table)
|
| 988 |
+
stale = sorted(c for c in have if c.startswith("competitive_"))
|
| 989 |
+
if stale:
|
| 990 |
+
con.execute(f"UPDATE {table} SET " + ", ".join(f'"{c}"=NULL' for c in stale))
|
| 991 |
rows = _competitive_rows(responses)
|
| 992 |
if not rows:
|
| 993 |
return 0, 0
|
|
|
|
| 997 |
raise ValueError("Pricing mirror has no ASIN column; sync Sales and Traffic before price enrichment")
|
| 998 |
types = _ensure(con, table, cols, rows)
|
| 999 |
assignments = ", ".join(f'"{c}"=?' for c in cols)
|
| 1000 |
+
# A parent ASIN's summary prices only a row with no child ASIN, never that parent's SKU rows.
|
| 1001 |
+
where = ('"child_asin"=? OR ("child_asin" IS NULL AND "parent_asin"=?)' if len(match_columns) == 2
|
| 1002 |
+
else f'"{match_columns[0]}"=?')
|
| 1003 |
updated = 0
|
| 1004 |
for row in rows:
|
| 1005 |
asin = row["competitive_body_asin"]
|
| 1006 |
values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
|
| 1007 |
+
result = con.execute(f'UPDATE {table} SET {assignments} WHERE {where}',
|
| 1008 |
+
values + [asin] * len(match_columns))
|
|
|
|
| 1009 |
try:
|
| 1010 |
updated += int(result.fetchone()[0])
|
| 1011 |
except Exception: # noqa: BLE001
|
|
|
|
| 1051 |
return "VARCHAR"
|
| 1052 |
if not rows:
|
| 1053 |
return "VARCHAR"
|
| 1054 |
+
if isinstance(rows, _DocumentRows):
|
| 1055 |
+
return rows.sniffed(name)
|
| 1056 |
seen, kinds = 0, set()
|
| 1057 |
for r in rows:
|
| 1058 |
v = r.get(name)
|
| 1059 |
if v is None or v == "":
|
| 1060 |
continue
|
| 1061 |
+
kinds.add(_value_kind(v))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
seen += 1
|
| 1063 |
if seen >= 500:
|
| 1064 |
break
|
| 1065 |
+
return _kinds_type(kinds)
|
|
|
|
|
|
|
| 1066 |
|
| 1067 |
|
| 1068 |
def _cell(value, kind):
|
|
|
|
| 1141 |
or type(r.get("_report_row_number")) is not int
|
| 1142 |
or r["_report_row_number"] < 1 for r in rows):
|
| 1143 |
raise ValueError("refusing Amazon rows without complete source attribution")
|
| 1144 |
+
ph = "(" + ", ".join("?" for _ in cols) + ")"
|
| 1145 |
+
insert = f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES '
|
| 1146 |
+
per = max(1, min(INSERT_BATCH_ROWS, INSERT_BATCH_VALUES // len(cols)))
|
| 1147 |
+
full = insert + ", ".join([ph] * per)
|
| 1148 |
+
|
| 1149 |
+
def write(batch):
|
| 1150 |
+
con.execute(full if len(batch) == per else insert + ", ".join([ph] * len(batch)),
|
| 1151 |
+
[value for row in batch for value in row])
|
| 1152 |
+
|
| 1153 |
+
# Multi-row statements in batches: no copy of every row exists at once, the caller's transaction
|
| 1154 |
+
# keeps the window all-or-nothing, and values bind and cast exactly as `executemany` did.
|
| 1155 |
+
count, batch = 0, []
|
| 1156 |
+
for r in rows:
|
| 1157 |
+
batch.append([_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols])
|
| 1158 |
+
if len(batch) == per:
|
| 1159 |
+
write(batch)
|
| 1160 |
+
count, batch = count + len(batch), []
|
| 1161 |
+
if batch:
|
| 1162 |
+
write(batch)
|
| 1163 |
+
count += len(batch)
|
| 1164 |
+
return count
|
| 1165 |
|
| 1166 |
|
| 1167 |
def replace_keys_for(table):
|
|
|
|
| 1181 |
return _replace(con, table, cols, types, rows), None
|
| 1182 |
quote = lambda name: '"' + str(name).replace('"', '""') + '"'
|
| 1183 |
target = quote(table)
|
| 1184 |
+
# The window's rows are staged in batches inside the transaction, typed as the mirror and
|
| 1185 |
+
# converted by the insert's own cells, so NULL and numeric keys agree and no key list exists.
|
| 1186 |
+
temporary = quote("amazon_window_rows_" + uuid.uuid4().hex)
|
| 1187 |
+
columns = ", ".join(quote(c) for c in cols)
|
|
|
|
| 1188 |
con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
|
| 1189 |
try:
|
|
|
|
|
|
|
| 1190 |
predicate = " AND ".join(
|
| 1191 |
f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
|
| 1192 |
con.execute("BEGIN TRANSACTION")
|
| 1193 |
try:
|
| 1194 |
+
count = _replace(con, temporary, cols, types, rows)
|
| 1195 |
con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
|
| 1196 |
+
con.execute(f"INSERT INTO {target} ({columns}) SELECT {columns} FROM {temporary}")
|
| 1197 |
con.execute("COMMIT")
|
| 1198 |
except Exception:
|
| 1199 |
con.execute("ROLLBACK")
|
| 1200 |
raise
|
| 1201 |
return count, None
|
| 1202 |
finally:
|
| 1203 |
+
con.execute(f"DROP TABLE IF EXISTS {temporary}")
|
| 1204 |
|
| 1205 |
|
| 1206 |
def _windows(start, end, size=MAX_WINDOW_DAYS):
|
|
|
|
| 1546 |
return 0
|
| 1547 |
|
| 1548 |
# Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
|
| 1549 |
+
text = _document_text(text)
|
| 1550 |
doc = json.loads(text)
|
| 1551 |
raw = doc.get(spec["json_root"])
|
| 1552 |
if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
|
|
|
|
| 2109 |
def _connect(self, tenant_key, rt):
|
| 2110 |
return _connect_tenant_mirror(tenant_key, rt)
|
| 2111 |
|
| 2112 |
+
def _publication_holds(self, report, waiting):
|
| 2113 |
+
"""Publish what a refresh pass completed; hold each unfinished table at its previous rows.
|
| 2114 |
+
|
| 2115 |
+
A first-load pass (it reports a state) keeps SDLC 73's whole-source publication. A problem no table owns
|
| 2116 |
+
holds the pass.
|
| 2117 |
+
"""
|
| 2118 |
+
if report.get("state") in {"partial", "failed", "done"}:
|
| 2119 |
+
return None
|
| 2120 |
+
rows = report.get("tables") or {}
|
| 2121 |
+
unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
|
| 2122 |
+
unfinished |= {name for name, row in rows.items() if isinstance(row, dict) and row.get("state") != "disabled"
|
| 2123 |
+
and not _mirrors._table_sync_complete("amazon", row)}
|
| 2124 |
+
# Head and lines come from one document, so neither publishes without the other.
|
| 2125 |
+
pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}}
|
| 2126 |
+
partner = {pairs[name]: name for name in unfinished if name in pairs and pairs[name] not in unfinished}
|
| 2127 |
+
unfinished |= set(partner)
|
| 2128 |
+
owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))}
|
| 2129 |
+
blocked = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []
|
| 2130 |
+
if str(problem).split(":", 1)[0].strip() not in owners]
|
| 2131 |
+
held = {}
|
| 2132 |
+
for name in sorted(unfinished):
|
| 2133 |
+
if name in partner:
|
| 2134 |
+
held[name] = f"held with {partner[name]}, which reads the same report document"
|
| 2135 |
+
continue
|
| 2136 |
+
row = rows.get(name) if isinstance(rows.get(name), dict) else {}
|
| 2137 |
+
state = " ".join(str(part) for part in (row.get("state") or "not reached in this pass",
|
| 2138 |
+
row.get("amazonStatus")) if part)
|
| 2139 |
+
held[name] = state + (f": {row['reason']}" if row.get("reason") else "")
|
| 2140 |
+
return {"held": held, "tables": {table_name(name) for name in held}, "blocked": blocked}
|
| 2141 |
+
|
| 2142 |
+
def _held_state_rows(self, rows, tables):
|
| 2143 |
+
"""A held table's data markers: watermark, floor, proofs, limits, role results and documents.
|
| 2144 |
+
|
| 2145 |
+
Its status, tick, refusal and pending-report markers describe the pass, so they publish with it.
|
| 2146 |
+
"""
|
| 2147 |
+
tables = set(tables)
|
| 2148 |
+
names = sorted({table_name(t) for t in TABLES} | tables, key=lambda name: (-len(name), name))
|
| 2149 |
+
|
| 2150 |
+
def owner(entity):
|
| 2151 |
+
if entity.startswith(entity_key("settlement_doc_")):
|
| 2152 |
+
return table_name("settlements")
|
| 2153 |
+
for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_"):
|
| 2154 |
+
for name in names:
|
| 2155 |
+
marker = entity_key(family + name[len(PREFIX):])
|
| 2156 |
+
if entity == marker or entity.startswith(marker + "_"):
|
| 2157 |
+
return name
|
| 2158 |
+
if entity in names:
|
| 2159 |
+
return None
|
| 2160 |
+
return next((name for name in names if entity.startswith(name + "_")), None)
|
| 2161 |
+
|
| 2162 |
+
return [row for row in rows if owner(str(row[0])) in tables]
|
| 2163 |
+
|
| 2164 |
def _before_restore(self, tenant_key, rt):
|
| 2165 |
obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
|
| 2166 |
if obligation["earliestTriggerAt"]:
|
|
|
|
| 3105 |
|
| 3106 |
def document_row_count(text, roots=()):
|
| 3107 |
"""Count source records before flattening, privacy filtering or mirror transformations."""
|
| 3108 |
+
if _spooled(text):
|
| 3109 |
+
try:
|
| 3110 |
+
if roots:
|
| 3111 |
+
return sum(sum(1 for _item in _json_root_items(text, root)) for root in roots)
|
| 3112 |
+
with text.open_text() as handle:
|
| 3113 |
+
reader = csv.reader(handle, delimiter="\t", quoting=csv.QUOTE_NONE)
|
| 3114 |
+
next(reader, None)
|
| 3115 |
+
return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader)
|
| 3116 |
+
except _JsonFallback:
|
| 3117 |
+
return document_row_count(text.read_text(), roots)
|
| 3118 |
if roots:
|
| 3119 |
doc = json.loads(text)
|
| 3120 |
return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
|
|
|
|
| 3159 |
stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
|
| 3160 |
if not stamp and (table == "listings" or not rows):
|
| 3161 |
raise ValueError("Amazon snapshot returned no data date or report creation date")
|
| 3162 |
+
def stamped(row):
|
|
|
|
| 3163 |
period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
|
| 3164 |
row["_snapshot_date"] = period
|
| 3165 |
if table == "inventory":
|
| 3166 |
row["date"] = period
|
| 3167 |
row["inventory_source"] = "FBA"
|
| 3168 |
+
return row
|
| 3169 |
+
|
| 3170 |
+
lazy = isinstance(rows, _DocumentRows)
|
| 3171 |
+
rows = rows.map(stamped) if lazy else rows
|
| 3172 |
+
counts, identities, total = {}, set(), 0
|
| 3173 |
+
for row in rows:
|
| 3174 |
+
period = (row if lazy else stamped(row))["_snapshot_date"]
|
| 3175 |
identity = _snapshot_identity(table, row)
|
| 3176 |
if identity in identities:
|
| 3177 |
raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
|
| 3178 |
identities.add(identity)
|
| 3179 |
counts[period] = counts.get(period, 0) + 1
|
| 3180 |
+
total += 1
|
| 3181 |
+
if total != source_count:
|
| 3182 |
raise ValueError(f"Amazon {table} snapshot row count differs from its document")
|
| 3183 |
if not counts:
|
| 3184 |
counts[_utc(stamp).date().isoformat()] = 0
|
| 3185 |
+
columns = sorted(set(columns) | {"_snapshot_date"}
|
| 3186 |
+
| (rows.keys() if lazy else {k for row in rows for k in row}))
|
| 3187 |
if table == "inventory":
|
| 3188 |
columns = sorted(set(columns) | {"inventory_source"})
|
| 3189 |
mirror = table_name(table)
|
|
|
|
| 3330 |
if options:
|
| 3331 |
kwargs["options"] = options
|
| 3332 |
text, meta = _report_document(con, table, fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
|
| 3333 |
+
_DocumentProbe.downloaded(text)
|
| 3334 |
parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
|
| 3335 |
+
if spec.get("json_root") and meta.get("cancelled") and _blank(text):
|
| 3336 |
# ⛔ HOTFIX 2026-09-29 (D-3496): CANCELLED is Amazon's "no data" (the client already
|
| 3337 |
# returns an empty document for it, S66-T42), but the JSON parser read "" as a document
|
| 3338 |
# with no row root and REFUSED it. So the two B2B opportunity reports never earned a
|
|
|
|
| 3342 |
elif spec.get("json_root"):
|
| 3343 |
columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
|
| 3344 |
if spec.get("json_extra_roots"):
|
| 3345 |
+
rows = _stamp_rows(rows, {"_json_root": spec["json_root"]})
|
| 3346 |
+
parts = [rows]
|
| 3347 |
for root in spec["json_extra_roots"]:
|
| 3348 |
extra_columns, extra = parse_json_report(text, root, **parser_args)
|
| 3349 |
+
parts.append(_stamp_rows(extra, {"_json_root": root}))
|
|
|
|
| 3350 |
columns = merge_columns(columns, extra_columns)
|
| 3351 |
+
rows = (_DocumentRows.chain(*parts) if any(isinstance(part, _DocumentRows) for part in parts)
|
| 3352 |
+
else [row for part in parts for row in part])
|
| 3353 |
else:
|
| 3354 |
columns, rows = parse_flat_file(text, **parser_args)
|
| 3355 |
if table in LEGACY_SNAPSHOT_TABLES:
|
|
|
|
| 3363 |
period, column = _utc(w0).strftime("%Y-%m"), "month"
|
| 3364 |
else:
|
| 3365 |
period, column = _iso(w0), "_period_start"
|
| 3366 |
+
def stamped(row):
|
| 3367 |
row[column] = period
|
| 3368 |
if w1 is not None:
|
| 3369 |
row["_period_end"] = _iso(w1)
|
|
|
|
| 3371 |
row["fulfillment_center"] = (row.get("fulfillment_center") or
|
| 3372 |
row.get("fulfillment_center_id") or
|
| 3373 |
row.get("fulfilment_center") or "all")
|
| 3374 |
+
return row
|
| 3375 |
+
|
| 3376 |
+
rows = rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
|
| 3377 |
+
columns = sorted(set(columns) | (rows.keys() if isinstance(rows, _DocumentRows)
|
| 3378 |
+
else {k for row in rows for k in row}))
|
| 3379 |
roots = (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ()
|
| 3380 |
# HOTFIX 2026-09-29: a CANCELLED JSON report is an empty document with no roots to count.
|
| 3381 |
+
empty = bool(roots) and ((meta.get("cancelled") and _blank(text)) or
|
| 3382 |
(table in ("b2b_recommended", "b2b_not_yet_on_amazon")
|
| 3383 |
+
and isinstance(json.loads(_document_text(text) or "{}"), dict)
|
| 3384 |
+
and spec["json_root"] not in json.loads(_document_text(text) or "{}")))
|
| 3385 |
marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
|
| 3386 |
"column": column, "count": 0 if empty else document_row_count(text, roots)}
|
| 3387 |
return _replace_period(con, table, rows, columns, column, period, marker=marker,
|
| 3388 |
snapshot=bool(spec.get("snapshot")))
|
| 3389 |
|
| 3390 |
|
| 3391 |
+
def _stamp_rows(rows, values):
|
| 3392 |
+
"""Set `values` on every row, on each pass when the rows are lazy."""
|
| 3393 |
+
def stamped(row):
|
| 3394 |
+
row.update(values)
|
| 3395 |
+
return row
|
| 3396 |
+
return rows.map(stamped) if isinstance(rows, _DocumentRows) else [stamped(row) for row in rows]
|
| 3397 |
+
|
| 3398 |
+
|
| 3399 |
def _order_metric_totals(records, *, flattened=False):
|
| 3400 |
totals = {"orders": 0, "units": 0, "sales": {}}
|
| 3401 |
for row in records:
|
|
|
|
| 3422 |
if not marker.get("count"):
|
| 3423 |
return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"}
|
| 3424 |
expected = marker["metrics"]["orders"]
|
| 3425 |
+
# getOrderMetrics counts the US marketplace only; the order report also carries CA, MX and MCF orders.
|
| 3426 |
+
columns = {row[0] for row in con.execute(
|
| 3427 |
+
"SELECT column_name FROM information_schema.columns WHERE table_name='amzn_orders'").fetchall()}
|
| 3428 |
+
channel = " AND sales_channel = 'Amazon.com'" if "sales_channel" in columns else ""
|
| 3429 |
actual = con.execute(
|
| 3430 |
"SELECT COUNT(DISTINCT amazon_order_id) FROM amzn_orders "
|
| 3431 |
"WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) "
|
| 3432 |
"AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ) "
|
| 3433 |
+
"AND lower(COALESCE(order_status, '')) NOT IN ('cancelled', 'canceled')" + channel,
|
| 3434 |
[marker["start"], marker["end"]]).fetchone()[0]
|
| 3435 |
return {"state": "ok" if expected == actual else "refused",
|
| 3436 |
"expected": expected, "actual": actual,
|
|
|
|
| 3553 |
expected.append("settlements")
|
| 3554 |
with _mirrors.sync_mirror("amazon", tenant_key, rt, report=report, log=log, expected_tables=expected):
|
| 3555 |
report.update(_sync_candidate(tenant_key, **options))
|
| 3556 |
+
_refuse_unreconciled(tenant_key, rt, report, log)
|
| 3557 |
return report
|
| 3558 |
|
| 3559 |
|
| 3560 |
+
def _refuse_unreconciled(tenant_key, rt, report, log):
|
| 3561 |
+
"""Standing rule 8: a table synced in this pass whose reconciliation refuses keeps its previous rows."""
|
| 3562 |
+
synced = [table for table, row in (report.get("tables") or {}).items()
|
| 3563 |
+
if isinstance(row, dict) and row.get("state", "done") == "done" and "rows" in row]
|
| 3564 |
+
if not synced or report.get("state") in {"partial", "failed", "done"}:
|
| 3565 |
+
return
|
| 3566 |
+
try:
|
| 3567 |
+
checked = validate(tenant_key, rt=rt, log=lambda *_: None)
|
| 3568 |
+
except Exception as exc: # noqa: BLE001
|
| 3569 |
+
report["problems"].append(f"Amazon reconciliation unavailable ({type(exc).__name__}); "
|
| 3570 |
+
"this pass publishes nothing new until it runs.")
|
| 3571 |
+
log(report["problems"][-1])
|
| 3572 |
+
return
|
| 3573 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 3574 |
+
for table in synced:
|
| 3575 |
+
if (checked.get(table) or {}).get("state") != "refused":
|
| 3576 |
+
continue
|
| 3577 |
+
reason = "reconciliation refused: " + str(checked[table].get("reason") or "no reason given")
|
| 3578 |
+
report["tables"][table] = {**report["tables"][table], "state": "refused", "reason": reason}
|
| 3579 |
+
report["problems"].append(f"{table}: {reason}")
|
| 3580 |
+
_record_state(con, entity_key(table), "refused", 0, reason)
|
| 3581 |
+
log(f"{table}: {reason}")
|
| 3582 |
+
|
| 3583 |
+
|
| 3584 |
def _sync_credential(tenant_key, rt):
|
| 3585 |
"""Refuse unusable credentials before storage or vendor work at either sync entry."""
|
| 3586 |
fields = credential(tenant_key, rt)
|
|
|
|
| 3702 |
or table in completed
|
| 3703 |
or (table == "settlements" and not refresh_loaded)]
|
| 3704 |
if scheduled and not first_load:
|
| 3705 |
+
ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
|
| 3706 |
+
# A table standing rule 8 held is asked again at this pass, with its pair, inside Amazon's own floor.
|
| 3707 |
+
pairs = {**SHARED_REPORT_SIBLINGS, **{line: head for head, line in SHARED_REPORT_SIBLINGS.items()}}
|
| 3708 |
+
for table in list(want):
|
| 3709 |
+
mark = _read_state(con, entity_key(table)) or {}
|
| 3710 |
+
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 3711 |
+
and not _interval_reason(table, ticks.get(table), end)):
|
| 3712 |
+
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
|
| 3713 |
+
plan = sync_plan(ticks, want, now=end)
|
| 3714 |
want = plan["tables"]
|
| 3715 |
for table, reason in plan["deferred"].items():
|
| 3716 |
report["tables"][table] = {"state": "deferred", "reason": reason,
|
|
|
|
| 3748 |
return count
|
| 3749 |
|
| 3750 |
work, plans, paired, contexts = [], {}, {}, {}
|
| 3751 |
+
documents, settlement_remaining, pass_rows = 0, 0, 0
|
| 3752 |
failed, landed = set(), set()
|
| 3753 |
pending_reports = set()
|
| 3754 |
sqp_asins, sqp_limit = [], None
|
|
|
|
| 3768 |
"refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []),
|
| 3769 |
"partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."}
|
| 3770 |
|
| 3771 |
+
def spend_rows(rows):
|
| 3772 |
+
"""A first-load pass past its row bound spends its document budget; the next pass continues."""
|
| 3773 |
+
nonlocal documents, pass_rows
|
| 3774 |
+
pass_rows += rows
|
| 3775 |
+
if first_load and isinstance(report_budget, int) and pass_rows >= FIRST_LOAD_ROWS_PER_PASS:
|
| 3776 |
+
documents = max(documents, report_budget)
|
| 3777 |
+
|
| 3778 |
def sqp_problem(message):
|
| 3779 |
report["problems"].append(message)
|
| 3780 |
log(message)
|
|
|
|
| 3938 |
settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
|
| 3939 |
for item in discovered[issued_index:])
|
| 3940 |
break
|
| 3941 |
+
probe = _DocumentProbe(table, rtype)
|
| 3942 |
operation = "getReportDocument"
|
| 3943 |
documents += 1
|
| 3944 |
_sync_event("amazon.sync", tenant_key, table,
|
| 3945 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 3946 |
text = cx.settlement_document(fields, document_id)
|
| 3947 |
+
_DocumentProbe.downloaded(text)
|
| 3948 |
cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
|
| 3949 |
count = 0
|
| 3950 |
if cols and rows:
|
|
|
|
| 3953 |
# A newly issued document still replaces its own settlement's lines.
|
| 3954 |
count = write_window(table, cols, types, rows)
|
| 3955 |
written += count
|
| 3956 |
+
probe.finish(count)
|
| 3957 |
# Record completion after the write, including header-only and blank files.
|
| 3958 |
_record_state(con, entity_key("settlement_doc_" + document_id), "done",
|
| 3959 |
count, issued.get("dataEndTime"))
|
|
|
|
| 4017 |
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
|
| 4018 |
for w0, w1 in margin_windows:
|
| 4019 |
# One detail window is one budget task, including its reconciliation query.
|
| 4020 |
+
probe = _DocumentProbe(table, spec.get("report") or operation)
|
| 4021 |
documents += 1
|
| 4022 |
_sync_event("amazon.sync", tenant_key, table,
|
| 4023 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 4024 |
+
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 4025 |
+
max_wait=max_wait, log=log)
|
| 4026 |
+
probe.finish(count)
|
| 4027 |
+
spend_rows(count)
|
| 4028 |
+
written += count
|
| 4029 |
landed_window = True
|
| 4030 |
if table in LEGACY_SNAPSHOT_TABLES:
|
| 4031 |
# The dated replacement already landed; AWD must not clear that
|
|
|
|
| 4045 |
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
|
| 4046 |
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
|
| 4047 |
documents += 1
|
| 4048 |
+
probe = _DocumentProbe(table, rtype)
|
| 4049 |
with _report_resume(
|
| 4050 |
con, table, rtype, start=_iso(w0),
|
| 4051 |
end=_iso(w1 - timedelta(seconds=1)) if cx.REPORT_TYPES[table].get("monthly") else _iso(w1), max_wait=max_wait,
|
|
|
|
| 4056 |
_sync_event("amazon.sync", tenant_key, table,
|
| 4057 |
{"state": "access", "operation": "report_document"}, rt, report, log)
|
| 4058 |
text, meta = cx.report_document(fields, rtype, **resume_kwargs)
|
| 4059 |
+
_DocumentProbe.downloaded(text)
|
| 4060 |
landed_window = True
|
| 4061 |
if root:
|
| 4062 |
parsed_cols, rows = parse_json_report(
|
|
|
|
| 4132 |
if cx.REPORT_TYPES[table].get("monthly"):
|
| 4133 |
_record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
|
| 4134 |
"count": len(rows)})
|
| 4135 |
+
probe.finish(len(rows) if cols else 0)
|
| 4136 |
+
spend_rows(len(rows) if cols else 0)
|
| 4137 |
if first_load:
|
| 4138 |
for reached in (table, sibling):
|
| 4139 |
if reached:
|
|
|
|
| 4154 |
if table == "pricing":
|
| 4155 |
if first_load:
|
| 4156 |
# Completion can be in another process/pass than the recent ASIN report.
|
| 4157 |
+
# One ASIN per row, as `_pricing_asins` picks it: a parent beside its children
|
| 4158 |
+
# would answer for no row and `validate()` would refuse the pass.
|
| 4159 |
available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
|
| 4160 |
+
picked = [c for c in ("child_asin", "parent_asin") if c in available]
|
| 4161 |
+
select = ", ".join(f'"{c}"' for c in picked)
|
| 4162 |
+
mirror_rows = [dict(zip(picked, row)) for row in con.execute(
|
| 4163 |
+
f"SELECT DISTINCT {select} FROM {mirror}").fetchall()] if picked else []
|
| 4164 |
+
pricing_asins = list(dict.fromkeys(pricing_asins + _pricing_asins(mirror_rows)))
|
| 4165 |
state_key = ENTITY_STATE_KEYS[table]
|
| 4166 |
if not pricing_asins:
|
| 4167 |
_record_state(con, state_key, "unproven", 0,
|
|
|
|
| 4368 |
lines by construction, so this drops line detail rather than losing an order. `order_items`
|
| 4369 |
keeps every row, which is why both tables read the same document instead of costing two.
|
| 4370 |
"""
|
| 4371 |
+
if isinstance(rows, _DocumentRows):
|
| 4372 |
+
return rows.pipe(_first_row_per_order)
|
| 4373 |
+
return list(_first_row_per_order(rows))
|
| 4374 |
+
|
| 4375 |
+
|
| 4376 |
+
def _first_row_per_order(rows):
|
| 4377 |
+
seen = set()
|
| 4378 |
for r in rows:
|
| 4379 |
oid = str(r.get("amazon_order_id") or r.get("order_id") or "")
|
| 4380 |
if oid and oid in seen:
|
| 4381 |
continue
|
| 4382 |
if oid:
|
| 4383 |
seen.add(oid)
|
| 4384 |
+
yield r
|
|
|
|
| 4385 |
|
| 4386 |
|
| 4387 |
def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
|
|
| 6055 |
"('O3','2026-07-04T00:00:00Z','Cancelled')")
|
| 6056 |
ck("orders independently reconcile Sales API counts and exclude cancelled orders",
|
| 6057 |
red and _validate_orders(con)["state"] == "ok")
|
| 6058 |
+
con.execute("ALTER TABLE amzn_orders ADD COLUMN sales_channel VARCHAR")
|
| 6059 |
+
con.execute("UPDATE amzn_orders SET sales_channel='Amazon.com'")
|
| 6060 |
+
con.execute("INSERT INTO amzn_orders VALUES ('O4','2026-07-03T00:00:00Z','Shipped','Amazon.ca'), "
|
| 6061 |
+
"('O5','2026-07-03T00:00:00Z','Shipped','Non-Amazon')")
|
| 6062 |
+
ck("orders reconcile only the US marketplace that Amazon Sales counts",
|
| 6063 |
+
_validate_orders(con)["state"] == "ok")
|
| 6064 |
spec = cx.REPORT_TYPES["reimbursements"]
|
| 6065 |
with patch.object(cx, "report_document", return_value=(
|
| 6066 |
"sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})):
|
|
|
|
| 6558 |
ck("NC-AZ19 competitive summary enriches every matching report row without appending",
|
| 6559 |
response_count == 1 and updated == 2 and enriched == 2,
|
| 6560 |
(response_count, updated, enriched))
|
| 6561 |
+
held = duckdb.connect(":memory:")
|
| 6562 |
+
held.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, "
|
| 6563 |
+
"competitive_body_asin VARCHAR, competitive_status_status_code BIGINT)")
|
| 6564 |
+
held.execute("INSERT INTO amzn_pricing VALUES ('B01','P01',NULL,NULL), ('B02','P02','B02',200)")
|
| 6565 |
+
response_count, _ = _apply_competitive(held, "amzn_pricing", competitive)
|
| 6566 |
+
left = held.execute("SELECT competitive_body_asin, competitive_status_status_code FROM amzn_pricing "
|
| 6567 |
+
"WHERE child_asin='B02'").fetchone()
|
| 6568 |
+
carried = held.execute("SELECT COUNT(DISTINCT competitive_body_asin) FROM amzn_pricing "
|
| 6569 |
+
"WHERE competitive_body_asin IS NOT NULL").fetchone()[0]
|
| 6570 |
+
held.close()
|
| 6571 |
+
ck("NC-AZ32 an ASIN absent from this pass keeps no earlier competitive price",
|
| 6572 |
+
left == (None, None) and carried == response_count == 1, (left, carried, response_count))
|
| 6573 |
+
sku = duckdb.connect(":memory:")
|
| 6574 |
+
sku.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
|
| 6575 |
+
sku.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'), ('B03','P01'), (NULL,'P04')")
|
| 6576 |
+
family = [{"body": {"asin": a, "price": p}} for a, p in (("B01", 1), ("B03", 3), ("P01", 9), ("P04", 4))]
|
| 6577 |
+
response_count, _ = _apply_competitive(sku, "amzn_pricing", family)
|
| 6578 |
+
priced = sku.execute("SELECT COALESCE(child_asin, parent_asin), competitive_body_asin, competitive_body_price "
|
| 6579 |
+
"FROM amzn_pricing ORDER BY 1").fetchall()
|
| 6580 |
+
sku.close()
|
| 6581 |
+
ck("NC-AZ33 a parent ASIN's summary never overwrites its SKU rows",
|
| 6582 |
+
priced == [("B01", "B01", 1), ("B03", "B03", 3), ("P04", "P04", 4)] and response_count == 4, priced)
|
| 6583 |
|
| 6584 |
mem.execute("""CREATE TABLE _sync_state (
|
| 6585 |
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
|
|
|
| 6715 |
ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
|
| 6716 |
ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
|
| 6717 |
ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
|
| 6718 |
+
same, detail = _spooled_parse_selftest()
|
| 6719 |
+
ck("NC-AZ34 a spooled document parses and counts exactly as its text does", same, detail)
|
| 6720 |
+
bounded, detail = _bounded_document_selftest()
|
| 6721 |
+
ck("NC-AZ35 a large synthetic report parses and writes in bounded memory", bounded, detail)
|
| 6722 |
+
log(f"\n{'GREEN' if not fails else 'RED'}: {35 - len(fails)}/35")
|
| 6723 |
return 0 if not fails else 1
|
| 6724 |
|
| 6725 |
|
| 6726 |
+
def _spool_fixture(text, *, gz=False, encoding="utf-8"):
|
| 6727 |
+
"""A `ReportDocument` made by the client's own spool path from local bytes, no network."""
|
| 6728 |
+
import gzip
|
| 6729 |
+
raw = text.encode(encoding) if isinstance(text, str) else text
|
| 6730 |
+
handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
|
| 6731 |
+
with os.fdopen(handle, "wb") as out:
|
| 6732 |
+
out.write(gzip.compress(raw) if gz else raw)
|
| 6733 |
+
return _client()._spooled_document(path, "GZIP" if gz else "")
|
| 6734 |
+
|
| 6735 |
+
|
| 6736 |
+
def _spooled_parse_selftest():
|
| 6737 |
+
"""D-7081: the streaming readers answer every fixture exactly as the whole-text parsers do."""
|
| 6738 |
+
origin = {"document_id": "selftest-document", "report_type": "selftest-report"}
|
| 6739 |
+
|
| 6740 |
+
def outcome(parse):
|
| 6741 |
+
try:
|
| 6742 |
+
cols, rows = parse()
|
| 6743 |
+
return "ok", list(cols), [dict(row) for row in rows]
|
| 6744 |
+
except Exception as exc: # noqa: BLE001
|
| 6745 |
+
return "error", type(exc).__name__, str(exc)
|
| 6746 |
+
|
| 6747 |
+
def count(text, roots):
|
| 6748 |
+
try:
|
| 6749 |
+
return document_row_count(text, roots)
|
| 6750 |
+
except Exception as exc: # noqa: BLE001
|
| 6751 |
+
return type(exc).__name__
|
| 6752 |
+
|
| 6753 |
+
flat = ["", " \n\t", "a\tb\r\n1\t2\r\n\r\n3\t4", "\ufeffsku\tsku\tbuyer-email\nA\tB\tx\n\"q\t1\n",
|
| 6754 |
+
"x\ty\r1\r2\t3\r\r", "sku\tname\nA\tsmart \u201cquote\u201d\n"]
|
| 6755 |
+
blobs = [(t, {}) for t in flat] + [(t, {"gz": True}) for t in flat] + [(flat[-1], {"encoding": "cp1252"})]
|
| 6756 |
+
misses = []
|
| 6757 |
+
for index, (text, how) in enumerate(blobs):
|
| 6758 |
+
expected = text if how.get("encoding") != "cp1252" else text.encode("cp1252").decode("cp1252")
|
| 6759 |
+
got = outcome(lambda: parse_flat_file(_spool_fixture(text, **how), **origin))
|
| 6760 |
+
if got != outcome(lambda: parse_flat_file(expected, **origin)):
|
| 6761 |
+
misses.append(f"flat {index}")
|
| 6762 |
+
if expected.strip() and count(_spool_fixture(text, **how), ()) != count(expected, ()):
|
| 6763 |
+
misses.append(f"flat count {index}")
|
| 6764 |
+
documents = ['', '[]', '{"rows": []}', '{"rows": {"a": 1}}', '{"rows": [1]}', '{"rows": [{"a": 1},]}',
|
| 6765 |
+
'{"rows": [{"a": 1}]} x', '{"rows": [{"a": 1}], "rows": [{"b": 2}]}', '{"other": 1}',
|
| 6766 |
+
' {"spec": {"q": [1, [2]]}, "rows": [{"a": 1, "b": null, "n": {"c": "\u201c"}}, '
|
| 6767 |
+
'{"a": 2.5, "l": [1, 2]}], "tail": [[1], []]} ',
|
| 6768 |
+
'{"rows": [{"_report_document_id": "vendor"}]}', '{"rows": [{"buyerEmail": "e", "a": 123456789012345678901}]}',
|
| 6769 |
+
'{"x": NaN, "rows": [{"v": Infinity, "w": -0.0, "s": "\\ud83d"}]}']
|
| 6770 |
+
for index, text in enumerate(documents):
|
| 6771 |
+
if outcome(lambda: parse_json_report(_spool_fixture(text), "rows", **origin)) != outcome(
|
| 6772 |
+
lambda: parse_json_report(text, "rows", **origin)):
|
| 6773 |
+
misses.append(f"json {index}")
|
| 6774 |
+
if count(_spool_fixture(text), ("rows",)) != count(text, ("rows",)):
|
| 6775 |
+
misses.append(f"json count {index}")
|
| 6776 |
+
return not misses, misses
|
| 6777 |
+
|
| 6778 |
+
|
| 6779 |
+
def _bounded_document_selftest(rows=24_000):
|
| 6780 |
+
"""D-7081: a report larger than the bound parses, sniffs and writes with a bounded Python peak.
|
| 6781 |
+
|
| 6782 |
+
The whole-text path holds the decoded text and every row at once, so it fails this bound; the
|
| 6783 |
+
spooled path holds one read piece and one insert batch.
|
| 6784 |
+
"""
|
| 6785 |
+
import gzip
|
| 6786 |
+
import tracemalloc
|
| 6787 |
+
bound = 10 << 20
|
| 6788 |
+
handle, path = tempfile.mkstemp(prefix="amazon-selftest-")
|
| 6789 |
+
os.close(handle)
|
| 6790 |
+
with gzip.open(path, "wt", encoding="utf-8") as out:
|
| 6791 |
+
out.write('{"reportSpecification": {"reportType": "selftest"}, "dataByDepartmentAndSearchTerm": [')
|
| 6792 |
+
for i in range(rows):
|
| 6793 |
+
out.write(("," if i else "") + json.dumps({
|
| 6794 |
+
"departmentName": "Amazon.com", "searchTerm": f"term {i} \u2122 caf\u00e9",
|
| 6795 |
+
"searchFrequencyRank": i + 1, "clickedAsin": f"B0{i:08d}",
|
| 6796 |
+
"clickedItemName": f"Product title number {i} with several words", "clickShareRank": i % 3 + 1,
|
| 6797 |
+
"clickShare": round(i / 7 % 1, 4), "conversionShare": round(i / 11 % 1, 4)}))
|
| 6798 |
+
out.write("]}")
|
| 6799 |
+
document = _client()._spooled_document(path, "GZIP")
|
| 6800 |
+
text_bytes = 2 * len(document)
|
| 6801 |
+
con = datastore.duckdb.connect(":memory:")
|
| 6802 |
+
table = "amzn_brand_analytics"
|
| 6803 |
+
# A first bound statement imports DuckDB's lazy modules; that one-time cost is not a row's.
|
| 6804 |
+
con.execute("SELECT ?", [1]).fetchall()
|
| 6805 |
+
tracemalloc.start()
|
| 6806 |
+
try:
|
| 6807 |
+
cols, parsed = parse_json_report(document, "dataByDepartmentAndSearchTerm",
|
| 6808 |
+
document_id="selftest-document", report_type="selftest")
|
| 6809 |
+
cols = merge_columns([c for c, _t in amazon_schema.columns_for("brand_analytics")], cols)
|
| 6810 |
+
types = _ensure(con, table, cols, parsed, declared=dict(amazon_schema.columns_for("brand_analytics")))
|
| 6811 |
+
first, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
|
| 6812 |
+
peak = tracemalloc.get_traced_memory()[1]
|
| 6813 |
+
finally:
|
| 6814 |
+
tracemalloc.stop()
|
| 6815 |
+
again, _note = replace_window(con, table, cols, types, parsed, replace_keys_for("brand_analytics"))
|
| 6816 |
+
stored = con.execute(f"SELECT COUNT(*), COUNT(DISTINCT _report_row_number) FROM {table}").fetchone()
|
| 6817 |
+
con.close()
|
| 6818 |
+
document.close()
|
| 6819 |
+
detail = {"rows": len(parsed), "first": first, "again": again, "stored": stored,
|
| 6820 |
+
"text_bytes": text_bytes, "peak": peak, "bound": bound}
|
| 6821 |
+
return (len(parsed) == first == again == rows and stored == (rows, rows)
|
| 6822 |
+
and text_bytes > bound and peak < bound), detail
|
| 6823 |
+
|
| 6824 |
+
|
| 6825 |
def main(argv=None):
|
| 6826 |
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 6827 |
ap.add_argument("--sync", action="store_true")
|
platform/harness/datastore.py
CHANGED
|
@@ -546,7 +546,9 @@ def check_odoo_snapshot(rt, candidate, published, row_counts):
|
|
| 546 |
checks = []
|
| 547 |
|
| 548 |
def record(name, diffs=(), **details):
|
| 549 |
-
|
|
|
|
|
|
|
| 550 |
"diffs": list(diffs), **details})
|
| 551 |
|
| 552 |
def columns(con, table):
|
|
@@ -592,6 +594,19 @@ def check_odoo_snapshot(rt, candidate, published, row_counts):
|
|
| 592 |
balance_diffs.append(item)
|
| 593 |
elif not balanced and company in exceptions:
|
| 594 |
accepted.append({**item, "reason": exceptions[company]["reason"]})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 595 |
if invalid:
|
| 596 |
balance_diffs.append({"invalid_ledger_lines": invalid})
|
| 597 |
record("ledger_balance", balance_diffs, exceptions=accepted)
|
|
|
|
| 546 |
checks = []
|
| 547 |
|
| 548 |
def record(name, diffs=(), **details):
|
| 549 |
+
# A check of Odoo's own ledger reports a warning and never holds a sync. Only our copy's row drop fails.
|
| 550 |
+
found = "failed" if name == "row_count_drop" else "warning"
|
| 551 |
+
checks.append({"check": name, "state": found if diffs else "passed",
|
| 552 |
"diffs": list(diffs), **details})
|
| 553 |
|
| 554 |
def columns(con, table):
|
|
|
|
| 594 |
balance_diffs.append(item)
|
| 595 |
elif not balanced and company in exceptions:
|
| 596 |
accepted.append({**item, "reason": exceptions[company]["reason"]})
|
| 597 |
+
move_cols = ledger & {"move_id", "move_name"}
|
| 598 |
+
if "move_id" in move_cols:
|
| 599 |
+
# Name the posted moves whose own lines do not sum to zero: that is the cause in Odoo.
|
| 600 |
+
named = ", any_value(move_name)" if "move_name" in move_cols else ""
|
| 601 |
+
for item in balance_diffs:
|
| 602 |
+
moves = candidate.execute(
|
| 603 |
+
f"SELECT move_id, sum(CAST(balance AS DECIMAL(38,6))) AS total{named} FROM account_move_line "
|
| 604 |
+
f"WHERE {where} AND isfinite(balance) AND company_id=? AND move_id IS NOT NULL "
|
| 605 |
+
"GROUP BY move_id HAVING abs(sum(CAST(balance AS DECIMAL(38,6)))) > ? ORDER BY abs(total) DESC, move_id",
|
| 606 |
+
[item["company_id"], Decimal(str(policy["balance_tolerance"]))]).fetchall()
|
| 607 |
+
item["unbalanced_moves"] = len(moves)
|
| 608 |
+
item["moves"] = [{"move_id": row[0], **({"move_name": row[2]} if named else {}),
|
| 609 |
+
"balance": float(row[1])} for row in moves[:10]]
|
| 610 |
if invalid:
|
| 611 |
balance_diffs.append({"invalid_ledger_lines": invalid})
|
| 612 |
record("ledger_balance", balance_diffs, exceptions=accepted)
|
platform/harness/mirror_store.py
CHANGED
|
@@ -66,7 +66,8 @@ def _catch_up_source(rt, connector):
|
|
| 66 |
connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
|
| 67 |
return _CATCH_UP.setdefault(identity, {}).setdefault(connector, {
|
| 68 |
"current": False, "parts": set(), "part_total": 0,
|
| 69 |
-
"passes": 0, "pass_pending": False, "newest": {}, "refused": False
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
def connected_sources(rt):
|
|
@@ -90,17 +91,68 @@ def catch_up_status(rt, sources=None):
|
|
| 90 |
if sources is None:
|
| 91 |
sources = connected_sources(rt)
|
| 92 |
with _CATCH_UP_LOCK:
|
| 93 |
-
states =
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
|
| 100 |
def source_catching_up(rt, connector):
|
| 101 |
return catch_up_status(rt, [connector])["active"]
|
| 102 |
|
| 103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
def _record_newest(con, connector):
|
| 105 |
"""Measure record dates, never checkpoint time or the sync cursor."""
|
| 106 |
preferred = ("invoice_date", "date_order", "date", "purchase_date", "posted_date",
|
|
@@ -139,7 +191,12 @@ def _sync_result(connector, report):
|
|
| 139 |
for row in report.values())
|
| 140 |
return completed, completed and all(row["phase"] == "live" for row in report.values())
|
| 141 |
if report.get("problems") or report.get("audit_problems"):
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
rows = report.get("tables") or {}
|
| 144 |
if connector == "meta":
|
| 145 |
completed = bool(rows) and not any(row.get("capped") for row in rows.values())
|
|
@@ -160,12 +217,21 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
|
|
| 160 |
with _CATCH_UP_LOCK:
|
| 161 |
state = _catch_up_source(rt, connector)
|
| 162 |
if not state["current"]:
|
| 163 |
-
state
|
| 164 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
completed, current = _sync_result(connector, report)
|
| 166 |
current = current and _catch_up_complete and Path(rt.datastore_path).exists()
|
| 167 |
newest = None
|
| 168 |
-
|
|
|
|
| 169 |
try:
|
| 170 |
con, _ = datastore._mirror_connection(rt.key, rt)
|
| 171 |
try:
|
|
@@ -179,8 +245,18 @@ def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwa
|
|
| 179 |
state = _catch_up_source(rt, connector)
|
| 180 |
if completed:
|
| 181 |
state["passes"] += 1
|
|
|
|
|
|
|
| 182 |
if current and not state["refused"]:
|
| 183 |
-
state.update(current=True, pass_pending=False, newest=newest or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
return report
|
| 185 |
|
| 186 |
|
|
@@ -300,6 +376,15 @@ class MirrorStore:
|
|
| 300 |
def _log_restored(self, paths, total, rows, log):
|
| 301 |
log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
|
| 302 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
def _mirror_cache_key(self, tenant_key, rt):
|
| 304 |
key = _tenant_key(tenant_key)
|
| 305 |
if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
|
|
@@ -570,7 +655,8 @@ class MirrorStore:
|
|
| 570 |
newest = _record_newest(con, self.connector)
|
| 571 |
with _CATCH_UP_LOCK:
|
| 572 |
_catch_up_source(rt, self.connector).update(
|
| 573 |
-
current=current, pass_pending=not current, newest=newest
|
|
|
|
| 574 |
except Exception as exc:
|
| 575 |
result["state"] = "refused"
|
| 576 |
message = (f"{self.label} mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
|
|
@@ -770,8 +856,6 @@ class MirrorStore:
|
|
| 770 |
return # The enclosing source pass owns publication after every table completes.
|
| 771 |
uploaded = 0
|
| 772 |
with self.lock:
|
| 773 |
-
if _held_manifest(self, tenant_key, rt):
|
| 774 |
-
raise PublicationHeld("Release the held snapshot before syncing this source.")
|
| 775 |
cache = self.restore_mirror(tenant_key, rt, log=log)
|
| 776 |
cache["dirty"].update(changed_tables)
|
| 777 |
try:
|
|
@@ -855,13 +939,22 @@ class MirrorStore:
|
|
| 855 |
**(_publication or {}))
|
| 856 |
if _publication is not None:
|
| 857 |
manifest["completed_at"] = stamp
|
| 858 |
-
published = _mirror_json(manifest)
|
| 859 |
if _hold is not None:
|
| 860 |
manifest["hold"] = _hold
|
| 861 |
rt.upload_bytes(self.prefix + _HELD_KEY, _mirror_json(manifest))
|
| 862 |
return manifest
|
|
|
|
|
|
|
|
|
|
|
|
|
| 863 |
rt.upload_bytes(self.prefix + _MANIFEST_KEY, published)
|
| 864 |
uploaded += len(published)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 865 |
if _publication is None:
|
| 866 |
datastore._write_publication(con, self.connector, manifest)
|
| 867 |
cache["manifest"] = manifest
|
|
@@ -930,9 +1023,31 @@ def _held_manifest(store, tenant_key, rt):
|
|
| 930 |
if (published.get("tenant") == tenant_key and published.get("snapshot_id") == held["snapshot_id"]
|
| 931 |
and published.get("release")):
|
| 932 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 933 |
return held
|
| 934 |
|
| 935 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 936 |
def _publication_summary(manifest):
|
| 937 |
return {**{key: manifest.get(key) or "unknown" for key in
|
| 938 |
("snapshot_id", "started_at", "completed_at", "state")},
|
|
@@ -1044,6 +1159,42 @@ def _open_candidate(store, tenant_key, rt, published):
|
|
| 1044 |
raise
|
| 1045 |
|
| 1046 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1047 |
@contextmanager
|
| 1048 |
def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
|
| 1049 |
expected_tables=None):
|
|
@@ -1072,14 +1223,13 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1072 |
with store.lock:
|
| 1073 |
if getattr(datastore._SYNC_TLS, "candidate", None) is not None:
|
| 1074 |
raise RuntimeError("A source publication is already active on this thread")
|
| 1075 |
-
if _held_manifest(store, tenant_key, rt):
|
| 1076 |
-
raise PublicationHeld("Release the held snapshot before syncing this source.")
|
| 1077 |
restored = store.restore_mirror(tenant_key, rt, log=log)
|
| 1078 |
if restored["state"] == "refused":
|
| 1079 |
raise RuntimeError("The saved mirror could not be restored. Repair store access and restart.")
|
| 1080 |
published, _path = store._connect(tenant_key, rt)
|
| 1081 |
candidate = None
|
| 1082 |
installed = False
|
|
|
|
| 1083 |
save_started = None
|
| 1084 |
try:
|
| 1085 |
opened = time.monotonic()
|
|
@@ -1107,20 +1257,31 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1107 |
or (row.get("state") == "disabled" and name not in pending))}
|
| 1108 |
if done:
|
| 1109 |
candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
|
| 1110 |
-
|
|
|
|
|
|
|
|
|
|
| 1111 |
if odoo:
|
| 1112 |
note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
|
| 1113 |
"SELECT name FROM _candidate_pending").fetchall()])
|
| 1114 |
return
|
| 1115 |
if report is not None:
|
| 1116 |
completed, current = _sync_result(store.connector, report)
|
| 1117 |
-
if not (completed if store.connector == "meta" else current):
|
| 1118 |
if odoo:
|
| 1119 |
note(f"held, the pass is not current (completed={completed}), entities not live",
|
| 1120 |
names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
|
| 1121 |
for name, row in report.items()
|
| 1122 |
if not (isinstance(row, dict) and row.get("phase") == "live")])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1124 |
if store.connector == "odoo" and any(row[1] not in {"live", "done"}
|
| 1125 |
for row in store._state_rows(candidate)):
|
| 1126 |
note("held, sync state not live or done", names=lambda: [
|
|
@@ -1139,8 +1300,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1139 |
validating = time.monotonic()
|
| 1140 |
metadata["validation"] = datastore.check_odoo_snapshot(rt, candidate, published, metadata["rows"])
|
| 1141 |
failed = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "failed"]
|
| 1142 |
-
|
| 1143 |
-
|
|
|
|
| 1144 |
if failed:
|
| 1145 |
prior = _publication_summary(datastore._publication(published, store.connector))
|
| 1146 |
hold = {"previous_snapshot_id": prior["snapshot_id"], "previous_publication": prior,
|
|
@@ -1152,7 +1314,8 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1152 |
if key != "rows"}, _hold=hold)
|
| 1153 |
if not held:
|
| 1154 |
raise RuntimeError("The failed candidate could not be saved. " + "; ".join(result["problems"]))
|
| 1155 |
-
raise PublicationHeld("Snapshot held: " + ", ".join(failed) + ".
|
|
|
|
| 1156 |
metadata.pop("rows")
|
| 1157 |
metadata["completed_at"] = _iso(datetime.now(timezone.utc))
|
| 1158 |
result = {"problems": []}
|
|
@@ -1182,6 +1345,13 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1182 |
raise
|
| 1183 |
if odoo:
|
| 1184 |
note("install", installing)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1185 |
finally:
|
| 1186 |
if save_started is not None and store.connector == "odoo":
|
| 1187 |
status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
|
|
@@ -1189,7 +1359,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 1189 |
print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
|
| 1190 |
if candidate is not None:
|
| 1191 |
candidate.close()
|
| 1192 |
-
if installed:
|
| 1193 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1194 |
path.unlink(missing_ok=True)
|
| 1195 |
published.close()
|
|
|
|
| 66 |
connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
|
| 67 |
return _CATCH_UP.setdefault(identity, {}).setdefault(connector, {
|
| 68 |
"current": False, "parts": set(), "part_total": 0,
|
| 69 |
+
"passes": 0, "pass_pending": False, "newest": {}, "refused": False, "misses": 0, "behind": "",
|
| 70 |
+
"attempt_at": 0.0, "missed_at": 0.0, "miss_reason": "", "since": time.time()})
|
| 71 |
|
| 72 |
|
| 73 |
def connected_sources(rt):
|
|
|
|
| 91 |
if sources is None:
|
| 92 |
sources = connected_sources(rt)
|
| 93 |
with _CATCH_UP_LOCK:
|
| 94 |
+
states = {connector_spec.CONNECTOR_SPEC_KEYS.get(name, name): _catch_up_source(rt, name)
|
| 95 |
+
for name in dict.fromkeys(sources)}
|
| 96 |
+
# A behind source has settled: its pending pass no longer holds the tenant or its count.
|
| 97 |
+
behind = {name: _behind(s) for name, s in states.items()}
|
| 98 |
+
per = {name: {"active": not s["current"] and not behind[name],
|
| 99 |
+
"done": len(s["parts"]) + s["passes"],
|
| 100 |
+
"total": s["part_total"] + s["passes"] + int(s["pass_pending"] and not behind[name]),
|
| 101 |
+
"behind": behind[name]} for name, s in states.items()}
|
| 102 |
+
return {"active": any(p["active"] for p in per.values()),
|
| 103 |
+
"done": sum(p["done"] for p in per.values()),
|
| 104 |
+
"total": sum(p["total"] for p in per.values()),
|
| 105 |
+
"newest": {k: v for s in states.values() for k, v in s["newest"].items()},
|
| 106 |
+
"sources": per}
|
| 107 |
|
| 108 |
|
| 109 |
def source_catching_up(rt, connector):
|
| 110 |
return catch_up_status(rt, [connector])["active"]
|
| 111 |
|
| 112 |
|
| 113 |
+
#: Consecutive unsettled passes before a source reads behind. Odoo retries within minutes;
|
| 114 |
+
#: Amazon and Meta pass about daily, so one unsettled pass names them behind.
|
| 115 |
+
BEHIND_AFTER_PASSES = {"odoo": 3}
|
| 116 |
+
#: A miss that no new pass followed within this many seconds also reads behind.
|
| 117 |
+
BEHIND_AFTER_SECONDS = 900
|
| 118 |
+
#: A restored source that no pass has started on within this many seconds reads behind.
|
| 119 |
+
IDLE_BEHIND_SECONDS = 1800
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _behind(state):
|
| 123 |
+
"""The settled reason, or a plain reason once no pass followed a miss or a restore in time."""
|
| 124 |
+
if state["behind"] or state["current"]:
|
| 125 |
+
return state["behind"]
|
| 126 |
+
now = time.time()
|
| 127 |
+
if state["misses"]:
|
| 128 |
+
waited = state["missed_at"] >= state["attempt_at"] and now - state["missed_at"] >= BEHIND_AFTER_SECONDS
|
| 129 |
+
return state["miss_reason"] if waited else ""
|
| 130 |
+
# A restored source whose pass was skipped (a resumed pass already pulled it) waits a whole cadence.
|
| 131 |
+
idle = (state["pass_pending"] and not state.get("attempt_at") and len(state["parts"]) >= state["part_total"]
|
| 132 |
+
and now - state.get("since", now) >= IDLE_BEHIND_SECONDS)
|
| 133 |
+
return "Its sync has not run since the restart." if idle else ""
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _record_miss(connector, state, reason):
|
| 137 |
+
"""Count one unsettled pass on a source that is not current, and name it behind at the limit."""
|
| 138 |
+
if state["current"]:
|
| 139 |
+
return
|
| 140 |
+
state["misses"] += 1
|
| 141 |
+
if state["refused"]:
|
| 142 |
+
reason = "Its saved copy could not be restored."
|
| 143 |
+
state.update(missed_at=time.time(), miss_reason=reason)
|
| 144 |
+
if state["misses"] >= BEHIND_AFTER_PASSES.get(connector, 1):
|
| 145 |
+
state["behind"] = reason
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _skipped_pass(report):
|
| 149 |
+
"""An empty report ran no pass, so it neither settles nor misses."""
|
| 150 |
+
if not isinstance(report, dict) or not report:
|
| 151 |
+
return True
|
| 152 |
+
return ("tables" in report and not report.get("tables") and not report.get("problems")
|
| 153 |
+
and not report.get("audit_problems"))
|
| 154 |
+
|
| 155 |
+
|
| 156 |
def _record_newest(con, connector):
|
| 157 |
"""Measure record dates, never checkpoint time or the sync cursor."""
|
| 158 |
preferred = ("invoice_date", "date_order", "date", "purchase_date", "posted_date",
|
|
|
|
| 191 |
for row in report.values())
|
| 192 |
return completed, completed and all(row["phase"] == "live" for row in report.values())
|
| 193 |
if report.get("problems") or report.get("audit_problems"):
|
| 194 |
+
# A problem owned by one held table publishes the rest; it never makes the pass current.
|
| 195 |
+
holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
|
| 196 |
+
if holds is None or holds["blocked"]:
|
| 197 |
+
return False, False
|
| 198 |
+
rows = report.get("tables") or {}
|
| 199 |
+
return bool(rows) and any(_table_sync_complete(connector, row) for row in rows.values()), False
|
| 200 |
rows = report.get("tables") or {}
|
| 201 |
if connector == "meta":
|
| 202 |
completed = bool(rows) and not any(row.get("capped") for row in rows.values())
|
|
|
|
| 217 |
with _CATCH_UP_LOCK:
|
| 218 |
state = _catch_up_source(rt, connector)
|
| 219 |
if not state["current"]:
|
| 220 |
+
state.update(pass_pending=True, attempt_at=time.time())
|
| 221 |
+
try:
|
| 222 |
+
report = pull(*args, **kwargs)
|
| 223 |
+
except Exception as exc:
|
| 224 |
+
if _catch_up_complete:
|
| 225 |
+
with _CATCH_UP_LOCK:
|
| 226 |
+
_record_miss(connector, _catch_up_source(rt, connector),
|
| 227 |
+
"Its last update is held for review." if isinstance(exc, PublicationHeld)
|
| 228 |
+
else "Its last sync failed.")
|
| 229 |
+
raise
|
| 230 |
completed, current = _sync_result(connector, report)
|
| 231 |
current = current and _catch_up_complete and Path(rt.datastore_path).exists()
|
| 232 |
newest = None
|
| 233 |
+
# An Amazon pass that published part of the source still moves the measured record dates.
|
| 234 |
+
if current or (completed and connector == "amazon" and Path(rt.datastore_path).exists()):
|
| 235 |
try:
|
| 236 |
con, _ = datastore._mirror_connection(rt.key, rt)
|
| 237 |
try:
|
|
|
|
| 245 |
state = _catch_up_source(rt, connector)
|
| 246 |
if completed:
|
| 247 |
state["passes"] += 1
|
| 248 |
+
if newest and not state["refused"]:
|
| 249 |
+
state["newest"] = newest
|
| 250 |
if current and not state["refused"]:
|
| 251 |
+
state.update(current=True, pass_pending=False, newest=newest or {}, misses=0, behind="")
|
| 252 |
+
elif not _catch_up_complete or _skipped_pass(report):
|
| 253 |
+
pass
|
| 254 |
+
elif completed and connector == "odoo" and not state["refused"]:
|
| 255 |
+
# A bounded Odoo backfill pass is progress: the next pass resumes it.
|
| 256 |
+
state["misses"] = 0
|
| 257 |
+
else:
|
| 258 |
+
_record_miss(connector, state, "Its last sync left some tables unavailable." if completed
|
| 259 |
+
else "Its last sync did not finish.")
|
| 260 |
return report
|
| 261 |
|
| 262 |
|
|
|
|
| 376 |
def _log_restored(self, paths, total, rows, log):
|
| 377 |
log(f"{self.label} mirror restored from the store: {len(paths)} tables, {total:,} rows")
|
| 378 |
|
| 379 |
+
def _publication_holds(self, report, waiting):
|
| 380 |
+
"""None publishes the whole source or nothing. A connector may return
|
| 381 |
+
{"held": {name: reason}, "tables": held mirror tables, "blocked": [problems that hold the pass]}."""
|
| 382 |
+
return None
|
| 383 |
+
|
| 384 |
+
def _held_state_rows(self, rows, tables):
|
| 385 |
+
"""The state rows that describe a held table's data, so they ride with its previous rows."""
|
| 386 |
+
return [row for row in rows if row[0] in tables or row[0].split(".", 1)[0] in tables]
|
| 387 |
+
|
| 388 |
def _mirror_cache_key(self, tenant_key, rt):
|
| 389 |
key = _tenant_key(tenant_key)
|
| 390 |
if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
|
|
|
|
| 655 |
newest = _record_newest(con, self.connector)
|
| 656 |
with _CATCH_UP_LOCK:
|
| 657 |
_catch_up_source(rt, self.connector).update(
|
| 658 |
+
current=current, pass_pending=not current, newest=newest,
|
| 659 |
+
**({"misses": 0, "behind": ""} if current else {}))
|
| 660 |
except Exception as exc:
|
| 661 |
result["state"] = "refused"
|
| 662 |
message = (f"{self.label} mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
|
|
|
|
| 856 |
return # The enclosing source pass owns publication after every table completes.
|
| 857 |
uploaded = 0
|
| 858 |
with self.lock:
|
|
|
|
|
|
|
| 859 |
cache = self.restore_mirror(tenant_key, rt, log=log)
|
| 860 |
cache["dirty"].update(changed_tables)
|
| 861 |
try:
|
|
|
|
| 939 |
**(_publication or {}))
|
| 940 |
if _publication is not None:
|
| 941 |
manifest["completed_at"] = stamp
|
|
|
|
| 942 |
if _hold is not None:
|
| 943 |
manifest["hold"] = _hold
|
| 944 |
rt.upload_bytes(self.prefix + _HELD_KEY, _mirror_json(manifest))
|
| 945 |
return manifest
|
| 946 |
+
superseded = _superseded_hold(self, tenant_key, rt)
|
| 947 |
+
if superseded:
|
| 948 |
+
manifest["superseded_hold"] = superseded
|
| 949 |
+
published = _mirror_json(manifest)
|
| 950 |
rt.upload_bytes(self.prefix + _MANIFEST_KEY, published)
|
| 951 |
uploaded += len(published)
|
| 952 |
+
if superseded is not None:
|
| 953 |
+
# A newer publication replaces a stored hold. Nobody has to release it.
|
| 954 |
+
try:
|
| 955 |
+
rt.delete_path(self.prefix + _HELD_KEY)
|
| 956 |
+
except Exception: # noqa: BLE001
|
| 957 |
+
log(f"{self.label} superseded hold pointer stays until the next publication")
|
| 958 |
if _publication is None:
|
| 959 |
datastore._write_publication(con, self.connector, manifest)
|
| 960 |
cache["manifest"] = manifest
|
|
|
|
| 1023 |
if (published.get("tenant") == tenant_key and published.get("snapshot_id") == held["snapshot_id"]
|
| 1024 |
and published.get("release")):
|
| 1025 |
return None
|
| 1026 |
+
# A publication that already superseded this hold leaves only a stale pointer.
|
| 1027 |
+
if (published.get("tenant") == _tenant_key(tenant_key) and isinstance(published.get("superseded_hold"), dict)
|
| 1028 |
+
and published["superseded_hold"].get("snapshot_id") == held["snapshot_id"]):
|
| 1029 |
+
return None
|
| 1030 |
return held
|
| 1031 |
|
| 1032 |
|
| 1033 |
+
def _superseded_hold(store, tenant_key, rt):
|
| 1034 |
+
"""Describe the stored hold a new publication replaces: None if there is none, {} for a stale pointer."""
|
| 1035 |
+
try:
|
| 1036 |
+
if rt.download_bytes(store.prefix + _HELD_KEY) is None:
|
| 1037 |
+
return None
|
| 1038 |
+
except Exception: # noqa: BLE001
|
| 1039 |
+
return None
|
| 1040 |
+
try:
|
| 1041 |
+
held = _held_manifest(store, tenant_key, rt)
|
| 1042 |
+
except Exception: # noqa: BLE001
|
| 1043 |
+
# An unreadable hold is deleted with the publication, never a crash.
|
| 1044 |
+
return {"snapshot_id": "unknown", "held_at": "unknown", "failed_checks": [], "unreadable": True}
|
| 1045 |
+
if held is None:
|
| 1046 |
+
return {}
|
| 1047 |
+
return {"snapshot_id": held["snapshot_id"], "held_at": held["hold"].get("held_at") or "unknown",
|
| 1048 |
+
"failed_checks": list(held["hold"].get("failed_checks") or [])}
|
| 1049 |
+
|
| 1050 |
+
|
| 1051 |
def _publication_summary(manifest):
|
| 1052 |
return {**{key: manifest.get(key) or "unknown" for key in
|
| 1053 |
("snapshot_id", "started_at", "completed_at", "state")},
|
|
|
|
| 1159 |
raise
|
| 1160 |
|
| 1161 |
|
| 1162 |
+
def _hold_previous(store, candidate, published, tables):
|
| 1163 |
+
"""Give each held table its published rows and data markers, inside the candidate's open transaction."""
|
| 1164 |
+
prior = store._tables(published)
|
| 1165 |
+
with tempfile.TemporaryDirectory(prefix="mirror-held-") as tmp:
|
| 1166 |
+
for name in sorted(tables):
|
| 1167 |
+
if name not in prior:
|
| 1168 |
+
candidate.execute(f"DROP TABLE IF EXISTS {_quoted_identifier(name)}")
|
| 1169 |
+
continue
|
| 1170 |
+
path = Path(tmp) / (name + ".parquet")
|
| 1171 |
+
published.execute(f"COPY {_quoted_identifier(name)} TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
|
| 1172 |
+
candidate.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 1173 |
+
f"SELECT * FROM {_parquet_source([path])}")
|
| 1174 |
+
private = store._held_state_rows(store._state_rows(candidate), tables)
|
| 1175 |
+
if private:
|
| 1176 |
+
candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in private])
|
| 1177 |
+
previous = store._held_state_rows(store._state_rows(published), tables)
|
| 1178 |
+
if previous:
|
| 1179 |
+
candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
|
| 1180 |
+
|
| 1181 |
+
|
| 1182 |
+
def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
|
| 1183 |
+
"""One log line: what a pass published and what it held, each with its reason."""
|
| 1184 |
+
rows = report.get("tables") or {}
|
| 1185 |
+
synced = sorted(name for name in done if name not in holds["held"] and "rows" in (rows.get(name) or {}))
|
| 1186 |
+
synced_text = ", ".join(f"{name} {rows[name]['rows']:,} rows" if isinstance(rows[name]["rows"], int)
|
| 1187 |
+
else name for name in synced) or "none"
|
| 1188 |
+
held = "; ".join(f"{name} ({str(reason)[:160]})" for name, reason in sorted(holds["held"].items())) or "none"
|
| 1189 |
+
if snapshot_id:
|
| 1190 |
+
text = (f"{store.label} publication {snapshot_id}: published {len(synced)} synced ({synced_text}) and "
|
| 1191 |
+
f"{len(set(done) - set(synced) - set(holds['held']))} current; "
|
| 1192 |
+
f"held {len(holds['held'])} at their previous rows: {held}")
|
| 1193 |
+
else:
|
| 1194 |
+
text = f"{store.label} publication held, the previous snapshot stays: {why}; unfinished: {held}"
|
| 1195 |
+
return text.replace("—", "-").replace("–", "-").replace("\n", " ").replace("\r", " ")
|
| 1196 |
+
|
| 1197 |
+
|
| 1198 |
@contextmanager
|
| 1199 |
def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_publish=None,
|
| 1200 |
expected_tables=None):
|
|
|
|
| 1223 |
with store.lock:
|
| 1224 |
if getattr(datastore._SYNC_TLS, "candidate", None) is not None:
|
| 1225 |
raise RuntimeError("A source publication is already active on this thread")
|
|
|
|
|
|
|
| 1226 |
restored = store.restore_mirror(tenant_key, rt, log=log)
|
| 1227 |
if restored["state"] == "refused":
|
| 1228 |
raise RuntimeError("The saved mirror could not be restored. Repair store access and restart.")
|
| 1229 |
published, _path = store._connect(tenant_key, rt)
|
| 1230 |
candidate = None
|
| 1231 |
installed = False
|
| 1232 |
+
kept = False
|
| 1233 |
save_started = None
|
| 1234 |
try:
|
| 1235 |
opened = time.monotonic()
|
|
|
|
| 1257 |
or (row.get("state") == "disabled" and name not in pending))}
|
| 1258 |
if done:
|
| 1259 |
candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", [[name] for name in done])
|
| 1260 |
+
# A connector policy may publish what completed and hold each unfinished table at its previous rows.
|
| 1261 |
+
holds = None if report is None or odoo else store._publication_holds(report, {
|
| 1262 |
+
row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()})
|
| 1263 |
+
if holds is None and candidate.execute("SELECT 1 FROM _candidate_pending LIMIT 1").fetchone():
|
| 1264 |
if odoo:
|
| 1265 |
note("held, tables still pending", names=lambda: [row[0] for row in candidate.execute(
|
| 1266 |
"SELECT name FROM _candidate_pending").fetchall()])
|
| 1267 |
return
|
| 1268 |
if report is not None:
|
| 1269 |
completed, current = _sync_result(store.connector, report)
|
| 1270 |
+
if not (completed if store.connector == "meta" or holds is not None else current):
|
| 1271 |
if odoo:
|
| 1272 |
note(f"held, the pass is not current (completed={completed}), entities not live",
|
| 1273 |
names=lambda: [f"{name}={row.get('phase') if isinstance(row, dict) else row}"
|
| 1274 |
for name, row in report.items()
|
| 1275 |
if not (isinstance(row, dict) and row.get("phase") == "live")])
|
| 1276 |
+
if holds is not None:
|
| 1277 |
+
log(_publication_line(store, report, done, holds, why=(
|
| 1278 |
+
"a problem no table owns: " + "; ".join(str(p)[:200] for p in holds["blocked"][:3])
|
| 1279 |
+
if holds["blocked"] else "no table completed in this pass")))
|
| 1280 |
return
|
| 1281 |
+
if holds is not None and holds["tables"]:
|
| 1282 |
+
# Rolled back after publication, so the candidate keeps each held table's private progress.
|
| 1283 |
+
candidate.execute("BEGIN TRANSACTION")
|
| 1284 |
+
_hold_previous(store, candidate, published, holds["tables"])
|
| 1285 |
if store.connector == "odoo" and any(row[1] not in {"live", "done"}
|
| 1286 |
for row in store._state_rows(candidate)):
|
| 1287 |
note("held, sync state not live or done", names=lambda: [
|
|
|
|
| 1300 |
validating = time.monotonic()
|
| 1301 |
metadata["validation"] = datastore.check_odoo_snapshot(rt, candidate, published, metadata["rows"])
|
| 1302 |
failed = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "failed"]
|
| 1303 |
+
warned = [row["check"] for row in metadata["validation"]["checks"] if row["state"] == "warning"]
|
| 1304 |
+
note("validation " + ("failed" if failed else "passed with warnings" if warned else "passed"),
|
| 1305 |
+
validating, names=(lambda: failed or warned) if failed or warned else None)
|
| 1306 |
if failed:
|
| 1307 |
prior = _publication_summary(datastore._publication(published, store.connector))
|
| 1308 |
hold = {"previous_snapshot_id": prior["snapshot_id"], "previous_publication": prior,
|
|
|
|
| 1314 |
if key != "rows"}, _hold=hold)
|
| 1315 |
if not held:
|
| 1316 |
raise RuntimeError("The failed candidate could not be saved. " + "; ".join(result["problems"]))
|
| 1317 |
+
raise PublicationHeld("Snapshot held: " + ", ".join(failed) + ". The previous snapshot is kept "
|
| 1318 |
+
"and the next pass checks again.")
|
| 1319 |
metadata.pop("rows")
|
| 1320 |
metadata["completed_at"] = _iso(datetime.now(timezone.utc))
|
| 1321 |
result = {"problems": []}
|
|
|
|
| 1345 |
raise
|
| 1346 |
if odoo:
|
| 1347 |
note("install", installing)
|
| 1348 |
+
if holds is not None and holds["tables"]:
|
| 1349 |
+
candidate.execute("ROLLBACK")
|
| 1350 |
+
candidate.execute("DELETE FROM _candidate_base")
|
| 1351 |
+
candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
|
| 1352 |
+
kept = True
|
| 1353 |
+
if holds is not None:
|
| 1354 |
+
log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
|
| 1355 |
finally:
|
| 1356 |
if save_started is not None and store.connector == "odoo":
|
| 1357 |
status = datastore.ODOO_MIRROR_STATUS.setdefault(str(Path(rt.datastore_path).resolve()), {})
|
|
|
|
| 1359 |
print(f"[datastore] Odoo mirror save {tenant_key}: {status['save']}")
|
| 1360 |
if candidate is not None:
|
| 1361 |
candidate.close()
|
| 1362 |
+
if installed and not kept:
|
| 1363 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1364 |
path.unlink(missing_ok=True)
|
| 1365 |
published.close()
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
a278ac86384656722f918a76c838415bdcb9e46155b910f9335eb24f7b292c8d
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -54,8 +54,11 @@ import type { ConnectorTab } from "./ConnectorModal";
|
|
| 54 |
import MonidRunDialog from "./MonidRunDialog";
|
| 55 |
import {
|
| 56 |
DatabaseSyncSummary,
|
|
|
|
| 57 |
catchUpLabel,
|
|
|
|
| 58 |
parseDatabaseSyncHealth,
|
|
|
|
| 59 |
useConnectorCatchUp,
|
| 60 |
type ConnectorCatchUp,
|
| 61 |
type DatabaseSyncHealth,
|
|
@@ -801,10 +804,13 @@ export default function ConnectorsPage({
|
|
| 801 |
const hasOdooCard =
|
| 802 |
load.phase === "ready" &&
|
| 803 |
load.rows.some((row) => row.key === "odoo" && row.state === "connected" && !!row.manage);
|
| 804 |
-
const
|
| 805 |
load.phase === "ready" &&
|
| 806 |
-
load.rows.some((row) =>
|
| 807 |
-
const catchUpStatus = useConnectorCatchUp(
|
|
|
|
|
|
|
|
|
|
| 808 |
const [odooWarnings, setOdooWarnings] = useState<string[]>([]);
|
| 809 |
useEffect(() => {
|
| 810 |
if (!hasOdooCard) return;
|
|
@@ -933,7 +939,7 @@ export default function ConnectorsPage({
|
|
| 933 |
onOpen={() => openConnector(row)}
|
| 934 |
onPullReviews={pullHandler(row)}
|
| 935 |
syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
|
| 936 |
-
catchUp={row
|
| 937 |
catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
|
| 938 |
catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
|
| 939 |
/>
|
|
@@ -956,7 +962,7 @@ export default function ConnectorsPage({
|
|
| 956 |
onOpen={() => openConnector(row)}
|
| 957 |
onPullReviews={pullHandler(row)}
|
| 958 |
syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
|
| 959 |
-
catchUp={row
|
| 960 |
catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
|
| 961 |
catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
|
| 962 |
/>
|
|
@@ -1138,8 +1144,9 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
|
|
| 1138 |
});
|
| 1139 |
return () => controller.abort();
|
| 1140 |
}, []);
|
|
|
|
| 1141 |
const shownDatabaseHealth = databaseHealth.filter((database) => database.health
|
| 1142 |
-
||
|
| 1143 |
if (problem) return <section className="w45-contract-card"><h4>Sync</h4><p className="set-error">{problem}</p></section>;
|
| 1144 |
if (!status || !schema) return <section className="w45-contract-card"><h4>Sync</h4><p>Loading connector status…</p></section>;
|
| 1145 |
return (
|
|
@@ -1148,9 +1155,10 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
|
|
| 1148 |
<OdooFreshnessNote lines={freshnessWarnings(freshness)} />
|
| 1149 |
<dl className="voice-agent-details">
|
| 1150 |
<div><dt>Schedule</dt><dd>{status.schedule || "Not scheduled"}</dd></div>
|
| 1151 |
-
<div><dt>Health</dt><dd>{
|
| 1152 |
-
: catchUpStatus.error || (catchUpStatus.loading ? "Checking sync"
|
| 1153 |
-
|
|
|
|
| 1154 |
<div><dt>Last run</dt><dd>{status.lastRun || "No run reported"}</dd></div>
|
| 1155 |
{freshness ? (
|
| 1156 |
<div>
|
|
@@ -1165,8 +1173,8 @@ export function OdooSyncStatus({ onOpenDatabases }: { onOpenDatabases?: () => vo
|
|
| 1165 |
<div key={database.key}>
|
| 1166 |
<strong>{database.label}</strong>
|
| 1167 |
<DatabaseSyncSummary health={database.health} databaseKey={database.key}
|
| 1168 |
-
catchUp={catchUpStatus.catchUp
|
| 1169 |
-
catchUpLoading={catchUpStatus.loading} />
|
| 1170 |
</div>
|
| 1171 |
)) : <p>No database sync state has been published.</p>}
|
| 1172 |
<h4>Field catalog</h4>
|
|
@@ -1371,9 +1379,10 @@ export function ConnectorCard({
|
|
| 1371 |
return (
|
| 1372 |
<div className="conn-card" data-connector={row.key}>
|
| 1373 |
<ConnectorCardHead row={row} />
|
| 1374 |
-
{catchUp?.active || catchUpError || catchUpLoading ? (
|
| 1375 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1376 |
-
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
|
|
|
| 1377 |
{catchUp?.active && catchUpError ? `, ${catchUpError}` : ""}
|
| 1378 |
</span>
|
| 1379 |
) : syncWarnings?.length ? (
|
|
|
|
| 54 |
import MonidRunDialog from "./MonidRunDialog";
|
| 55 |
import {
|
| 56 |
DatabaseSyncSummary,
|
| 57 |
+
behindLabel,
|
| 58 |
catchUpLabel,
|
| 59 |
+
catchUpSourceKey,
|
| 60 |
parseDatabaseSyncHealth,
|
| 61 |
+
sourceCatchUp,
|
| 62 |
useConnectorCatchUp,
|
| 63 |
type ConnectorCatchUp,
|
| 64 |
type DatabaseSyncHealth,
|
|
|
|
| 804 |
const hasOdooCard =
|
| 805 |
load.phase === "ready" &&
|
| 806 |
load.rows.some((row) => row.key === "odoo" && row.state === "connected" && !!row.manage);
|
| 807 |
+
const hasConnectedSourceCard =
|
| 808 |
load.phase === "ready" &&
|
| 809 |
+
load.rows.some((row) => ["odoo", "amazon_sp", "meta_ads"].includes(row.key) && row.state === "connected");
|
| 810 |
+
const catchUpStatus = useConnectorCatchUp(hasConnectedSourceCard);
|
| 811 |
+
// Each connected source card shows its own catch-up; the Odoo card alone reports a status read error.
|
| 812 |
+
const sourceCard = (row: ConnectorRow) => ["odoo", "amazon_sp", "meta_ads"].includes(row.key)
|
| 813 |
+
&& row.state === "connected" ? sourceCatchUp(catchUpStatus.catchUp, catchUpSourceKey(row.key)) : undefined;
|
| 814 |
const [odooWarnings, setOdooWarnings] = useState<string[]>([]);
|
| 815 |
useEffect(() => {
|
| 816 |
if (!hasOdooCard) return;
|
|
|
|
| 939 |
onOpen={() => openConnector(row)}
|
| 940 |
onPullReviews={pullHandler(row)}
|
| 941 |
syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
|
| 942 |
+
catchUp={sourceCard(row)}
|
| 943 |
catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
|
| 944 |
catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
|
| 945 |
/>
|
|
|
|
| 962 |
onOpen={() => openConnector(row)}
|
| 963 |
onPullReviews={pullHandler(row)}
|
| 964 |
syncWarnings={row.key === "odoo" ? odooWarnings : undefined}
|
| 965 |
+
catchUp={sourceCard(row)}
|
| 966 |
catchUpError={row.key === "odoo" ? catchUpStatus.error : undefined}
|
| 967 |
catchUpLoading={row.key === "odoo" && catchUpStatus.loading}
|
| 968 |
/>
|
|
|
|
| 1144 |
});
|
| 1145 |
return () => controller.abort();
|
| 1146 |
}, []);
|
| 1147 |
+
const odooCatchUp = sourceCatchUp(catchUpStatus.catchUp, "odoo");
|
| 1148 |
const shownDatabaseHealth = databaseHealth.filter((database) => database.health
|
| 1149 |
+
|| odooCatchUp?.active || odooCatchUp?.newest[database.key] || catchUpStatus.error);
|
| 1150 |
if (problem) return <section className="w45-contract-card"><h4>Sync</h4><p className="set-error">{problem}</p></section>;
|
| 1151 |
if (!status || !schema) return <section className="w45-contract-card"><h4>Sync</h4><p>Loading connector status…</p></section>;
|
| 1152 |
return (
|
|
|
|
| 1155 |
<OdooFreshnessNote lines={freshnessWarnings(freshness)} />
|
| 1156 |
<dl className="voice-agent-details">
|
| 1157 |
<div><dt>Schedule</dt><dd>{status.schedule || "Not scheduled"}</dd></div>
|
| 1158 |
+
<div><dt>Health</dt><dd>{odooCatchUp?.active ? catchUpLabel(odooCatchUp)
|
| 1159 |
+
: catchUpStatus.error || (catchUpStatus.loading ? "Checking sync"
|
| 1160 |
+
: odooCatchUp?.behind ? behindLabel(odooCatchUp.behind) : status.health)}
|
| 1161 |
+
{odooCatchUp?.active && catchUpStatus.error ? `, ${catchUpStatus.error}` : ""}</dd></div>
|
| 1162 |
<div><dt>Last run</dt><dd>{status.lastRun || "No run reported"}</dd></div>
|
| 1163 |
{freshness ? (
|
| 1164 |
<div>
|
|
|
|
| 1173 |
<div key={database.key}>
|
| 1174 |
<strong>{database.label}</strong>
|
| 1175 |
<DatabaseSyncSummary health={database.health} databaseKey={database.key}
|
| 1176 |
+
catchUp={sourceCatchUp(catchUpStatus.catchUp, catchUpSourceKey(database.key))}
|
| 1177 |
+
catchUpError={catchUpStatus.error} catchUpLoading={catchUpStatus.loading} />
|
| 1178 |
</div>
|
| 1179 |
)) : <p>No database sync state has been published.</p>}
|
| 1180 |
<h4>Field catalog</h4>
|
|
|
|
| 1379 |
return (
|
| 1380 |
<div className="conn-card" data-connector={row.key}>
|
| 1381 |
<ConnectorCardHead row={row} />
|
| 1382 |
+
{catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
|
| 1383 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1384 |
+
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
| 1385 |
+
|| (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
|
| 1386 |
{catchUp?.active && catchUpError ? `, ${catchUpError}` : ""}
|
| 1387 |
</span>
|
| 1388 |
) : syncWarnings?.length ? (
|
web/src/connectors/DatabasesTab.tsx
CHANGED
|
@@ -124,11 +124,22 @@ function syncStamp(value: unknown): string | null {
|
|
| 124 |
return text && Number.isFinite(Date.parse(text)) ? text : null;
|
| 125 |
}
|
| 126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
export interface ConnectorCatchUp {
|
| 128 |
active: boolean;
|
| 129 |
done: number;
|
| 130 |
total: number;
|
| 131 |
newest: Record<string, string>;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
}
|
| 133 |
|
| 134 |
/** Missing catchUp is compatible; a malformed status is not a settled sync. */
|
|
@@ -149,7 +160,40 @@ export function parseConnectorCatchUp(value: unknown): ConnectorCatchUp | null {
|
|
| 149 |
if (!stamp) throw new Error("Sync status unavailable");
|
| 150 |
newest[key] = stamp;
|
| 151 |
}
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
}
|
| 154 |
|
| 155 |
export async function readConnectorCatchUp(signal: AbortSignal): Promise<ConnectorCatchUp | null> {
|
|
@@ -162,6 +206,10 @@ export function catchUpLabel(catchUp: ConnectorCatchUp): string {
|
|
| 162 |
return `Updating ${catchUp.done.toLocaleString()} of ${catchUp.total.toLocaleString()}`;
|
| 163 |
}
|
| 164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
export function useConnectorCatchUp(enabled: boolean) {
|
| 166 |
const [state, setState] = useState<{
|
| 167 |
catchUp: ConnectorCatchUp | null; error: string; loading: boolean;
|
|
|
|
| 124 |
return text && Number.isFinite(Date.parse(text)) ? text : null;
|
| 125 |
}
|
| 126 |
|
| 127 |
+
interface ConnectorSourceCatchUp {
|
| 128 |
+
active: boolean;
|
| 129 |
+
done: number;
|
| 130 |
+
total: number;
|
| 131 |
+
behind: string;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
export interface ConnectorCatchUp {
|
| 135 |
active: boolean;
|
| 136 |
done: number;
|
| 137 |
total: number;
|
| 138 |
newest: Record<string, string>;
|
| 139 |
+
/** Per-source progress; null when the server reports only the tenant total. */
|
| 140 |
+
sources: Record<string, ConnectorSourceCatchUp> | null;
|
| 141 |
+
/** Set when this status is scoped to one source that settled behind. */
|
| 142 |
+
behind?: string;
|
| 143 |
}
|
| 144 |
|
| 145 |
/** Missing catchUp is compatible; a malformed status is not a settled sync. */
|
|
|
|
| 160 |
if (!stamp) throw new Error("Sync status unavailable");
|
| 161 |
newest[key] = stamp;
|
| 162 |
}
|
| 163 |
+
let sources: Record<string, ConnectorSourceCatchUp> | null = null;
|
| 164 |
+
if (Object.prototype.hasOwnProperty.call(raw, "sources")) {
|
| 165 |
+
const rawSources = syncObject(raw.sources);
|
| 166 |
+
sources = {};
|
| 167 |
+
for (const [name, value] of Object.entries(rawSources)) {
|
| 168 |
+
const entry = syncObject(value);
|
| 169 |
+
const sourceDone = syncCount(entry.done);
|
| 170 |
+
const sourceTotal = syncCount(entry.total);
|
| 171 |
+
if (typeof entry.active !== "boolean" || sourceDone === null || sourceTotal === null
|
| 172 |
+
|| sourceDone > sourceTotal || typeof entry.behind !== "string") {
|
| 173 |
+
throw new Error("Sync status unavailable");
|
| 174 |
+
}
|
| 175 |
+
sources[name] = { active: entry.active, done: sourceDone, total: sourceTotal,
|
| 176 |
+
behind: syncText(entry.behind) };
|
| 177 |
+
}
|
| 178 |
+
}
|
| 179 |
+
return { active: raw.active, done, total, newest, sources };
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
/** The catch-up source a connector card or database waits on. */
|
| 183 |
+
export function catchUpSourceKey(key: string): string {
|
| 184 |
+
if (key === "amazon_sp" || key === "amazon" || key.startsWith("ut_amzn_")) return "amazon";
|
| 185 |
+
if (key === "meta_ads" || key === "meta" || key.startsWith("ut_meta_")) return "meta";
|
| 186 |
+
return "odoo";
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
/** Scope the tenant status to one source; an older server keeps the tenant total on Odoo only. */
|
| 190 |
+
export function sourceCatchUp(catchUp: ConnectorCatchUp | null, source: string): ConnectorCatchUp | null {
|
| 191 |
+
if (!catchUp) return null;
|
| 192 |
+
if (catchUp.sources === null) return source === "odoo" ? catchUp : null;
|
| 193 |
+
const own = catchUp.sources[source];
|
| 194 |
+
if (!own) return null;
|
| 195 |
+
return { ...catchUp, active: catchUp.active && own.active, done: own.done, total: own.total,
|
| 196 |
+
behind: own.behind || undefined };
|
| 197 |
}
|
| 198 |
|
| 199 |
export async function readConnectorCatchUp(signal: AbortSignal): Promise<ConnectorCatchUp | null> {
|
|
|
|
| 206 |
return `Updating ${catchUp.done.toLocaleString()} of ${catchUp.total.toLocaleString()}`;
|
| 207 |
}
|
| 208 |
|
| 209 |
+
export function behindLabel(reason: string): string {
|
| 210 |
+
return `Behind. ${reason}`;
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
export function useConnectorCatchUp(enabled: boolean) {
|
| 214 |
const [state, setState] = useState<{
|
| 215 |
catchUp: ConnectorCatchUp | null; error: string; loading: boolean;
|
web/src/customer-grid/GridActivity.tsx
CHANGED
|
@@ -135,11 +135,16 @@ export function GridActivity({ database }: { database: string }) {
|
|
| 135 |
if (!response.ok) throw new Error("status unavailable");
|
| 136 |
const body = await response.json() as {
|
| 137 |
tables?: Record<string, unknown>;
|
| 138 |
-
catchUp?: { active: boolean; done: number; total: number; newest: Record<string, string>
|
|
|
|
| 139 |
};
|
| 140 |
connected = Object.hasOwn(body.tables ?? {}, database);
|
| 141 |
-
// C1 owns catch-up;
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
} catch {
|
| 144 |
if (live) setCatchUp({ database, active: false });
|
| 145 |
} finally {
|
|
|
|
| 135 |
if (!response.ok) throw new Error("status unavailable");
|
| 136 |
const body = await response.json() as {
|
| 137 |
tables?: Record<string, unknown>;
|
| 138 |
+
catchUp?: { active: boolean; done: number; total: number; newest: Record<string, string>;
|
| 139 |
+
sources?: Record<string, { active?: boolean }> };
|
| 140 |
};
|
| 141 |
connected = Object.hasOwn(body.tables ?? {}, database);
|
| 142 |
+
// C1 owns catch-up for this database's own source; an unscoped status holds every source.
|
| 143 |
+
const source = database.startsWith("ut_amzn_") ? "amazon"
|
| 144 |
+
: database.startsWith("ut_meta_") ? "meta" : "odoo";
|
| 145 |
+
const own = body.catchUp?.sources?.[source];
|
| 146 |
+
if (live) setCatchUp({ database, active: connected && body.catchUp?.active === true
|
| 147 |
+
&& (own ? own.active === true : true) });
|
| 148 |
} catch {
|
| 149 |
if (live) setCatchUp({ database, active: false });
|
| 150 |
} finally {
|
web/src/sheet/SheetCatchUp.tsx
CHANGED
|
@@ -35,7 +35,15 @@ export default function SheetCatchUp({ workbookId, revision, children }: {
|
|
| 35 |
};
|
| 36 |
}, [workbookId, revision]);
|
| 37 |
|
| 38 |
-
if (!status.active)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
const { done, total } = status;
|
| 40 |
const measured = typeof done === "number" && Number.isSafeInteger(done) && done >= 0
|
| 41 |
&& typeof total === "number" && Number.isSafeInteger(total) && total > 0 && done <= total;
|
|
|
|
| 35 |
};
|
| 36 |
}, [workbookId, revision]);
|
| 37 |
|
| 38 |
+
if (!status.active) {
|
| 39 |
+
if (!status.behind?.length) return children;
|
| 40 |
+
return <>
|
| 41 |
+
<span className="sheet-catch-up-behind" role="status">
|
| 42 |
+
{status.behind.map((entry) => `${entry.source} is behind. ${entry.reason}`).join(" ")}
|
| 43 |
+
</span>
|
| 44 |
+
{children}
|
| 45 |
+
</>;
|
| 46 |
+
}
|
| 47 |
const { done, total } = status;
|
| 48 |
const measured = typeof done === "number" && Number.isSafeInteger(done) && done >= 0
|
| 49 |
&& typeof total === "number" && Number.isSafeInteger(total) && total > 0 && done <= total;
|
web/src/sheet/sheetApi.ts
CHANGED
|
@@ -63,12 +63,18 @@ export interface SheetCatchUpStatus {
|
|
| 63 |
active: boolean;
|
| 64 |
done?: number;
|
| 65 |
total?: number;
|
|
|
|
|
|
|
| 66 |
}
|
| 67 |
|
| 68 |
export const readSheetCatchUp = (workbookId: string, signal?: AbortSignal): Promise<Result<SheetCatchUpStatus>> =>
|
| 69 |
call(`/sheets/${encodeURIComponent(workbookId)}/catch-up`, { method: "GET", signal }, body => {
|
| 70 |
const status = (body as { catchUp?: SheetCatchUpStatus } | null)?.catchUp;
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
});
|
| 73 |
|
| 74 |
const errorOf = (body: unknown): { code: string; message: string; rev_current?: number } => {
|
|
|
|
| 63 |
active: boolean;
|
| 64 |
done?: number;
|
| 65 |
total?: number;
|
| 66 |
+
/** Sources this workbook binds that settled behind, each with a plain reason. */
|
| 67 |
+
behind?: { source: string; reason: string }[];
|
| 68 |
}
|
| 69 |
|
| 70 |
export const readSheetCatchUp = (workbookId: string, signal?: AbortSignal): Promise<Result<SheetCatchUpStatus>> =>
|
| 71 |
call(`/sheets/${encodeURIComponent(workbookId)}/catch-up`, { method: "GET", signal }, body => {
|
| 72 |
const status = (body as { catchUp?: SheetCatchUpStatus } | null)?.catchUp;
|
| 73 |
+
if (status?.active === true) return status;
|
| 74 |
+
const listed = status?.behind;
|
| 75 |
+
const behind = (Array.isArray(listed) ? listed : []).filter((entry) =>
|
| 76 |
+
typeof entry?.source === "string" && entry.source && typeof entry.reason === "string" && entry.reason);
|
| 77 |
+
return behind.length ? { active: false, behind } : { active: false };
|
| 78 |
});
|
| 79 |
|
| 80 |
const errorOf = (body: unknown): { code: string; message: string; rev_current?: number } => {
|
web/src/styles/sdlc73/sheet.css
CHANGED
|
@@ -160,6 +160,13 @@
|
|
| 160 |
font-variant-numeric: tabular-nums;
|
| 161 |
}
|
| 162 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
.sheet-last-edit {
|
| 164 |
appearance: none;
|
| 165 |
border: 0;
|
|
|
|
| 160 |
font-variant-numeric: tabular-nums;
|
| 161 |
}
|
| 162 |
|
| 163 |
+
.sheet-catch-up-behind {
|
| 164 |
+
padding: 8px 0;
|
| 165 |
+
color: var(--lp-muted);
|
| 166 |
+
font-size: var(--lp-fs-2xs);
|
| 167 |
+
white-space: nowrap;
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
.sheet-last-edit {
|
| 171 |
appearance: none;
|
| 172 |
border: 0;
|