fsanyoto commited on
Commit
2c49277
·
verified ·
1 Parent(s): a2e9a9c

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/amazon_quotes_loop.py +5 -93
  4. api/amazon_relational.py +1 -6
  5. api/app_templates.py +1 -11
  6. api/application_registry.py +6 -28
  7. api/connectors_amazon.py +10 -5
  8. api/external_freshness.py +55 -150
  9. api/main.py +5 -52
  10. api/process_role.py +35 -101
  11. api/routes_amazon_projects.py +3 -9
  12. api/routes_brand.py +2 -52
  13. api/routes_changes.py +0 -3
  14. api/routes_competitors.py +10 -32
  15. api/routes_grid.py +0 -7
  16. api/routes_odoo_tables.py +2 -10
  17. api/routes_shares.py +32 -27
  18. api/routes_starred.py +0 -5
  19. api/routes_tables.py +3 -25
  20. api/sync_runtime.py +10 -55
  21. api/usage_ledger.py +53 -59
  22. platform/core/amazon_projects.py +21 -90
  23. platform/core/amazon_rivals.py +117 -575
  24. platform/core/connector_spec.py +26 -69
  25. platform/core/external_tools.py +1 -1
  26. platform/core/shares.py +2 -24
  27. platform/harness/amazon_ads_client.py +470 -158
  28. platform/harness/amazon_ads_schema.py +91 -38
  29. platform/harness/amazon_ads_store.py +238 -171
  30. platform/harness/amazon_client.py +3 -6
  31. platform/harness/amazon_schema.py +13 -22
  32. platform/harness/amazon_store.py +184 -708
  33. platform/harness/datastore.py +34 -44
  34. platform/harness/mirror_store.py +152 -394
  35. platform/modules/amazon_profit.py +23 -207
  36. platform/modules/amazon_profit_account.py +7 -122
  37. platform/modules/competitor_watch.py +3 -19
  38. requirements.txt +0 -1
  39. web/package-lock.json +0 -485
  40. web/package-lock.linux.sha256 +2 -2
  41. web/package.json +0 -1
  42. web/public/AIOS_DEPLOY_RECEIPT +1 -1
  43. web/src/apps/amazon-profit/AppView.tsx +6 -81
  44. web/src/apps/amazon-profit/ProjectOverview.tsx +9 -32
  45. web/src/apps/amazon-profit/ProjectSkuPicker.tsx +14 -43
  46. web/src/apps/amazon-profit/Projects.tsx +20 -53
  47. web/src/apps/amazon-profit/ProjectsShared.tsx +3 -20
  48. web/src/apps/templateMarks.tsx +0 -4
  49. web/src/automation/AutomationDetail.tsx +23 -1
  50. web/src/automation/AutomationSurface.tsx +22 -2
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "84d02c9227",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "8962a7d2b8",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 84d02c9227
 
1
+ 8962a7d2b8
api/amazon_quotes_loop.py CHANGED
@@ -9,12 +9,10 @@ import logging
9
  from pathlib import Path
10
  import threading
11
  import time
12
- import uuid
13
 
14
  JOB = "amazon-rival-quotes"
15
  #: The loop asks each tenant whether its daily run is due this often.
16
  POLL_SECONDS = 600
17
- SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")
18
 
19
  #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
20
  #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
@@ -28,29 +26,17 @@ _thread = None
28
  _stop = None
29
 
30
 
31
- class SnapshotBlocked(RuntimeError):
32
- """A safe admission cause that can be recorded without exposing provider details."""
33
-
34
-
35
  def _say(message):
36
  print(f"[{JOB}] {message}", flush=True)
37
 
38
 
39
  def readers(fields):
40
- """Read prices and complete facts with the tenant credential and the shared Amazon pacers."""
41
  from harness import amazon_client as cx
42
- from harness import amazon_competitor_facts as facts
43
-
44
- def complete(asins):
45
- catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
46
- return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
47
- "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
48
- for asin in asins}
49
-
50
  return {"offers": lambda asins: cx.item_offers(fields, asins),
51
  # The pricing sync draws this same `competitive_summary` pacer in this worker process.
52
  "summaries": lambda asins: cx.competitive_summaries(fields, asins),
53
- "facts": complete}
54
 
55
 
56
  @contextmanager
@@ -78,69 +64,9 @@ def _after_quotes(tenant, changed):
78
  "the run result is unchanged.")
79
 
80
 
81
- def _snapshot_cause(runtime, *, check_lease=True):
82
- from harness import amazon_store
83
-
84
- enabled = amazon_store.read_database_enablement(runtime)
85
- if not enabled["available"]:
86
- return "Competitor database settings could not be read. Restore settings access and retry the run."
87
- if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
88
- return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
89
- if check_lease and amazon_store.first_load_job_live(runtime):
90
- return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
91
- return None
92
-
93
-
94
- def _publish_snapshots(tenant, runtime, result):
95
- from harness import amazon_competitor_facts as facts, amazon_store, mirror_store
96
-
97
- lock = mirror_store.for_connector("amazon").lock
98
- if not lock.acquire(blocking=False):
99
- raise SnapshotBlocked("The Amazon mirror writer is busy.")
100
- holder = f"{JOB}:{result['id']}"
101
- leased = False
102
- try:
103
- leased = amazon_store.acquire_amazon_lease(runtime, holder)
104
- if not leased:
105
- raise SnapshotBlocked("The Amazon mirror writer lease is held.")
106
- cause = _snapshot_cause(runtime, check_lease=False)
107
- if cause:
108
- raise SnapshotBlocked(cause)
109
- report = {"tables": {}, "problems": []}
110
- with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
111
- con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
112
- try:
113
- stored = facts.import_snapshots(con, result["snapshots"])
114
- for table in SNAPSHOT_TABLES:
115
- total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
116
- report["tables"][table] = {"state": "done", "rows": total}
117
- finally:
118
- con.close()
119
- if report["problems"]:
120
- raise RuntimeError("Competitor mirror publication failed")
121
- result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
122
- finally:
123
- try:
124
- if leased:
125
- amazon_store.release_amazon_lease(runtime, holder)
126
- finally:
127
- lock.release()
128
-
129
-
130
- def _run_blocked(runtime, cause):
131
- from core import amazon_rivals as rivals
132
-
133
- run_id = uuid.uuid4().hex
134
- rivals.start_run(runtime, run_id, asin_total=0)
135
- return {"id": run_id, "progress": 0, "cause": cause,
136
- "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}
137
-
138
-
139
  def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
140
  """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
141
  from core import amazon_rivals as rivals
142
- if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
143
- raise ValueError("Competitor quotes require the matching tenant runtime")
144
  try:
145
  if not rivals.due(runtime, now=now):
146
  return None
@@ -154,23 +80,9 @@ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
154
  fields = amazon_store.credential(tenant, runtime)
155
  if not fields:
156
  return None
157
- started = time.monotonic()
158
- cause = _snapshot_cause(runtime)
159
- result = (_run_blocked(runtime, cause) if cause else
160
- rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
161
- seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
162
- log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
163
- if result["snapshots"]:
164
- try:
165
- _publish_snapshots(tenant, runtime, result)
166
- except Exception as exc:
167
- cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
168
- f"Competitor snapshots could not be published ({type(exc).__name__}).")
169
- result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
170
- f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
171
- result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
172
- rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
173
- counts=result["counts"], cause=result["cause"])
174
  if result["cause"] is None:
175
  _after_quotes(tenant, result["changed"])
176
  return result
 
9
  from pathlib import Path
10
  import threading
11
  import time
 
12
 
13
  JOB = "amazon-rival-quotes"
14
  #: The loop asks each tenant whether its daily run is due this often.
15
  POLL_SECONDS = 600
 
16
 
17
  #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
18
  #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
 
26
  _stop = None
27
 
28
 
 
 
 
 
29
  def _say(message):
30
  print(f"[{JOB}] {message}", flush=True)
31
 
32
 
33
  def readers(fields):
34
+ """The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
35
  from harness import amazon_client as cx
 
 
 
 
 
 
 
 
36
  return {"offers": lambda asins: cx.item_offers(fields, asins),
37
  # The pricing sync draws this same `competitive_summary` pacer in this worker process.
38
  "summaries": lambda asins: cx.competitive_summaries(fields, asins),
39
+ "ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
40
 
41
 
42
  @contextmanager
 
64
  "the run result is unchanged.")
65
 
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
68
  """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
69
  from core import amazon_rivals as rivals
 
 
70
  try:
71
  if not rivals.due(runtime, now=now):
72
  return None
 
80
  fields = amazon_store.credential(tenant, runtime)
81
  if not fields:
82
  return None
83
+ result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
84
+ seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
85
+ log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  if result["cause"] is None:
87
  _after_quotes(tenant, result["changed"])
88
  return result
api/amazon_relational.py CHANGED
@@ -650,8 +650,6 @@ def _fields_for(cur, key, *, read_through=False):
650
  "pinned": name == pinned,
651
  **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
652
  if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
653
- **({"description": amazon_schema.SQP_LIMIT_NOTE}
654
- if name == "asin" and key == PREFIX + "search_query_performance" else {}),
655
  **({"format": fmt} if fmt else {}),
656
  }, flow="amazon_relational"))
657
  for lookup in derived.get(name, ()):
@@ -725,8 +723,7 @@ READ_THROUGH = frozenset(
725
  and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
726
  PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
727
  # Change history grows without a cap (standing rule 1), so it is never stored.
728
- PREFIX + table for table in amazon_store.HISTORY_TABLES) | frozenset((
729
- PREFIX + "competitor_snapshots", PREFIX + "competitor_images"))
730
 
731
 
732
  def _quoted(name):
@@ -1601,8 +1598,6 @@ def apply_plan(rt, built, username="amazon", today=None):
1601
  doc[key]["description"] = ads_descriptions[key]
1602
  if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
1603
  doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
1604
- if key == PREFIX + "search_query_performance":
1605
- doc[key]["description"] = amazon_schema.SQP_LIMIT_NOTE
1606
  if key == PREFIX + "listing_items":
1607
  doc[key]["amazonListingIdentity"] = listing_version
1608
  return doc
 
650
  "pinned": name == pinned,
651
  **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
652
  if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
 
 
653
  **({"format": fmt} if fmt else {}),
654
  }, flow="amazon_relational"))
655
  for lookup in derived.get(name, ()):
 
723
  and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
724
  PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
725
  # Change history grows without a cap (standing rule 1), so it is never stored.
726
+ PREFIX + table for table in amazon_store.HISTORY_TABLES)
 
727
 
728
 
729
  def _quoted(name):
 
1598
  doc[key]["description"] = ads_descriptions[key]
1599
  if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
1600
  doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
 
 
1601
  if key == PREFIX + "listing_items":
1602
  doc[key]["amazonListingIdentity"] = listing_version
1603
  return doc
api/app_templates.py CHANGED
@@ -127,22 +127,12 @@ TEMPLATES = (
127
  # Project management reads its own module tables (ut_amazon_*), so it declares no
128
  # source: a cold mirror must not refuse an App whose rows live in the store.
129
  _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
130
- "Create value", (),
131
- optional_sources=("ut_amzn_search_query_performance", "ut_amzn_order_items",
132
- "ut_amzn_brand_analytics", "ut_amzn_listings", "ut_amzn_catalog_items",
133
- "ut_amzn_listing_items", "ut_amzn_competitor_snapshots",
134
- "ut_amzn_competitor_images"),
135
- optional_connectors=("odoo",), proposes=False),
136
  {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
137
  "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
138
  "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
139
  "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
140
  "tools": (), "proposes": False},
141
- {"kind": "reports", "label": "Reports", "purpose": "Download reports shared with you.",
142
- "connector": "odoo", "connectors": ("odoo",), "tags": ["Odoo"], "sources": (),
143
- "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "reports",
144
- "module": "modules.reports", "router": "routes_reports", "view": "reports",
145
- "tools": (), "proposes": False},
146
  )
147
  if not all(validate_connectors(row) for row in TEMPLATES):
148
  raise ValueError("An App template omits a source connector.")
 
127
  # Project management reads its own module tables (ut_amazon_*), so it declares no
128
  # source: a cold mirror must not refuse an App whose rows live in the store.
129
  _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
130
+ "Create value", (), optional_connectors=("odoo",), proposes=False),
 
 
 
 
 
131
  {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
132
  "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
133
  "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
134
  "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
135
  "tools": (), "proposes": False},
 
 
 
 
 
136
  )
137
  if not all(validate_connectors(row) for row in TEMPLATES):
138
  raise ValueError("An App template omits a source connector.")
api/application_registry.py CHANGED
@@ -624,16 +624,6 @@ def _source_rows_exist(session, sources):
624
  return False
625
 
626
 
627
- def _reports_readable(runtime, user, is_admin):
628
- """Report grants admit the Reports App independently of its App sharing record."""
629
- try:
630
- from modules import reports
631
- except Exception as exc: # A failed Reports import withholds its App.
632
- raise_if_resource_error(exc)
633
- return False
634
- return bool(is_admin or reports.granted_keys(runtime, user))
635
-
636
-
637
  def _readable_template_app(session, app_id):
638
  """Resolve App identity before reading any source or exposing a refusal."""
639
  from deps import err
@@ -645,10 +635,7 @@ def _readable_template_app(session, app_id):
645
  except Exception:
646
  raise err(503, "app_unavailable", "This App cannot be read right now.") from None
647
  if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
648
- or app.get("retired") is True
649
- or not (_reports_readable(session.runtime, session.uname, session.admin)
650
- if app.get("templateKind") == "reports"
651
- else may_read(app, session.uname, session.admin))):
652
  raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
653
  kind = app.get("templateKind")
654
  catalogue = _app_templates.BY_KIND.get(str(kind or ""))
@@ -1318,7 +1305,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
1318
 
1319
 
1320
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
1321
- is_default=False, readable=None):
1322
  row=copy.deepcopy(_app_projection(app))
1323
  row.setdefault("origin", None)
1324
  if row.get("type") == "code":
@@ -1327,10 +1314,9 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
1327
  compatible = [database for database in row.get("compatibleDatabases") or ()
1328
  if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
1329
  row["compatibleDatabases"] = compatible
1330
- app_readable = may_read(app, user, is_admin)
1331
- row["capabilities"]={"read":app_readable if readable is None else readable,"edit":may_edit(app,user,is_admin),
1332
  "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
1333
- "duplicate":app_readable,"star":app_readable,
1334
  "default":bool(is_admin)}
1335
  row["starred"] = user in (app.get("starredBy") or ())
1336
  row["isDefault"] = bool(is_default)
@@ -1391,7 +1377,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1391
  """List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
1392
  doc = load(runtime)
1393
  rows=[]
1394
- reports_reader = None
1395
  for app in doc["applications"].values():
1396
  if (isinstance(app, dict) and getattr(runtime, "key", None)
1397
  and app.get("tenant") != runtime.key):
@@ -1401,19 +1386,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1401
  if (isinstance(app, dict) and app.get("templateKind") == "alerts"
1402
  and not (may_open or _user_wall(runtime, user))("automation")):
1403
  continue
1404
- report_app = isinstance(app, dict) and app.get("templateKind") == "reports"
1405
- if report_app:
1406
- if reports_reader is None:
1407
- reports_reader = _reports_readable(runtime, user, is_admin)
1408
- if not reports_reader or app.get("retired") is True:
1409
- continue
1410
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
1411
  continue
1412
  if database:
1413
  # A View resolves its App only through this read, so a retired seed stays listed here.
1414
- if not report_app and not may_read(app,user,is_admin): continue
1415
  if database not in (app.get("compatibleDatabases") or ()): continue
1416
- elif not report_app and not _listed(app, user): continue
1417
  if (not _database_free(app)
1418
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
1419
  catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
@@ -1425,7 +1404,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1425
  default_key = f"{database}::{app.get('type')}" if database else ""
1426
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
1427
  target_database=database,
1428
- readable=True if report_app else None,
1429
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
1430
  # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
1431
  # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
 
624
  return False
625
 
626
 
 
 
 
 
 
 
 
 
 
 
627
  def _readable_template_app(session, app_id):
628
  """Resolve App identity before reading any source or exposing a refusal."""
629
  from deps import err
 
635
  except Exception:
636
  raise err(503, "app_unavailable", "This App cannot be read right now.") from None
637
  if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
638
+ or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
 
 
 
639
  raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
640
  kind = app.get("templateKind")
641
  catalogue = _app_templates.BY_KIND.get(str(kind or ""))
 
1305
 
1306
 
1307
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
1308
+ is_default=False):
1309
  row=copy.deepcopy(_app_projection(app))
1310
  row.setdefault("origin", None)
1311
  if row.get("type") == "code":
 
1314
  compatible = [database for database in row.get("compatibleDatabases") or ()
1315
  if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
1316
  row["compatibleDatabases"] = compatible
1317
+ row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin),
 
1318
  "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
1319
+ "duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin),
1320
  "default":bool(is_admin)}
1321
  row["starred"] = user in (app.get("starredBy") or ())
1322
  row["isDefault"] = bool(is_default)
 
1377
  """List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
1378
  doc = load(runtime)
1379
  rows=[]
 
1380
  for app in doc["applications"].values():
1381
  if (isinstance(app, dict) and getattr(runtime, "key", None)
1382
  and app.get("tenant") != runtime.key):
 
1386
  if (isinstance(app, dict) and app.get("templateKind") == "alerts"
1387
  and not (may_open or _user_wall(runtime, user))("automation")):
1388
  continue
 
 
 
 
 
 
1389
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
1390
  continue
1391
  if database:
1392
  # A View resolves its App only through this read, so a retired seed stays listed here.
1393
+ if not may_read(app,user,is_admin): continue
1394
  if database not in (app.get("compatibleDatabases") or ()): continue
1395
+ elif not _listed(app, user): continue
1396
  if (not _database_free(app)
1397
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
1398
  catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
 
1404
  default_key = f"{database}::{app.get('type')}" if database else ""
1405
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
1406
  target_database=database,
 
1407
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
1408
  # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
1409
  # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
api/connectors_amazon.py CHANGED
@@ -611,11 +611,16 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
611
  answer[table]["warning"] = warning
612
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
613
  answer[table]["limit"] = limit
614
- for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"):
615
- answer[table] = {
616
- "rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False,
617
- "unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]],
618
- }
 
 
 
 
 
619
  # A history table is written in its source's transaction, so it shares the source's last success.
620
  for table in az.HISTORY_TABLES:
621
  rows = mirrored.get(table)
 
611
  answer[table]["warning"] = warning
612
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
613
  answer[table]["limit"] = limit
614
+ answer["sku_keywords"] = {
615
+ "rowsLoaded": mirrored.get("sku_keywords"),
616
+ "lastSuccess": None,
617
+ "inProgress": False,
618
+ "unknownCause": az.amazon_schema.PROVENANCE_COPY["derived"],
619
+ }
620
+ answer["keyword_ranks"] = {
621
+ "rowsLoaded": mirrored.get("keyword_ranks"), "lastSuccess": None, "inProgress": False,
622
+ "unknownCause": az.amazon_schema.PROVENANCE_COPY["apify_dataset"],
623
+ }
624
  # A history table is written in its source's transaction, so it shares the source's last success.
625
  for table in az.HISTORY_TABLES:
626
  rows = mirrored.get(table)
api/external_freshness.py CHANGED
@@ -82,24 +82,18 @@ def _amazon_raw(table_key, source=None):
82
  return None
83
 
84
 
85
- def amazon_sync_state(cur, rt=None, *, entry=None):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
- if rt is not None or entry is not None:
88
  from core import connector_sync, connector_spec
89
- if entry is None:
90
- entry = connector_sync.status(rt, "amazon_sp", fresh=True)
91
  if entry is not None:
92
  last_pass = entry.get("pass") or {}
93
  held = last_pass.get("held") or {}
94
  stamps = last_pass.get("tableSync") or {}
95
  tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
96
- state = {"amzn_sync_mark_" + name: stamp for name in tables
97
- if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
98
- if cur is not None and any(name not in stamps for name in held):
99
- previous = amazon_sync_state(cur)
100
- state.update({"amzn_sync_mark_" + name: stamp for name in held if name not in stamps
101
- and (stamp := _amazon_table_stamp(previous, name)) is not None})
102
- return state
103
  return {str(entity): at for entity, at in cur.execute(
104
  "SELECT entity, updated_at FROM _sync_state "
105
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
@@ -141,8 +135,8 @@ def amazon_sync_stamp(cur, table_key, state=None):
141
  return _amazon_table_stamp(state, table)
142
 
143
 
144
- def _amazon_refreshes(session, keys, *, entry, served=False):
145
- """Use the supplied canonical entry or served receipts for authorized Amazon databases."""
146
  import assistant_mirror
147
 
148
  cur, state, refreshed = None, None, {}
@@ -154,7 +148,7 @@ def _amazon_refreshes(session, keys, *, entry, served=False):
154
  cur = None
155
  continue
156
  if state is None:
157
- state = amazon_sync_state(cur, entry=None if served else entry)
158
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
159
  except Exception:
160
  continue
@@ -244,7 +238,7 @@ def _mirror_refreshes(session, definitions, keys):
244
  return refreshed
245
 
246
 
247
- def _served_behind(session, key, *, worker=None):
248
  """process_role's install lag for the connector behind one mirror database, or None."""
249
  key = str(key or "")
250
  connector = ("amazon" if key.startswith("ut_amzn_") else
@@ -253,82 +247,49 @@ def _served_behind(session, key, *, worker=None):
253
  return None
254
  try:
255
  import process_role
256
- if worker is None:
257
- worker = process_role.stored_worker(session.runtime)
258
- return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector,
259
- worker=worker)
260
  except Exception:
261
  return None
262
 
263
 
264
- def _retained_fact(served, state, reason):
265
- """Keep the successful source receipt while naming why its rows have not advanced."""
266
- fact = {"source_synced_at": utc_stamp(served), "source_sync": state, "source_sync_reason": reason}
267
- if (day := _sync_date(served)) is not None:
268
- fact["source_synced_on"] = day
269
- return fact
270
-
271
-
272
- def _stale_fact(served, lag=None):
273
  """A fact for a mirror whose newest published sync this deployment could not install."""
274
  day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
275
  reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
276
  if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
277
- if lag and lag.get("cause"):
278
- reason += " " + lag["cause"]
279
- if lag and (retry := utc_stamp(lag.get("nextRetryAt"))):
280
- reason += f" Next retry at {retry}."
281
- return _retained_fact(served, "stale", reason)
282
-
283
-
284
- def _amazon_holds(entry, keys):
285
- """Map each authorized table to its own canonical hold or its held input tables."""
286
- held = (entry.get("pass") or {}).get("held") or {}
287
- result = {}
288
- for key in keys:
289
- raw = _amazon_raw(key)
290
- table = raw.removeprefix("amzn_") if raw else ""
291
- dependencies = (table, *_AMAZON_DERIVED_INPUTS.get(table, ()))
292
- rows = [held[name] for name in dependencies if isinstance(held.get(name), dict)]
293
- if rows:
294
- result[key] = rows
295
- return result
296
-
297
-
298
- def _held_fact(served, entries):
299
- """Name the kept rows and each stored hold cause without changing their source date."""
300
- import dash_norm
301
 
302
- day = _sync_date(served) or (utc_stamp(served) or "")[:10]
303
- reason = f"Held at the rows synced on {day}." if day else "Held at the previously synced rows."
304
- causes = dict.fromkeys(dash_norm.plain(entry.get("cause") or "The last sync did not finish this table.")
305
- for entry in entries)
306
- reason += " " + " ".join(causes)
307
- retries = sorted({stamp for entry in entries if (stamp := utc_stamp(entry.get("nextTryAt")))})
308
- if retries:
309
- reason += f" Next retry at {retries[0]}."
310
- return _retained_fact(served, "stale", reason)
311
 
 
 
 
312
 
313
- def _edit_receipts(rt, edited):
314
- """Read the latest edit for all personal databases in one history aggregate."""
 
 
 
 
 
 
 
 
 
 
 
315
  edits = {}
316
  if edited:
 
 
 
317
  try:
318
  from core import history_rows
319
  latest = history_rows.backend(rt).latest(sorted(edited))
320
- except Exception: # Unavailable history keeps the definition's recorded edit time.
321
  latest = {}
322
  for key, at in latest.items() if isinstance(latest, dict) else ():
323
  if key in edited:
324
  edits[key] = _latest(edits.get(key), at)
325
- return edits
326
-
327
-
328
- def _connector_receipts(rt, connected, definitions):
329
- """Read matching jobs without borrowing another connection's successful sync."""
330
- from core import connector_sync
331
-
332
  receipts = {}
333
  if connected:
334
  try:
@@ -344,84 +305,27 @@ def _connector_receipts(rt, connected, definitions):
344
  or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
345
  continue
346
  receipts[key] = job
347
- return receipts
348
-
349
-
350
- class _StatusSnapshot:
351
- """Keep the request's fresh canonical status available to readers that request freshness again."""
352
-
353
- def __init__(self, runtime, statuses):
354
- self._runtime, self._statuses = runtime, statuses
355
-
356
- def get(self, name, fresh=False):
357
- from core import connector_sync
358
-
359
- if name == connector_sync.STATUS_KEY:
360
- return self._statuses
361
- return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)
362
-
363
- def __getattr__(self, name):
364
- return getattr(self._runtime, name)
365
-
366
-
367
- def _odoo_freshness(rt):
368
- """Read Odoo's metadata without restoring or querying its mirror."""
369
- import odoo_relational
370
- from core import user_tables
371
-
372
- docs = {}
373
- for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
374
- try:
375
- docs[name] = rt.get(name) or {}
376
- except Exception: # A missing status must not turn a discovery read into an error.
377
- docs[name] = {}
378
- return odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
379
-
380
-
381
- def _synced_fact(values):
382
- """Use a successful source timestamp or preserve an explicitly date-only receipt."""
383
- stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
384
- if stamp is not None:
385
- return {"source_synced_at": stamp, "source_sync": "synced"}
386
- day = next((day for value in values if (day := _sync_date(value)) is not None), None)
387
- if day is not None:
388
- return {"source_synced_on": day, "source_sync": "synced_date_only",
389
- "source_sync_reason": "The source reports only a sync date, without a time."}
390
- return None
391
-
392
-
393
- def snapshot(session, databases):
394
- """Index source receipts and edit times once for this authorized discovery call."""
395
- from core import connector_sync, user_tables
396
-
397
- wanted, rt = set(databases), session.runtime
398
- try:
399
- definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
400
- if not isinstance(definitions, dict):
401
- raise ValueError("source definitions unavailable")
402
- except Exception:
403
- return {key: {"source_synced_at": None, "source_sync": "never_synced",
404
- "source_sync_reason": "The source's connector status could not be verified."}
405
- for key in wanted}
406
- lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
407
- connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
408
- edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
409
- edits, receipts = _edit_receipts(rt, edited), _connector_receipts(rt, connected, definitions)
410
  odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
 
 
 
 
 
 
 
 
 
 
 
411
  amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
412
- statuses = connector_sync.status_all(rt, fresh=True) if amazon_keys or odoo_keys else {}
413
- odoo = _odoo_freshness(_StatusSnapshot(rt, statuses)) if odoo_keys else {}
414
- worker = (statuses.get("sync_worker") or {}).get("default") or {}
415
- amazon_entry = (statuses.get("amazon_sp") or {}).get("default")
416
- behind = {group: _served_behind(session, next(iter(keys)), worker=worker)
417
- for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
418
- amazon = _amazon_refreshes(session, amazon_keys, entry=amazon_entry,
419
- served=behind.get("amazon") is not None) if amazon_keys else {}
420
- held = _amazon_holds(amazon_entry or {}, amazon_keys)
421
  mirror_keys = {key for key in connected - amazon_keys
422
  if not user_tables.materialises(key, st=lent, defn=definitions[key])}
423
  mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
424
  facts = {}
 
 
 
425
  odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
426
  if isinstance(item, dict) and item.get("key")}
427
  for key in wanted:
@@ -432,10 +336,7 @@ def snapshot(session, databases):
432
  elif key == "ut_activity":
433
  fact["source_sync"] = "not_applicable"
434
  elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
435
- fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key),
436
- behind["amazon" if key in amazon_keys else "odoo"])
437
- elif key in held:
438
- fact = _held_fact(amazon.get(key), held[key])
439
  else:
440
  health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
441
  job = receipts.get(key) or {}
@@ -447,9 +348,13 @@ def snapshot(session, databases):
447
  if job.get("state") == "ready":
448
  successes.append(job.get("at"))
449
  values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
450
- synced = _synced_fact(values)
451
- if synced is not None:
452
- fact.update(synced)
 
 
 
 
453
  elif (own_failed or job.get("state") == "failed"
454
  or key in odoo_keys and odoo.get("phase") == "error"):
455
  fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
 
82
  return None
83
 
84
 
85
+ def amazon_sync_state(cur, rt=None):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
+ if rt is not None:
88
  from core import connector_sync, connector_spec
89
+ entry = connector_sync.status(rt, "amazon_sp", fresh=True)
 
90
  if entry is not None:
91
  last_pass = entry.get("pass") or {}
92
  held = last_pass.get("held") or {}
93
  stamps = last_pass.get("tableSync") or {}
94
  tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
95
+ return {"amzn_sync_mark_" + name: stamp for name in tables
96
+ if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
 
 
 
 
 
97
  return {str(entity): at for entity, at in cur.execute(
98
  "SELECT entity, updated_at FROM _sync_state "
99
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
 
135
  return _amazon_table_stamp(state, table)
136
 
137
 
138
+ def _amazon_refreshes(session, keys):
139
+ """Read only sync metadata for the authorized Amazon databases on this tenant's cursor."""
140
  import assistant_mirror
141
 
142
  cur, state, refreshed = None, None, {}
 
148
  cur = None
149
  continue
150
  if state is None:
151
+ state = amazon_sync_state(cur, session.runtime)
152
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
153
  except Exception:
154
  continue
 
238
  return refreshed
239
 
240
 
241
+ def _served_behind(session, key):
242
  """process_role's install lag for the connector behind one mirror database, or None."""
243
  key = str(key or "")
244
  connector = ("amazon" if key.startswith("ut_amzn_") else
 
247
  return None
248
  try:
249
  import process_role
250
+ return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector)
 
 
 
251
  except Exception:
252
  return None
253
 
254
 
255
+ def _stale_fact(served):
 
 
 
 
 
 
 
 
256
  """A fact for a mirror whose newest published sync this deployment could not install."""
257
  day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
258
  reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
259
  if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
260
+ return {"source_synced_at": None, "source_sync": "stale", "source_sync_reason": reason}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
 
 
 
 
 
 
 
 
 
 
262
 
263
+ def snapshot(session, databases):
264
+ """Index source receipts and edit times once for this authorized discovery call."""
265
+ from core import connector_sync, user_tables
266
 
267
+ wanted = set(databases)
268
+ rt = session.runtime
269
+ try:
270
+ definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
271
+ if not isinstance(definitions, dict):
272
+ raise ValueError("source definitions unavailable")
273
+ except Exception:
274
+ return {key: {"source_synced_at": None, "source_sync": "never_synced",
275
+ "source_sync_reason": "The source's connector status could not be verified."}
276
+ for key in wanted}
277
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
278
+ connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
279
+ edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
280
  edits = {}
281
  if edited:
282
+ # D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
283
+ # newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
284
+ # definition's `updated` stamp below and never fails discovery.
285
  try:
286
  from core import history_rows
287
  latest = history_rows.backend(rt).latest(sorted(edited))
288
+ except Exception:
289
  latest = {}
290
  for key, at in latest.items() if isinstance(latest, dict) else ():
291
  if key in edited:
292
  edits[key] = _latest(edits.get(key), at)
 
 
 
 
 
 
 
293
  receipts = {}
294
  if connected:
295
  try:
 
305
  or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
306
  continue
307
  receipts[key] = job
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
309
+ odoo = {}
310
+ if odoo_keys:
311
+ import odoo_relational
312
+
313
+ docs = {}
314
+ for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
315
+ try:
316
+ docs[name] = rt.get(name) or {}
317
+ except Exception:
318
+ docs[name] = {}
319
+ odoo = odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
320
  amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
321
+ amazon = _amazon_refreshes(session, amazon_keys) if amazon_keys else {}
 
 
 
 
 
 
 
 
322
  mirror_keys = {key for key in connected - amazon_keys
323
  if not user_tables.materialises(key, st=lent, defn=definitions[key])}
324
  mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
325
  facts = {}
326
+ # One install-lag read per connector: a secondary web worker reads it from the primary's file.
327
+ behind = {group: _served_behind(session, next(iter(keys)))
328
+ for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
329
  odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
330
  if isinstance(item, dict) and item.get("key")}
331
  for key in wanted:
 
336
  elif key == "ut_activity":
337
  fact["source_sync"] = "not_applicable"
338
  elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
339
+ fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key))
 
 
 
340
  else:
341
  health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
342
  job = receipts.get(key) or {}
 
348
  if job.get("state") == "ready":
349
  successes.append(job.get("at"))
350
  values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
351
+ stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
352
+ synced_on = next((day for value in values if (day := _sync_date(value)) is not None), None)
353
+ if stamp is not None:
354
+ fact.update(source_synced_at=stamp, source_sync="synced")
355
+ elif synced_on is not None:
356
+ fact.update(source_synced_on=synced_on, source_sync="synced_date_only",
357
+ source_sync_reason="The source reports only a sync date, without a time.")
358
  elif (own_failed or job.get("state") == "failed"
359
  or key in odoo_keys and odoo.get("phase") == "error"):
360
  fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
api/main.py CHANGED
@@ -758,13 +758,6 @@ import app_routes
758
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
759
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
760
  _mount_router("routes_amazon_projects")
761
- _mount_router("routes_amazon_pdp")
762
- _mount_router("routes_pdp_text")
763
- _mount_router("routes_pdp_images")
764
- if _mount_router("routes_pdp_publish"):
765
- # Amazon fetches published image URLs without a session, so the media router carries no auth.
766
- from routes_pdp_publish import media_router as _pdp_media_router
767
- app.include_router(_pdp_media_router)
768
  _mount_router("routes_competitors")
769
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
770
  _mount_router("routes_sku_actions")
@@ -1350,8 +1343,6 @@ def _pull_amazon(why, vendor=True):
1350
  # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
1351
  _install_amazon_publication(tenant_key, rt, why)
1352
  if _amazon.first_load_job_live(rt):
1353
- from harness import mirror_store
1354
- mirror_store.record_sync_attempt("amazon", rt)
1355
  print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
1356
  return False
1357
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
@@ -1387,7 +1378,6 @@ def _pull_amazon(why, vendor=True):
1387
  except Exception as exc:
1388
  # A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
1389
  if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
1390
- mirror_store.record_sync_attempt("amazon", rt)
1391
  print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
1392
  flush=True)
1393
  return False
@@ -1759,16 +1749,8 @@ def _amazon_sync_receipt(report, rt=None):
1759
 
1760
 
1761
  def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1762
- """Name a classified failure or a completed publication's typed entity hold."""
1763
  if ok:
1764
- if provider == "odoo":
1765
- from harness import mirror_store
1766
- held = mirror_store.odoo_moving_entities(cause)
1767
- if held:
1768
- entity = sorted(held)[0]
1769
- others = f" {len(held) - 1} other entities were held." if len(held) > 1 else ""
1770
- return (f"Odoo {entity} changed during all 3 full reads. Its previous rows were kept."
1771
- + others + " Other completed entities published.")
1772
  return detail
1773
  classifier, sentence = _sync_now_cause(cause)
1774
  if classifier == "nothing_due":
@@ -1969,35 +1951,9 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1969
  **({"after_entity": _after_entity} if _after_entity else {}))
1970
  if not report and selected_entities != []:
1971
  raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
1972
- held = mirror_store.odoo_moving_entities(report)
1973
- if any(row.get("phase") != "live" and name not in held for name, row in report.items()):
1974
  raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
1975
- waiting = mirror_store.pending_sync_entities("odoo", tenant, rt)
1976
- if held and waiting == []:
1977
- import json
1978
- store = mirror_store.for_connector("odoo")
1979
- with store.lock:
1980
- published, _ = store._connect(tenant, rt)
1981
- candidate = None
1982
- try:
1983
- path = rt.datastore_path.with_suffix(".odoo.candidate.duckdb")
1984
- candidate = datastore.apply_limits(datastore.duckdb.connect(str(path), read_only=True))
1985
- retained = {row[0] for row in candidate.execute("SELECT name FROM _candidate_held").fetchall()}
1986
- publication = json.loads(store._download(rt, mirror_store._PASS_KEY) or b"{}")
1987
- installed = datastore._publication(published, "odoo")
1988
- if (retained == set(held) and publication.get("state") == "published"
1989
- and publication.get("tenant") == tenant and publication.get("connector") == "odoo"
1990
- and publication.get("completed_at") and installed.get("state") == "complete"
1991
- and publication.get("snapshot_id")
1992
- and publication.get("snapshot_id") == installed.get("snapshot_id")):
1993
- waiting = retained
1994
- finally:
1995
- if candidate is not None:
1996
- candidate.close()
1997
- published.close()
1998
- if ((held and (set(waiting or ()) != set(held)
1999
- or not any(row.get("phase") == "live" for row in report.values())))
2000
- or (not held and waiting is not None)):
2001
  raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
2002
  problems = []
2003
  elif provider == "amazon_sp":
@@ -2090,15 +2046,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
2090
  detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
2091
  cause=amazon_failure or problems[0])
2092
  else:
2093
- ok, detail = True, _sync_now_detail(provider, tenant, ok=True, detail="Sync finished.",
2094
- cause=report if provider == "odoo" else None)
2095
  if amazon_detail and not problems:
2096
  detail += " " + amazon_detail
2097
  except _SyncNowRefused as exc:
2098
  detail = str(exc)
2099
  except Exception as exc: # noqa: BLE001
2100
  detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
2101
- if progress and not amazon_detail and not (provider == "odoo" and ok and held):
2102
  import re
2103
  suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
2104
  cause_code = suffix[0] if suffix else ""
@@ -3565,8 +3520,6 @@ def _amazon_desk_pass(tenant, *, project=True):
3565
  job.join()
3566
  state["job"] = None
3567
  _project_desk(state["outcome"], tenant)
3568
- import sync_runtime
3569
- sync_runtime.relay_at_boundary()
3570
  receipt = connector_sync.sync_request("amazon_sp", tenant)
3571
  if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
3572
  or max(receipt.get("notBefore") or 0,
 
758
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
759
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
760
  _mount_router("routes_amazon_projects")
 
 
 
 
 
 
 
761
  _mount_router("routes_competitors")
762
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
763
  _mount_router("routes_sku_actions")
 
1343
  # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
1344
  _install_amazon_publication(tenant_key, rt, why)
1345
  if _amazon.first_load_job_live(rt):
 
 
1346
  print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
1347
  return False
1348
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
 
1378
  except Exception as exc:
1379
  # A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
1380
  if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
 
1381
  print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
1382
  flush=True)
1383
  return False
 
1749
 
1750
 
1751
  def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1752
+ """Record one classified failure; leave successful receipt text untouched."""
1753
  if ok:
 
 
 
 
 
 
 
 
1754
  return detail
1755
  classifier, sentence = _sync_now_cause(cause)
1756
  if classifier == "nothing_due":
 
1951
  **({"after_entity": _after_entity} if _after_entity else {}))
1952
  if not report and selected_entities != []:
1953
  raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
1954
+ if any(row.get("phase") != "live" for row in report.values()):
 
1955
  raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
1956
+ if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1957
  raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
1958
  problems = []
1959
  elif provider == "amazon_sp":
 
2046
  detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
2047
  cause=amazon_failure or problems[0])
2048
  else:
2049
+ ok, detail = True, "Sync finished."
 
2050
  if amazon_detail and not problems:
2051
  detail += " " + amazon_detail
2052
  except _SyncNowRefused as exc:
2053
  detail = str(exc)
2054
  except Exception as exc: # noqa: BLE001
2055
  detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
2056
+ if progress and not amazon_detail:
2057
  import re
2058
  suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
2059
  cause_code = suffix[0] if suffix else ""
 
3520
  job.join()
3521
  state["job"] = None
3522
  _project_desk(state["outcome"], tenant)
 
 
3523
  receipt = connector_sync.sync_request("amazon_sp", tenant)
3524
  if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
3525
  or max(receipt.get("notBefore") or 0,
api/process_role.py CHANGED
@@ -56,20 +56,6 @@ READY_FILE = "READY.json"
56
  PRIMARY_LOCK = "web-primary.lock"
57
  PRIMARY_FILE = "web-primary.json"
58
  INSTALL_ATTEMPTS = 3
59
- INSTALL_RETRY_SECONDS = 3600
60
- _INSTALL_CAUSES = (
61
- (TimeoutError, "The install timed out."),
62
- (ConnectionError, "The saved sync could not be downloaded."),
63
- (PermissionError, "The install could not access the saved sync."),
64
- (FileNotFoundError, "The saved sync is missing a required file."),
65
- (OSError, "The install could not read or write its files."),
66
- (MemoryError, "The install ran out of memory."),
67
- (ValueError, "The saved sync failed validation."),
68
- (TypeError, "The saved sync has an invalid structure."),
69
- (LookupError, "The saved sync is missing required data."),
70
- (RuntimeError, "The install stopped with an internal error."),
71
- (Exception, "The install failed unexpectedly."),
72
- )
73
  INCOMPLETE_GRACE_SECONDS = 3600
74
  #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
75
  WORKER_STATUS = "sync_worker"
@@ -550,34 +536,37 @@ def pending_handoffs(root=None):
550
 
551
 
552
  def _failure_map(root=None):
553
- """Return each published snapshot's own failed install receipt."""
554
  if not is_primary():
555
  stored = primary_state(root).get("installFailures") or {}
556
  return {tuple(key.split("/", 1)): row for key, row in stored.items()
557
  if "/" in key and isinstance(row, dict)}
558
- return {key: dict(row) for key, snapshot in dict(_STORED["mirrors"]).items()
559
- if (row := _INSTALL_FAILURES.get((key, snapshot)))}
560
-
561
-
562
- def served_behind(tenant, connector, root=None, *, worker=None):
 
 
 
 
 
 
 
563
  """`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
564
 
565
  None when nothing is published, the published snapshot is served, or no install of it failed yet.
566
  """
567
  key = (str(tenant or "").strip().lower(), str(connector))
568
- worker = worker or {}
569
- published = ((worker.get("mirrors") or {}).get(key[1]) or {}).get("snapshotId") or _known_mirrors(root).get(key)
570
  if not published:
571
  return None
572
  served = _installed_map(root).get(key)
573
- failure = _failure_map(root).get(key) or (worker.get("installs") or {}).get(_install_key(key, published)) or {}
574
- if (failure.get("tenant"), failure.get("connector"), failure.get("snapshotId")) != (*key, published):
575
- return None
576
  attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
577
  if served == published or attempts < 1:
578
  return None
579
- return {"published": published, "served": served or failure.get("served"), "attempts": attempts,
580
- "error": failure.get("type"), "cause": failure.get("cause"), "nextRetryAt": failure.get("nextRetryAt")}
581
 
582
 
583
  # ----------------------------------------------------------------------------- several web workers
@@ -785,44 +774,6 @@ def _remove_dir(path):
785
  shutil.rmtree(path, ignore_errors=True)
786
 
787
 
788
- def _install_key(key, snapshot):
789
- return "/".join((*key, snapshot))
790
-
791
-
792
- def _install_due(key, snapshot, stamp):
793
- failure = _INSTALL_FAILURES.get((key, snapshot)) or {}
794
- return failure.get("attempts", 0) < INSTALL_ATTEMPTS or stamp >= failure["nextRetryAt"]
795
-
796
-
797
- def _record_install(key, snapshot, stamp, *, exc=None, rt=None, changed=True):
798
- """Keep one scoped install receipt and persist its safe cause and retry deadline."""
799
- from core import connector_sync
800
-
801
- identity = (key, snapshot)
802
- was_failed = identity in _INSTALL_FAILURES
803
- row = {"tenant": key[0], "connector": key[1], "snapshotId": snapshot, "at": stamp,
804
- "state": "failed" if exc is not None else "installed"}
805
- if exc is not None:
806
- kind, cause = next((kind, cause) for kind, cause in _INSTALL_CAUSES if isinstance(exc, kind))
807
- attempts = (_INSTALL_FAILURES.get(identity) or {}).get("attempts", 0) + 1
808
- row.update(attempts=attempts, type=kind.__name__, cause=cause, served=_INSTALLED.get(key),
809
- nextRetryAt=stamp + (INSTALL_RETRY_SECONDS if attempts >= INSTALL_ATTEMPTS else POLL_SECONDS))
810
- _INSTALL_FAILURES[identity] = row
811
- LAST_INSTALL_ERROR.clear()
812
- LAST_INSTALL_ERROR.update(row)
813
- else:
814
- row["changed"] = changed is not False
815
- _INSTALL_FAILURES.pop(identity, None)
816
- if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector"),
817
- LAST_INSTALL_ERROR.get("snapshotId")) == (*key, snapshot):
818
- LAST_INSTALL_ERROR.clear()
819
- if rt is not None and (exc is not None or was_failed):
820
- written = connector_sync.update_status(rt, WORKER_STATUS, "installs", {_install_key(key, snapshot): row})
821
- if not written:
822
- print(f"[process-role] install status not stored for {key[0]}/{key[1]}", flush=True)
823
- return row
824
-
825
-
826
  def install_ready(root=None, installer=None, *, now=None):
827
  """Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
828
 
@@ -835,22 +786,25 @@ def install_ready(root=None, installer=None, *, now=None):
835
  for key, rows in ready_handoffs(root).items():
836
  completed_at, newest_dir, ready = rows[-1]
837
  snapshot = ready.get("snapshotId") or newest_dir.name
838
- if _INSTALLED.get(key) != snapshot and _install_due(key, snapshot, stamp):
839
  try:
840
  changed = installer(key[1], key[0], ready)
841
- except Exception as exc: # Retry failed installs without exposing their diagnostic text.
842
- failure = _record_install(key, snapshot, stamp, exc=exc)
 
 
843
  print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
844
- f"({failure['type']})", flush=True)
845
  continue
846
  _INSTALLED[key] = snapshot
847
- _record_install(key, snapshot, stamp, changed=changed)
848
  if changed is False:
849
  # Already resident: nothing was rebuilt, so this is not an install.
850
  for _at, older, _ready in rows[:-1]:
851
  _remove_dir(older)
852
  continue
853
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
 
 
854
  installed.append(key)
855
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
856
  for _at, older, _ready in rows[:-1]:
@@ -908,31 +862,29 @@ def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
908
  installer = installer or _install_from_store
909
  stamp = time.time() if now is None else now
910
  installed = []
911
- stored = (stored_worker(rt).get("installs") or {}) if rt is not None else {}
912
  for connector, row in sorted((mirrors or {}).items()):
913
  snapshot = row.get("snapshotId") if isinstance(row, dict) else None
914
  if not snapshot:
915
  continue
916
- key = (str(tenant).strip().lower(), str(connector))
917
  _STORED["mirrors"][key] = snapshot
918
- previous = stored.get(_install_key(key, snapshot)) or {}
919
- if (previous.get("state") == "failed" and (key, snapshot) not in _INSTALL_FAILURES
920
- and (previous.get("tenant"), previous.get("connector"), previous.get("snapshotId")) == (*key, snapshot)):
921
- _INSTALL_FAILURES[(key, snapshot)] = previous
922
- if _INSTALLED.get(key) == snapshot or not _install_due(key, snapshot, stamp):
923
  continue
924
  try:
925
  changed = installer(key[1], key[0], rt, snapshot)
926
- except Exception as exc: # Preserve the serving copy and retry this snapshot on its own deadline.
927
- failure = _record_install(key, snapshot, stamp, exc=exc, rt=rt)
 
 
928
  print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
929
- f"({failure['type']})", flush=True)
930
  continue
931
  _INSTALLED[key] = snapshot
932
- _record_install(key, snapshot, stamp, rt=rt, changed=changed)
933
  if changed is False:
934
  continue
935
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
 
 
936
  installed.append(key)
937
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
938
  return installed
@@ -1283,23 +1235,6 @@ def start_web_services():
1283
  return True
1284
 
1285
 
1286
- def _install_diagnostics(stored):
1287
- """Read this tenant's current failed snapshot and last changed install across web processes."""
1288
- tenant = writer_tenant()
1289
- rows = {**(stored.get("installs") or {}), **{
1290
- _install_key(key, snapshot): row for (key, snapshot), row in dict(_INSTALL_FAILURES).items()}}
1291
- rows = sorted((row for row in rows.values() if isinstance(row, dict) and row.get("tenant") == tenant),
1292
- key=lambda row: row.get("at", 0), reverse=True)
1293
- mirrors = {key[1]: snapshot for key, snapshot in _known_mirrors().items() if key[0] == tenant}
1294
- mirrors.update({connector: row.get("snapshotId") for connector, row in (stored.get("mirrors") or {}).items()})
1295
- failure = next((row for row in rows if row.get("state") == "failed"
1296
- and mirrors.get(row["connector"]) == row["snapshotId"]
1297
- and _INSTALLED.get((tenant, row["connector"])) != row["snapshotId"]), None)
1298
- installed = next((row for row in rows if row.get("state") == "installed" and row.get("changed")), None)
1299
- local = dict(LAST_INSTALL) if LAST_INSTALL.get("tenant") == tenant else None
1300
- return local or installed, failure
1301
-
1302
-
1303
  def status():
1304
  """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
1305
 
@@ -1324,13 +1259,12 @@ def status():
1324
  fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
1325
  "lastExit": None, "jobId": row.get("jobId")}
1326
  import sync_runtime
1327
- last_install, last_error = _install_diagnostics(stored)
1328
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
1329
  "syncRuntime": sync_runtime.runtime_name()[0],
1330
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
1331
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1332
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1333
- "restarts": _WORKER["restarts"], "lastInstall": last_install,
1334
- "lastInstallError": last_error,
1335
  "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
1336
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
 
56
  PRIMARY_LOCK = "web-primary.lock"
57
  PRIMARY_FILE = "web-primary.json"
58
  INSTALL_ATTEMPTS = 3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  INCOMPLETE_GRACE_SECONDS = 3600
60
  #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
61
  WORKER_STATUS = "sync_worker"
 
536
 
537
 
538
  def _failure_map(root=None):
539
+ """{(tenant, connector): {snapshotId, attempts, type, at}} for each published snapshot that failed to install."""
540
  if not is_primary():
541
  stored = primary_state(root).get("installFailures") or {}
542
  return {tuple(key.split("/", 1)): row for key, row in stored.items()
543
  if "/" in key and isinstance(row, dict)}
544
+ last, failures = dict(LAST_INSTALL_ERROR), {}
545
+ for key, snapshot in dict(_STORED["mirrors"]).items():
546
+ attempts = _INSTALL_FAILURES.get((key, snapshot), 0)
547
+ if attempts:
548
+ # LAST_INSTALL_ERROR is one slot: its type and time belong only to the snapshot it names.
549
+ same = (last.get("tenant"), last.get("connector"), last.get("snapshotId")) == (*key, snapshot)
550
+ failures[key] = {"snapshotId": snapshot, "attempts": attempts,
551
+ "type": last.get("type") if same else None, "at": last.get("at") if same else None}
552
+ return failures
553
+
554
+
555
+ def served_behind(tenant, connector, root=None):
556
  """`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
557
 
558
  None when nothing is published, the published snapshot is served, or no install of it failed yet.
559
  """
560
  key = (str(tenant or "").strip().lower(), str(connector))
561
+ published = _known_mirrors(root).get(key)
 
562
  if not published:
563
  return None
564
  served = _installed_map(root).get(key)
565
+ failure = _failure_map(root).get(key) or {}
 
 
566
  attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
567
  if served == published or attempts < 1:
568
  return None
569
+ return {"published": published, "served": served, "attempts": attempts, "error": failure.get("type")}
 
570
 
571
 
572
  # ----------------------------------------------------------------------------- several web workers
 
774
  shutil.rmtree(path, ignore_errors=True)
775
 
776
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
777
  def install_ready(root=None, installer=None, *, now=None):
778
  """Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
779
 
 
786
  for key, rows in ready_handoffs(root).items():
787
  completed_at, newest_dir, ready = rows[-1]
788
  snapshot = ready.get("snapshotId") or newest_dir.name
789
+ if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
790
  try:
791
  changed = installer(key[1], key[0], ready)
792
+ except Exception as exc: # noqa: BLE001
793
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
794
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
795
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
796
  print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
797
+ f"({type(exc).__name__}: {exc})", flush=True)
798
  continue
799
  _INSTALLED[key] = snapshot
 
800
  if changed is False:
801
  # Already resident: nothing was rebuilt, so this is not an install.
802
  for _at, older, _ready in rows[:-1]:
803
  _remove_dir(older)
804
  continue
805
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
806
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
807
+ LAST_INSTALL_ERROR.clear()
808
  installed.append(key)
809
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
810
  for _at, older, _ready in rows[:-1]:
 
862
  installer = installer or _install_from_store
863
  stamp = time.time() if now is None else now
864
  installed = []
 
865
  for connector, row in sorted((mirrors or {}).items()):
866
  snapshot = row.get("snapshotId") if isinstance(row, dict) else None
867
  if not snapshot:
868
  continue
869
+ key = (str(tenant), str(connector))
870
  _STORED["mirrors"][key] = snapshot
871
+ if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
 
 
 
 
872
  continue
873
  try:
874
  changed = installer(key[1], key[0], rt, snapshot)
875
+ except Exception as exc: # noqa: BLE001
876
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
877
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
878
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
879
  print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
880
+ f"({type(exc).__name__}: {exc})", flush=True)
881
  continue
882
  _INSTALLED[key] = snapshot
 
883
  if changed is False:
884
  continue
885
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
886
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
887
+ LAST_INSTALL_ERROR.clear()
888
  installed.append(key)
889
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
890
  return installed
 
1235
  return True
1236
 
1237
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1238
  def status():
1239
  """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
1240
 
 
1259
  fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
1260
  "lastExit": None, "jobId": row.get("jobId")}
1261
  import sync_runtime
 
1262
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
1263
  "syncRuntime": sync_runtime.runtime_name()[0],
1264
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
1265
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1266
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1267
+ "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
1268
+ "lastInstallError": dict(LAST_INSTALL_ERROR) or None,
1269
  "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
1270
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
api/routes_amazon_projects.py CHANGED
@@ -38,7 +38,7 @@ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "
38
  "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
39
  #: Project JSON keys a client may echo back. They are ignored, never written.
40
  READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
41
- "rollup", "impact", "via", "figures", "type", "listed"))
42
 
43
 
44
  def _call(fn, *args, **kwargs):
@@ -116,7 +116,6 @@ def _project(record, today):
116
  impact = dict(record.get("impact") or {})
117
  impact.pop(projects.RESULT_CONFIGS_KEY, None)
118
  return {"id": record["id"], "name": record["name"], "status": record["status"],
119
- "type": record.get("type", "project"), "listed": record.get("listed", True),
120
  "priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
121
  "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
122
  "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
@@ -268,7 +267,7 @@ def _row_ids(cells):
268
  return sorted(cells)
269
 
270
 
271
- def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_profit.VIA):
272
  import core.user_tables as ut
273
 
274
  def _set(cur):
@@ -284,7 +283,7 @@ def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_pr
284
  row[key] = value
285
  return cur
286
 
287
- ut.update_with_activity(table_key, _set, st=st, by=by, via=via,
288
  row_ids=_row_ids(cells))
289
 
290
 
@@ -875,11 +874,6 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi
875
  def delete_project(project_id: str, session: Session = Depends(require_session)):
876
  """Delete one project and its targets."""
877
  runtime = _writer(session)
878
- record = _call(projects.get, runtime, project_id)
879
- if record.get("type") == "pdp":
880
- from routes_amazon_pdp import delete_workflow
881
- delete_workflow(project_id, session=session)
882
- return {"ok": True, "id": project_id}
883
  pairs = _target_pairs(runtime, project_id)
884
  _call(projects.delete, runtime, project_id, by=_by(session))
885
  _repaint(runtime, pairs)
 
38
  "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
39
  #: Project JSON keys a client may echo back. They are ignored, never written.
40
  READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
41
+ "rollup", "impact", "via", "figures"))
42
 
43
 
44
  def _call(fn, *args, **kwargs):
 
116
  impact = dict(record.get("impact") or {})
117
  impact.pop(projects.RESULT_CONFIGS_KEY, None)
118
  return {"id": record["id"], "name": record["name"], "status": record["status"],
 
119
  "priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
120
  "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
121
  "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
 
267
  return sorted(cells)
268
 
269
 
270
+ def _write_cells(st, table_key, cells):
271
  import core.user_tables as ut
272
 
273
  def _set(cur):
 
283
  row[key] = value
284
  return cur
285
 
286
+ ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
287
  row_ids=_row_ids(cells))
288
 
289
 
 
874
  def delete_project(project_id: str, session: Session = Depends(require_session)):
875
  """Delete one project and its targets."""
876
  runtime = _writer(session)
 
 
 
 
 
877
  pairs = _target_pairs(runtime, project_id)
878
  _call(projects.delete, runtime, project_id, by=_by(session))
879
  _repaint(runtime, pairs)
api/routes_brand.py CHANGED
@@ -35,16 +35,13 @@ from __future__ import annotations
35
 
36
  import os
37
  import re
38
- from hashlib import sha256
39
  from xml.etree import ElementTree
40
  from pathlib import Path
41
 
42
- from fastapi import APIRouter, Depends, Request
43
- from fastapi.concurrency import run_in_threadpool
44
- from fastapi.responses import FileResponse, Response
45
 
46
  from deps import Session, err, require_session
47
- from core import tenant_brand
48
 
49
  router = APIRouter(prefix="/api/v1")
50
 
@@ -80,53 +77,6 @@ _MEDIA = {".png": "image/png", ".svg": "image/svg+xml"}
80
  _HERE = Path(__file__).resolve().parent
81
 
82
 
83
- def _logo_call(fn, *args):
84
- try:
85
- return fn(*args)
86
- except tenant_brand.LogoStorageError as error:
87
- raise err(503, "logo_storage_unavailable", str(error)) from None
88
- except ValueError as error:
89
- raise err(422, "invalid_logo", str(error)) from None
90
-
91
-
92
- def _logo_admin(session):
93
- if not session.admin:
94
- raise err(403, "admin_required", "Only an administrator can change the tenant logo.")
95
-
96
-
97
- @router.get("/tenant-brand/logo")
98
- def tenant_logo(session: Session = Depends(require_session)):
99
- """Serve this tenant's logo to any signed-in user without changing state."""
100
- result = _logo_call(tenant_brand.logo, session.runtime)
101
- if result is None:
102
- raise err(404, "no_logo", "This workspace has no logo.")
103
- data, mime = result
104
- return Response(data, media_type=mime, headers={
105
- "ETag": f'"{sha256(data).hexdigest()}"', "Cache-Control": "private, no-cache",
106
- "X-Content-Type-Options": "nosniff"})
107
-
108
-
109
- @router.put("/tenant-brand/logo")
110
- async def put_tenant_logo(request: Request, session: Session = Depends(require_session)):
111
- """Accept at most 2 MiB of raw PNG or JPEG bytes from an administrator."""
112
- _logo_admin(session)
113
- data = bytearray()
114
- async for chunk in request.stream():
115
- if len(data) + len(chunk) > tenant_brand.MAX_BYTES:
116
- raise err(413, "logo_too_large", tenant_brand.SIZE_LIMIT)
117
- data.extend(chunk)
118
- return await run_in_threadpool(
119
- _logo_call, tenant_brand.put_logo, session.runtime, bytes(data), session.uname)
120
-
121
-
122
- @router.delete("/tenant-brand/logo", status_code=204)
123
- def delete_tenant_logo(session: Session = Depends(require_session)):
124
- """Remove this tenant's logo for an administrator."""
125
- _logo_admin(session)
126
- _logo_call(tenant_brand.delete_logo, session.runtime)
127
- return Response(status_code=204)
128
-
129
-
130
  def _roots():
131
  env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
132
  out = [Path(env)] if env else []
 
35
 
36
  import os
37
  import re
 
38
  from xml.etree import ElementTree
39
  from pathlib import Path
40
 
41
+ from fastapi import APIRouter, Depends
42
+ from fastapi.responses import FileResponse
 
43
 
44
  from deps import Session, err, require_session
 
45
 
46
  router = APIRouter(prefix="/api/v1")
47
 
 
77
  _HERE = Path(__file__).resolve().parent
78
 
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  def _roots():
81
  env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
82
  out = [Path(env)] if env else []
api/routes_changes.py CHANGED
@@ -62,7 +62,6 @@ import core.store as store
62
  import core.user_tables as user_tables
63
  import modules.customer_data as customer_data
64
  import modules.product_data as product_data
65
- import virtual_sources
66
 
67
  from deps import Session, err, require_changes_session, require_session
68
 
@@ -201,8 +200,6 @@ def _may_watch(session, scope):
201
  @router.get("/changes")
202
  def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
203
  """Serve legacy change checks without making Pg a resident client."""
204
- if virtual_sources.get(_clean_scope(scope)) is not None:
205
- return {"scope": _clean_scope(scope), "tokens": {}, "backend": store.backend()}
206
  names = buckets_for(scope)
207
  if names is None:
208
  raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
 
62
  import core.user_tables as user_tables
63
  import modules.customer_data as customer_data
64
  import modules.product_data as product_data
 
65
 
66
  from deps import Session, err, require_changes_session, require_session
67
 
 
200
  @router.get("/changes")
201
  def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
202
  """Serve legacy change checks without making Pg a resident client."""
 
 
203
  names = buckets_for(scope)
204
  if names is None:
205
  raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
api/routes_competitors.py CHANGED
@@ -39,19 +39,6 @@ def _open_mirror(runtime):
39
  cur.close()
40
 
41
 
42
- @contextmanager
43
- def _boundary_errors():
44
- try:
45
- yield
46
- except rivals.MirrorUnreadable as exc:
47
- raise err(503, "amazon_mirror_unreadable",
48
- f"The Amazon data could not be read ({exc}). Try again shortly.") from None
49
- except rivals.BackendUnavailable:
50
- raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
51
- except RuntimeError:
52
- raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
53
-
54
-
55
  def _rt(session):
56
  session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
57
  try:
@@ -164,23 +151,6 @@ def competitors_list(grain: str, key: str, session: Session = Depends(require_se
164
  return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
165
 
166
 
167
- @router.get("/competitors/pool")
168
- def competitors_pool(grain: str, key: str, session: Session = Depends(require_session)):
169
- """Read the eligible Setup pool and its stored daily facts through the caller's tenant session."""
170
- runtime = _rt(session)
171
- key = _grain(grain, key)
172
- with _boundary_errors(), _open_mirror(runtime) as cur:
173
- target, parent = _target(cur, grain, key)
174
- if parent is not None:
175
- raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
176
- pool = rivals.pool(runtime, cur, target)
177
- payload = {field: pool[field] for field in ('asin', 'productType', 'dataAsOf', 'week', 'month',
178
- 'keywordRanksOn', 'cause', 'held', 'removed', 'competitors')}
179
- if not payload['competitors']:
180
- payload['cause'] = rivals.NO_POOL
181
- return {**payload, 'canEdit': routes_amazon_profit._may_edit(session)}
182
-
183
-
184
  def _writable(session, body):
185
  runtime = _rt(session)
186
  if not routes_amazon_profit._may_edit(session):
@@ -188,8 +158,16 @@ def _writable(session, body):
188
  body = body if isinstance(body, dict) else {}
189
  grain = body.get("grain")
190
  key = _grain(grain, body.get("key"))
191
- with _boundary_errors(), _open_mirror(runtime) as cur:
192
- target, parent = _target(cur, grain, key)
 
 
 
 
 
 
 
 
193
  if parent is not None:
194
  raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
195
  try:
 
39
  cur.close()
40
 
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  def _rt(session):
43
  session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
44
  try:
 
151
  return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
152
 
153
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
  def _writable(session, body):
155
  runtime = _rt(session)
156
  if not routes_amazon_profit._may_edit(session):
 
158
  body = body if isinstance(body, dict) else {}
159
  grain = body.get("grain")
160
  key = _grain(grain, body.get("key"))
161
+ try:
162
+ with _open_mirror(runtime) as cur:
163
+ target, parent = _target(cur, grain, key)
164
+ except rivals.MirrorUnreadable as exc:
165
+ raise err(503, "amazon_mirror_unreadable",
166
+ f"The Amazon data could not be read ({exc}). Try again shortly.") from None
167
+ except rivals.BackendUnavailable:
168
+ raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
169
+ except RuntimeError:
170
+ raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
171
  if parent is not None:
172
  raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
173
  try:
api/routes_grid.py CHANGED
@@ -20,7 +20,6 @@ from inspect import signature
20
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
21
 
22
  from deps import Session, err, module_gate, perms, require_session
23
- import virtual_sources
24
 
25
  router = APIRouter(prefix="/api/v1")
26
 
@@ -50,7 +49,6 @@ def row_patch_receipt(scope):
50
  raise err(400, 'bad_op_id', str(exc)) from exc
51
  session = bound.arguments['session']
52
  scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
53
- virtual_sources.refuse_write(scope_key)
54
  bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
55
 
56
  def apply():
@@ -235,7 +233,6 @@ def grid_choice_remove(scope: str = "customer", body: dict = Body(default=None),
235
 
236
  body = body if isinstance(body, dict) else {}
237
  scope = _scope_or_400(scope)
238
- virtual_sources.refuse_write(scope)
239
  key = str(body.get("key") or "").strip()[:80]
240
  options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
241
  confirm = body.get("confirm") is True
@@ -478,8 +475,6 @@ def workspace(scope: str = "customer",
478
  from routes_customers import grid_assembly
479
 
480
  scope = _scope_or_400(scope)
481
- if virtual_sources.get(scope) is not None:
482
- return {"workspace": virtual_sources.assembly(session, scope, with_rows=False)["workspace"]}
483
  # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
484
  # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
485
  # session (its wall is the table's own, inside the assembly).
@@ -1529,7 +1524,6 @@ def bulk_cells(body: dict = Body(default=None),
1529
  from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
1530
 
1531
  scope = _scope_or_400((body or {}).get("scopeKey"))
1532
- virtual_sources.refuse_write(scope)
1533
  if scope != "product":
1534
  # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
1535
  raise err(400, "scope_not_bulk_writable",
@@ -2132,7 +2126,6 @@ def grid_events_route(body: dict = Body(default=None),
2132
  # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
2133
  # what a scope is, or the surface you read is not the surface you wrote.
2134
  scope = _scope_or_400((body or {}).get("scopeKey"))
2135
- virtual_sources.refuse_write(scope)
2136
  if scope.startswith("ut_") and any(
2137
  isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
2138
  # Managed records require Route/Zone tools, including keys without stored definitions.
 
20
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
21
 
22
  from deps import Session, err, module_gate, perms, require_session
 
23
 
24
  router = APIRouter(prefix="/api/v1")
25
 
 
49
  raise err(400, 'bad_op_id', str(exc)) from exc
50
  session = bound.arguments['session']
51
  scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
 
52
  bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
53
 
54
  def apply():
 
233
 
234
  body = body if isinstance(body, dict) else {}
235
  scope = _scope_or_400(scope)
 
236
  key = str(body.get("key") or "").strip()[:80]
237
  options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
238
  confirm = body.get("confirm") is True
 
475
  from routes_customers import grid_assembly
476
 
477
  scope = _scope_or_400(scope)
 
 
478
  # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
479
  # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
480
  # session (its wall is the table's own, inside the assembly).
 
1524
  from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
1525
 
1526
  scope = _scope_or_400((body or {}).get("scopeKey"))
 
1527
  if scope != "product":
1528
  # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
1529
  raise err(400, "scope_not_bulk_writable",
 
2126
  # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
2127
  # what a scope is, or the surface you read is not the surface you wrote.
2128
  scope = _scope_or_400((body or {}).get("scopeKey"))
 
2129
  if scope.startswith("ut_") and any(
2130
  isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
2131
  # Managed records require Route/Zone tools, including keys without stored definitions.
api/routes_odoo_tables.py CHANGED
@@ -5096,12 +5096,10 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
5096
  def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
5097
  groupBy=None, groupKey=None, ids=None, cursor=None):
5098
  from harness import datastore
5099
- import virtual_sources
5100
 
5101
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
5102
  "sorts": _json_arg(sorts, "sorts"), "search": search})
5103
- virtual = virtual_sources.get(table_key) is not None
5104
- payload = virtual_sources.rows_payload(session, table_key, related=related) if virtual else None
5105
  window_offset = offset
5106
  if table_key == 'ut_activity':
5107
  if cursor is not None:
@@ -5140,9 +5138,7 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
5140
  options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
5141
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
5142
  cur=read['cur'], **read['frm'])
5143
- if table_key == 'ut_activity' or virtual:
5144
- if virtual and cursor is not None:
5145
- raise err(400, 'window_cursor_unsupported', 'Continue this source with an offset.')
5146
  win = datastore.window(**options)
5147
  win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
5148
  else:
@@ -5289,10 +5285,6 @@ def odoo_table_rows(table_key: str,
5289
  "limit, filters, sorts and search in the query string.")
5290
  ids = _record_ids(body)
5291
 
5292
- import virtual_sources
5293
- if virtual_sources.get(table_key) is not None:
5294
- return _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
5295
- sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor)
5296
  # Admission resolves this Amazon database without publishing a process-wide binding.
5297
  page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
5298
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
 
5096
  def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
5097
  groupBy=None, groupKey=None, ids=None, cursor=None):
5098
  from harness import datastore
 
5099
 
5100
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
5101
  "sorts": _json_arg(sorts, "sorts"), "search": search})
5102
+ payload = None
 
5103
  window_offset = offset
5104
  if table_key == 'ut_activity':
5105
  if cursor is not None:
 
5138
  options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
5139
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
5140
  cur=read['cur'], **read['frm'])
5141
+ if table_key == 'ut_activity':
 
 
5142
  win = datastore.window(**options)
5143
  win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
5144
  else:
 
5285
  "limit, filters, sorts and search in the query string.")
5286
  ids = _record_ids(body)
5287
 
 
 
 
 
5288
  # Admission resolves this Amazon database without publishing a process-wide binding.
5289
  page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
5290
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
api/routes_shares.py CHANGED
@@ -1221,8 +1221,6 @@ def get_cohort(scope: str, cohort_id: str, session: Session = Depends(require_se
1221
  @router.get("/share/{kind}/{oid}")
1222
  def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1223
  kind = _kind_or_400(kind)
1224
- if kind == "report" and not session.admin:
1225
- raise err(404, "no_object", "no such item, or it is not shared with this account")
1226
  if kind == "field":
1227
  table_key, field_key = shares.split_field_oid(oid)
1228
  if table_key and field_key:
@@ -1281,9 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1281
  **rec,
1282
  "role": role,
1283
  "mayAdminister": may_admin,
1284
- "maxGrantableRole": (
1285
- "view" if kind == "report" else "edit" if claiming else shares.max_grantable_role(
1286
- kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)),
1287
  "ownerLabel": owner_label,
1288
  # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
1289
  # them. `assignable_people` serves bare display names because `user`-kind CELLS store
@@ -1305,19 +1302,35 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1305
 
1306
 
1307
  def _people(tenant):
1308
- """Active tenant identities, excluding test accounts, ordered by display name."""
1309
- return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
1310
-
1311
-
1312
- def _report_entries_or_400(entries):
1313
- """A report grants download to named people alone."""
1314
- for entry in entries:
1315
- if (not isinstance(entry, dict) or not str(entry.get("user") or "").strip()
1316
- or str(entry.get("user")).strip() == shares.EVERYONE):
1317
- raise err(400, "bad_entries",
1318
- "Add people by name. A report is shared one person at a time.")
1319
- if str(entry.get("role") or "view").strip().lower() != "view":
1320
- raise err(400, "bad_role", "A report grant can only allow download.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1321
 
1322
 
1323
  def _stranger(session, kind, oid):
@@ -1342,11 +1355,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1342
  _notify=True):
1343
  """Apply the shared share admission, ceiling and atomic writer for both doors."""
1344
  kind = _kind_or_400(kind)
1345
- if kind == "report" and not session.admin:
1346
- if _stranger(session, kind, oid):
1347
- raise err(404, "no_object", "no such item, or it is not shared with this account")
1348
- raise err(403, "not_owner",
1349
- "Only an administrator can change who can download this report.")
1350
  if kind == "field":
1351
  table_key, field_key = shares.split_field_oid(oid)
1352
  if table_key and field_key:
@@ -1420,8 +1428,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1420
  raise err(400, "bad_entries",
1421
  "entries must be a list of {user, role}. Send [] to un-share, which is how "
1422
  "revoking is expressed")
1423
- if kind == "report":
1424
- _report_entries_or_400(entries)
1425
  _entries_or_400(session, entries)
1426
  # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
1427
  # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
@@ -1617,8 +1623,7 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1617
 
1618
  try:
1619
  out = shares.set_grants(kind, oid, entries,
1620
- owner=(None if kind == "report" else
1621
- session.uname if claiming else rec["owner"]),
1622
  granter=session.uname,
1623
  st=session.runtime,
1624
  expect=rec["entries"])
 
1221
  @router.get("/share/{kind}/{oid}")
1222
  def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1223
  kind = _kind_or_400(kind)
 
 
1224
  if kind == "field":
1225
  table_key, field_key = shares.split_field_oid(oid)
1226
  if table_key and field_key:
 
1279
  **rec,
1280
  "role": role,
1281
  "mayAdminister": may_admin,
1282
+ "maxGrantableRole": "edit" if claiming else shares.max_grantable_role(
1283
+ kind, oid, session.uname, is_admin=_admin_share, st=session.runtime),
 
1284
  "ownerLabel": owner_label,
1285
  # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
1286
  # them. `assignable_people` serves bare display names because `user`-kind CELLS store
 
1302
 
1303
 
1304
  def _people(tenant):
1305
+ """[{username, name}] for this tenant — same population as `assignable_people`, with the
1306
+ BINDING identity alongside the display one.
1307
+
1308
+ ⛔⛔ W49-T23 / D-235 — A BLANK TENANT RETURNS NOBODY, NOT EVERYBODY. This read
1309
+ `if want and str(u.get('tenant') …) != want`, so a blank or `None` slug skipped the filter
1310
+ entirely and handed one workspace every tenant's roster: the exact `if want and …` fail-OPEN
1311
+ that W33-T37 closed in `core.users.assignable_people` / `avatar_map` one layer down, left
1312
+ behind in the copy that lives here. The three callers all pass `session.tenant`, which is
1313
+ `claims["t"]` and is already proven non-blank by `require_session` (an unresolvable slug is a
1314
+ 401 before any route runs), so this changes no answer any live caller gets today — it removes
1315
+ the shape, before a fourth caller arrives that does not carry that proof. Fail-closed, because
1316
+ from this function a blank slug and "every tenant at once" are the same observation, and one
1317
+ of the two callers below turns this roster into who may be GRANTED access.
1318
+ """
1319
+ try:
1320
+ reg = users.registry() or {}
1321
+ except Exception:
1322
+ return []
1323
+ want = str(tenant or '').strip().lower()
1324
+ if not want:
1325
+ return []
1326
+ out = []
1327
+ for uname, u in reg.items():
1328
+ if not isinstance(u, dict) or u.get('active') is False:
1329
+ continue
1330
+ if str(u.get('tenant') or 'royal-imports').strip().lower() != want:
1331
+ continue
1332
+ out.append({"username": str(uname), "name": str(u.get('name') or uname)})
1333
+ return sorted(out, key=lambda p: p["name"].lower())
1334
 
1335
 
1336
  def _stranger(session, kind, oid):
 
1355
  _notify=True):
1356
  """Apply the shared share admission, ceiling and atomic writer for both doors."""
1357
  kind = _kind_or_400(kind)
 
 
 
 
 
1358
  if kind == "field":
1359
  table_key, field_key = shares.split_field_oid(oid)
1360
  if table_key and field_key:
 
1428
  raise err(400, "bad_entries",
1429
  "entries must be a list of {user, role}. Send [] to un-share, which is how "
1430
  "revoking is expressed")
 
 
1431
  _entries_or_400(session, entries)
1432
  # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
1433
  # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
 
1623
 
1624
  try:
1625
  out = shares.set_grants(kind, oid, entries,
1626
+ owner=session.uname if claiming else rec["owner"],
 
1627
  granter=session.uname,
1628
  st=session.runtime,
1629
  expect=rec["entries"])
api/routes_starred.py CHANGED
@@ -1231,9 +1231,6 @@ def starred_records(database: str = "", session: Session = Depends(require_sessi
1231
  key = _clean_id(database)
1232
  if not key:
1233
  raise err(400, "bad_request", "no database was named")
1234
- import virtual_sources
1235
- if virtual_sources.get(key) is not None:
1236
- return {"database": key, "ids": [], "count": 0, "view": None}
1237
  return _records_payload(session, key, _database_or_refuse(session, key))
1238
 
1239
 
@@ -1252,8 +1249,6 @@ def set_starred_record(body: dict = Body(default=None),
1252
  raw_id = _clean_id(body.get("id"))
1253
  if not key:
1254
  raise err(400, "bad_request", "no database was named")
1255
- import virtual_sources
1256
- virtual_sources.refuse_write(key)
1257
  if not raw_id.isdigit():
1258
  # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
1259
  raise err(400, "bad_request", "a record id is a number")
 
1231
  key = _clean_id(database)
1232
  if not key:
1233
  raise err(400, "bad_request", "no database was named")
 
 
 
1234
  return _records_payload(session, key, _database_or_refuse(session, key))
1235
 
1236
 
 
1249
  raw_id = _clean_id(body.get("id"))
1250
  if not key:
1251
  raise err(400, "bad_request", "no database was named")
 
 
1252
  if not raw_id.isdigit():
1253
  # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
1254
  raise err(400, "bad_request", "a record id is a number")
api/routes_tables.py CHANGED
@@ -56,7 +56,6 @@ from fastapi import HTTPException, Request, Response
56
  from deps import Session, err, require_session
57
  from harness.runtime import DatastoreMismatch
58
  from routes_grid import row_patch_receipt
59
- import virtual_sources
60
 
61
  router = APIRouter(prefix="/api/v1")
62
 
@@ -286,8 +285,6 @@ def related_metrics(table_key: str, body: dict = Body(default=None),
286
  if (not isinstance(pin, dict) or not isinstance(pids, list)
287
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
288
  raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
289
- if virtual_sources.get(table_key) is not None:
290
- return virtual_sources.related_metrics(session, table_key, pin, pids)
291
  session.require(table_key)
292
  viewer = copy.copy(session)
293
  viewer.runtime = _MetricReadRuntime(session.runtime)
@@ -393,8 +390,6 @@ def _table_lend(runtime, table_key, defs_only=False):
393
  def _managed_or_refuse(session, table_key, *, mutation=False):
394
  """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
395
  from core import route_system
396
- if mutation:
397
- virtual_sources.refuse_write(table_key)
398
  definition = _ut().get_one(table_key, st=session.runtime)
399
  # Route/Zone are virtual system descriptors until their first dedicated write, so looking
400
  # only in `user_tables` would leave a generic mutation path open during that empty state.
@@ -589,7 +584,6 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=
589
  proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
590
  asserts the fast path was TAKEN.
591
  """
592
- virtual_sources.refuse_write(table_key)
593
  ut = _ut()
594
  # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
595
  # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
@@ -672,8 +666,8 @@ def _records_lock_message(table_key, definition, st):
672
  return f"This {name} is locked, so only user-created field values can be edited."
673
 
674
 
675
- def _editor_or_refuse(session, table_key, st=None):
676
- """Require editor authority on a database this session may see. `(defn, st)`."""
677
  # Share one snapshot across the read, editor and record-mode checks.
678
  st = st if st is not None else _ut().lend(session.runtime)
679
  # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
@@ -683,12 +677,6 @@ def _editor_or_refuse(session, table_key, st=None):
683
  defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
684
  if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
685
  raise err(403, "database_read_only", "Can edit access is required to change this database.")
686
- return defn, st
687
-
688
-
689
- def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
690
- """Require editor authority and reserve locked database edits for authored field values."""
691
- defn, st = _editor_or_refuse(session, table_key, st)
692
  if not _ut().records_mutable(table_key, st=st):
693
  if cell_updates and table_key != "ut_activity":
694
  from core import shared_overlay
@@ -1063,9 +1051,6 @@ def _activity_grid_rows(session):
1063
  @router.get('/tables/{table_key}/activity')
1064
  def table_activity(table_key: str, cursor: str = None, limit: int = 50,
1065
  session: Session = Depends(require_session)):
1066
- if virtual_sources.get(table_key) is not None:
1067
- from core import activity_log
1068
- return {'records': [], 'next_cursor': None, 'fields': activity_log.definition()['fields']}
1069
  if table_key.startswith('ut_'):
1070
  _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
1071
  elif table_key not in ('customer_data', 'product_data'):
@@ -2137,9 +2122,6 @@ def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None):
2137
  def ut_assembly(session: Session, table_key: str, storage_key: str = "",
2138
  consume_corrections: bool = False, with_rows: bool = True, st=None,
2139
  *, fields_only: bool = False, needed=None, cursor=None, limit=None):
2140
- if virtual_sources.get(table_key) is not None:
2141
- return virtual_sources.assembly(session, table_key, with_rows=with_rows and not fields_only,
2142
- storage_key=storage_key)
2143
  from core import grid_events
2144
  actor = grid_events.request_admission(session, table_key + '_table_workspace')
2145
  result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
@@ -3365,8 +3347,6 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
3365
  (found by this route's own gate check, not by luck)."""
3366
  import aios_grid
3367
 
3368
- if virtual_sources.get(table_key) is not None:
3369
- return virtual_sources.rows_payload(session, table_key, related=related)
3370
  if table_key == 'ut_activity':
3371
  if related:
3372
  raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
@@ -4541,8 +4521,7 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
4541
  from core import grid_events
4542
 
4543
  _managed_or_refuse(session, table_key, mutation=True)
4544
- # A locked database admits authored values only, so its record wall waits for the retained values.
4545
- _editor_or_refuse(session, table_key)
4546
  body = body if isinstance(body, dict) else {}
4547
  row_id = str(body.get("rowId") or "").strip()
4548
  if not row_id:
@@ -4572,7 +4551,6 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
4572
  raise err(404, "history_event_not_found", "that retained value version is unavailable")
4573
  if any(key in (g.get("hidden") or ()) for key in updates):
4574
  raise err(403, "field_hidden", "that retained version includes a hidden field")
4575
- _records_or_refuse(session, table_key, cell_updates=updates)
4576
  prior_history_ids = {
4577
  str(item.get("id")) for item in _ut().value_history(
4578
  table_key, row_id, st=session.runtime, retention_days=days,
 
56
  from deps import Session, err, require_session
57
  from harness.runtime import DatastoreMismatch
58
  from routes_grid import row_patch_receipt
 
59
 
60
  router = APIRouter(prefix="/api/v1")
61
 
 
285
  if (not isinstance(pin, dict) or not isinstance(pids, list)
286
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
287
  raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
 
 
288
  session.require(table_key)
289
  viewer = copy.copy(session)
290
  viewer.runtime = _MetricReadRuntime(session.runtime)
 
390
  def _managed_or_refuse(session, table_key, *, mutation=False):
391
  """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
392
  from core import route_system
 
 
393
  definition = _ut().get_one(table_key, st=session.runtime)
394
  # Route/Zone are virtual system descriptors until their first dedicated write, so looking
395
  # only in `user_tables` would leave a generic mutation path open during that empty state.
 
584
  proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
585
  asserts the fast path was TAKEN.
586
  """
 
587
  ut = _ut()
588
  # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
589
  # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
 
666
  return f"This {name} is locked, so only user-created field values can be edited."
667
 
668
 
669
+ def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
670
+ """Require editor authority and reserve locked database edits for authored field values."""
671
  # Share one snapshot across the read, editor and record-mode checks.
672
  st = st if st is not None else _ut().lend(session.runtime)
673
  # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
 
677
  defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
678
  if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
679
  raise err(403, "database_read_only", "Can edit access is required to change this database.")
 
 
 
 
 
 
680
  if not _ut().records_mutable(table_key, st=st):
681
  if cell_updates and table_key != "ut_activity":
682
  from core import shared_overlay
 
1051
  @router.get('/tables/{table_key}/activity')
1052
  def table_activity(table_key: str, cursor: str = None, limit: int = 50,
1053
  session: Session = Depends(require_session)):
 
 
 
1054
  if table_key.startswith('ut_'):
1055
  _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
1056
  elif table_key not in ('customer_data', 'product_data'):
 
2122
  def ut_assembly(session: Session, table_key: str, storage_key: str = "",
2123
  consume_corrections: bool = False, with_rows: bool = True, st=None,
2124
  *, fields_only: bool = False, needed=None, cursor=None, limit=None):
 
 
 
2125
  from core import grid_events
2126
  actor = grid_events.request_admission(session, table_key + '_table_workspace')
2127
  result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
 
3347
  (found by this route's own gate check, not by luck)."""
3348
  import aios_grid
3349
 
 
 
3350
  if table_key == 'ut_activity':
3351
  if related:
3352
  raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
 
4521
  from core import grid_events
4522
 
4523
  _managed_or_refuse(session, table_key, mutation=True)
4524
+ _records_or_refuse(session, table_key)
 
4525
  body = body if isinstance(body, dict) else {}
4526
  row_id = str(body.get("rowId") or "").strip()
4527
  if not row_id:
 
4551
  raise err(404, "history_event_not_found", "that retained value version is unavailable")
4552
  if any(key in (g.get("hidden") or ()) for key in updates):
4553
  raise err(403, "field_hidden", "that retained version includes a hidden field")
 
4554
  prior_history_ids = {
4555
  str(item.get("id")) for item in _ut().value_history(
4556
  table_key, row_id, st=session.runtime, retention_days=days,
api/sync_runtime.py CHANGED
@@ -6,7 +6,7 @@ publish is 878.8 s inside the 1,200 s limit, so a pass fits one job.
6
  Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
7
  every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
8
  inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
9
- live only in the worker. A resident job hands over to a new job only at a pass boundary of the
10
  resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
11
  timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
12
  The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
@@ -17,7 +17,7 @@ AIOS_SYNC_RUNTIME picks the runtime: `hf-job`, `external` (a separately run resi
17
  HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
18
 
19
  Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
20
- its work is due when per-pass. Worker side: `relay_at_boundary` hands queued work to the store.
21
  The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
22
  `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
23
  of RESYNC_STATE's.
@@ -29,7 +29,6 @@ import math
29
  import os
30
  import threading
31
  import time
32
- from copy import deepcopy
33
 
34
  RUNTIMES = ("hf-job", "external", "off")
35
  RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
@@ -234,51 +233,8 @@ def relay_due(now=None):
234
  return ""
235
 
236
 
237
- def _handoff_queued_requests():
238
- """Persist ordinary queued receipts without changing durable events or interrupting a claim."""
239
- from core import connector_sync
240
- import process_role
241
- with connector_sync._SYNC_REQUEST_LOCK:
242
- receipts = deepcopy(connector_sync._SYNC_REQUESTS)
243
- runtimes = dict(connector_sync._SYNC_REQUEST_RUNTIMES)
244
- if any(row.get("state") == "running" for row in receipts.values()):
245
- return None
246
- try:
247
- for row in receipts.values():
248
- if row.get("state") != "queued" or row.get("tables") is not None:
249
- continue
250
- tenant = row["tenant"]
251
- rt = runtimes.get(tenant) or process_role._status_runtime(tenant)
252
- if rt.key != tenant:
253
- raise ValueError("The relay desk does not match the receipt tenant.")
254
- connector_sync.queue_remote_request(rt, row, row.get("connectionId"))
255
- except Exception as exc:
256
- print(f"[sync-worker] job: relay handoff failed ({type(exc).__name__}); keeping this job running", flush=True)
257
- return None
258
- return receipts
259
-
260
-
261
- def relay_at_boundary(next_at=None, *, exit_fn=None):
262
- """Relay a resident job after its current write and projection finish, before another claim."""
263
- from core import connector_sync
264
- if not ONCE["on"] or ONCE["mode"] != "resident":
265
- return False
266
- reason = relay_due()
267
- if not reason:
268
- return False
269
- receipts = _handoff_queued_requests()
270
- if receipts is None:
271
- return False
272
- with connector_sync._SYNC_REQUEST_LOCK:
273
- # No receipt can be admitted or claimed between the final handoff check and exit.
274
- if connector_sync._SYNC_REQUESTS != receipts:
275
- return False
276
- finish(time.time() if next_at is None else next_at, reason, exit_fn=exit_fn)
277
- return True
278
-
279
-
280
  def idle_wait(event, seconds, *, exit_fn=None):
281
- """Wait between passes, relaying a resident job before a queued wake can start another pass."""
282
  if not ONCE["on"]:
283
  return event.wait(seconds)
284
  if seconds is None or not math.isfinite(float(seconds)):
@@ -286,21 +242,18 @@ def idle_wait(event, seconds, *, exit_fn=None):
286
  deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
287
  resident = ONCE["mode"] == "resident"
288
  while True:
289
- if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
290
- return False
291
  remaining = deadline - time.time()
292
  if remaining <= IDLE_GRACE_SECONDS:
293
  return event.wait(max(0.0, remaining))
294
  # Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
295
  if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
296
- if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
297
- return False
298
  return True
299
- if resident:
 
300
  continue
301
  busy, wake_at = _desk_state(time.time())
302
  if not busy:
303
- finish(deadline if wake_at is None else min(deadline, wake_at), "idle", exit_fn=exit_fn)
304
  return False
305
 
306
 
@@ -345,7 +298,7 @@ def no_lease_line(tenant, seconds=None):
345
 
346
 
347
  def _pass_in_flight():
348
- """True while the store resync loop runs and owns its pass boundaries."""
349
  return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
350
 
351
 
@@ -356,7 +309,9 @@ def run_once(*, exit_fn=None, alive=None):
356
  if alive():
357
  continue
358
  if ONCE["mode"] == "resident":
359
- if relay_at_boundary(exit_fn=exit_fn):
 
 
360
  return False
361
  continue
362
  finish(time.time() + MAX_IDLE_SECONDS,
 
6
  Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
7
  every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
8
  inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
9
+ live only in the worker. A resident job hands over to a new job only at an idle point of the
10
  resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
11
  timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
12
  The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
 
17
  HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
18
 
19
  Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
20
+ its work is due when per-pass. Worker side: `idle_wait` is the resync loop's only exit point.
21
  The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
22
  `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
23
  of RESYNC_STATE's.
 
29
  import os
30
  import threading
31
  import time
 
32
 
33
  RUNTIMES = ("hf-job", "external", "off")
34
  RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
 
233
  return ""
234
 
235
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
236
  def idle_wait(event, seconds, *, exit_fn=None):
237
+ """The resync loop's wait, and a job's only exit point. Outside a job it is `event.wait(seconds)`."""
238
  if not ONCE["on"]:
239
  return event.wait(seconds)
240
  if seconds is None or not math.isfinite(float(seconds)):
 
242
  deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
243
  resident = ONCE["mode"] == "resident"
244
  while True:
 
 
245
  remaining = deadline - time.time()
246
  if remaining <= IDLE_GRACE_SECONDS:
247
  return event.wait(max(0.0, remaining))
248
  # Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
249
  if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
 
 
250
  return True
251
+ reason = relay_due() if resident else "idle"
252
+ if not reason:
253
  continue
254
  busy, wake_at = _desk_state(time.time())
255
  if not busy:
256
+ finish(deadline if wake_at is None else min(deadline, wake_at), reason, exit_fn=exit_fn)
257
  return False
258
 
259
 
 
298
 
299
 
300
  def _pass_in_flight():
301
+ """True while the store resync loop runs: it relays at its own idle point."""
302
  return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
303
 
304
 
 
309
  if alive():
310
  continue
311
  if ONCE["mode"] == "resident":
312
+ reason = relay_due()
313
+ if reason and not _desk_state(time.time())[0]:
314
+ finish(time.time(), reason, exit_fn=exit_fn)
315
  return False
316
  continue
317
  finish(time.time() + MAX_IDLE_SECONDS,
api/usage_ledger.py CHANGED
@@ -3,13 +3,16 @@
3
  R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
4
  does NOT cut anyone off this wave; over the allowance it still works, shown red."*
5
 
6
- ⛔ THIS FILE MEASURES AND NEVER ENFORCES. Token allowances remain report-only.
7
- Image-making limits live in `image_budget`, which reads measured image dollars and its own holds.
 
 
8
 
9
- **THE SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
10
  enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
11
- (a sentence becoming a draft flow) · `image_gen` (image making, edits and supporting reads or briefs).
12
- The gate in `verify_api.py` derives LLM entry points from the code rather than this list.
 
13
 
14
  ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
15
  obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
@@ -29,17 +32,17 @@ that hides the gap is worse than one that admits it.
29
  """
30
  from __future__ import annotations
31
 
32
- from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
33
  import os
34
  import time
35
 
36
- #: Weekly counters and image-settlement retry receipts contain no prompts or generated content.
 
37
  STORE_KEY = "ai_usage"
38
 
39
- #: The AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
40
  #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
41
  #: row nobody looks at.
42
- SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft", "image_gen", "pdp_text")
43
 
44
  #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
45
  #: Env-tunable so a deployment can set a real number without a release.
@@ -129,39 +132,26 @@ def _int_or_none(value):
129
  return None
130
 
131
 
132
- def usd_micros(value):
133
- """Return nonnegative dollars as whole micro-dollars, rounded half up, or None if unknown."""
134
- if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)):
135
- return None
136
- try:
137
- amount = Decimal(str(value))
138
- if amount.is_finite() and amount >= 0:
139
- return int((amount * 1_000_000).to_integral_value(rounding=ROUND_HALF_UP))
140
- except (InvalidOperation, ValueError, OverflowError):
141
- return None
142
- return None
143
-
144
-
145
- def _book_dollars(row, micros, calls, entry_id):
146
- """Book measured dollars or a separate unknown count, resolving a single-call receipt once."""
147
- receipts = row.setdefault("_cost_entries", {}) if entry_id else {}
148
- repeated = bool(entry_id and entry_id in receipts)
149
- if repeated and (receipts[entry_id] is not None or micros is None):
150
- return False
151
- if repeated:
152
- row["usd_unmeasured"] = max(0, int(row.get("usd_unmeasured") or 0) - 1)
153
- if micros is None:
154
- row["usd_unmeasured"] = int(row.get("usd_unmeasured") or 0) + calls
155
- else:
156
- row["usd_micros"] = int(row.get("usd_micros") or 0) + micros
157
- if entry_id:
158
- receipts[entry_id] = micros
159
- return not repeated
160
-
161
-
162
  def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
163
- total=None, calls=1, cost_usd=None, st=None, user="", week=None, entry_id=""):
164
- """Report a call, with optional held week and single-call retry id, without enforcing a limit."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  key = str(surface or "").strip()
166
  if key not in SURFACES:
167
  # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
@@ -172,16 +162,15 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
172
  if tot is None and (ins is not None or outs is not None):
173
  tot = (ins or 0) + (outs or 0)
174
  n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
175
- if not n or (entry_id and n != 1):
176
  return False
177
  uname = str(user or "").strip().lower()
178
  if st is None:
179
  UNATTRIBUTED["calls"] += n
180
  UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
181
  return False
182
- week = week or week_of()
183
  prov, mdl = str(provider or "")[:40], str(model or "")[:80]
184
- micros = usd_micros(cost_usd)
185
 
186
  def _up(data):
187
  data = data if isinstance(data, dict) else {}
@@ -189,16 +178,18 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
189
  row = ((weeks.setdefault(week, {})
190
  .setdefault(uname or "-", {}))
191
  .setdefault(key, {}))
192
- if _book_dollars(row, micros, n, entry_id):
193
- row["calls"] = int(row.get("calls") or 0) + n
194
- if tot is None:
195
- row["unmeasured"] = int(row.get("unmeasured") or 0) + n
196
- else:
197
- row["tokens"] = int(row.get("tokens") or 0) + tot
198
- if ins is not None:
199
- row["tokens_in"] = int(row.get("tokens_in") or 0) + ins
200
- if outs is not None:
201
- row["tokens_out"] = int(row.get("tokens_out") or 0) + outs
 
 
202
  if prov:
203
  row["provider"] = prov
204
  if mdl:
@@ -210,8 +201,12 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
210
  return weeks
211
 
212
  try:
213
- # Image dollars must persist before their budget holds can disappear.
214
- st.update(STORE_KEY, _up, flush="sync" if key == "image_gen" else "async")
 
 
 
 
215
  return True
216
  except Exception: # noqa: BLE001
217
  UNATTRIBUTED["calls"] += n
@@ -220,8 +215,7 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
220
 
221
 
222
  def _blank_row():
223
- return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0,
224
- "usd_micros": 0, "usd_unmeasured": 0}
225
 
226
 
227
  def _fold(rows):
@@ -229,7 +223,7 @@ def _fold(rows):
229
  for row in rows:
230
  if not isinstance(row, dict):
231
  continue
232
- for field in out:
233
  got = row.get(field)
234
  if isinstance(got, int) and not isinstance(got, bool):
235
  out[field] += max(0, got)
 
3
  R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
4
  does NOT cut anyone off this wave; over the allowance it still works, shown red."*
5
 
6
+ ⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of
7
+ this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives
8
+ where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two
9
+ would make an over-allowance week silently stop somebody's enrichment run.
10
 
11
+ **THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
12
  enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
13
+ (a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate
14
+ in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point
15
+ is caught by the gate and not by somebody remembering to edit a tuple.
16
 
17
  ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
18
  obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
 
32
  """
33
  from __future__ import annotations
34
 
 
35
  import os
36
  import time
37
 
38
+ #: The store bucket. Small by construction — counters only, never a per-call log: a log would grow
39
+ #: without bound on a route that runs per row, and nothing in R9 asks a question a log answers.
40
  STORE_KEY = "ai_usage"
41
 
42
+ #: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
43
  #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
44
  #: row nobody looks at.
45
+ SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft")
46
 
47
  #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
48
  #: Env-tunable so a deployment can set a real number without a release.
 
132
  return None
133
 
134
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
  def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
136
+ total=None, calls=1, st=None, user=""):
137
+ """Book ONE AI call. Returns True when the line landed durably, False when it could not.
138
+
139
+ C7's signature is the five positional parameters; everything after `*` is this module's own
140
+ extension and each one has a reason:
141
+
142
+ · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field`
143
+ accumulates a per-run total through an `_ask` contract a gate injects against, and changing
144
+ that contract to carry a split would break the injected shape for no gain here).
145
+ · `calls` — so a RUN over many rows books one line with its real call count instead of one
146
+ store write per row.
147
+ · `st` / `user` — the target. Required in practice; see this module's header for the measured
148
+ reason a contextvar cannot supply them.
149
+
150
+ ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a
151
+ store blip during an enrichment run would abort the run and lose the cells.
152
+ ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which
153
+ `GET /usage` shows an admin.
154
+ """
155
  key = str(surface or "").strip()
156
  if key not in SURFACES:
157
  # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
 
162
  if tot is None and (ins is not None or outs is not None):
163
  tot = (ins or 0) + (outs or 0)
164
  n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
165
+ if not n:
166
  return False
167
  uname = str(user or "").strip().lower()
168
  if st is None:
169
  UNATTRIBUTED["calls"] += n
170
  UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
171
  return False
172
+ week = week_of()
173
  prov, mdl = str(provider or "")[:40], str(model or "")[:80]
 
174
 
175
  def _up(data):
176
  data = data if isinstance(data, dict) else {}
 
178
  row = ((weeks.setdefault(week, {})
179
  .setdefault(uname or "-", {}))
180
  .setdefault(key, {}))
181
+ row["calls"] = int(row.get("calls") or 0) + n
182
+ if tot is None:
183
+ # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be
184
+ # representable or the meter quietly under-reports. The dashboard shows it as a count of
185
+ # unmeasured calls beside the token total.
186
+ row["unmeasured"] = int(row.get("unmeasured") or 0) + n
187
+ else:
188
+ row["tokens"] = int(row.get("tokens") or 0) + tot
189
+ if ins is not None:
190
+ row["tokens_in"] = int(row.get("tokens_in") or 0) + ins
191
+ if outs is not None:
192
+ row["tokens_out"] = int(row.get("tokens_out") or 0) + outs
193
  if prov:
194
  row["provider"] = prov
195
  if mdl:
 
201
  return weeks
202
 
203
  try:
204
+ # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of
205
+ # every AI surface in the product (an enrichment run books one line per RUN, but the
206
+ # assistant books one per question), and a blocking hub upload per call would put a network
207
+ # round trip inside the answer a person is waiting for. The mutation applies to the
208
+ # in-process cache immediately, so `GET /usage` reads its own writes.
209
+ st.update(STORE_KEY, _up, flush="async")
210
  return True
211
  except Exception: # noqa: BLE001
212
  UNATTRIBUTED["calls"] += n
 
215
 
216
 
217
  def _blank_row():
218
+ return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0}
 
219
 
220
 
221
  def _fold(rows):
 
223
  for row in rows:
224
  if not isinstance(row, dict):
225
  continue
226
+ for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"):
227
  got = row.get(field)
228
  if isinstance(got, int) and not isinstance(got, bool):
229
  out[field] += max(0, got)
platform/core/amazon_projects.py CHANGED
@@ -26,7 +26,6 @@ STATUSES = ('Backlog', 'Planned', 'In progress', 'Completed', 'Canceled')
26
  COMPLETED = 'Completed'
27
  CANCELED = 'Canceled'
28
  PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
29
- TYPES = ('project', 'pdp')
30
  #: Health turns At risk this many days before the target date.
31
  AT_RISK_DAYS = 7
32
  CHUNK = 200
@@ -244,8 +243,6 @@ def _record(values, columns=PROJECT_COLUMNS):
244
  value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
245
  members = json.loads(value) if isinstance(value, str) else value
246
  result['contributors'] = sorted(members or [], key=lambda member: member['username'])
247
- result['type'] = values[len(columns) + 4] if len(values) > len(columns) + 4 else 'project'
248
- result['listed'] = bool(values[len(columns) + 5]) if len(values) > len(columns) + 5 else True
249
  return result
250
 
251
 
@@ -751,53 +748,11 @@ def _activity_target_preview(value, truncated):
751
  return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
752
 
753
 
754
- def _pdp_item_stages(value, pdp):
755
- """Accept complete typed workflow item identities before comparing historical stages."""
756
- if not isinstance(value, dict) or value.get('type') != 'pdp' or not isinstance(value.get('items'), list):
757
- return None
758
- stages = {}
759
- for item in value['items']:
760
- if (not isinstance(item, dict) or not isinstance(item.get('asin'), str)
761
- or not isinstance(item.get('stage'), str) or item['stage'] not in pdp.STAGE_PROGRESS
762
- or item['asin'] in stages):
763
- return None
764
- stages[item['asin']] = item['stage']
765
- return stages
766
-
767
-
768
- def _pdp_activity_facts(event, field, before, after):
769
- """Project only C4 action facts retained in the bounded stored event."""
770
- from core import amazon_pdp as pdp
771
- if event.get('via') != pdp.VIA:
772
- return None
773
- unavailable = {'action': 'unavailable'}
774
- if event.get('changes_truncated'):
775
- return unavailable
776
- old, new = (_pdp_item_stages(value, pdp) for value in (before, after))
777
- if field == 'setup' and before is None and new is not None:
778
- scope = after.get('scope')
779
- scope = scope if isinstance(scope, list) and scope and all(key in pdp.SCOPE_PARTS for key in scope) else None
780
- return {'action': 'created', 'scope': scope}
781
- if old is not None and new is not None:
782
- changed = {stage for asin, stage in new.items() if asin in old and old[asin] != stage}
783
- if len(changed) == 1:
784
- stage = next(iter(changed))
785
- if field == ('publish' if stage == 'published' else stage):
786
- return {'action': 'moved', 'stage': stage} if stage != 'published' else {'action': 'published'}
787
- return {'action': 'setup'} if field in ('setup', 'competitors') and not changed else unavailable
788
- if isinstance(after, dict) and isinstance(after.get('slot'), str) and after['slot'] in pdp.SLOTS:
789
- part = pdp.SLOTS[after['slot']]['key']
790
- if after.get('part') == part and field in ('draft', 'approve', 'request'):
791
- return {'action': {'draft': 'saved', 'approve': 'approved', 'request': 'requested'}[field]}
792
- return {'action': 'ai'} if field == 'ai' else unavailable
793
-
794
-
795
  def _activity_entries(event, actor_names, task_names):
796
  """Project one log event into named activity entries."""
797
  out = []
798
  for index, (field, change) in enumerate((event.get('changes') or {}).items()):
799
  before, after = change.get('before'), change.get('after')
800
- pdp_facts = _pdp_activity_facts(event, field, before, after)
801
  if field == 'project':
802
  action, field = ('created' if before is None else 'deleted'), None
803
  elif field == 'targets':
@@ -820,8 +775,6 @@ def _activity_entries(event, actor_names, task_names):
820
  'by': event['by'], 'byUsername': event['by'],
821
  'byName': actor_names.get(event['by'], event['by']),
822
  'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
823
- if pdp_facts is not None:
824
- out[-1]['pdp'] = pdp_facts
825
  return out
826
 
827
 
@@ -1142,8 +1095,6 @@ class PgProjectStore:
1142
  notes TEXT, completed_on DATE, source_id TEXT,
1143
  created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
1144
  rollup JSONB, impact JSONB,
1145
- type TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)})),
1146
- listed BOOLEAN NOT NULL DEFAULT TRUE,
1147
  PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
1148
  CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
1149
  OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
@@ -1161,11 +1112,8 @@ class PgProjectStore:
1161
  # One row's lookup is one indexed read on this index.
1162
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
1163
  f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
1164
- additions = {'rollup': 'JSONB', 'impact': 'JSONB',
1165
- 'type': f"TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)}))",
1166
- 'listed': 'BOOLEAN NOT NULL DEFAULT TRUE'}
1167
- for field, declaration in additions.items():
1168
- con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} {declaration}')
1169
 
1170
  @contextmanager
1171
  def _write(self):
@@ -1178,20 +1126,13 @@ class PgProjectStore:
1178
  cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
1179
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1180
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1181
- return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members}, '
1182
- f'{self._type_projection()}, {self._listed_projection()} '
1183
  f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
1184
 
1185
  def _rollup_projection(self):
1186
  # Old schemas read blank figures until a guarded write adds the columns.
1187
  return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
1188
 
1189
- def _type_projection(self):
1190
- return "COALESCE(to_jsonb(p)->>'type', 'project')"
1191
-
1192
- def _listed_projection(self):
1193
- return "COALESCE((to_jsonb(p)->>'listed')::boolean, TRUE)"
1194
-
1195
  def _members_projection(self, con):
1196
  if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
1197
  return 'NULL'
@@ -1237,19 +1178,15 @@ class PgProjectStore:
1237
  'WHERE tenant = %s AND id = %s',
1238
  (stamp, json.dumps(impact), self.tenant_slug, project_id))
1239
 
1240
- def _insert(self, con, record, pairs, stamp):
1241
- columns = (*PROJECT_COLUMNS, 'impact', 'type', 'listed')
1242
- values = (*[record[c] for c in PROJECT_COLUMNS],
1243
- json.dumps(_with_result_configs(None, record.get('result_configs', []))),
1244
- record.get('type', 'project'), record.get('listed', True))
1245
- con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
1246
- f'VALUES ({_marks(len(PROJECT_COLUMNS))}, %s::jsonb, %s, %s)', values)
1247
- self._insert_targets(con, record['id'], pairs, stamp)
1248
- self._replace_members(con, record['id'], record.get('contributors', []), [], stamp)
1249
-
1250
  def insert(self, record, pairs, stamp):
1251
  with self._write() as con:
1252
- self._insert(con, record, pairs, stamp)
 
 
 
 
 
 
1253
  return self._one(con, record['id'])
1254
 
1255
  def get(self, project_id):
@@ -1261,8 +1198,6 @@ class PgProjectStore:
1261
  def update(self, project_id, changes, stamp):
1262
  with self._write() as con:
1263
  current = self._one(con, project_id, lock=True)
1264
- if current['type'] == 'pdp' and set(changes) & {'progress', 'status', 'completed_on'}:
1265
- raise ValueError('PDP item stages control project progress and completion.')
1266
  candidate = _fields(changes, previous=current)
1267
  if 'contributors' in changes:
1268
  candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
@@ -1278,25 +1213,23 @@ class PgProjectStore:
1278
  self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
1279
  return self._one(con, project_id)
1280
 
1281
- def _delete(self, con, project_id):
1282
- for table in (MEMBERS, TARGETS):
1283
- con.execute(f'DELETE FROM {self.t(table)} WHERE tenant = %s AND project_id = %s',
1284
- (self.tenant_slug, project_id))
1285
- row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
1286
- (self.tenant_slug, project_id)).fetchone()
1287
- if row is None:
1288
- raise KeyError('Project not found.')
1289
-
1290
  def delete(self, project_id):
1291
  with self._write() as con:
1292
- self._delete(con, project_id)
 
 
 
 
 
 
 
1293
  return {'deleted': project_id}
1294
 
1295
  def list_all(self):
1296
  with self._connection() as con:
1297
  if not self._exists(con):
1298
  return []
1299
- rows = con.execute(self._select_sql(f' AND {self._listed_projection()} ORDER BY p.created_at, p.id',
1300
  members=self._members_projection(con)),
1301
  (self.tenant_slug,)).fetchall()
1302
  return [_record(row) for row in rows]
@@ -1320,12 +1253,10 @@ class PgProjectStore:
1320
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1321
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1322
  rows = con.execute(
1323
- f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members}, '
1324
- f'{self._type_projection()}, {self._listed_projection()} '
1325
  f'FROM {self.t(TARGETS)} AS t '
1326
  f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
1327
- f'WHERE t.tenant = %s AND {self._listed_projection()} AND ({match}) '
1328
- 'ORDER BY p.created_at, p.id',
1329
  (self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
1330
  for row in rows:
1331
  out[(row[0], row[1])].append(_record(row[2:]))
 
26
  COMPLETED = 'Completed'
27
  CANCELED = 'Canceled'
28
  PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
 
29
  #: Health turns At risk this many days before the target date.
30
  AT_RISK_DAYS = 7
31
  CHUNK = 200
 
243
  value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
244
  members = json.loads(value) if isinstance(value, str) else value
245
  result['contributors'] = sorted(members or [], key=lambda member: member['username'])
 
 
246
  return result
247
 
248
 
 
748
  return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
749
 
750
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
751
  def _activity_entries(event, actor_names, task_names):
752
  """Project one log event into named activity entries."""
753
  out = []
754
  for index, (field, change) in enumerate((event.get('changes') or {}).items()):
755
  before, after = change.get('before'), change.get('after')
 
756
  if field == 'project':
757
  action, field = ('created' if before is None else 'deleted'), None
758
  elif field == 'targets':
 
775
  'by': event['by'], 'byUsername': event['by'],
776
  'byName': actor_names.get(event['by'], event['by']),
777
  'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
 
 
778
  return out
779
 
780
 
 
1095
  notes TEXT, completed_on DATE, source_id TEXT,
1096
  created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
1097
  rollup JSONB, impact JSONB,
 
 
1098
  PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
1099
  CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
1100
  OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
 
1112
  # One row's lookup is one indexed read on this index.
1113
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
1114
  f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
1115
+ for field in ('rollup', 'impact'):
1116
+ con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} JSONB')
 
 
 
1117
 
1118
  @contextmanager
1119
  def _write(self):
 
1126
  cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
1127
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1128
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1129
+ return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} '
 
1130
  f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
1131
 
1132
  def _rollup_projection(self):
1133
  # Old schemas read blank figures until a guarded write adds the columns.
1134
  return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
1135
 
 
 
 
 
 
 
1136
  def _members_projection(self, con):
1137
  if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
1138
  return 'NULL'
 
1178
  'WHERE tenant = %s AND id = %s',
1179
  (stamp, json.dumps(impact), self.tenant_slug, project_id))
1180
 
 
 
 
 
 
 
 
 
 
 
1181
  def insert(self, record, pairs, stamp):
1182
  with self._write() as con:
1183
+ columns = (*PROJECT_COLUMNS, 'impact')
1184
+ values = (*[record[c] for c in PROJECT_COLUMNS],
1185
+ json.dumps(_with_result_configs(None, record.get('result_configs', []))))
1186
+ con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
1187
+ f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values)
1188
+ self._insert_targets(con, record['id'], pairs, stamp)
1189
+ self._replace_members(con, record['id'], record['contributors'], [], stamp)
1190
  return self._one(con, record['id'])
1191
 
1192
  def get(self, project_id):
 
1198
  def update(self, project_id, changes, stamp):
1199
  with self._write() as con:
1200
  current = self._one(con, project_id, lock=True)
 
 
1201
  candidate = _fields(changes, previous=current)
1202
  if 'contributors' in changes:
1203
  candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
 
1213
  self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
1214
  return self._one(con, project_id)
1215
 
 
 
 
 
 
 
 
 
 
1216
  def delete(self, project_id):
1217
  with self._write() as con:
1218
+ con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s',
1219
+ (self.tenant_slug, project_id))
1220
+ con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s',
1221
+ (self.tenant_slug, project_id))
1222
+ row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
1223
+ (self.tenant_slug, project_id)).fetchone()
1224
+ if row is None:
1225
+ raise KeyError('Project not found.')
1226
  return {'deleted': project_id}
1227
 
1228
  def list_all(self):
1229
  with self._connection() as con:
1230
  if not self._exists(con):
1231
  return []
1232
+ rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id',
1233
  members=self._members_projection(con)),
1234
  (self.tenant_slug,)).fetchall()
1235
  return [_record(row) for row in rows]
 
1253
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1254
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1255
  rows = con.execute(
1256
+ f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} '
 
1257
  f'FROM {self.t(TARGETS)} AS t '
1258
  f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
1259
+ f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id',
 
1260
  (self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
1261
  for row in rows:
1262
  out[(row[0], row[1])].append(_record(row[2:]))
platform/core/amazon_rivals.py CHANGED
@@ -1,11 +1,10 @@
1
- """Per-tenant PostgreSQL money-search rival pools and daily quotes.
2
 
3
  Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
4
  module (verify_layering) and no App module (rule 13). Tables are created on the first write.
5
  """
6
 
7
  from contextlib import contextmanager
8
- from dataclasses import dataclass
9
  from datetime import date, datetime, timedelta, timezone
10
  import json
11
  import re
@@ -20,6 +19,8 @@ RUNS = 'amazon_rival_runs'
20
  WANTED = 'amazon_rival_wanted'
21
  TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
22
 
 
 
23
  #: Brand Analytics keeps the top 3 clicked ASINs of each search term.
24
  TOP_CLICKED = 3
25
  SOURCES = frozenset(('auto', 'added'))
@@ -28,35 +29,16 @@ CHUNK = 200
28
  BA_TABLE = 'amzn_brand_analytics'
29
  LISTINGS_TABLE = 'amzn_listings'
30
  PRICING_TABLE = 'amzn_pricing'
31
- SQP_TABLE = 'amzn_search_query_performance'
32
- RANKS_TABLE = 'amzn_apify_rank_observations'
33
- SNAPSHOTS_TABLE = 'amzn_competitor_snapshots'
34
- IMAGES_TABLE = 'amzn_competitor_images'
35
- POOL_SNAPSHOT_FIELDS = ('snapshot_date', 'brand', 'title', 'bullets', 'price', 'currency', 'prime', 'offers')
36
- POOL_IMAGE_FIELDS = ('variant', 'link', 'width', 'height')
37
 
38
  NO_TOP3 = "Not in any search term's top 3 clicked"
39
  NO_WEEK = 'Brand Analytics has no dated weekly reports'
40
- NO_MONTH = 'No completed monthly Search Query Performance report for this product yet.'
41
- NO_MONEY = 'No searches with market money for this product yet.'
42
- NO_KEYWORD_RANKS = "No keyword ranks for this product's searches yet."
43
- NO_PRODUCT_TYPE = 'Product type not read yet. The next daily run reads it.'
44
- WAITING_PRODUCT_TYPE = '{count} found, waiting for the daily read of their product type.'
45
  NOT_OURS = 'Not an ASIN in your Amazon listings'
46
  PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
47
- NO_POOL = 'No search report or keyword ranks for this product yet.'
48
  ASIN_RE = re.compile(r'[A-Z0-9]{10}')
49
 
50
- POOL_FIELDS = {
51
- RIVALS: {'origin': ('TEXT', "'click'"), 'money_won': ('DOUBLE PRECISION', '0'),
52
- 'terms_won': ('INTEGER', '0'), 'term_rank': ('INTEGER', 'NULL'),
53
- 'eligible': ('BOOLEAN', 'TRUE')},
54
- DISCOVERY: {'month': ('DATE', 'NULL'), 'keyword_ranks_on': ('DATE', 'NULL'),
55
- 'product_type': ('TEXT', 'NULL'), 'held': ('INTEGER', '0')},
56
- }
57
  RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
58
- 'added_by', 'updated_at', *POOL_FIELDS[RIVALS])
59
- DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count', *POOL_FIELDS[DISCOVERY])
60
  QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
61
  'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
62
  RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
@@ -193,355 +175,96 @@ def children(cur, parent):
193
  return [r[0] for r in rows if r[0]]
194
 
195
 
196
- def _money_sql(cur):
197
- """Newest completed monthly searches per ASIN, including zero-money searches for freshness."""
198
- required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
199
- 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
200
- if not required <= _columns(cur, SQP_TABLE):
201
- return ('SELECT NULL::VARCHAR AS asin, NULL::VARCHAR AS term, NULL::DATE AS month, '
202
- 'NULL::DOUBLE AS market_money WHERE FALSE')
203
- return f'''
204
- WITH reports AS (
205
- SELECT upper(trim(CAST(asin AS VARCHAR))) AS asin,
206
- lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS term,
207
- TRY_CAST(start_date AS DATE) AS start_on, TRY_CAST(end_date AS DATE) AS end_on,
208
- TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
209
- TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS market_money
210
- FROM {SQP_TABLE}
211
- WHERE date_diff('day', TRY_CAST(start_date AS DATE), TRY_CAST(end_date AS DATE)) > 7
212
- AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
213
- latest AS (
214
- SELECT * FROM reports QUALIFY end_on = max(end_on) OVER (PARTITION BY asin))
215
- SELECT asin, term, CAST(date_trunc('month', start_on) AS DATE) AS month,
216
- max(market_money) AS market_money
217
- FROM latest GROUP BY asin, term, month'''
218
-
219
-
220
- def _click_sql(cur):
221
- """Normalized dated top-three clicks, with percentage shares kept in their source units."""
222
  cols = _columns(cur, BA_TABLE)
223
- if not {'search_term', 'clicked_asin', 'click_share_rank', 'click_share',
224
- '_period_start', '_period_end'} <= cols:
225
- return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DOUBLE AS share, '
226
- 'NULL::VARCHAR AS title, NULL::INTEGER AS term_rank, NULL::DATE AS week WHERE FALSE')
227
- title = 'CAST(clicked_item_name AS VARCHAR)' if 'clicked_item_name' in cols else 'NULL::VARCHAR'
228
- return f'''SELECT lower(trim(CAST(search_term AS VARCHAR))) AS term,
229
- upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
230
- TRY_CAST(click_share AS DOUBLE) AS share, {title} AS title,
231
- TRY_CAST(click_share_rank AS INTEGER) AS term_rank, {_week_sql()} AS week
232
- FROM {BA_TABLE} WHERE {weekly_period_sql()}
233
- AND TRY_CAST(click_share_rank AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}'''
234
-
235
-
236
- def _rank_rows_sql(cur):
237
- """Raw Keyword ranks observations include rivals absent from the own-placement view."""
238
- if not {'keyword', 'asin', 'date', 'organic_rank', 'is_sponsored'} <= _columns(cur, RANKS_TABLE):
239
- return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DATE AS observed_on, '
240
- 'NULL::INTEGER AS term_rank, NULL::BOOLEAN AS sponsored WHERE FALSE')
241
- return f'''SELECT lower(trim(CAST(keyword AS VARCHAR))) AS term,
242
- upper(trim(CAST(asin AS VARCHAR))) AS rival, TRY_CAST(date AS DATE) AS observed_on,
243
- TRY_CAST(organic_rank AS INTEGER) AS term_rank,
244
- TRY_CAST(is_sponsored AS BOOLEAN) AS sponsored FROM {RANKS_TABLE}'''
245
-
246
-
247
- def _search_dates(cur, targets):
248
- rows = _read(cur, f'''
249
- WITH money AS ({_money_sql(cur)}), observations AS ({_rank_rows_sql(cur)})
250
- SELECT m.asin, max(m.month), max(o.observed_on), count(DISTINCT m.term)
251
- FILTER (WHERE m.market_money > 0)
252
- FROM money m LEFT JOIN observations o ON o.term = m.term AND m.market_money > 0
253
- WHERE m.asin IN {_in(targets)} GROUP BY m.asin''', targets)
254
- return {a: dict(month=_day(m), keyword_ranks_on=_day(k), money_terms=n) for a, m, k, n in rows}
255
-
256
-
257
- def _newest_snapshots_sql(cur, fields=('product_type', 'brand')):
258
- if not {'asin', 'snapshot_date', 'product_type', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
259
- return 'SELECT NULL::VARCHAR AS asin, ' + ', '.join(f'NULL::VARCHAR AS {f}' for f in fields) + ' WHERE FALSE'
260
- return (f'SELECT upper(trim(asin)) AS asin, {", ".join(fields)} FROM {SNAPSHOTS_TABLE} '
261
- 'QUALIFY row_number() OVER (PARTITION BY upper(trim(asin)) ORDER BY snapshot_date DESC) = 1')
262
-
263
-
264
- def _brand(value):
265
- return ''.join((value or '').casefold().split())
266
-
267
-
268
- def _same_type(target_type, rival_type):
269
- return bool(target_type and rival_type and target_type == rival_type)
270
-
271
-
272
- def _other_brand(brand, own_brands):
273
- return not _brand(brand) or _brand(brand) not in own_brands
274
-
275
-
276
- @dataclass
277
- class PoolFacts:
278
- """Newest ASIN facts and all historical own brands determine pool eligibility."""
279
-
280
- latest: dict
281
- own_brands: set
282
- own_asins: set
283
-
284
- def product_type(self, target):
285
- return self.latest.get(target, {}).get('product_type')
286
-
287
- def eligible(self, target, rival):
288
- fact = self.latest.get(rival, {})
289
- return (rival not in self.own_asins and _same_type(self.product_type(target), fact.get('product_type'))
290
- and _other_brand(fact.get('brand'), self.own_brands))
291
-
292
-
293
- def _pool_facts(cur, values, own, prefetched=None, candidate_rows=()):
294
- latest = {r[1]: dict(product_type=r[-2], brand=r[-1]) for r in candidate_rows}
295
- for part in _chunks(sorted(values - latest.keys())):
296
- rows = _read(cur, f'SELECT asin, product_type, brand FROM ({_newest_snapshots_sql(cur)}) '
297
- f'WHERE asin IN {_in(part)}', part)
298
- latest.update({a: dict(product_type=t, brand=b) for a, t, b in rows})
299
- brands = set()
300
- if {'asin', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
301
- rows = _read(cur, f'SELECT DISTINCT s.brand FROM {SNAPSHOTS_TABLE} s '
302
- f'JOIN {LISTINGS_TABLE} l ON upper(trim(s.asin)) = upper(trim(l.asin1))')
303
- brands.update(_brand(row[0]) for row in rows if _brand(row[0]))
304
- latest.update(prefetched or {})
305
- brands.update(_brand(latest[a].get('brand')) for a in own if a in latest and _brand(latest[a].get('brand')))
306
- return PoolFacts(latest, brands, own)
307
-
308
-
309
- def _candidates(cur, targets, week):
310
- """Every non-own clicked or organic page-one ASIN on each target's monthly money searches."""
311
  sql = f'''
312
- WITH money AS ({_money_sql(cur)}), clicks AS ({_click_sql(cur)}),
313
- observations AS ({_rank_rows_sql(cur)}),
314
- newest_observations AS (
315
- SELECT * FROM observations
316
- QUALIFY observed_on = max(observed_on) OVER (PARTITION BY term)),
317
- click AS (
318
- SELECT term, rival, max(share) AS share, min(term_rank) AS term_rank, max(title) AS title
319
- FROM clicks WHERE week = CAST(? AS DATE) GROUP BY term, rival),
320
- page1 AS (
321
- SELECT term, rival, min(term_rank) AS term_rank FROM newest_observations
322
- WHERE sponsored = FALSE GROUP BY term, rival),
323
- placements AS (
324
- SELECT coalesce(c.term, p.term) AS term, coalesce(c.rival, p.rival) AS rival,
325
- c.share, c.title, coalesce(c.term_rank, p.term_rank) AS term_rank,
326
- c.term IS NOT NULL AS clicked, p.term IS NOT NULL AS page1
327
- FROM click c FULL OUTER JOIN page1 p ON p.term = c.term AND p.rival = c.rival),
328
  hits AS (
329
- SELECT m.asin, p.*, m.market_money * coalesce(p.share, 0) / 100 AS money_won
330
- FROM money m JOIN placements p ON p.term = m.term
331
- WHERE m.asin IN {_in(targets)} AND m.market_money > 0 AND p.rival <> ''
332
- AND p.rival NOT IN (SELECT upper(trim(CAST(asin1 AS VARCHAR)))
333
- FROM {LISTINGS_TABLE} WHERE asin1 IS NOT NULL)),
334
- totals AS (
335
- SELECT asin, rival, max(share) AS share, sum(money_won) AS money_won,
336
- count(*) FILTER (WHERE clicked AND term_rank = 1) AS terms_won,
337
- CASE WHEN bool_or(clicked) AND bool_or(page1) THEN 'both'
338
- WHEN bool_or(clicked) THEN 'click' ELSE 'page1' END AS origin
339
- FROM hits GROUP BY asin, rival),
340
- representative AS (
341
- SELECT asin, rival, term, title, term_rank FROM hits
342
- QUALIFY row_number() OVER (PARTITION BY asin, rival
343
- ORDER BY money_won DESC, term_rank, term) = 1)
344
- SELECT t.asin, t.rival, t.share, r.term, r.title, t.origin, t.money_won, t.terms_won, r.term_rank,
345
- s.product_type, s.brand
346
- FROM totals t JOIN representative r USING (asin, rival)
347
- LEFT JOIN ({_newest_snapshots_sql(cur)}) s ON s.asin = t.rival
348
- ORDER BY t.asin, t.money_won DESC, t.rival'''
349
- return _read(cur, sql, (week, *targets))
350
 
351
 
352
  # --------------------------------------------------------------------------- discovery
353
 
354
- def discover(rt, cur, asins=None, *, candidate_rows=None, prefetched=None):
355
- """Refresh uncapped money-search pools while preserving removed and hand-added rivals."""
 
 
356
  backend = _backend(rt)
357
  requested = None if asins is None else _asins(asins)
358
  targets = sorted(our_asins(cur, requested))
359
  stamp = _now()
360
  discovery = {}
361
  for missing in sorted(set(requested or ()) - set(targets)):
362
- discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS,
363
- auto_count=0, month=None, keyword_ranks_on=None, product_type=None, held=0)
364
  week = newest_week(cur) if targets else None
365
- dates = _search_dates(cur, targets) if targets else {}
366
- rows = (_candidates(cur, targets, week) if candidate_rows is None else candidate_rows) if targets else []
367
- stored = backend.pool_rows(targets)
368
- before = backend.rivals_of(targets)
369
- values = set(targets) | {r[1] for r in rows} | {r['rival_asin'] for r in stored}
370
- facts = _pool_facts(cur, values, our_asins(cur), prefetched, rows)
371
- found, held = _eligible_candidates(rows, stored, facts, week, stamp)
 
 
 
 
 
 
 
372
  for target in targets:
373
- count = (len(found.get(target, ())) if facts.product_type(target) else
374
- sum(r['source'] == 'auto' for r in stored if r['asin'] == target and not r['removed']))
375
- meta = dates.get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0)
376
- cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, count)
377
- discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count,
378
- month=meta['month'], keyword_ranks_on=meta['keyword_ranks_on'],
379
- product_type=facts.product_type(target), held=held.get(target, 0))
380
- ready = [t for t in targets if facts.product_type(t)]
381
- backend.replace_auto(ready, [row for t in ready for row in found.get(t, ())], list(discovery.values()))
382
- backend.set_eligibility(stored, facts)
383
- members = backend.rivals_of(targets)
384
- pools = {target: len(members.get(target, ())) for target in targets}
385
  return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
386
- 'pools': pools, 'rowsRead': len(rows), 'held': held,
387
- 'changed': tuple(t for t in targets if set(before.get(t, ())) != set(members.get(t, ()))),
388
- 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
389
-
390
-
391
- def _eligible_candidates(rows, stored, facts, week, stamp):
392
- kept = {(r['asin'], r['rival_asin']) for r in stored if r['removed'] or r['source'] == 'added'}
393
- candidates = {(r[0], r[1]) for r in rows} | {
394
- (r['asin'], r['rival_asin']) for r in stored if r['source'] == 'added' and not r['removed']}
395
- removed = {(r['asin'], r['rival_asin']) for r in stored if r['removed']}
396
- held = {}
397
- for target, rival in candidates - removed:
398
- if (rival not in facts.own_asins and not facts.product_type(rival)
399
- and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)):
400
- held[target] = held.get(target, 0) + 1
401
- found = {}
402
- for target, rival, share, term, title, origin, money, terms, term_rank, _type, _name in rows:
403
- if (target, rival) in kept or not facts.eligible(target, rival):
404
- continue
405
- found.setdefault(target, []).append(dict(
406
- asin=target, rival_asin=rival, source='auto', removed=False, eligible=True,
407
- click_share=None if share is None else float(share), term=term, title=title,
408
- week=week, added_by=None, updated_at=stamp, origin=origin,
409
- money_won=float(money), terms_won=int(terms), term_rank=term_rank))
410
- return found, held
411
-
412
-
413
- def _discovery_cause(week, meta, count):
414
- if meta['month'] is None:
415
- return NO_MONTH
416
- if not meta['money_terms']:
417
- return NO_MONEY
418
- if meta['keyword_ranks_on'] is None:
419
- return NO_KEYWORD_RANKS
420
- if week is None:
421
- return NO_WEEK
422
- return None if count else NO_TOP3
423
-
424
-
425
- def _pool_cause(product_type, held, week, meta, count):
426
- if not product_type:
427
- return NO_PRODUCT_TYPE
428
- if held:
429
- return WAITING_PRODUCT_TYPE.format(count=held)
430
- return _discovery_cause(week, meta, count)
431
 
432
 
433
  def stale(rt, cur, target):
434
- """True when the ASIN's discovery predates a weekly, monthly or Keyword ranks source."""
435
  row = _backend(rt).discovery([target]).get(target)
436
  if row is None:
437
  return True
438
- types = _read(cur, f'SELECT product_type FROM ({_newest_snapshots_sql(cur)}) WHERE asin = ?', [target])
439
- product_type = types[0][0] if types else None
440
- if row['product_type'] != product_type or (not product_type and row['cause'] != NO_PRODUCT_TYPE):
441
- return True
442
- newest = {'week': newest_week(cur), **(_search_dates(cur, [target]).get(target) or {})}
443
- return any(newest.get(key) is not None and
444
- (row.get(key) is None or row[key] < newest[key])
445
- for key in ('week', 'month', 'keyword_ranks_on'))
446
 
447
 
448
  # --------------------------------------------------------------------------- doors
449
 
450
  def rivals(rt, target):
451
- """One ASIN's rivals: hand-added first, then money won and rank average, with discovery dates."""
452
  target = asin(target)
453
  backend = _backend(rt)
454
  rows = backend.rivals(target)
455
  found = backend.discovery([target]).get(target)
456
  return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
457
  'week': found['week'] if found else None,
458
- 'month': found['month'] if found else None,
459
- 'keywordRanksOn': found['keyword_ranks_on'] if found else None,
460
  'discoveredAt': found['discovered_at'] if found else None,
461
- 'cause': found['cause'] if found else None,
462
- 'productType': found['product_type'] if found else None,
463
- 'held': found['held'] if found else 0}
464
-
465
-
466
- def pool(rt, cur, target):
467
- """Read eligible pool rows through the caller's mirror cursor without writes or provider reads."""
468
- target = asin(target)
469
- backend = _backend(rt)
470
- if cur is None:
471
- return {**rivals(rt, target), 'rivals': [], 'competitors': [], 'dataAsOf': None, 'cause': NO_POOL}
472
- own = our_asins(cur)
473
- week = newest_week(cur)
474
- sources = _candidates(cur, [target], week) if target in own else []
475
- stored = backend.pool_rows([target])
476
- facts = _pool_facts(cur, {target} | {r[1] for r in sources} | {r['rival_asin'] for r in stored},
477
- own, candidate_rows=sources)
478
- found, held = _eligible_candidates(sources, stored, facts, week, _now())
479
- if facts.product_type(target):
480
- rows = found.get(target, []) + [dict(r, eligible=True) for r in stored
481
- if r['source'] == 'added' and not r['removed']
482
- and facts.eligible(target, r['rival_asin'])]
483
- else:
484
- rows = [r for r in stored if r['eligible'] and not r['removed'] and r['rival_asin'] not in own
485
- and _other_brand(facts.latest.get(r['rival_asin'], {}).get('brand'), facts.own_brands)]
486
- if target not in own:
487
- rows = []
488
- averages = backend.rank_averages([r['rival_asin'] for r in rows])
489
- rows = [{**r, **averages.get(r['rival_asin'], {'rank_average': None, 'rank_days': 0})} for r in rows]
490
- meta = _search_dates(cur, [target]).get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0)
491
- cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, len(rows))
492
- rows = sorted(rows, key=_pool_order)
493
- snapshots = _pool_snapshots(cur, [target, *(r['rival_asin'] for r in rows)])
494
- images = _pool_images(cur, [r['rival_asin'] for r in rows])
495
- dates = [s['snapshot_date'] for s in snapshots.values() if s['snapshot_date'] is not None]
496
- return {**rivals(rt, target), 'rivals': rows, 'week': week,
497
- 'month': meta['month'], 'keywordRanksOn': meta['keyword_ranks_on'],
498
- 'productType': facts.product_type(target), 'held': held.get(target, 0),
499
- 'cause': cause if target in own else NOT_OURS, 'dataAsOf': max(dates, default=None),
500
- 'competitors': [_pool_competitor(r, snapshots.get(r['rival_asin'], {}),
501
- images.get(r['rival_asin'], [])) for r in rows]}
502
-
503
-
504
- def _pool_snapshots(cur, values):
505
- snapshots = {}
506
- for part in _chunks(sorted(set(values))):
507
- rows = _read(cur, f'SELECT * FROM ({_newest_snapshots_sql(cur, POOL_SNAPSHOT_FIELDS)}) '
508
- f'WHERE asin IN {_in(part)}', part)
509
- for target, *fields in rows:
510
- snapshot = dict(zip(POOL_SNAPSHOT_FIELDS, fields))
511
- snapshot['snapshot_date'] = _day(snapshot['snapshot_date'])
512
- snapshot['bullets'] = json.loads(snapshot['bullets'] or '[]')
513
- snapshots[target] = snapshot
514
- return snapshots
515
-
516
-
517
- def _pool_images(cur, values):
518
- """Resolve only the newest snapshot's variants against changed-link history."""
519
- images = {}
520
- if not {'asin', *POOL_IMAGE_FIELDS} <= _columns(cur, IMAGES_TABLE):
521
- return images
522
- for part in _chunks(values):
523
- rows = _read(cur, f'''
524
- SELECT s.asin, i.variant, i.link, i.width, i.height
525
- FROM ({_newest_snapshots_sql(cur, ('image_variants',))}) s,
526
- json_each(s.image_variants) v, {IMAGES_TABLE} i
527
- WHERE s.asin IN {_in(part)} AND i.asin = s.asin
528
- AND i.variant = json_extract_string(v.value, '$.variant')
529
- AND i.link = json_extract_string(v.value, '$.link')
530
- ORDER BY s.asin, CAST(v.key AS INTEGER)''', part)
531
- for target, *fields in rows:
532
- images.setdefault(target, []).append(dict(zip(POOL_IMAGE_FIELDS, fields)))
533
- return images
534
-
535
-
536
- def _pool_competitor(row, snapshot, images):
537
- days = row['rank_days']
538
- return {'asin': row['rival_asin'], 'brand': snapshot.get('brand'), 'title': snapshot.get('title'),
539
- 'origin': row['origin'], 'moneyWon': row['money_won'], 'termsWon': row['terms_won'],
540
- 'term': row['term'], 'termRank': row['term_rank'], 'bestSellerRank': row['rank_average'],
541
- 'rankDays': days, 'rankCause': f'Rank average needs 7 days of ranks. {days} so far.' if days < 7 else None,
542
- 'price': snapshot.get('price'), 'currency': snapshot.get('currency'), 'prime': snapshot.get('prime'),
543
- 'offers': snapshot.get('offers'), 'imageCount': len(images), 'images': images,
544
- 'bullets': snapshot.get('bullets', []), 'snapshotOn': snapshot.get('snapshot_date')}
545
 
546
 
547
  def add(rt, target, rival, *, user, title=None):
@@ -549,10 +272,9 @@ def add(rt, target, rival, *, user, title=None):
549
  target, rival = asin(target), asin(rival, 'rival')
550
  if rival == target:
551
  raise ValueError('An ASIN cannot be its own rival.')
552
- return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, eligible=False,
553
  click_share=None, term=None, title=title, week=None,
554
- added_by=str(user or '') or None, updated_at=_now(),
555
- origin='added', money_won=0, terms_won=0, term_rank=None))
556
 
557
 
558
  def remove(rt, target, rival, *, user):
@@ -616,19 +338,6 @@ def runs(rt, limit=10):
616
  return _backend(rt).runs(int(limit))
617
 
618
 
619
- def rank_average(rt, asins):
620
- """Mean daily rank in each ASIN's newest display group, requiring seven of its newest 30 days."""
621
- values = _asins(asins)
622
- found = _backend(rt).rank_averages(values) if values else {}
623
- return {a: (found.get(a) or {}).get('rank_average') for a in values}
624
-
625
-
626
- def _pool_order(row):
627
- average = row.get('rank_average')
628
- return (-float(row.get('money_won') or 0),
629
- average is None, average if average is not None else 0, row['rival_asin'])
630
-
631
-
632
  # --------------------------------------------------------------------------- the daily quotes run
633
 
634
  #: The run row gets a progress marker after each block of this many ASINs.
@@ -642,8 +351,7 @@ MAX_BACKOFF_SECONDS = 300.0
642
  #: An unfinished run, or a finished run with a cause, is retried after this many hours.
643
  RETRY_RUN_HOURS = 3
644
 
645
- READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'facts': 'catalog facts'}
646
- FACT_COUNT_COLUMNS = {'title': 'title', 'product_type': 'product_type', 'image': 'image', 'rank': 'sales_rank'}
647
  NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
648
  NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
649
  NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
@@ -741,7 +449,7 @@ def _summary_quote(response):
741
 
742
 
743
  def _keyed(kind, got):
744
- if kind == 'facts':
745
  return {str(k).upper(): v for k, v in (got or {}).items()}
746
  if kind == 'offers':
747
  return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
@@ -781,16 +489,7 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
781
  attempt = 0
782
  while True:
783
  try:
784
- got = _keyed(kind, fn(part))
785
- answers.update(got)
786
- if kind == 'facts':
787
- for target, fact in got.items():
788
- if fact.get('cause'):
789
- causes[target] = fact['cause']
790
- if '(429)' in fact['cause']:
791
- counts['throttled'] += 1
792
- elif 'Amazon refused' in fact['cause']:
793
- counts['refused'] += 1
794
  break
795
  except Exception as exc: # noqa: BLE001
796
  if _throttled(exc) and attempt < THROTTLE_RETRIES:
@@ -806,19 +505,14 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
806
  return answers, causes
807
 
808
 
809
- def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts,
810
- prefetched=None, fact_causes=None):
811
  """One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
812
  mine = [a for a in block if a in own]
813
  others = [a for a in block if a not in own]
814
  offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
815
  summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
816
  counts=counts)
817
- cached = prefetched or {}
818
- facts, rank_causes = _read_batches('facts', readers['facts'], [a for a in block if a not in cached],
819
- sleep=sleep, log=log, counts=counts)
820
- facts.update({a: cached[a] for a in block if a in cached})
821
- rank_causes.update(fact_causes or {})
822
  stamp = _now()
823
  rows = []
824
  for target in block:
@@ -837,31 +531,20 @@ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, c
837
  row.update(_summary_quote(summed[target]))
838
  else:
839
  row['buy_box_cause'] = NO_PRICE
840
- rank = facts.get(target) or {}
841
  value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
842
  row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
843
- rank_cause=None if value is not None else rank_causes.get(target) or NO_RANK)
844
  rows.append(row)
845
- return rows, [_snapshot(row, facts.get(row['asin']) or {}, row['asin'] in own) for row in rows]
846
-
847
-
848
- def _snapshot(quote, facts, own):
849
- """C3 facts with price and rank taken from the exact persisted quote row."""
850
- return {**{key: facts.get(key) for key in ('product_type', 'brand', 'title', 'bullets',
851
- 'image_variants', 'prime', 'offers', 'rank_cat', 'image')},
852
- 'asin': quote['asin'], 'snapshot_date': quote['quote_on'],
853
- 'price': quote['offer_price' if own else 'buy_box_price'], 'currency': quote['currency'],
854
- 'sales_rank': quote['sales_rank'], 'display_group': quote['display_group']}
855
 
856
 
857
- def _tally(counts, rows, own, snapshots):
858
  for row in rows:
859
  counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
860
  or row['buy_box_price'] is not None)
861
  counts['ranked'] += int(row['sales_rank'] is not None)
862
  counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
863
- for key, field in FACT_COUNT_COLUMNS.items():
864
- counts['facts'][key] += sum(row.get(field) is not None and row[field] != '' for row in snapshots)
865
 
866
 
867
  def _changed(before, row):
@@ -869,47 +552,35 @@ def _changed(before, row):
869
  return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
870
 
871
 
872
- def _prepare_pools(rt, open_mirror, facts_reader, sleep, log, counts):
873
- with open_mirror() as cur:
874
- if cur is None:
875
- return None
876
- own = our_asins(cur)
877
- sources = _candidates(cur, sorted(own), newest_week(cur)) if own else []
878
- stored = _backend(rt).pool_rows(sorted(own))
879
- values = own | {r[1] for r in sources} | {r['rival_asin'] for r in stored if not r['removed']}
880
- known = _pool_facts(cur, values, own, candidate_rows=sources)
881
- missing = sorted(a for a in values if not known.product_type(a))
882
- prefetched, causes = _read_batches('facts', facts_reader, missing, sleep=sleep, log=log, counts=counts)
883
- prefetched = {a: prefetched.get(a, {}) for a in missing}
884
- with open_mirror() as cur:
885
- if cur is None:
886
- return None
887
- found = discover(rt, cur, candidate_rows=sources, prefetched=prefetched)
888
- return own, found, prefetched, causes
889
-
890
-
891
- def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run_id=None, now=None,
892
  log=None, sleep=time.sleep):
893
- """Discover, read unknown types outside the mirror, filter pools, then quote with cached facts."""
 
 
 
 
 
 
894
  _backend(rt)
895
  log = log or (lambda _message: None)
896
  now = now or datetime.now(timezone.utc)
897
  quote_on = now.date().isoformat()
898
  run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
899
  counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
900
- backoffs=0, discovered=0, week=None, pools={}, facts=dict.fromkeys(FACT_COUNT_COLUMNS, 0))
901
  start_run(rt, run_id, asin_total=0)
902
  log(f'run {run_id} started')
903
  done = 0
904
  try:
905
  try:
906
- prepared = _prepare_pools(rt, open_mirror, facts, sleep, log, counts)
907
- if prepared is None:
908
- return _finish(rt, run_id, done, counts, NO_MIRROR, log)
909
- own, found, prefetched, fact_causes = prepared
 
910
  except MirrorUnreadable as exc:
911
  return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
912
- counts.update(discovered=found['rivals'], week=found['week'], pools=found['pools'])
913
  others = sorted(set(rival_asins(rt)) - own)
914
  extra = sorted(set(wanted(rt)) - own - set(others))
915
  targets = sorted(own) + others + extra
@@ -918,23 +589,18 @@ def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run
918
  log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
919
  f'{len(extra)} wanted')
920
  before = newest_quotes(rt, targets) if targets else {}
921
- readers = {'offers': offers, 'summaries': summaries, 'facts': facts}
922
- changed, snapshots = list(found['changed']), []
923
  for block in _chunks(targets, PROGRESS_EVERY):
924
- rows, captured = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts,
925
- prefetched, fact_causes)
926
  put_quotes(rt, rows)
927
- snapshots.extend(captured)
928
- _tally(counts, rows, own, captured)
929
  changed += [row['asin'] for row in rows if _changed(before, row)]
930
  done += len(block)
931
  mark_run(rt, run_id, progress=done, counts=counts)
932
  log(f'run {run_id} progress {done}/{len(targets)}')
933
- cause = (f'Amazon could not finish every read at its rate or refused a read. '
934
- f'Retry the run within {RETRY_RUN_HOURS} hours after the rate limit delay.'
935
- if counts['throttled'] or counts['refused'] else None)
936
- result = _finish(rt, run_id, done, counts, cause, log)
937
- result.update(changed=tuple(dict.fromkeys(changed)), snapshots=snapshots)
938
  return result
939
  except Exception as exc:
940
  mark_run(rt, run_id, progress=done, finished=True, counts=counts,
@@ -947,7 +613,7 @@ def _finish(rt, run_id, done, counts, cause, log):
947
  mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
948
  log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
949
  f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
950
- return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': (), 'snapshots': []}
951
 
952
 
953
  # --------------------------------------------------------------------------- competition status and rollups
@@ -1217,70 +883,6 @@ def validate(rt, tables, plan):
1217
  'rows': len(parents), 'mismatches': bad['parent']}]
1218
 
1219
 
1220
- # --------------------------------------------------------------------------- pool validation
1221
-
1222
- def _pool_expected(cur, targets):
1223
- """Recompute raw click money independently of discovery's candidate and money queries."""
1224
- required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
1225
- 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
1226
- clicks = {'search_term', 'clicked_asin', 'click_share', 'click_share_rank', '_period_start', '_period_end'}
1227
- if not required <= _columns(cur, SQP_TABLE) or not clicks <= _columns(cur, BA_TABLE):
1228
- return {}
1229
- rows = _read(cur, f'''
1230
- WITH monthly AS (
1231
- SELECT upper(trim(CAST(asin AS VARCHAR))) AS owner,
1232
- lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS query,
1233
- TRY_CAST(end_date AS DATE) AS ends,
1234
- TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
1235
- TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS money
1236
- FROM {SQP_TABLE}
1237
- WHERE TRY_CAST(end_date AS DATE) > TRY_CAST(start_date AS DATE) + INTERVAL '7 days'
1238
- AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
1239
- newest_month AS (SELECT owner, max(ends) AS ends FROM monthly GROUP BY owner),
1240
- searches AS (
1241
- SELECT m.owner, m.query, max(m.money) AS money FROM monthly m
1242
- JOIN newest_month n ON n.owner = m.owner AND n.ends = m.ends
1243
- GROUP BY m.owner, m.query HAVING max(m.money) > 0),
1244
- report AS (
1245
- SELECT lower(trim(CAST(search_term AS VARCHAR))) AS query,
1246
- upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
1247
- TRY_CAST(click_share AS DOUBLE) AS share,
1248
- TRY_CAST(click_share_rank AS INTEGER) AS position, {_week_sql()} AS starts
1249
- FROM {BA_TABLE} WHERE {weekly_period_sql()}),
1250
- term_clicks AS (
1251
- SELECT query, rival, max(share) AS share, min(position) AS position FROM report
1252
- WHERE starts = (SELECT max(starts) FROM report) AND position BETWEEN 1 AND 3
1253
- GROUP BY query, rival)
1254
- SELECT s.owner, c.rival, sum(s.money * coalesce(c.share, 0) * 0.01),
1255
- sum(CASE WHEN c.position = 1 THEN 1 ELSE 0 END)
1256
- FROM searches s JOIN term_clicks c ON c.query = s.query
1257
- WHERE s.owner IN {_in(targets)} GROUP BY s.owner, c.rival''', targets)
1258
- return {(a, r): (float(m), int(n)) for a, r, m, n in rows}
1259
-
1260
-
1261
- def validate_pool(cur, rt, targets):
1262
- """Tie stored auto-pool money and term wins to an independent raw-source aggregation."""
1263
- targets = _asins(targets)
1264
- expected = _pool_expected(cur, targets) if targets else {}
1265
- bad_money, bad_terms, count = [], [], 0
1266
- backend = _backend(rt)
1267
- for target in targets:
1268
- for row in backend.rivals(target):
1269
- if row['source'] != 'auto':
1270
- continue
1271
- count += 1
1272
- money, terms = expected.get((target, row['rival_asin']), (0, 0))
1273
- label = f'{target} {row["rival_asin"]}'
1274
- if _differs(row['money_won'], money, 0.000001):
1275
- bad_money.append(label)
1276
- if row['terms_won'] != terms:
1277
- bad_terms.append(label)
1278
- return [{'check': 'pool money won ties to Brand Analytics and search report rows',
1279
- 'ok': not bad_money, 'rows': count, 'mismatches': bad_money},
1280
- {'check': 'pool terms won ties to Brand Analytics and search report rows',
1281
- 'ok': not bad_terms, 'rows': count, 'mismatches': bad_terms}]
1282
-
1283
-
1284
  # --------------------------------------------------------------------------- storage
1285
 
1286
  def _ddl(table):
@@ -1351,10 +953,6 @@ class PgRivalStore:
1351
  """Create every rival table and index, only on a write."""
1352
  for name in TABLES:
1353
  con.execute(_ddl(self.t(name))[name])
1354
- for name, fields in POOL_FIELDS.items():
1355
- for column, (kind, default) in fields.items():
1356
- con.execute(f'ALTER TABLE {self.t(name)} ADD COLUMN IF NOT EXISTS {column} {kind} '
1357
- f'DEFAULT {default}')
1358
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
1359
 
1360
  @contextmanager
@@ -1364,49 +962,29 @@ class PgRivalStore:
1364
  self.ensure_schema(con)
1365
  yield con
1366
 
1367
- def _select(self, con, name, columns, where, args, order='', *, include_ineligible=False):
1368
  if not self._exists(con, name):
1369
  return []
1370
- projection = columns
1371
- fields = POOL_FIELDS.get(name, {})
1372
- if fields.keys() & set(columns) or name == RIVALS:
1373
- existing = {r[0] for r in con.execute(
1374
- 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
1375
- (self.schema, name)).fetchall()}
1376
- projection = [f'{fields[c][1]} AS {c}' if c in fields and c not in existing else c for c in columns]
1377
- if name == RIVALS and 'eligible' in existing and not include_ineligible:
1378
- where = f'({where}) AND eligible = TRUE' if where else 'eligible = TRUE'
1379
- rows = con.execute(f'SELECT {", ".join(projection)} FROM {self.t(name)} WHERE tenant = %s'
1380
  f'{" AND " + where if where else ""}{order}',
1381
  (self.tenant_slug, *args)).fetchall()
1382
  return [dict(zip(columns, row)) for row in rows]
1383
 
1384
  @staticmethod
1385
  def _rival(row):
1386
- return {**row, 'removed': bool(row['removed']), 'eligible': bool(row['eligible']), 'week': _day(row['week']),
1387
- 'origin': 'added' if row['source'] == 'added' else row['origin'],
1388
  'click_share': None if row['click_share'] is None else float(row['click_share'])}
1389
 
1390
- def pool_rows(self, targets):
1391
- """Stored choices include hidden additions and removal tombstones."""
 
1392
  with self._connection() as con:
1393
- return [self._rival(row) for part in _chunks(list(targets)) for row in self._select(
1394
- con, RIVALS, RIVAL_COLUMNS, f'asin IN ({_marks(len(part))})', part,
1395
- include_ineligible=True)]
1396
-
1397
- def set_eligibility(self, rows, facts):
1398
- with self._write() as con:
1399
- for row in rows:
1400
- if row['removed']:
1401
- continue
1402
- target, rival = row['asin'], row['rival_asin']
1403
- allowed = (facts.eligible(target, rival) if facts.product_type(target) else
1404
- row['eligible'] and rival not in facts.own_asins
1405
- and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands))
1406
- if allowed != row['eligible']:
1407
- con.execute(f'UPDATE {self.t(RIVALS)} SET eligible = %s '
1408
- 'WHERE tenant = %s AND asin = %s AND rival_asin = %s',
1409
- (allowed, self.tenant_slug, target, rival))
1410
 
1411
  def replace_auto(self, targets, rows, discovery):
1412
  with self._write() as con:
@@ -1425,9 +1003,7 @@ class PgRivalStore:
1425
  + ', '.join(f'({_marks(len(cols))})' for _ in part)
1426
  + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
1427
  'discovered_at = excluded.discovered_at, cause = excluded.cause, '
1428
- 'auto_count = excluded.auto_count, month = excluded.month, '
1429
- 'keyword_ranks_on = excluded.keyword_ranks_on, '
1430
- 'product_type = excluded.product_type, held = excluded.held',
1431
  tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
1432
 
1433
  def discovery(self, targets):
@@ -1435,41 +1011,14 @@ class PgRivalStore:
1435
  rows = []
1436
  for part in _chunks(list(targets)):
1437
  rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
1438
- return {row['asin']: {**row, **{key: _day(row[key]) for key in
1439
- ('week', 'month', 'keyword_ranks_on')}} for row in rows}
1440
 
1441
  def rivals(self, target):
1442
  with self._connection() as con:
1443
- rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False))
1444
- averages = self.rank_averages([row['rival_asin'] for row in rows])
1445
- enriched = [{**self._rival(row), **averages.get(row['rival_asin'],
1446
- {'rank_average': None, 'rank_days': 0})} for row in rows]
1447
- return sorted(enriched, key=lambda row: (row['source'] != 'added', *_pool_order(row)))
1448
-
1449
- def rank_averages(self, values):
1450
- out = {}
1451
- with self._connection() as con:
1452
- if not self._exists(con, QUOTES):
1453
- return out
1454
- for part in _chunks(values):
1455
- rows = con.execute(f'''
1456
- WITH newest AS (
1457
- SELECT asin, max(quote_on) AS newest FROM {self.t(QUOTES)}
1458
- WHERE tenant = %s AND asin IN ({_marks(len(part))}) GROUP BY asin),
1459
- groups AS (
1460
- SELECT n.asin, n.newest, q.display_group FROM newest n
1461
- JOIN {self.t(QUOTES)} q ON q.asin = n.asin AND q.quote_on = n.newest
1462
- WHERE q.tenant = %s)
1463
- SELECT q.asin, CASE WHEN count(q.sales_rank) >= 7 THEN avg(q.sales_rank) END,
1464
- count(q.sales_rank)
1465
- FROM groups m JOIN {self.t(QUOTES)} q ON q.asin = m.asin
1466
- AND q.display_group = m.display_group AND q.quote_on >= m.newest - 29
1467
- AND q.quote_on <= m.newest
1468
- WHERE q.tenant = %s GROUP BY q.asin''',
1469
- (self.tenant_slug, *part, self.tenant_slug, self.tenant_slug)).fetchall()
1470
- out.update({a: dict(rank_average=None if avg is None else float(avg), rank_days=n)
1471
- for a, avg, n in rows})
1472
- return out
1473
 
1474
  def removed_count(self, target):
1475
  with self._connection() as con:
@@ -1483,15 +1032,11 @@ class PgRivalStore:
1483
  with self._write() as con:
1484
  got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
1485
  "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
1486
- "origin = 'added', money_won = 0, terms_won = 0, term_rank = NULL, "
1487
  'removed = excluded.removed, added_by = excluded.added_by, '
1488
  'updated_at = excluded.updated_at, '
1489
  'title = coalesce(excluded.title, title) '
1490
  f'RETURNING {", ".join(RIVAL_COLUMNS)}',
1491
  (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
1492
- if not dict(zip(RIVAL_COLUMNS, got))['eligible']:
1493
- con.execute(f'UPDATE {self.t(DISCOVERY)} SET week = NULL, month = NULL, keyword_ranks_on = NULL '
1494
- 'WHERE tenant = %s AND asin = %s', (self.tenant_slug, row['asin']))
1495
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1496
 
1497
  def remove(self, target, rival, user, stamp):
@@ -1505,13 +1050,15 @@ class PgRivalStore:
1505
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1506
 
1507
  def rivals_of(self, targets):
1508
- """`{asin: [rival ASIN]}` of the rivals not removed."""
1509
  out = {}
1510
  with self._connection() as con:
1511
  for part in _chunks(list(targets)):
1512
  for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
1513
  f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
1514
- ' ORDER BY asin, rival_asin'):
 
 
1515
  out.setdefault(row['asin'], []).append(row['rival_asin'])
1516
  return out
1517
 
@@ -1532,10 +1079,6 @@ class PgRivalStore:
1532
  with self._connection() as con:
1533
  if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
1534
  return own, rows
1535
- columns = {r[0] for r in con.execute(
1536
- 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
1537
- (self.schema, RIVALS)).fetchall()}
1538
- eligibility = 'AND r.eligible = TRUE ' if 'eligible' in columns else ''
1539
  for part in _chunks(list(targets)):
1540
  marks = _marks(len(part))
1541
  for target, rank, group in con.execute(
@@ -1546,8 +1089,7 @@ class PgRivalStore:
1546
  rows += [tuple(r) for r in con.execute(
1547
  f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
1548
  f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
1549
- f'WHERE r.tenant = %s AND r.removed = %s {eligibility}'
1550
- f'AND r.asin IN ({marks}) AND {newest}',
1551
  (self.tenant_slug, False, *part)).fetchall()]
1552
  return own, rows
1553
 
 
1
+ """Per-tenant PostgreSQL rivals of our ASINs, found from Brand Analytics, plus their daily quotes.
2
 
3
  Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
4
  module (verify_layering) and no App module (rule 13). Tables are created on the first write.
5
  """
6
 
7
  from contextlib import contextmanager
 
8
  from datetime import date, datetime, timedelta, timezone
9
  import json
10
  import re
 
19
  WANTED = 'amazon_rival_wanted'
20
  TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
21
 
22
+ #: R23: at most this many auto-found rivals per ASIN, by click share.
23
+ AUTO_LIMIT = 10
24
  #: Brand Analytics keeps the top 3 clicked ASINs of each search term.
25
  TOP_CLICKED = 3
26
  SOURCES = frozenset(('auto', 'added'))
 
29
  BA_TABLE = 'amzn_brand_analytics'
30
  LISTINGS_TABLE = 'amzn_listings'
31
  PRICING_TABLE = 'amzn_pricing'
 
 
 
 
 
 
32
 
33
  NO_TOP3 = "Not in any search term's top 3 clicked"
34
  NO_WEEK = 'Brand Analytics has no dated weekly reports'
 
 
 
 
 
35
  NOT_OURS = 'Not an ASIN in your Amazon listings'
36
  PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
 
37
  ASIN_RE = re.compile(r'[A-Z0-9]{10}')
38
 
 
 
 
 
 
 
 
39
  RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
40
+ 'added_by', 'updated_at')
41
+ DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count')
42
  QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
43
  'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
44
  RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
 
175
  return [r[0] for r in rows if r[0]]
176
 
177
 
178
+ def _candidates(cur, targets, week, per_asin):
179
+ """Ranked co-clicked ASINs per target, bounded in SQL to `per_asin` rows each."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  cols = _columns(cur, BA_TABLE)
181
+ share = 'TRY_CAST("click_share" AS DOUBLE)' if 'click_share' in cols else 'CAST(NULL AS DOUBLE)'
182
+ title = 'CAST("clicked_item_name" AS VARCHAR)' if 'clicked_item_name' in cols else 'CAST(NULL AS VARCHAR)'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  sql = f'''
184
+ WITH ours AS (
185
+ SELECT DISTINCT upper(trim(CAST("asin1" AS VARCHAR))) AS asin FROM {LISTINGS_TABLE}
186
+ WHERE "asin1" IS NOT NULL),
187
+ week AS (
188
+ SELECT "search_term" AS term, upper(trim(CAST("clicked_asin" AS VARCHAR))) AS clicked,
189
+ {share} AS share, {title} AS title
190
+ FROM {BA_TABLE}
191
+ WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}
192
+ AND ({weekly_period_sql()}) AND {_week_sql()} = CAST(? AS DATE)),
 
 
 
 
 
 
 
193
  hits AS (
194
+ SELECT DISTINCT term, clicked AS asin FROM week WHERE clicked IN {_in(targets)}),
195
+ cands AS (
196
+ SELECT h.asin, w.clicked AS rival, w.share, w.term, w.title
197
+ FROM hits h JOIN week w ON w.term = h.term
198
+ WHERE w.clicked <> '' AND w.clicked NOT IN (SELECT asin FROM ours)),
199
+ best AS (
200
+ SELECT asin, rival, max(share) AS share,
201
+ arg_max(term, coalesce(share, -1)) AS term, arg_max(title, coalesce(share, -1)) AS title
202
+ FROM cands GROUP BY asin, rival),
203
+ ranked AS (
204
+ SELECT *, row_number() OVER (PARTITION BY asin ORDER BY share DESC NULLS LAST, rival) AS rn
205
+ FROM best)
206
+ SELECT asin, rival, share, term, title FROM ranked WHERE rn <= ? ORDER BY asin, rn'''
207
+ return _read(cur, sql, (week, *targets, per_asin))
 
 
 
 
 
 
 
208
 
209
 
210
  # --------------------------------------------------------------------------- discovery
211
 
212
+ def discover(rt, cur, asins=None):
213
+ """Rewrite the auto rivals of our ASINs (or of `asins`) from the newest dated week.
214
+
215
+ A removed rival stays removed and a hand-added one stays added. Returns counts and `rowsRead`."""
216
  backend = _backend(rt)
217
  requested = None if asins is None else _asins(asins)
218
  targets = sorted(our_asins(cur, requested))
219
  stamp = _now()
220
  discovery = {}
221
  for missing in sorted(set(requested or ()) - set(targets)):
222
+ discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, auto_count=0)
 
223
  week = newest_week(cur) if targets else None
224
+ found = {}
225
+ rows_read = 0
226
+ if targets and week is not None:
227
+ kept = backend.kept(targets)
228
+ per_asin = AUTO_LIMIT + max((len(v) for v in kept.values()), default=0)
229
+ rows = _candidates(cur, targets, week, per_asin)
230
+ rows_read = len(rows)
231
+ for target, rival, share, term, title in rows:
232
+ if rival in kept.get(target, ()) or len(found.get(target, ())) >= AUTO_LIMIT:
233
+ continue
234
+ found.setdefault(target, []).append(dict(
235
+ asin=target, rival_asin=rival, source='auto', removed=False,
236
+ click_share=None if share is None else float(share), term=term, title=title,
237
+ week=week, added_by=None, updated_at=stamp))
238
  for target in targets:
239
+ count = len(found.get(target, ()))
240
+ cause = NO_WEEK if week is None else (None if count else NO_TOP3)
241
+ discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count)
242
+ backend.replace_auto(targets, [row for t in targets for row in found.get(t, ())], list(discovery.values()))
 
 
 
 
 
 
 
 
243
  return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
244
+ 'rowsRead': rows_read, 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
 
246
 
247
  def stale(rt, cur, target):
248
+ """True when this ASIN's discovery is missing or predates the newest dated week."""
249
  row = _backend(rt).discovery([target]).get(target)
250
  if row is None:
251
  return True
252
+ newest = newest_week(cur)
253
+ return newest is not None and (row['week'] is None or row['week'] < newest)
 
 
 
 
 
 
254
 
255
 
256
  # --------------------------------------------------------------------------- doors
257
 
258
  def rivals(rt, target):
259
+ """One ASIN's rivals: hand-added first, then auto by click share, with its discovery row."""
260
  target = asin(target)
261
  backend = _backend(rt)
262
  rows = backend.rivals(target)
263
  found = backend.discovery([target]).get(target)
264
  return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
265
  'week': found['week'] if found else None,
 
 
266
  'discoveredAt': found['discovered_at'] if found else None,
267
+ 'cause': found['cause'] if found else None}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
 
270
  def add(rt, target, rival, *, user, title=None):
 
272
  target, rival = asin(target), asin(rival, 'rival')
273
  if rival == target:
274
  raise ValueError('An ASIN cannot be its own rival.')
275
+ return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False,
276
  click_share=None, term=None, title=title, week=None,
277
+ added_by=str(user or '') or None, updated_at=_now()))
 
278
 
279
 
280
  def remove(rt, target, rival, *, user):
 
338
  return _backend(rt).runs(int(limit))
339
 
340
 
 
 
 
 
 
 
 
 
 
 
 
 
 
341
  # --------------------------------------------------------------------------- the daily quotes run
342
 
343
  #: The run row gets a progress marker after each block of this many ASINs.
 
351
  #: An unfinished run, or a finished run with a cause, is retried after this many hours.
352
  RETRY_RUN_HOURS = 3
353
 
354
+ READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'ranks': 'sales rank'}
 
355
  NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
356
  NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
357
  NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
 
449
 
450
 
451
  def _keyed(kind, got):
452
+ if kind == 'ranks':
453
  return {str(k).upper(): v for k, v in (got or {}).items()}
454
  if kind == 'offers':
455
  return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
 
489
  attempt = 0
490
  while True:
491
  try:
492
+ answers.update(_keyed(kind, fn(part)))
 
 
 
 
 
 
 
 
 
493
  break
494
  except Exception as exc: # noqa: BLE001
495
  if _throttled(exc) and attempt < THROTTLE_RETRIES:
 
505
  return answers, causes
506
 
507
 
508
+ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts):
 
509
  """One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
510
  mine = [a for a in block if a in own]
511
  others = [a for a in block if a not in own]
512
  offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
513
  summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
514
  counts=counts)
515
+ ranked, rank_causes = _read_batches('ranks', readers['ranks'], block, sleep=sleep, log=log, counts=counts)
 
 
 
 
516
  stamp = _now()
517
  rows = []
518
  for target in block:
 
531
  row.update(_summary_quote(summed[target]))
532
  else:
533
  row['buy_box_cause'] = NO_PRICE
534
+ rank = ranked.get(target) or {}
535
  value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
536
  row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
537
+ rank_cause=rank_causes.get(target) or (None if value is not None else NO_RANK))
538
  rows.append(row)
539
+ return rows
 
 
 
 
 
 
 
 
 
540
 
541
 
542
+ def _tally(counts, rows, own):
543
  for row in rows:
544
  counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
545
  or row['buy_box_price'] is not None)
546
  counts['ranked'] += int(row['sales_rank'] is not None)
547
  counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
 
 
548
 
549
 
550
  def _changed(before, row):
 
552
  return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
553
 
554
 
555
+ def run_quotes(rt, open_mirror, *, offers, summaries, ranks, seller_id=None, run_id=None, now=None,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
556
  log=None, sleep=time.sleep):
557
+ """Rediscover rivals, then quote our ASINs, every rival and every wanted ASIN for today.
558
+
559
+ `open_mirror()` is a context manager yielding a mirror cursor or None. It closes before the first
560
+ SP-API read. `offers`, `summaries` and `ranks` each take a list of at most 20 ASINs and return
561
+ `item_offers` responses, `competitive_summaries` responses and `{asin: {rank, group}}`. A raised
562
+ read is a cause on each ASIN of its batch, and a 429 backs off first. Returns the run's counts
563
+ and `changed`, the ASINs whose quote differs from their previous newest quote."""
564
  _backend(rt)
565
  log = log or (lambda _message: None)
566
  now = now or datetime.now(timezone.utc)
567
  quote_on = now.date().isoformat()
568
  run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
569
  counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
570
+ backoffs=0, discovered=0, week=None)
571
  start_run(rt, run_id, asin_total=0)
572
  log(f'run {run_id} started')
573
  done = 0
574
  try:
575
  try:
576
+ with open_mirror() as cur:
577
+ if cur is None:
578
+ return _finish(rt, run_id, done, counts, NO_MIRROR, log)
579
+ found = discover(rt, cur)
580
+ own = our_asins(cur)
581
  except MirrorUnreadable as exc:
582
  return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
583
+ counts.update(discovered=found['rivals'], week=found['week'])
584
  others = sorted(set(rival_asins(rt)) - own)
585
  extra = sorted(set(wanted(rt)) - own - set(others))
586
  targets = sorted(own) + others + extra
 
589
  log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
590
  f'{len(extra)} wanted')
591
  before = newest_quotes(rt, targets) if targets else {}
592
+ readers = {'offers': offers, 'summaries': summaries, 'ranks': ranks}
593
+ changed = []
594
  for block in _chunks(targets, PROGRESS_EVERY):
595
+ rows = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts)
 
596
  put_quotes(rt, rows)
597
+ _tally(counts, rows, own)
 
598
  changed += [row['asin'] for row in rows if _changed(before, row)]
599
  done += len(block)
600
  mark_run(rt, run_id, progress=done, counts=counts)
601
  log(f'run {run_id} progress {done}/{len(targets)}')
602
+ result = _finish(rt, run_id, done, counts, None, log)
603
+ result['changed'] = tuple(changed)
 
 
 
604
  return result
605
  except Exception as exc:
606
  mark_run(rt, run_id, progress=done, finished=True, counts=counts,
 
613
  mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
614
  log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
615
  f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
616
+ return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()}
617
 
618
 
619
  # --------------------------------------------------------------------------- competition status and rollups
 
883
  'rows': len(parents), 'mismatches': bad['parent']}]
884
 
885
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
886
  # --------------------------------------------------------------------------- storage
887
 
888
  def _ddl(table):
 
953
  """Create every rival table and index, only on a write."""
954
  for name in TABLES:
955
  con.execute(_ddl(self.t(name))[name])
 
 
 
 
956
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
957
 
958
  @contextmanager
 
962
  self.ensure_schema(con)
963
  yield con
964
 
965
+ def _select(self, con, name, columns, where, args, order=''):
966
  if not self._exists(con, name):
967
  return []
968
+ rows = con.execute(f'SELECT {", ".join(columns)} FROM {self.t(name)} WHERE tenant = %s'
 
 
 
 
 
 
 
 
 
969
  f'{" AND " + where if where else ""}{order}',
970
  (self.tenant_slug, *args)).fetchall()
971
  return [dict(zip(columns, row)) for row in rows]
972
 
973
  @staticmethod
974
  def _rival(row):
975
+ return {**row, 'removed': bool(row['removed']), 'week': _day(row['week']),
 
976
  'click_share': None if row['click_share'] is None else float(row['click_share'])}
977
 
978
+ def kept(self, targets):
979
+ """`{asin: {rival}}` of removed and hand-added rivals, which discovery must not touch."""
980
+ out = {}
981
  with self._connection() as con:
982
+ for part in _chunks(targets):
983
+ for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
984
+ f"asin IN ({_marks(len(part))}) AND (removed = %s OR source = 'added')",
985
+ (*part, True)):
986
+ out.setdefault(row['asin'], set()).add(row['rival_asin'])
987
+ return out
 
 
 
 
 
 
 
 
 
 
 
988
 
989
  def replace_auto(self, targets, rows, discovery):
990
  with self._write() as con:
 
1003
  + ', '.join(f'({_marks(len(cols))})' for _ in part)
1004
  + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
1005
  'discovered_at = excluded.discovered_at, cause = excluded.cause, '
1006
+ 'auto_count = excluded.auto_count',
 
 
1007
  tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
1008
 
1009
  def discovery(self, targets):
 
1011
  rows = []
1012
  for part in _chunks(list(targets)):
1013
  rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
1014
+ return {row['asin']: {**row, 'week': _day(row['week'])} for row in rows}
 
1015
 
1016
  def rivals(self, target):
1017
  with self._connection() as con:
1018
+ rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False),
1019
+ " ORDER BY CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
1020
+ 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, rival_asin')
1021
+ return [self._rival(row) for row in rows]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1022
 
1023
  def removed_count(self, target):
1024
  with self._connection() as con:
 
1032
  with self._write() as con:
1033
  got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
1034
  "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
 
1035
  'removed = excluded.removed, added_by = excluded.added_by, '
1036
  'updated_at = excluded.updated_at, '
1037
  'title = coalesce(excluded.title, title) '
1038
  f'RETURNING {", ".join(RIVAL_COLUMNS)}',
1039
  (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
 
 
 
1040
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1041
 
1042
  def remove(self, target, rival, user, stamp):
 
1050
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1051
 
1052
  def rivals_of(self, targets):
1053
+ """`{asin: [rival ASIN]}` of the rivals not removed, hand-added first then by click share."""
1054
  out = {}
1055
  with self._connection() as con:
1056
  for part in _chunks(list(targets)):
1057
  for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
1058
  f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
1059
+ " ORDER BY asin, CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
1060
+ 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, '
1061
+ 'rival_asin'):
1062
  out.setdefault(row['asin'], []).append(row['rival_asin'])
1063
  return out
1064
 
 
1079
  with self._connection() as con:
1080
  if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
1081
  return own, rows
 
 
 
 
1082
  for part in _chunks(list(targets)):
1083
  marks = _marks(len(part))
1084
  for target, rank, group in con.execute(
 
1089
  rows += [tuple(r) for r in con.execute(
1090
  f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
1091
  f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
1092
+ f'WHERE r.tenant = %s AND r.removed = %s AND r.asin IN ({marks}) AND {newest}',
 
1093
  (self.tenant_slug, False, *part)).fetchall()]
1094
  return own, rows
1095
 
platform/core/connector_spec.py CHANGED
@@ -843,16 +843,6 @@ AMAZON_TABLES = (
843
  identity=("rank_id",),
844
  links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
845
  ),
846
- ConnectorTableSpec(
847
- key=_AMZN + "competitor_snapshots", native="competitor_run:amazon_quotes_loop.run_tenant",
848
- source="amzn_competitor_snapshots", label="Amazon Marketing: Competitor snapshots", pinned="asin",
849
- identity=("asin", "snapshot_date"),
850
- ),
851
- ConnectorTableSpec(
852
- key=_AMZN + "competitor_images", native="competitor_run:amazon_quotes_loop.run_tenant",
853
- source="amzn_competitor_images", label="Amazon Marketing: Competitor images", pinned="asin",
854
- identity=("asin", "variant", "link"),
855
- ),
856
  )
857
 
858
  #: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
@@ -883,38 +873,6 @@ AMAZON_TABLES = AMAZON_TABLES + tuple(
883
 
884
  # Local projections supplement measured vendor columns without changing the table contract.
885
  AMAZON_FIELDS = {
886
- _AMZN + "competitor_snapshots": tuple(
887
- {"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
888
- for name, kind, label in (
889
- ("asin", "VARCHAR", "ASIN"),
890
- ("snapshot_date", "DATE", "Snapshot date"),
891
- ("product_type", "VARCHAR", "Product type"),
892
- ("brand", "VARCHAR", "Brand"),
893
- ("title", "VARCHAR", "Title"),
894
- ("bullets", "VARCHAR", "Bullet points"),
895
- ("image_variants", "VARCHAR", "Image variants"),
896
- ("price", "DOUBLE", "Price"),
897
- ("currency", "VARCHAR", "Currency"),
898
- ("prime", "BOOLEAN", "Prime"),
899
- ("offers", "BIGINT", "Offers"),
900
- ("sales_rank", "BIGINT", "Sales rank"),
901
- ("display_group", "VARCHAR", "Display group"),
902
- ("rank_cat", "VARCHAR", "Rank category"),
903
- ("image", "VARCHAR", "Image"),
904
- )
905
- ),
906
- _AMZN + "competitor_images": tuple(
907
- {"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
908
- for name, kind, label in (
909
- ("asin", "VARCHAR", "ASIN"),
910
- ("variant", "VARCHAR", "Variant"),
911
- ("link", "VARCHAR", "Link"),
912
- ("sha256", "VARCHAR", "SHA256"),
913
- ("width", "BIGINT", "Width"),
914
- ("height", "BIGINT", "Height"),
915
- ("snapshot_date", "DATE", "Snapshot date"),
916
- )
917
- ),
918
  _AMZN + "keyword_ranks": tuple(
919
  {"name": name, "type": kind, "label": label, "source": source}
920
  for name, kind, label, source in (
@@ -3637,9 +3595,6 @@ AMAZON_DEFAULT_COLUMNS = {
3637
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3638
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
3639
  "sponsored_rank_basis", "sku", "asin"),
3640
- _AMZN + "competitor_snapshots": ("asin", "snapshot_date", "brand", "title", "product_type",
3641
- "price", "currency", "prime", "offers", "sales_rank", "rank_cat"),
3642
- _AMZN + "competitor_images": ("asin", "variant", "link", "sha256", "width", "height", "snapshot_date"),
3643
  }
3644
  # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
3645
  AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
@@ -3767,7 +3722,7 @@ AMAZON_GROUPS = {
3767
  **{_AMZN + t: "Advertising" for t in (
3768
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3769
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
3770
- "sku_keywords", "keyword_ranks", "competitor_snapshots", "competitor_images")},
3771
  }
3772
  AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
3773
 
@@ -3785,15 +3740,15 @@ AMAZON_ADS_REPORTS = ("campaigns", "targeting", "advertised_products", "search_t
3785
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3786
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3787
  "advertised_products": "advertised products", "search_terms": "search terms"}
3788
- #: `native` is the Ads report type id. Unknown: the exact v1 spelling; the first credentialed
3789
- #: report settles it (PRD P17), and each id is one literal here.
3790
  _ADS_NATIVE = {
3791
- "sp": {"campaigns": "spCampaigns", "targeting": "spTargeting",
3792
- "advertised_products": "spAdvertisedProduct", "search_terms": "spSearchTerm"},
3793
- "sb": {"campaigns": "sbCampaigns", "targeting": "sbTargeting",
3794
- "advertised_products": "sbAds", "search_terms": "sbSearchTerm"},
3795
- "sd": {"campaigns": "sdCampaigns", "targeting": "sdTargeting",
3796
- "advertised_products": "sdAdvertisedProduct", "search_terms": "sdSearchTerm"},
3797
  }
3798
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3799
  "advertised_products": "asin", "search_terms": "search_term"}
@@ -3822,21 +3777,22 @@ AMAZON_ADS_TABLES = tuple(
3822
  #: which verify_connectors holds equal to the `"amazon"` registration.
3823
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3824
 
3825
- #: Days the first sync reaches back, per table: the longest window Amazon documents per ad type
3826
- #: (SP 95, SB 60, SD 65, search terms 60). Unknown: v1 retention (primer F12). The first
3827
- #: credentialed sync measures the oldest day Amazon returns.
3828
- AMAZON_ADS_BACKFILL_DAYS = {
3829
- spec.key: (60 if spec.key.endswith("_search_terms")
3830
- else {"sp": 95, "sb": 60, "sd": 65}[spec.key[len(_ADS):len(_ADS) + 2]])
3831
- for spec in AMAZON_ADS_TABLES}
3832
 
3833
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3834
  AMAZON_ADS_THROUGHPUT = (
3835
- "Amazon Reporting v1 runs one report at a time per advertiser and about 100 reports a day. "
3836
- "Sponsored Products, Brands and Display need 11 report types, and one report can take up to "
3837
- "3 hours, so one day of refresh may not finish in a day. Loopable runs the reports one at a "
3838
- "time and keeps every day it already has. If the refresh falls behind, ask Amazon Ads support "
3839
- "to raise the report limit for this advertiser.")
 
 
 
 
 
3840
 
3841
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3842
  AMAZON_ADS_NOT_OFFERED = {
@@ -3859,9 +3815,10 @@ def amazon_ads_description(key):
3859
  if unoffered:
3860
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3861
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3862
- return (f"{spec.label} by day, from Amazon Ads Reporting v1. History is kept from the day you "
3863
- f"connect, and the first sync reaches back {days} days, the longest Amazon keeps. "
3864
- f"{AMAZON_ADS_THROUGHPUT}")
 
3865
 
3866
 
3867
  def table_group(key):
 
843
  identity=("rank_id",),
844
  links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
845
  ),
 
 
 
 
 
 
 
 
 
 
846
  )
847
 
848
  #: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
 
873
 
874
  # Local projections supplement measured vendor columns without changing the table contract.
875
  AMAZON_FIELDS = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
876
  _AMZN + "keyword_ranks": tuple(
877
  {"name": name, "type": kind, "label": label, "source": source}
878
  for name, kind, label, source in (
 
3595
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3596
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
3597
  "sponsored_rank_basis", "sku", "asin"),
 
 
 
3598
  }
3599
  # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
3600
  AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
 
3722
  **{_AMZN + t: "Advertising" for t in (
3723
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3724
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
3725
+ "sku_keywords", "keyword_ranks")},
3726
  }
3727
  AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
3728
 
 
3740
  _ADS_PRODUCT = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
3741
  _ADS_REPORT_LABEL = {"campaigns": "campaigns", "targeting": "targeting",
3742
  "advertised_products": "advertised products", "search_terms": "search terms"}
3743
+ #: `native` is the Reporting v1 report group that fills the table (`amazon_ads_schema.REPORT_GROUPS`).
3744
+ #: v1 has no report types. SD search terms has no group, because v1 does not offer it.
3745
  _ADS_NATIVE = {
3746
+ "sp": {"campaigns": "campaigns", "targeting": "targeting",
3747
+ "advertised_products": "advertised_products", "search_terms": "search_terms"},
3748
+ "sb": {"campaigns": "campaigns", "targeting": "targeting",
3749
+ "advertised_products": "advertised_products_sb", "search_terms": "search_terms"},
3750
+ "sd": {"campaigns": "campaigns", "targeting": "targeting",
3751
+ "advertised_products": "advertised_products", "search_terms": "not_offered"},
3752
  }
3753
  _ADS_PINNED = {"campaigns": "campaign_name", "targeting": "targeting",
3754
  "advertised_products": "asin", "search_terms": "search_term"}
 
3777
  #: which verify_connectors holds equal to the `"amazon"` registration.
3778
  AMAZON_ADS_GROUPS = {spec.key: "Advertising" for spec in AMAZON_ADS_TABLES}
3779
 
3780
+ #: Days the first sync reaches back: Reporting v1 keeps 24 months of daily data, and standing
3781
+ #: rule 1 takes all of it.
3782
+ AMAZON_ADS_BACKFILL_DAYS = {spec.key: 730 for spec in AMAZON_ADS_TABLES}
 
 
 
 
3783
 
3784
  #: Standing rule 1: the throughput limit that cannot be removed, with cause and fix. On screen.
3785
  AMAZON_ADS_THROUGHPUT = (
3786
+ "Amazon Ads runs one report at a time for each Loopable app login (its client ID) and about "
3787
+ "100 reports a day, shared by every advertiser on that login. Sponsored Products, Brands and "
3788
+ "Display need 5 reports for each refresh, and one report can take up to 3 hours, so one day of "
3789
+ "refresh may not finish in a day. Loopable runs the reports one at a time and keeps every day "
3790
+ "it already has. If the refresh falls behind, ask Amazon Ads support to raise the report limit "
3791
+ "for this client ID.")
3792
+
3793
+ #: On screen: the attribution window Amazon applies to Sponsored Products seller metrics.
3794
+ AMAZON_ADS_SP_ATTRIBUTION = ("Sponsored Products sales, orders and units count purchases within 7 "
3795
+ "days of a click, Amazon's default for sellers.")
3796
 
3797
  #: Standing rule 1: the report v1 does not offer, with cause and fix. On screen.
3798
  AMAZON_ADS_NOT_OFFERED = {
 
3815
  if unoffered:
3816
  return f"{unoffered} {AMAZON_ADS_THROUGHPUT}"
3817
  days = AMAZON_ADS_BACKFILL_DAYS[spec.key]
3818
+ attribution = AMAZON_ADS_SP_ATTRIBUTION + " " if spec.key.startswith(_ADS + "sp_") else ""
3819
+ return (f"{spec.label} by day, from Amazon Ads Reporting v1. {attribution}History is kept from "
3820
+ f"the day you connect, and the first sync reaches back {days} days, the longest Amazon "
3821
+ f"keeps. {AMAZON_ADS_THROUGHPUT}")
3822
 
3823
 
3824
  def table_group(key):
platform/core/external_tools.py CHANGED
@@ -267,7 +267,7 @@ _APPLICATION = {
267
  "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
268
  "amazon_operator", "restock_planner", "reimbursements", "aged_stock",
269
  "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
270
- "amazon_dashboard", "amazon_profit", "alerts", "reports"]},
271
  "fieldMap": {"type": "object", "additionalProperties": _STRING},
272
  "navPlacement": _object({"section": {"const": "app"},
273
  "order": {"type": "integer", "minimum": 0, "maximum": 999}},
 
267
  "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
268
  "amazon_operator", "restock_planner", "reimbursements", "aged_stock",
269
  "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
270
+ "amazon_dashboard", "amazon_profit", "alerts"]},
271
  "fieldMap": {"type": "object", "additionalProperties": _STRING},
272
  "navPlacement": _object({"section": {"const": "app"},
273
  "order": {"type": "integer", "minimum": 0, "maximum": 999}},
platform/core/shares.py CHANGED
@@ -80,7 +80,7 @@ SHARES_KEY = 'object_shares'
80
  #: here is this tuple member. That is the point of one registry: the new object's WALL is written
81
  #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
82
  #: permission decision in a fourth file ([[one-evaluator-per-question]]).
83
- KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook', 'report')
84
 
85
  #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
86
  #: Spelled as a single character so it can never collide with a username (usernames are lower-case
@@ -692,32 +692,13 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
692
  return repaired[0]
693
 
694
 
695
- def _synthetic_qa_publication(target, table_keys):
696
- """Identify private scratch publications outside the legacy account plane."""
697
- from core import user_tables, users
698
-
699
- qa = users.qa_identity(users._QA_USERNAME)
700
- if (not qa or not table_keys or getattr(target, 'key', None) != qa['tenant']
701
- or getattr(getattr(target, 'tenant', None), 'key', None) != qa['tenant']):
702
- return False
703
- for key in table_keys:
704
- if not isinstance(key, str) or not key.startswith(user_tables.KEY_PREFIX):
705
- return False
706
- definition = user_tables.get_definition(key, st=target)
707
- if (not isinstance(definition, dict) or definition.get('key') != key
708
- or definition.get('createdBy') != qa['username']
709
- or definition.get('source') != user_tables.BLANK_SOURCE):
710
- return False
711
- return True
712
-
713
-
714
  def _strict_accounts():
715
  """Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
716
  from core import account_rows, users
717
  # On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
718
  if account_rows._pg_source(users.store):
719
  return dict(account_rows._all(users.store))
720
- registry = users.registry()
721
  if account_rows.is_marker(registry):
722
  raise RuntimeError('the account registry is unavailable')
723
  return registry
@@ -727,9 +708,6 @@ def repair_published_database_grants(st=None, *, table_keys):
727
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
728
  from core import freeze_notice
729
  try:
730
- table_keys = tuple(table_keys)
731
- if _synthetic_qa_publication(_st(st), table_keys):
732
- return 0
733
  return reconcile_database_grants(st, table_keys=table_keys)
734
  except freeze_notice.DataInputFrozen:
735
  raise
 
80
  #: here is this tuple member. That is the point of one registry: the new object's WALL is written
81
  #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
82
  #: permission decision in a fourth file ([[one-evaluator-per-question]]).
83
+ KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook')
84
 
85
  #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
86
  #: Spelled as a single character so it can never collide with a username (usernames are lower-case
 
692
  return repaired[0]
693
 
694
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
695
  def _strict_accounts():
696
  """Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
697
  from core import account_rows, users
698
  # On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
699
  if account_rows._pg_source(users.store):
700
  return dict(account_rows._all(users.store))
701
+ registry = users.store._read_strict('users')
702
  if account_rows.is_marker(registry):
703
  raise RuntimeError('the account registry is unavailable')
704
  return registry
 
708
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
709
  from core import freeze_notice
710
  try:
 
 
 
711
  return reconcile_database_grants(st, table_keys=table_keys)
712
  except freeze_notice.DataInputFrozen:
713
  raise
platform/harness/amazon_ads_client.py CHANGED
@@ -4,7 +4,7 @@
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
- reports create, status and row pages on Reporting v1 (R8)
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
@@ -17,8 +17,11 @@ harness module never imports `modules` at load). A missing approval source refus
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
- stored with the credential, so the client needs no profile listing call.
 
 
21
  """
 
22
  import importlib
23
  import json
24
  import logging
@@ -31,6 +34,7 @@ import urllib.parse
31
  import urllib.request
32
  from contextlib import contextmanager
33
  from datetime import date, datetime, timedelta, timezone
 
34
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
35
 
36
  KEY = "amazon_ads"
@@ -46,22 +50,28 @@ DEFAULT_REGION = "na"
46
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
47
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
48
 
49
- #: Reporting v1 limits (primer F11, F12): one report in flight per advertiser, about 100 a day,
50
- #: up to 3 hours per report, at most 31 days per request.
51
  REPORT_CONCURRENCY = 1
52
  REPORTS_PER_DAY = 100
53
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
54
- MAX_DAYS_PER_REQUEST = 31
55
- REPORT_POLL_SECONDS = 30.0
 
 
56
  _TIMEOUT = 60
57
 
58
- #: The only non-write calls: Reporting v1 create, status and row pages. Unknown: the exact v1
59
- #: paths; the first credentialed call settles them, and each is one literal here.
60
  REPORT_OPERATIONS = {
61
- "createReport": ("POST", "reporting/v1/reports"),
62
- "getReport": ("GET", "reporting/v1/reports/{reportId}"),
63
- "getReportRows": ("GET", "reporting/v1/reports/{reportId}/rows"),
64
  }
 
 
 
 
65
 
66
  # Subscription management never admits an advertising write.
67
  STREAM_OPERATIONS = {
@@ -208,7 +218,15 @@ class AdsWriteRefused(AdsError):
208
  f"the named writes: bid, budget, negative keyword and scheduled change.")
209
 
210
 
211
- class AdsReportUnavailable(AdsError):
 
 
 
 
 
 
 
 
212
  """Amazon does not offer this report to this advertiser."""
213
 
214
 
@@ -225,25 +243,10 @@ def fixture_mode():
225
  return os.environ.get(FIXTURE_ENV) == "1"
226
 
227
 
228
- def _template(path):
229
- return tuple(path.strip("/").split("/"))
230
-
231
-
232
- def _matches(template, path):
233
- parts = str(path or "").strip("/").split("/")
234
- if len(parts) != len(template):
235
- return False
236
- return all((bool(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part)))
237
- if seg.startswith("{") and seg.endswith("}") else part == seg
238
- for part, seg in zip(parts, template))
239
-
240
-
241
  def report_operation(method, path):
242
- """The Reporting v1 operation name for one call, or None."""
243
- for name, (verb, tmpl) in REPORT_OPERATIONS.items():
244
- if str(method).upper() == verb and _matches(_template(tmpl), path):
245
- return name
246
- return None
247
 
248
 
249
  class _Prepared:
@@ -503,12 +506,17 @@ def _send(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
503
  if not _admitted(prepared):
504
  raise AdsWriteRefused("the request did not come from the request builder")
505
  if fixture_mode():
506
- return FIXTURE.answer(prepared, tenant_key, profile_id)
 
507
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
508
 
509
 
510
  # --------------------------------------------------------------------------- live transport
511
 
 
 
 
 
512
  def credential(tenant_key, rt=None, connection_id=None):
513
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
514
  try:
@@ -538,11 +546,50 @@ def _token(fields):
538
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
539
 
540
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
541
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
542
  fields = credential(tenant_key, rt, connection_id=connection_id)
543
  if fields is None:
544
- raise AdsError("No Amazon Ads credential is stored for this tenant. "
545
- "Connect Amazon Ads under Amazon Ads Setup.")
546
  if prepared.operation in STREAM_OPERATIONS and (
547
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
548
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
@@ -551,13 +598,7 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
551
  url = f"{host}/{prepared.path}"
552
  if prepared.query:
553
  url += "?" + urllib.parse.urlencode(prepared.query)
554
- headers = {"Authorization": f"Bearer {_token(fields)}",
555
- "Amazon-Advertising-API-ClientId": fields["client_id"],
556
- "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
- "Content-Type": "application/json", "Accept": "application/json"}
558
- if prepared.operation in STREAM_OPERATIONS:
559
- headers["Content-Type"] = STREAM_CONTENT_TYPE
560
- headers["Accept"] = STREAM_CONTENT_TYPE
561
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
562
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
563
  try:
@@ -566,12 +607,10 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
566
  except urllib.error.HTTPError as exc:
567
  try:
568
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
569
- message = str(detail.get("details") or detail.get("message") or "")[:200]
570
- except Exception: # noqa: BLE001
571
- message = ""
572
- if exc.code == 400 and re.search(r"not (?:supported|offered|available)", message, re.I):
573
- raise AdsReportUnavailable(f"Amazon does not offer this report: {message}") from None
574
- raise AdsError(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}: {message}") from None
575
  except Exception as exc: # noqa: BLE001
576
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
577
 
@@ -580,89 +619,276 @@ def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None)
580
 
581
  _REPORT_LOCKS = {}
582
  _REPORT_LOCKS_GUARD = threading.Lock()
 
 
 
 
 
 
 
 
 
 
 
 
583
 
584
 
585
- def _report_lock(tenant_key, profile_id):
 
586
  with _REPORT_LOCKS_GUARD:
587
- return _REPORT_LOCKS.setdefault((tenant_key, str(profile_id or "")), threading.Lock())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
588
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
589
 
590
- def run_report(tenant_key, table, start, end, *, rt=None, profile_id=None, connection_id=None, sleep=time.sleep,
591
- now=time.monotonic, max_wait=None, report_id=None):
592
- """Create one Reporting v1 report, wait for it, and read every row page. -> (report_id, rows).
593
 
594
- `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without createReport.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
595
  """
596
  from harness import amazon_ads_schema as schema
597
- spec = schema.TABLE_SPECS[schema.short_name(table)]
598
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
599
- raise ValueError("a report window is at most 31 days")
600
- body = {"reportTypeId": spec["report_type"], "adProduct": spec["ad_type"].upper(),
601
- "startDate": start.isoformat(), "endDate": end.isoformat(),
602
- "timeUnit": "DAILY", "columns": list(schema.vendor_columns(table)), "format": "JSON"}
603
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
604
- # One report in flight per advertiser (F11): the lock serialises this process's reports.
605
- with _report_lock(tenant_key, profile_id):
606
  if report_id is None:
607
- created = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1], body=body,
608
- rt=rt, profile_id=profile_id, connection_id=connection_id)
609
- report_id = str(created.get("reportId") or "")
610
- else:
611
- report_id = str(report_id)
612
- if not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", report_id):
613
- raise AdsError("Amazon Ads returned no report id")
614
- started = now()
615
- status_path = REPORT_OPERATIONS["getReport"][1].replace("{reportId}", report_id)
616
- while True:
617
- status = request(tenant_key, "GET", status_path, rt=rt, profile_id=profile_id, connection_id=connection_id)
618
- state = str(status.get("status") or "").upper()
619
- if state == "COMPLETED":
620
- break
621
- if state in ("FAILED", "FAILURE", "CANCELLED"):
622
- reason = str(status.get("failureReason") or state)[:200]
623
- if re.search(r"not (?:supported|offered|available)", reason, re.I):
624
- raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
625
- raise AdsError(f"Amazon Ads report {report_id} ended {state}: {reason}")
626
- elapsed = now() - started
627
- if elapsed > REPORT_MAX_WAIT_SECONDS:
628
- raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
629
- if limit is not None and elapsed >= limit:
630
- raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
631
- pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
632
- sleep(pause if limit is None else min(pause, limit - elapsed))
633
- rows, token = [], None
634
- rows_path = REPORT_OPERATIONS["getReportRows"][1].replace("{reportId}", report_id)
635
- while True:
636
- page = request(tenant_key, "GET", rows_path, query={"nextToken": token} if token else None,
637
- rt=rt, profile_id=profile_id, connection_id=connection_id)
638
- rows.extend(page.get("rows") or [])
639
- token = page.get("nextToken")
640
- if not token:
641
- return report_id, rows
642
 
643
 
644
  # --------------------------------------------------------------------------- recording fixture
645
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
646
  class _Fixture:
647
- """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to. Records each call; answers reports
648
- with deterministic rows, two report polls, and paged rows."""
649
 
650
- PAGE_SIZE = 25
 
 
 
 
 
651
 
652
  def __init__(self):
653
  self.reset()
654
 
655
  def reset(self):
656
- self.calls, self.reports = [], {}
657
- self.subscriptions = {}
658
  self.in_flight = self.max_in_flight = 0
659
- self.drop_days, self.unsupported = set(), set()
 
660
  self.responder = None
661
 
662
- def answer(self, prepared, tenant_key, profile_id):
 
 
 
 
663
  self.calls.append({"method": prepared.method, "path": prepared.path,
664
  "operation": prepared.operation, "body": prepared.body,
665
- "query": dict(prepared.query), "tenant": tenant_key,
666
  "named": dict(prepared.named) if prepared.named else None})
667
  if self.responder is not None:
668
  return self.responder(prepared)
@@ -678,88 +904,174 @@ class _Fixture:
678
  if prepared.named is not None:
679
  items = next(iter(prepared.body.values()))
680
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
 
 
681
  if prepared.operation == "createReport":
682
- report_id = f"fx{len(self.reports) + 1:05d}"
683
- self.in_flight += 1
684
- self.max_in_flight = max(self.max_in_flight, self.in_flight)
685
- self.reports[report_id] = {"body": prepared.body, "polls": 0, "rows": None}
686
- return {"reportId": report_id, "status": "PENDING"}
687
- report_id = prepared.path.split("/")[3]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  report = self.reports[report_id]
689
- if prepared.operation == "getReport":
690
- report["polls"] += 1
691
- if report["body"]["reportTypeId"] in self.unsupported:
692
- self.in_flight -= 1
693
- return {"status": "FAILED", "failureReason": "Report type is not supported for this advertiser"}
694
- if report["polls"] < 2:
695
- return {"status": "PROCESSING"}
696
- return {"status": "COMPLETED"}
697
- if report["rows"] is None:
698
- report["rows"] = fixture_rows(report["body"], drop_days=self.drop_days)
699
- start = int(prepared.query.get("nextToken") or 0)
700
- page = report["rows"][start:start + self.PAGE_SIZE]
701
- end = start + len(page)
702
- if end >= len(report["rows"]):
 
 
 
 
 
 
703
  self.in_flight -= 1
704
- return {"rows": page}
705
- return {"rows": page, "nextToken": str(end)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
706
 
707
 
708
  def fixture_rows(body, *, drop_days=()):
709
- """Deterministic Ads report rows for one request body, every day in its window."""
710
- report_type, columns = body["reportTypeId"], body["columns"]
711
- start, end = date.fromisoformat(body["startDate"]), date.fromisoformat(body["endDate"])
712
- ad = report_type[:2]
713
- out, day = [], start
714
- while day <= end:
715
- if day.isoformat() not in drop_days:
716
- for c in (1, 2):
717
- for g in ((0,) if report_type.endswith("Campaigns") else (1, 2)):
718
- n = day.toordinal() % 7 + c + g
719
- row = {
720
- "date": day.isoformat(), "campaignId": f"{ad.encode().hex()}10{c}",
721
- "campaignName": f"{ad.upper()} campaign {c}", "campaignStatus": "ENABLED",
722
- "campaignBudgetAmount": 25.0 * c, "campaignBudgetCurrencyCode": "USD",
723
- "adGroupId": f"20{c}{g}", "adGroupName": f"Ad group {c}.{g}",
724
- "keywordId": f"30{c}{g}", "targetingId": f"30{c}{g}",
725
- "targeting": f"term {c}{g}", "targetingExpression": f"asin is B00FX{c}{g}",
726
- "matchType": "exact", "keywordBid": 0.5 + g / 10,
727
- "adId": f"40{c}{g}", "advertisedSku": f"SKU-{c}{g}",
728
- "advertisedAsin": f"B00FX0{c}{g}0", "promotedSku": f"SKU-{c}{g}",
729
- "promotedAsin": f"B00FX0{c}{g}0", "searchTerm": f"search {c}{g}",
730
- "impressions": 100 * n, "clicks": 3 * n, "cost": round(1.25 * n, 2),
731
- "sales": round(9.5 * n, 2), "sales14d": round(9.5 * n, 2),
732
- "purchases": n, "purchases14d": n, "unitsSold": n, "unitsSoldClicks14d": n,
733
- }
734
- out.append({k: row.get(k) for k in columns})
735
- day += timedelta(days=1)
736
  return out
737
 
738
 
739
  FIXTURE = _Fixture()
740
 
741
 
 
 
 
 
 
 
742
  @contextmanager
743
  def recording_fixture():
744
- """Route every call to a fresh recording fixture for the block. -> the fixture."""
745
  previous = os.environ.get(FIXTURE_ENV)
746
  os.environ[FIXTURE_ENV] = "1"
747
  FIXTURE.reset()
 
748
  try:
749
  yield FIXTURE
750
  finally:
751
  FIXTURE.reset()
 
752
  if previous is None:
753
  os.environ.pop(FIXTURE_ENV, None)
754
  else:
755
  os.environ[FIXTURE_ENV] = previous
756
 
757
 
758
- __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "NAMED_WRITES",
759
- "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
760
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
761
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
762
- "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsReportUnavailable",
763
- "AdsReportPending", "fixture_mode", "build_request", "report_operation",
764
- "send_named_write", "request", "credential", "run_report", "FIXTURE",
765
- "recording_fixture", "fixture_rows"]
 
4
  read-only scope: `advertising::campaign_management` can change any campaign (primer F10). So the
5
  refusal lives here, in the request builder, before any credential read, token exchange or socket:
6
 
7
+ reports Reporting v1: create, retrieve, the advertiser account query, and the report files
8
  streams create and list subscriptions for the dedicated V1 destination
9
  named writes `send_named_write(tenant_key, kind, payload, *, approval_id)`, kind in
10
  `bid`, `budget`, `negative_keyword`, `scheduled_change`, each operation one path constant
 
17
  made until Royal's Direct Advertiser credential arrives (R7, R17).
18
 
19
  NA host `advertising-api.amazon.com`, one profile id per marketplace (F13). The profile id is
20
+ stored with the credential. v1 names the account by `advertiserAccountId`, which the client looks
21
+ up once per connection from `query/advertiserAccounts`. Every path, header and field comes from
22
+ Amazon's saved contracts in `AutoWiki/research/amazon-ads-v1-schema/`.
23
  """
24
+ import gzip
25
  import importlib
26
  import json
27
  import logging
 
34
  import urllib.request
35
  from contextlib import contextmanager
36
  from datetime import date, datetime, timedelta, timezone
37
+ from typing import NamedTuple
38
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
39
 
40
  KEY = "amazon_ads"
 
50
  LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
51
  CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "profile_id")
52
 
53
+ #: Reporting v1 limits (limits.md, dates.md): one report in flight per client ID, about 100 a day,
54
+ #: up to 3 hours per report, at most 120 days per request.
55
  REPORT_CONCURRENCY = 1
56
  REPORTS_PER_DAY = 100
57
  REPORT_MAX_WAIT_SECONDS = 3 * 3600
58
+ MAX_DAYS_PER_REQUEST = 120
59
+ REPORT_POLL_SECONDS = 60.0
60
+ #: Amazon's error text reaches the sync report verbatim, up to this many characters.
61
+ ERROR_TEXT_LIMIT = 600
62
  _TIMEOUT = 60
63
 
64
+ #: The only non-write calls: the Reporting v1 operations, each a POST to one literal path from the
65
+ #: Reports and AdvertiserAccounts contracts. Every other `adsApi` path refuses, a GET included.
66
  REPORT_OPERATIONS = {
67
+ "createReport": ("POST", "adsApi/v1/create/reports"),
68
+ "retrieveReport": ("POST", "adsApi/v1/retrieve/reports"),
69
+ "queryAdvertiserAccounts": ("POST", "adsApi/v1/query/advertiserAccounts"),
70
  }
71
+ #: The contract names `Amazon-Ads-ClientId`. The limits page names the older header. v1 sends both.
72
+ V1_CLIENT_HEADERS = ("Amazon-Ads-ClientId", "Amazon-Advertising-API-ClientId")
73
+ #: A report file is a presigned S3 URL: https, on this host suffix, GET with no Ads header.
74
+ REPORT_FILE_HOST_SUFFIX = ".amazonaws.com"
75
 
76
  # Subscription management never admits an advertising write.
77
  STREAM_OPERATIONS = {
 
218
  f"the named writes: bid, budget, negative keyword and scheduled change.")
219
 
220
 
221
+ class AdsRejected(AdsError):
222
+ """Amazon answered an error. `errors` keeps each entry's code, message and fieldLocation."""
223
+
224
+ def __init__(self, message, errors=()):
225
+ super().__init__(message)
226
+ self.errors = list(errors)
227
+
228
+
229
+ class AdsReportUnavailable(AdsRejected):
230
  """Amazon does not offer this report to this advertiser."""
231
 
232
 
 
243
  return os.environ.get(FIXTURE_ENV) == "1"
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def report_operation(method, path):
247
+ """The Reporting v1 operation name for one call, or None. A path matches only its literal."""
248
+ return next((name for name, (verb, endpoint) in REPORT_OPERATIONS.items()
249
+ if str(method).upper() == verb and str(path or "").strip("/") == endpoint), None)
 
 
250
 
251
 
252
  class _Prepared:
 
506
  if not _admitted(prepared):
507
  raise AdsWriteRefused("the request did not come from the request builder")
508
  if fixture_mode():
509
+ headers = _headers(prepared, FIXTURE.credential(tenant_key), FIXTURE_TOKEN, profile_id)
510
+ return FIXTURE.answer(prepared, tenant_key, profile_id, headers)
511
  return _live(prepared, tenant_key, rt=rt, profile_id=profile_id, connection_id=connection_id)
512
 
513
 
514
  # --------------------------------------------------------------------------- live transport
515
 
516
+ _NO_CREDENTIAL = ("No Amazon Ads credential is stored for this tenant. "
517
+ "Connect Amazon Ads under Amazon Ads Setup.")
518
+
519
+
520
  def credential(tenant_key, rt=None, connection_id=None):
521
  """The tenant's stored Amazon Ads credential fields, or None. The keychain is the only source."""
522
  try:
 
546
  raise AdsError(f"Amazon Ads sign in failed ({type(exc).__name__})") from None
547
 
548
 
549
+ def _headers(prepared, fields, token, profile_id=None):
550
+ """The headers one admitted call sends. A v1 call names its account in the body, not a scope."""
551
+ if prepared.operation in REPORT_OPERATIONS:
552
+ return {"Authorization": f"Bearer {token}", **dict.fromkeys(V1_CLIENT_HEADERS, fields["client_id"]),
553
+ "Content-Type": "application/json", "Accept": "application/json"}
554
+ headers = {"Authorization": f"Bearer {token}",
555
+ "Amazon-Advertising-API-ClientId": fields["client_id"],
556
+ "Amazon-Advertising-API-Scope": str(profile_id or fields["profile_id"]),
557
+ "Content-Type": "application/json", "Accept": "application/json"}
558
+ if prepared.operation in STREAM_OPERATIONS:
559
+ headers["Content-Type"] = STREAM_CONTENT_TYPE
560
+ headers["Accept"] = STREAM_CONTENT_TYPE
561
+ return headers
562
+
563
+
564
+ def _errors(detail):
565
+ """Amazon's error entries from an error body, each with its code, message and fieldLocation."""
566
+ if not isinstance(detail, dict):
567
+ return []
568
+ entries = detail["errors"] if isinstance(detail.get("errors"), list) else [detail]
569
+ return [{"code": str(e.get("code") or ""), "message": str(e.get("details") or e.get("message") or ""),
570
+ "fieldLocation": str(e.get("fieldLocation") or "")}
571
+ for e in entries if isinstance(e, dict) and (e.get("code") or e.get("details") or e.get("message"))]
572
+
573
+
574
+ def _error_text(errors):
575
+ """Amazon's errors as one line a person reads, verbatim, cut to `ERROR_TEXT_LIMIT`."""
576
+ return " ".join(" ".join(filter(None, (e["code"] + ":" if e["code"] else "", e["message"],
577
+ f"(field {e['fieldLocation']})" if e["fieldLocation"] else "")))
578
+ for e in errors)[:ERROR_TEXT_LIMIT]
579
+
580
+
581
+ def _rejection(prefix, errors, *, may_be_unoffered=True):
582
+ """The exception for Amazon's errors: not offered when Amazon says so, else rejected."""
583
+ text = _error_text(errors)
584
+ if may_be_unoffered and re.search(r"not (?:supported|offered|available)", text, re.I):
585
+ return AdsReportUnavailable(f"Amazon does not offer this report: {text}", errors)
586
+ return AdsRejected(f"{prefix}: {text}" if text else prefix, errors)
587
+
588
+
589
  def _live(prepared, tenant_key, *, rt=None, profile_id=None, connection_id=None):
590
  fields = credential(tenant_key, rt, connection_id=connection_id)
591
  if fields is None:
592
+ raise AdsError(_NO_CREDENTIAL)
 
593
  if prepared.operation in STREAM_OPERATIONS and (
594
  str(profile_id or fields["profile_id"]) != str(fields["profile_id"])
595
  or str(fields.get("region") or DEFAULT_REGION).lower() != "na"):
 
598
  url = f"{host}/{prepared.path}"
599
  if prepared.query:
600
  url += "?" + urllib.parse.urlencode(prepared.query)
601
+ headers = _headers(prepared, fields, _token(fields), profile_id)
 
 
 
 
 
 
602
  data = json.dumps(prepared.body).encode() if prepared.body is not None else None
603
  req = urllib.request.Request(url, data=data, method=prepared.method, headers=headers)
604
  try:
 
607
  except urllib.error.HTTPError as exc:
608
  try:
609
  detail = json.loads(exc.read().decode("utf-8", "replace") or "{}")
610
+ except Exception: # noqa: BLE001 an unreadable body still raises its status
611
+ detail = {}
612
+ raise _rejection(f"Amazon Ads answered HTTP {exc.code} on {prepared.operation}", _errors(detail),
613
+ may_be_unoffered=exc.code == 400) from None
 
 
614
  except Exception as exc: # noqa: BLE001
615
  raise AdsError(f"Amazon Ads could not be reached ({type(exc).__name__})") from None
616
 
 
619
 
620
  _REPORT_LOCKS = {}
621
  _REPORT_LOCKS_GUARD = threading.Lock()
622
+ #: Process caches: the advertiser account per connection, the optional fields Amazon refused per
623
+ #: (client id, group) with Amazon's text, and the report file URLs a retrieve returned.
624
+ _ACCOUNTS = {}
625
+ _REFUSED = {}
626
+ _ISSUED_FILES = set()
627
+
628
+
629
+ class GroupReport(NamedTuple):
630
+ """One finished group report: its id, flat rows per member table, and notes a person reads."""
631
+ report_id: str
632
+ rows: dict
633
+ notes: tuple
634
 
635
 
636
+ def _report_lock(client_id):
637
+ """One lock per client ID: Amazon runs one report at a time for each app login."""
638
  with _REPORT_LOCKS_GUARD:
639
+ return _REPORT_LOCKS.setdefault(str(client_id), threading.Lock())
640
+
641
+
642
+ def _report_credential(tenant_key, rt, connection_id):
643
+ """The credential a report runs under: its client id keys the lock, its profile names the account."""
644
+ if fixture_mode():
645
+ return FIXTURE.credential(tenant_key)
646
+ fields = credential(tenant_key, rt, connection_id=connection_id)
647
+ if fields is None:
648
+ raise AdsError(_NO_CREDENTIAL)
649
+ return fields
650
+
651
+
652
+ def _lists_profile(account, profile_id):
653
+ return (isinstance(account, dict) and bool(account.get("advertiserAccountId"))
654
+ and any(isinstance(alt, dict) and str(alt.get("profileId")) == str(profile_id)
655
+ for alt in account.get("alternateIds") or ()))
656
+
657
 
658
+ def _advertiser_account(tenant_key, *, rt, profile_id, connection_id):
659
+ """The v1 `advertiserAccountId` whose alternate ids carry the stored profile id, cached per connection."""
660
+ key, token = (tenant_key, connection_id or "", str(profile_id)), None
661
+ while key not in _ACCOUNTS:
662
+ page = request(tenant_key, "POST", REPORT_OPERATIONS["queryAdvertiserAccounts"][1],
663
+ body={"maxResults": 100, **({"nextToken": token} if token else {})},
664
+ rt=rt, profile_id=profile_id, connection_id=connection_id)
665
+ match = next((a for a in page.get("advertiserAccounts") or () if _lists_profile(a, profile_id)), None)
666
+ token = page.get("nextToken")
667
+ if match is not None:
668
+ _ACCOUNTS[key] = str(match["advertiserAccountId"])
669
+ elif not token:
670
+ raise AdsError(f"No Amazon Ads advertiser account lists profile {profile_id}. "
671
+ f"Check the profile ID on this Amazon Ads connection.")
672
+ return _ACCOUNTS[key]
673
 
 
 
 
674
 
675
+ def _create_body(group, start, end, account, fields):
676
+ """A CreateReportRequest for one group and window, filtered to the group's ad products."""
677
+ from harness import amazon_ads_schema as schema
678
+ return {"accessRequestedAccounts": [{"advertiserAccountId": account}],
679
+ "reports": [{"format": "GZIP_JSON",
680
+ "periods": [{"datePeriod": {"startDate": start.isoformat(), "endDate": end.isoformat()}}],
681
+ "query": {"fields": list(fields),
682
+ "filter": {"on": {"field": schema.AD_PRODUCT_FIELD, "comparisonOperator": "IN",
683
+ "not": False, "values": list(group.tables)}}}}]}
684
+
685
+
686
+ def _first_report(reply, operation):
687
+ """The one report a 207 multi-status answer carries, or its errors raised verbatim."""
688
+ errors = [e for entry in reply.get("error") or () if isinstance(entry, dict)
689
+ for e in _errors({"errors": entry.get("errors") or []})]
690
+ if errors or reply.get("error"):
691
+ raise _rejection(f"Amazon Ads refused {operation}", errors)
692
+ for entry in reply.get("success") or ():
693
+ report = entry.get("report") if isinstance(entry, dict) else None
694
+ if isinstance(report, dict) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", str(report.get("reportId") or "")):
695
+ return report
696
+ raise AdsError(f"Amazon Ads returned no report on {operation}")
697
+
698
+
699
+ def _named_fields(exc, fields):
700
+ """The requested fields an error names, by id in its text or by index in its field location."""
701
+ text = " ".join([str(exc)] + [e["fieldLocation"] for e in exc.errors])
702
+ named = {f for f in fields if re.search(rf"(?<![\w.]){re.escape(f)}(?!\w)", text)}
703
+ named.update(fields[int(i)] for i in re.findall(r"fields\[(\d+)\]", text) if int(i) < len(fields))
704
+ return named
705
+
706
+
707
+ def _create(tenant_key, group, start, end, account, refused, opts):
708
+ """Create one group report. An error naming only optional fields drops them and creates again."""
709
+ while True:
710
+ fields = group.fields(drop=refused)
711
+ try:
712
+ reply = request(tenant_key, "POST", REPORT_OPERATIONS["createReport"][1],
713
+ body=_create_body(group, start, end, account, fields), **opts)
714
+ return _first_report(reply, "createReport")
715
+ except AdsRejected as exc:
716
+ named = _named_fields(exc, fields)
717
+ if not named or named & set(group.core):
718
+ raise
719
+ refused.update(dict.fromkeys(sorted(named), _error_text(exc.errors) or str(exc)))
720
+
721
+
722
+ def _await(tenant_key, report_id, opts, *, sleep, now, limit):
723
+ """Poll retrieve about once a minute until the report completes. -> its report file URLs."""
724
+ started = now()
725
+ while True:
726
+ report = _first_report(request(tenant_key, "POST", REPORT_OPERATIONS["retrieveReport"][1],
727
+ body={"reportIds": [report_id]}, **opts), "retrieveReport")
728
+ state = str(report.get("status") or "").upper()
729
+ if state == "COMPLETED":
730
+ urls = [str(part.get("url") or "") for part in report.get("completedReportParts") or ()
731
+ if isinstance(part, dict)]
732
+ _ISSUED_FILES.update(urls)
733
+ return urls
734
+ if state == "FAILED":
735
+ reason = " ".join(filter(None, (str(report.get("failureCode") or ""),
736
+ str(report.get("failureReason") or ""))))[:ERROR_TEXT_LIMIT] or state
737
+ if re.search(r"not (?:supported|offered|available)", reason, re.I):
738
+ raise AdsReportUnavailable(f"Amazon does not offer this report: {reason}")
739
+ raise AdsError(f"Amazon Ads report {report_id} ended FAILED: {reason}")
740
+ elapsed = now() - started
741
+ if elapsed > REPORT_MAX_WAIT_SECONDS:
742
+ raise AdsReportPending(f"Amazon Ads report {report_id} did not finish in 3 hours", report_id)
743
+ if limit is not None and elapsed >= limit:
744
+ raise AdsReportPending(f"Amazon Ads report {report_id} is still being prepared", report_id)
745
+ pause = 0 if fixture_mode() else REPORT_POLL_SECONDS
746
+ sleep(pause if limit is None else min(pause, limit - elapsed))
747
+
748
+
749
+ def _flat(row, prefix=""):
750
+ """A nested v1 row -> flat dotted field ids. A flat row stays as it is."""
751
+ out = {}
752
+ for key, value in row.items():
753
+ if isinstance(value, dict):
754
+ out.update(_flat(value, f"{prefix}{key}."))
755
+ else:
756
+ out[f"{prefix}{key}"] = value
757
+ return out
758
+
759
+
760
+ def _part_rows(raw):
761
+ """One report file -> flat rows. GZIP or plain, a JSON array or JSON lines."""
762
+ try:
763
+ text = (gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw).decode("utf-8", "replace").strip()
764
+ if not text:
765
+ return []
766
+ try:
767
+ data = json.loads(text)
768
+ except ValueError:
769
+ data = [json.loads(line) for line in text.splitlines() if line.strip()]
770
+ except (OSError, EOFError, ValueError) as exc:
771
+ raise AdsError(f"Amazon Ads sent a report file Loopable cannot read ({type(exc).__name__})") from None
772
+ return [_flat(row) for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
773
+
774
+
775
+ def _read_file(url):
776
+ """GET one report file with no Ads header. Only an https S3 URL a retrieve returned is admitted."""
777
+ parts = urllib.parse.urlsplit(str(url or ""))
778
+ host = parts.hostname or ""
779
+ if parts.scheme != "https" or not host.endswith(REPORT_FILE_HOST_SUFFIX) or url not in _ISSUED_FILES:
780
+ raise AdsWriteRefused(f"{parts.scheme or 'no scheme'} {_safe_path(host)} is not a report file Amazon returned")
781
+ _ISSUED_FILES.discard(url)
782
+ headers = {}
783
+ if fixture_mode():
784
+ return _part_rows(FIXTURE.download(url, headers))
785
+ try:
786
+ with urllib.request.urlopen(urllib.request.Request(url, method="GET", headers=headers),
787
+ timeout=_TIMEOUT) as r:
788
+ raw = r.read()
789
+ except Exception as exc: # noqa: BLE001 the URL carries a signature: never echo it
790
+ raise AdsError(f"Amazon Ads report file could not be read ({type(exc).__name__})") from None
791
+ return _part_rows(raw)
792
+
793
+
794
+ def _split(group, rows):
795
+ """Flat rows -> {member table: rows} by `adProduct.value`, and a note for rows no member takes."""
796
+ from harness import amazon_ads_schema as schema
797
+ out, tables, stray = {table: [] for table in group.members}, group.tables, 0
798
+ for row in rows:
799
+ table = tables.get(str(row.get(schema.AD_PRODUCT_FIELD) or "").upper())
800
+ if table is None or not row.get(schema.vendor_name("date")):
801
+ stray += 1
802
+ else:
803
+ out[table].append(row)
804
+ note = (f"Amazon Ads sent {stray} rows with no date or with another ad product. Loopable left them out.",)
805
+ return out, note if stray else ()
806
+
807
+
808
+ def run_group_report(tenant_key, group, start, end, *, rt=None, profile_id=None, connection_id=None,
809
+ sleep=time.sleep, now=time.monotonic, max_wait=None, report_id=None):
810
+ """Create one Reporting v1 group report, wait for it, and read every file. -> GroupReport.
811
+
812
+ `max_wait` caps this call's poll loop in seconds. `report_id` resumes that report without a create.
813
  """
814
  from harness import amazon_ads_schema as schema
815
+ group = schema.GROUPS[group] if isinstance(group, str) else group
816
  if (end - start).days + 1 > MAX_DAYS_PER_REQUEST:
817
+ raise ValueError(f"a report window is at most {MAX_DAYS_PER_REQUEST} days")
818
+ fields = _report_credential(tenant_key, rt, connection_id)
819
+ opts = {"rt": rt, "profile_id": str(profile_id or fields["profile_id"]), "connection_id": connection_id}
820
+ refused = _REFUSED.setdefault((str(fields["client_id"]), group.name), {})
821
  limit = None if max_wait is None else max(0.0, min(float(max_wait), REPORT_MAX_WAIT_SECONDS))
822
+ with _report_lock(fields["client_id"]):
 
823
  if report_id is None:
824
+ account = _advertiser_account(tenant_key, **opts)
825
+ report_id = _create(tenant_key, group, start, end, account, refused, opts)["reportId"]
826
+ urls = _await(tenant_key, str(report_id), opts, sleep=sleep, now=now, limit=limit)
827
+ rows = [row for url in urls for row in _read_file(url)]
828
+ split, stray = _split(group, rows)
829
+ notes = tuple(f"Amazon Ads refused the field {field} on this report, so Loopable reads it without that "
830
+ f"field. Amazon said: {text}" for field, text in refused.items())
831
+ return GroupReport(str(report_id), split, notes + stray)
832
+
833
+
834
+ def run_report(tenant_key, table, start, end, **options):
835
+ """One table's rows from its group report. -> (report_id, rows). The store reads `run_group_report`."""
836
+ from harness import amazon_ads_schema as schema
837
+ group = schema.group_of(table)
838
+ if group is None:
839
+ raise AdsReportUnavailable(f"Amazon Reporting v1 offers no report for {schema.short_name(table)}")
840
+ result = run_group_report(tenant_key, group, start, end, **options)
841
+ return result.report_id, result.rows[schema.short_name(table)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
842
 
843
 
844
  # --------------------------------------------------------------------------- recording fixture
845
 
846
+ FIXTURE_TOKEN = "fixture-access-token"
847
+ FIXTURE_CREDENTIAL = {"client_id": "amzn1.application-oa2-client.fixture", "client_secret": "fixture",
848
+ "refresh_token": "fixture", "profile_id": "1000000001", "region": "na"}
849
+ FIXTURE_FILE_HOST = "https://fixture-ads-reports.s3.amazonaws.com"
850
+ FIXTURE_CREATED = "2026-10-20T09:00:00Z"
851
+ _FIXTURE_PREFIX = {"SPONSORED_PRODUCTS": ("71", "SP"), "SPONSORED_BRANDS": ("72", "SB"),
852
+ "SPONSORED_DISPLAY": ("73", "SD")}
853
+ #: Per campaign and day: the metric totals every finer row splits evenly.
854
+ _FIXTURE_TOTALS = {"metric.impressions": 400, "metric.clicks": 12, "metric.totalCost": 5.0,
855
+ "metric.sales": 38.0, "metric.purchases": 4, "metric.unitsSold": 4}
856
+
857
+
858
+ def _multi_status(report=None, errors=None):
859
+ """A ReportMultiStatusResponse with one success or one error entry at index 0."""
860
+ return {"success": [{"index": 0, "report": report}] if report else [],
861
+ "error": [{"index": 0, "errors": errors}] if errors else []}
862
+
863
+
864
  class _Fixture:
865
+ """The recording fixture `AMAZON_ADS_FIXTURE=1` routes to, in the saved contracts' v1 shapes.
 
866
 
867
+ It records method, path, operation, headers and body of every call, report file downloads
868
+ included. Toggles: `drop_days`, `unsupported` (group names that end FAILED), `refuse_fields`
869
+ (a create naming one answers 400), `nested` (files carry nested JSON lines), `responder`.
870
+ """
871
+
872
+ FILES = 2
873
 
874
  def __init__(self):
875
  self.reset()
876
 
877
  def reset(self):
878
+ self.calls, self.reports, self.subscriptions, self.credentials = [], {}, {}, {}
 
879
  self.in_flight = self.max_in_flight = 0
880
+ self.drop_days, self.unsupported, self.refuse_fields = set(), set(), set()
881
+ self.nested = False
882
  self.responder = None
883
 
884
+ def credential(self, tenant_key):
885
+ """The fixture credential for one tenant. `credentials[tenant]` overrides its fields."""
886
+ return {**FIXTURE_CREDENTIAL, **self.credentials.get(tenant_key, {})}
887
+
888
+ def answer(self, prepared, tenant_key, profile_id, headers=None):
889
  self.calls.append({"method": prepared.method, "path": prepared.path,
890
  "operation": prepared.operation, "body": prepared.body,
891
+ "query": dict(prepared.query), "tenant": tenant_key, "headers": dict(headers or {}),
892
  "named": dict(prepared.named) if prepared.named else None})
893
  if self.responder is not None:
894
  return self.responder(prepared)
 
904
  if prepared.named is not None:
905
  items = next(iter(prepared.body.values()))
906
  return {"success": [{"index": i} for i in range(len(items))], "error": []}
907
+ if prepared.operation == "queryAdvertiserAccounts":
908
+ return self._accounts(prepared.body, profile_id)
909
  if prepared.operation == "createReport":
910
+ return self._create(prepared.body)
911
+ return self._retrieve(prepared.body["reportIds"][0])
912
+
913
+ def _accounts(self, body, profile_id):
914
+ """Two pages: an unrelated account, then the account that lists the caller's profile."""
915
+ if not body.get("nextToken"):
916
+ return {"advertiserAccounts": [{"advertiserAccountId": "amzn1.ads-account.g.fxother",
917
+ "alternateIds": [{"profileId": "999", "countryCode": "CA"}]}],
918
+ "nextToken": "fixture-page-2"}
919
+ return {"advertiserAccounts": [{"advertiserAccountId": f"amzn1.ads-account.g.fx{profile_id}",
920
+ "alternateIds": [{"profileId": str(profile_id), "countryCode": "US",
921
+ "entityId": "ENTITYFX"}]}]}
922
+
923
+ def _create(self, body):
924
+ if not (body.get("accessRequestedAccounts") or [{}])[0].get("advertiserAccountId"):
925
+ return _multi_status(errors=[{"code": "400001", "message": "An advertiser account is required.",
926
+ "fieldLocation": "accessRequestedAccounts"}])
927
+ refused = [f for f in body["reports"][0]["query"]["fields"] if f in self.refuse_fields]
928
+ if refused:
929
+ raise _rejection("Amazon Ads answered HTTP 400 on createReport",
930
+ [{"code": "400004", "message": f"Field {refused[0]} is invalid for this request.",
931
+ "fieldLocation": ""}], may_be_unoffered=False)
932
+ report_id = f"fx{len(self.reports) + 1:05d}"
933
+ self.in_flight += 1
934
+ self.max_in_flight = max(self.max_in_flight, self.in_flight)
935
+ self.reports[report_id] = {"body": body, "polls": 0, "files": None, "done": False}
936
+ return _multi_status(self._record(report_id, "PENDING"))
937
+
938
+ def _record(self, report_id, status, **extra):
939
+ """A contract `Report`: every required key, the request's format, periods and query echoed."""
940
+ request = self.reports[report_id]["body"]["reports"][0]
941
+ return {"reportId": report_id, "status": status, "format": request["format"],
942
+ "periods": request["periods"], "query": request["query"],
943
+ "creationDateTime": FIXTURE_CREATED, "lastUpdatedDateTime": FIXTURE_CREATED, **extra}
944
+
945
+ def _retrieve(self, report_id):
946
+ from harness import amazon_ads_schema as schema
947
  report = self.reports[report_id]
948
+ report["polls"] += 1
949
+ query = report["body"]["reports"][0]["query"]
950
+ group = schema.group_for(query["fields"], query["filter"]["on"]["values"])
951
+ if group is not None and group.name in self.unsupported:
952
+ self._finish(report)
953
+ return _multi_status(self._record(report_id, "FAILED", failureCode="REPORT_NOT_SUPPORTED",
954
+ failureReason="This report is not supported for this advertiser."))
955
+ if report["polls"] < 2:
956
+ return _multi_status(self._record(report_id, "PROCESSING"))
957
+ if report["files"] is None:
958
+ rows = fixture_rows(report["body"], drop_days=self.drop_days)
959
+ report["files"] = {f"{FIXTURE_FILE_HOST}/{report_id}/part-{i}.json.gz?X-Amz-Signature=fx{report_id}{i}":
960
+ _fixture_file(rows[i::self.FILES], self.nested) for i in range(self.FILES)}
961
+ return _multi_status(self._record(report_id, "COMPLETED", completedDateTime=FIXTURE_CREATED, completedReportParts=[
962
+ {"url": url, "sizeInBytes": len(raw), "urlExpirationDateTime": "2026-12-31T00:00:00Z"}
963
+ for url, raw in report["files"].items()]))
964
+
965
+ def _finish(self, report):
966
+ if not report["done"]:
967
+ report["done"] = True
968
  self.in_flight -= 1
969
+
970
+ def download(self, url, headers):
971
+ """Serve one report file and record the GET with the headers the client sent."""
972
+ self.calls.append({"method": "GET", "path": url, "operation": "downloadReportPart", "body": None,
973
+ "query": {}, "tenant": None, "headers": dict(headers), "named": None})
974
+ report = self.reports[urllib.parse.urlsplit(url).path.split("/")[1]]
975
+ raw = report["files"][url]
976
+ report.setdefault("read", set()).add(url)
977
+ if report["read"] == set(report["files"]):
978
+ self._finish(report)
979
+ return raw
980
+
981
+
982
+ def _fixture_file(rows, nested):
983
+ """GZIP JSON: an array of flat rows, or nested objects one per line."""
984
+ def nest(row):
985
+ out = {}
986
+ for field, value in row.items():
987
+ head, _, tail = field.partition(".")
988
+ out.setdefault(head, {})[tail] = value
989
+ return out
990
+ text = "\n".join(json.dumps(nest(r)) for r in rows) if nested else json.dumps(rows)
991
+ return gzip.compress(text.encode("utf-8"))
992
+
993
+
994
+ def _fixture_query(body):
995
+ report = body["reports"][0]
996
+ period = report["periods"][0]["datePeriod"]
997
+ return (report["query"]["fields"], report["query"]["filter"]["on"]["values"],
998
+ date.fromisoformat(period["startDate"]), date.fromisoformat(period["endDate"]))
999
+
1000
+
1001
+ def _fixture_days(start, end, drop_days):
1002
+ return [start + timedelta(days=i) for i in range((end - start).days + 1)
1003
+ if (start + timedelta(days=i)).isoformat() not in drop_days]
1004
+
1005
+
1006
+ def fixture_totals(body, *, drop_days=()):
1007
+ """The fixture's own totals for one create body. -> {(ad product, date, campaign id): {field: total}}."""
1008
+ _fields, products, start, end = _fixture_query(body)
1009
+ out = {}
1010
+ for day in _fixture_days(start, end, drop_days):
1011
+ for index, product in enumerate(products):
1012
+ for c in (1, 2):
1013
+ n = day.toordinal() % 7 + c + index
1014
+ out[(product, day.isoformat(), f"{_FIXTURE_PREFIX[product][0]}10{c}")] = {
1015
+ field: total * n for field, total in _FIXTURE_TOTALS.items()}
1016
+ return out
1017
 
1018
 
1019
  def fixture_rows(body, *, drop_days=()):
1020
+ """Deterministic v1 rows for one create body: one row per distinct value of every requested
1021
+ dimension and day, only the requested fields and ad products. Two ad groups split a campaign."""
1022
+ fields, _products, start, end = _fixture_query(body)
1023
+ groups = (1, 2) if "adGroup.id" in fields else (0,)
1024
+ out = []
1025
+ for (product, day, campaign), totals in fixture_totals(body, drop_days=drop_days).items():
1026
+ prefix, short = _FIXTURE_PREFIX[product]
1027
+ c = campaign[-1]
1028
+ for g in groups:
1029
+ row = {"date.value": day, "adProduct.value": product, "campaign.id": campaign,
1030
+ "campaign.name": f"{short} campaign {c}", "campaign.deliveryStatus": "DELIVERING",
1031
+ "campaign.budgetAmount": str(25.0 * int(c)), "budgetCurrency.value": "USD",
1032
+ "adGroup.id": f"{prefix}20{c}{g}", "adGroup.name": f"Ad group {c}.{g}",
1033
+ "adGroupTarget.id": f"{prefix}30{c}{g}", "targeting.value": f"term {c}{g}",
1034
+ "targetingMatchType.value": "EXACT", "adGroupTarget.bid": str(0.5 + g / 10),
1035
+ "ad.id": f"{prefix}40{c}{g}", "advertisedProduct.id": f"B0FX{prefix}{c}{g}00",
1036
+ "advertisedProductMarketplace.value": "ATVPDKIKX0DER", "advertisedProductSku.value": f"SKU-{c}{g}",
1037
+ "searchTerm.value": f"search {c}{g}",
1038
+ **{f: (t // len(groups) if isinstance(t, int) else t / len(groups)) for f, t in totals.items()}}
1039
+ out.append({f: row[f] for f in fields})
 
 
 
 
 
 
 
1040
  return out
1041
 
1042
 
1043
  FIXTURE = _Fixture()
1044
 
1045
 
1046
+ def _reset_caches():
1047
+ _ACCOUNTS.clear()
1048
+ _REFUSED.clear()
1049
+ _ISSUED_FILES.clear()
1050
+
1051
+
1052
  @contextmanager
1053
  def recording_fixture():
1054
+ """Route every call to a fresh recording fixture and fresh report caches for the block. -> the fixture."""
1055
  previous = os.environ.get(FIXTURE_ENV)
1056
  os.environ[FIXTURE_ENV] = "1"
1057
  FIXTURE.reset()
1058
+ _reset_caches()
1059
  try:
1060
  yield FIXTURE
1061
  finally:
1062
  FIXTURE.reset()
1063
+ _reset_caches()
1064
  if previous is None:
1065
  os.environ.pop(FIXTURE_ENV, None)
1066
  else:
1067
  os.environ[FIXTURE_ENV] = previous
1068
 
1069
 
1070
+ __all__ = ["KEY", "LABEL", "ENTRY_TYPE", "FIXTURE_ENV", "HOSTS", "REPORT_OPERATIONS", "V1_CLIENT_HEADERS",
1071
+ "NAMED_WRITES", "STREAM_OPERATIONS", "STREAM_CONTENT_TYPE", "stream_operation",
1072
  "FIELD_RULES", "APPROVAL_SOURCE", "SCHEDULE_APPROVAL_SOURCE", "SENT_WRITE_SOURCE",
1073
  "SCHEDULE_WEEKDAY_CONVENTION", "REPORT_CONCURRENCY", "REPORTS_PER_DAY",
1074
+ "MAX_DAYS_PER_REQUEST", "AdsError", "AdsWriteRefused", "AdsRejected", "AdsReportUnavailable",
1075
+ "AdsReportPending", "GroupReport", "fixture_mode", "build_request", "report_operation",
1076
+ "send_named_write", "request", "credential", "run_group_report", "run_report", "FIXTURE",
1077
+ "recording_fixture", "fixture_rows", "fixture_totals"]
platform/harness/amazon_ads_schema.py CHANGED
@@ -1,16 +1,17 @@
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
- One entry per ad type and report: the report type, the columns we store, the Ads column each one
4
- is read from, the row identity and the backfill window. The table keys, labels and backfill days
5
- come from `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
- Unknown: the exact Reporting v1 column spellings. The vendor names below are the Ads reporting
8
- vocabulary the primer records (F11, F12). The first credentialed report settles them (PRD P17),
9
- and a rename is one literal in `_VENDOR`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
 
14
 
15
  _HERE = Path(__file__).resolve().parent
16
  if str(_HERE.parent) not in sys.path:
@@ -22,8 +23,6 @@ AD_TYPES = _cs.AMAZON_ADS_AD_TYPES
22
  REPORTS = _cs.AMAZON_ADS_REPORTS
23
  DATABASE_PREFIX = "ut_amzn_"
24
  MIRROR_PREFIX = "amzn_ads_"
25
- #: Amazon accepts at most 31 days per report request.
26
- MAX_DAYS_PER_REQUEST = 31
27
 
28
  #: The C6 columns every Ads table carries, in this order.
29
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
@@ -54,22 +53,71 @@ _EXTRA = {
54
  }
55
  _TAIL = ("profile_id", "currency")
56
 
57
- #: Our column -> the Ads column, shared by all three ad types unless `_VENDOR_BY_TYPE` says not.
58
  _VENDOR = {
59
- "campaign_id": "campaignId", "campaign_name": "campaignName",
60
- "campaign_status": "campaignStatus", "campaign_budget_amount": "campaignBudgetAmount",
61
- "ad_group_id": "adGroupId", "ad_group_name": "adGroupName", "target_id": "keywordId",
62
- "targeting": "targeting", "match_type": "matchType", "bid": "keywordBid",
63
- "ad_id": "adId", "advertised_sku": "advertisedSku", "asin": "advertisedAsin",
64
- "search_term": "searchTerm", "date": "date", "impressions": "impressions",
65
- "clicks": "clicks", "cost": "cost", "sales": "sales", "orders": "purchases",
66
- "units": "unitsSold", "currency": "campaignBudgetCurrencyCode",
67
- }
68
- _VENDOR_BY_TYPE = {
69
- "sp": {"sales": "sales14d", "orders": "purchases14d", "units": "unitsSoldClicks14d"},
70
- "sd": {"target_id": "targetingId", "targeting": "targetingExpression",
71
- "advertised_sku": "promotedSku", "asin": "promotedAsin"},
72
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
 
75
  def _short(ad_type, report):
@@ -89,25 +137,24 @@ def coltype(name):
89
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
90
 
91
 
92
- def vendor_name(ad_type, column):
93
- """The Ads column one of our columns is read from, or None when the vendor has none."""
94
- if column == "profile_id":
95
- return None
96
- return _VENDOR_BY_TYPE.get(ad_type, {}).get(column) or _VENDOR.get(column)
97
 
98
 
99
  TABLE_SPECS = {}
100
  for _spec in _cs.AMAZON_ADS_TABLES:
101
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
 
102
  TABLE_SPECS[_short(_ad_type, _report)] = {
103
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
104
- "ad_type": _ad_type, "report": _report, "report_type": _spec.native,
105
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
106
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
107
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
108
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
109
  }
110
- del _spec, _ad_type, _report
111
 
112
 
113
  def short_name(table):
@@ -127,11 +174,16 @@ def columns_for(table):
127
  return TABLE_SPECS[short_name(table)]["columns"]
128
 
129
 
130
- def vendor_columns(table):
131
- """The Ads columns one report asks for, in our column order, each once."""
132
- spec = TABLE_SPECS[short_name(table)]
133
- names = (vendor_name(spec["ad_type"], c) for c in spec["columns"])
134
- return tuple(dict.fromkeys(n for n in names if n))
 
 
 
 
 
135
 
136
 
137
  def _cell(name, value):
@@ -149,11 +201,11 @@ def _cell(name, value):
149
 
150
 
151
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
152
- """One Ads report row -> one stored row, with its stable `_row_key` and report origin."""
153
  spec = TABLE_SPECS[short_name(table)]
154
  out = {}
155
  for column in spec["columns"]:
156
- source = vendor_name(spec["ad_type"], column)
157
  value = vendor_row.get(source) if source else None
158
  if column == "profile_id":
159
  value = profile_id
@@ -213,7 +265,8 @@ def stream_coltype(name):
213
  return "DOUBLE"
214
  return "VARCHAR"
215
 
216
- __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "MAX_DAYS_PER_REQUEST",
 
217
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
218
- "vendor_columns", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
219
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
 
1
  """amazon_ads_schema.py: the vendor schema artifact for the Amazon Ads tables (S79-T12, contract C6).
2
 
3
+ One entry per ad type and report: the columns we store, the Reporting v1 field each one is read
4
+ from, the row identity and the backfill window. The table keys, labels and backfill days come from
5
+ `core.connector_spec`, so the two files cannot disagree on a name.
6
 
7
+ Field ids come from Amazon's v1 field library, saved at
8
+ `AutoWiki/research/amazon-ads-v1-schema/reporting_v1_fields.json`. v1 has no report types: one
9
+ report per `REPORT_GROUPS` row covers several ad types, and its rows split by `adProduct.value`.
10
  """
11
  import hashlib
12
  import sys
13
  from pathlib import Path
14
+ from typing import NamedTuple
15
 
16
  _HERE = Path(__file__).resolve().parent
17
  if str(_HERE.parent) not in sys.path:
 
23
  REPORTS = _cs.AMAZON_ADS_REPORTS
24
  DATABASE_PREFIX = "ut_amzn_"
25
  MIRROR_PREFIX = "amzn_ads_"
 
 
26
 
27
  #: The C6 columns every Ads table carries, in this order.
28
  C6_COLUMNS = ("campaign_id", "ad_group_id", "target_id", "advertised_sku", "asin", "date",
 
53
  }
54
  _TAIL = ("profile_id", "currency")
55
 
56
+ #: Our column -> its Reporting v1 field id, one map for every ad type. The store stamps profile_id.
57
  _VENDOR = {
58
+ "date": "date.value", "campaign_id": "campaign.id", "campaign_name": "campaign.name",
59
+ "campaign_status": "campaign.deliveryStatus", "campaign_budget_amount": "campaign.budgetAmount",
60
+ "ad_group_id": "adGroup.id", "ad_group_name": "adGroup.name", "target_id": "adGroupTarget.id",
61
+ "targeting": "targeting.value", "match_type": "targetingMatchType.value", "bid": "adGroupTarget.bid",
62
+ "ad_id": "ad.id", "advertised_sku": "advertisedProductSku.value", "asin": "advertisedProduct.id",
63
+ "search_term": "searchTerm.value", "impressions": "metric.impressions", "clicks": "metric.clicks",
64
+ "cost": "metric.totalCost", "sales": "metric.sales", "orders": "metric.purchases",
65
+ "units": "metric.unitsSold", "currency": "budgetCurrency.value",
 
 
 
 
 
66
  }
67
+ #: The v1 field that names each row's ad type, and its value per ad type.
68
+ AD_PRODUCT_FIELD = "adProduct.value"
69
+ AD_PRODUCTS = {"sp": "SPONSORED_PRODUCTS", "sb": "SPONSORED_BRANDS", "sd": "SPONSORED_DISPLAY"}
70
+ #: The name the v1 field library lists each ad type under.
71
+ LIBRARY_AD_PRODUCTS = {"sp": "Sponsored Products", "sb": "Sponsored Brands", "sd": "Sponsored Display"}
72
+
73
+ METRICS = ("metric.impressions", "metric.clicks", "metric.totalCost", "metric.sales",
74
+ "metric.purchases", "metric.unitsSold")
75
+ _HEAD = ("date.value", AD_PRODUCT_FIELD, "campaign.id", "campaign.name")
76
+ _AD_GROUP = ("adGroup.id", "adGroup.name")
77
+ _TAIL_FIELDS = ("budgetCurrency.value",) + METRICS
78
+ _TARGET_TEXT = ("targeting.value", "targetingMatchType.value")
79
+
80
+
81
+ class ReportGroup(NamedTuple):
82
+ """One Reporting v1 report and the member tables its rows split into by `adProduct.value`.
83
+
84
+ `optional` fields list no ad product in Amazon's library: a create that refuses one drops it.
85
+ """
86
+ name: str
87
+ members: tuple
88
+ core: tuple
89
+ optional: tuple = ()
90
+
91
+ @property
92
+ def tables(self):
93
+ """`adProduct.value` -> the member table its rows land in."""
94
+ return {AD_PRODUCTS[table.split("_", 1)[0]]: table for table in self.members}
95
+
96
+ def fields(self, drop=()):
97
+ """The fields one create asks for, without the optional fields Amazon refused."""
98
+ return tuple(f for f in self.core + self.optional if f not in drop)
99
+
100
+
101
+ #: ⛔ Every requested dimension is a group-by and sets the row grain. A group asks only for its
102
+ #: tables' identity dimensions, attributes of those, metrics, the ad product and required partners.
103
+ #: Every core field lists every ad product of its group in the library: SB lacks the SKU field and
104
+ #: the marketplace field `advertisedProduct.id` requires, so SB advertised products is its own report.
105
+ REPORT_GROUPS = (
106
+ ReportGroup("campaigns", ("sp_campaigns", "sb_campaigns", "sd_campaigns"),
107
+ _HEAD + ("campaign.deliveryStatus", "campaign.budgetAmount") + _TAIL_FIELDS),
108
+ ReportGroup("targeting", ("sp_targeting", "sb_targeting", "sd_targeting"),
109
+ _HEAD + _AD_GROUP + ("adGroupTarget.id",) + _TAIL_FIELDS,
110
+ _TARGET_TEXT + ("adGroupTarget.bid",)),
111
+ ReportGroup("advertised_products", ("sp_advertised_products", "sd_advertised_products"),
112
+ _HEAD + _AD_GROUP + ("ad.id", "advertisedProduct.id", "advertisedProductMarketplace.value",
113
+ "advertisedProductSku.value") + _TAIL_FIELDS),
114
+ ReportGroup("advertised_products_sb", ("sb_advertised_products",),
115
+ _HEAD + _AD_GROUP + ("ad.id",) + _TAIL_FIELDS),
116
+ ReportGroup("search_terms", ("sp_search_terms", "sb_search_terms"),
117
+ _HEAD + _AD_GROUP + ("adGroupTarget.id", "searchTerm.value") + _TAIL_FIELDS, _TARGET_TEXT),
118
+ )
119
+ GROUPS = {group.name: group for group in REPORT_GROUPS}
120
+ _GROUP_OF = {table: group for group in REPORT_GROUPS for table in group.members}
121
 
122
 
123
  def _short(ad_type, report):
 
137
  return "BIGINT" if name in _INT else "DOUBLE" if name in _DBL else "VARCHAR"
138
 
139
 
140
+ def vendor_name(column):
141
+ """The v1 field one of our columns is read from, or None when the vendor has none."""
142
+ return _VENDOR.get(column)
 
 
143
 
144
 
145
  TABLE_SPECS = {}
146
  for _spec in _cs.AMAZON_ADS_TABLES:
147
  _ad_type, _report = _spec.key[len(DATABASE_PREFIX) + len("ads_"):].split("_", 1)
148
+ _group = _GROUP_OF.get(_short(_ad_type, _report))
149
  TABLE_SPECS[_short(_ad_type, _report)] = {
150
  "key": _spec.key, "source": _spec.source, "label": _spec.label,
151
+ "ad_type": _ad_type, "report": _report, "group": _group.name if _group else None,
152
  "columns": _columns(_report), "row_identity": tuple(_spec.identity),
153
  "backfill_days": _cs.AMAZON_ADS_BACKFILL_DAYS[_spec.key],
154
  "offered": _spec.key not in _cs.AMAZON_ADS_NOT_OFFERED,
155
  "not_offered_reason": _cs.AMAZON_ADS_NOT_OFFERED.get(_spec.key),
156
  }
157
+ del _spec, _ad_type, _report, _group
158
 
159
 
160
  def short_name(table):
 
174
  return TABLE_SPECS[short_name(table)]["columns"]
175
 
176
 
177
+ def group_of(table):
178
+ """The report group that fills one Ads table, or None for a table v1 does not offer."""
179
+ return _GROUP_OF.get(short_name(table))
180
+
181
+
182
+ def group_for(fields, ad_products):
183
+ """The report group a create request asks for, by its fields and ad product filter, or None."""
184
+ fields, ad_products = set(fields), set(ad_products)
185
+ return next((g for g in REPORT_GROUPS if set(g.tables) == ad_products
186
+ and set(g.core) <= fields <= set(g.core + g.optional)), None)
187
 
188
 
189
  def _cell(name, value):
 
201
 
202
 
203
  def row_from_vendor(table, vendor_row, *, profile_id=None, report_id=None):
204
+ """One flat v1 report row (dotted field ids) -> one stored row, with its `_row_key` and report origin."""
205
  spec = TABLE_SPECS[short_name(table)]
206
  out = {}
207
  for column in spec["columns"]:
208
+ source = vendor_name(column)
209
  value = vendor_row.get(source) if source else None
210
  if column == "profile_id":
211
  value = profile_id
 
265
  return "DOUBLE"
266
  return "VARCHAR"
267
 
268
+ __all__ = ["AD_TYPES", "REPORTS", "C6_COLUMNS", "LABELS", "TABLE_SPECS", "AD_PRODUCT_FIELD", "AD_PRODUCTS",
269
+ "LIBRARY_AD_PRODUCTS", "METRICS", "ReportGroup", "REPORT_GROUPS", "GROUPS",
270
  "ORIGIN_COLUMNS", "coltype", "vendor_name", "short_name", "columns_for",
271
+ "group_of", "group_for", "row_from_vendor", "row_key", "STREAM_WINDOWS", "STREAM_METRICS", "STREAM_GRAIN",
272
  "STREAM_COLUMNS", "STREAM_TABLE_SPECS", "stream_spec", "stream_coltype"]
platform/harness/amazon_ads_store.py CHANGED
@@ -10,8 +10,8 @@ back the longest window Amazon documents per table (`connector_spec.AMAZON_ADS_B
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
- ⛔ STANDING RULE 1. Every report is read whole and paged. The limits that cannot be removed (one
14
- report at a time, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
@@ -178,7 +178,7 @@ def _day_marker(con, key):
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
- """The report windows one sync asks for, oldest first, each at most 31 days."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
@@ -237,13 +237,13 @@ def _upsert(con, table, rows):
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
 
240
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
241
  part = keys[i:i + INSERT_BATCH_ROWS]
242
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
243
- con.executemany(
244
- f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
245
- f'VALUES ({", ".join("?" for _ in cols)})',
246
- [[by_key[k].get(c) for c in cols] for k in part])
247
  return len(by_key)
248
 
249
 
@@ -389,16 +389,158 @@ def _land(con, table, rows, report_id, start, end, profile, connection_id, label
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
- """Require named transport support instead of sending a sibling's credential."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
- try:
397
- return client.run_report(key, table, start, end, **options)
398
- except TypeError as exc:
399
- if connection_id and "unexpected keyword argument 'connection_id'" in str(exc):
400
- raise client.AdsError("The Amazon Ads client needs named connection support before this connection can sync.") from None
401
- raise
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
402
 
403
 
404
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
@@ -409,6 +551,7 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
409
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
410
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
411
  pending ledger, and the next pass reads it before it creates any other report.
 
412
  """
413
  key = amazon_store._tenant_key(tenant_key)
414
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
@@ -425,10 +568,9 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
425
  own = con is None
426
  if own:
427
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
428
- bounded = max_seconds is not None
429
  started = _clock()
430
- deadline = started + float(max_seconds) if bounded else None
431
- waits = report["waits"]
432
  try:
433
  datastore._ensure_sync_state(con)
434
  now = _utc_now(now)
@@ -437,164 +579,24 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
437
  def stamp():
438
  return now + timedelta(seconds=max(0.0, _clock() - started))
439
 
440
- def wait(label, spec, reason):
441
- waits.append(f"{label}: {spec['label']}: {reason}")
442
-
443
- synced = []
444
- checkpointed = []
445
- for cid, label, profile, legacy in connections:
446
  connect_day = _connect_day(con, today, cid, legacy=legacy)
447
  report.setdefault("connectDay", connect_day.isoformat())
448
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
449
  report["connections"].append(entry)
450
- used = _budget(con, today, cid, legacy=legacy)
451
- # Resume every report this connection left preparing before it creates another one.
452
- resumed, blocked = {}, False
453
- for table in TABLES:
454
- pending = _read_pending(con, table, cid)
455
- if pending is None:
456
- continue
457
- spec = schema.TABLE_SPECS[table]
458
- _ensure_table(con, table)
459
- if table_name(table) not in checkpointed:
460
- checkpointed.append(table_name(table))
461
- age = (stamp() - pending["created"]).total_seconds()
462
- expired = (f"Amazon Ads report {pending['report_id']} did not finish in 3 hours")
463
- if age >= client.REPORT_MAX_WAIT_SECONDS:
464
- _clear_pending(con, table, cid)
465
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
466
- resumed[table] = ("failed", expired, 0, None)
467
- continue
468
- limit = client.REPORT_MAX_WAIT_SECONDS - age
469
- if bounded:
470
- limit = min(limit, max(0.0, deadline - _clock()))
471
- try:
472
- report_id, rows = _run_report(key, table, pending["start"], pending["end"], rt, profile, cid,
473
- max_wait=limit, report_id=pending["report_id"], now=_clock)
474
- except client.AdsReportUnavailable as exc:
475
- _clear_pending(con, table, cid)
476
- resumed[table] = ("not_offered", _not_offered_reason(spec, exc), 0, None)
477
- continue
478
- except client.AdsReportPending:
479
- if (stamp() - pending["created"]).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS:
480
- _clear_pending(con, table, cid)
481
- report["problems"].append(f"{label}: {spec['label']}: {expired}")
482
- resumed[table] = ("failed", expired, 0, None)
483
- else:
484
- blocked = True
485
- wait(label, spec, PREPARING_REASON)
486
- resumed[table] = ("deferred", PREPARING_REASON, 0, None)
487
- continue
488
- except client.AdsError as exc:
489
- _clear_pending(con, table, cid)
490
- reason = str(exc)[:200]
491
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
492
- resumed[table] = ("failed", reason, 0, None)
493
- continue
494
- _clear_pending(con, table, cid)
495
- pulled = _land(con, table, rows, report_id, pending["start"], pending["end"], profile, cid, label)
496
- resumed[table] = ("done", "", pulled, (pending["start"], pending["end"]))
497
- requested = [schema.short_name(t) for t in (tables or TABLES)]
498
  for table in list(dict.fromkeys(requested + list(resumed))):
499
- spec = schema.TABLE_SPECS[table]
500
- _ensure_table(con, table)
501
- origin_changed = _refresh_origin(con, table, cid, label, legacy)
502
- pulled, state, mark, reason = 0, "done", _marker_table(table), ""
503
- prior = resumed.get(table)
504
- landed = None
505
- if prior is not None:
506
- state, reason, pulled, landed = prior
507
- if state != "done" or table not in requested:
508
- pass
509
- elif not spec["offered"]:
510
- state, reason = "not_offered", spec["not_offered_reason"]
511
- elif not _cadence_due(con, table, now, cid):
512
- state = "skipped"
513
- elif blocked:
514
- state, reason = "deferred", PREPARING_REASON
515
- wait(label, spec, reason)
516
- else:
517
- for start, end in windows(con, table, connect_day, today, cid):
518
- if landed and landed[0] <= start and end <= landed[1]:
519
- continue # The resumed report just read these days.
520
- if used >= client.REPORTS_PER_DAY:
521
- state = "deferred"
522
- reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
523
- f"reports are used, so days from {start.isoformat()} load on the next sync.")
524
- report["problems"].append(f"{label}: {spec['label']}: waiting for tomorrow's report limit")
525
- waits.append(report["problems"][-1])
526
- break
527
- if bounded and _clock() >= deadline:
528
- state, reason = "deferred", TIME_LIMIT_REASON
529
- wait(label, spec, reason)
530
- break
531
- used += 1
532
- _spend_budget(con, today, used, cid)
533
- created = stamp()
534
- try:
535
- if bounded:
536
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid,
537
- max_wait=max(0.0, deadline - _clock()), now=_clock)
538
- else:
539
- report_id, rows = _run_report(key, table, start, end, rt, profile, cid)
540
- except client.AdsReportUnavailable as exc:
541
- state = "not_offered"
542
- reason = _not_offered_reason(spec, exc)
543
- break
544
- except client.AdsError as exc:
545
- if (bounded and isinstance(exc, client.AdsReportPending) and exc.report_id
546
- and (stamp() - created).total_seconds() < client.REPORT_MAX_WAIT_SECONDS):
547
- _write_pending(con, table, cid, exc.report_id, start, end, created)
548
- if table_name(table) not in checkpointed:
549
- checkpointed.append(table_name(table))
550
- blocked = True
551
- state, reason = "deferred", PREPARING_REASON
552
- wait(label, spec, reason)
553
- break
554
- state = "failed"
555
- reason = str(exc)[:200]
556
- report["problems"].append(f"{label}: {spec['label']}: {reason}")
557
- break
558
- pulled += _land(con, table, rows, report_id, start, end, profile, cid, label)
559
- if table_name(table) not in checkpointed:
560
- checkpointed.append(table_name(table))
561
- if state == "done":
562
- _record_cadence(con, table, now, cid)
563
- if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
564
- report["partial"] = True
565
- if reason:
566
- _record_limit(con, table, reason, cid, state=state)
567
- shown = f"{label}: {reason}" if cid else reason
568
- report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
569
- elif state == "done":
570
- _clear_limit(con, table, cid)
571
- total = _count(con, table)
572
- entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
573
- if state != "skipped":
574
- amazon_store._record_state(con, _state_key(amazon_store.entity_key(mark), cid),
575
- "done" if state == "done" else "partial", _count(con, table, cid), state)
576
- previous = report["tables"].get(table, {})
577
- severity = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
578
- aggregate = max((state, previous.get("state", state)), key=severity.get)
579
- report["tables"][table] = {"state": aggregate, "pulled": previous.get("pulled", 0) + pulled,
580
- "in_mirror": total}
581
- if (pulled or origin_changed) and table_name(table) not in synced:
582
  synced.append(table_name(table))
583
- log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
584
- for table, state in report["tables"].items():
585
- if state["state"] == "skipped":
586
- continue
587
- amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
588
- "done" if state["state"] == "done" else "partial",
589
- state["in_mirror"], state["state"])
590
- if table in report["limits"]:
591
- _record_limit(con, table, report["limits"][table], state=state["state"])
592
- else:
593
- _clear_limit(con, table)
594
- report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
595
- for cid, *_ in connections for table in TABLES)
596
- if report["partial"]:
597
- report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
598
  if own and rt is not None and (synced or checkpointed):
599
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
600
  finally:
@@ -604,10 +606,75 @@ def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None
604
  return report
605
 
606
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
607
  def _not_offered_reason(spec, exc):
608
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
609
- f"({str(exc)[:160]}). The table keeps the days it has. To ask for it, "
610
- f"contact Amazon Ads support with the report type {spec['report_type']}.")
611
 
612
 
613
  def _persist(tenant_key, rt, con, synced, report, log):
 
10
  A later sync re-reads the table's daily or intraday cadence window and upserts by row identity.
11
  No sync deletes a day an earlier sync stored, so a day Amazon has aged out stays here.
12
 
13
+ ⛔ STANDING RULE 1. Every report is read whole, every file of it. The limits that cannot be removed
14
+ (one report at a time per client ID, about 100 a day, a report v1 does not offer) are written to `_sync_state` as
15
  `amzn_limit_ads_<table>` with the reason a person reads, and stated on the table description.
16
 
17
  Receipts use the Amazon mirror's markers: `amzn_sync_mark_ads_<table>` is the newest landed day,
 
178
 
179
 
180
  def windows(con, table, connect_day, today, connection_id=None):
181
+ """The report windows one sync asks for, oldest first, each at most 120 days (`client.MAX_DAYS_PER_REQUEST`)."""
182
  table = schema.short_name(table)
183
  spec = schema.TABLE_SPECS[table]
184
  cadence = amazon_client.CADENCE["ads_" + table]
 
237
  cols = _columns_for(table)
238
  by_key = {r["_row_key"]: r for r in rows}
239
  keys = list(by_key)
240
+ one = f'({", ".join("?" for _ in cols)})'
241
  for i in range(0, len(keys), INSERT_BATCH_ROWS):
242
  part = keys[i:i + INSERT_BATCH_ROWS]
243
  con.execute(f'DELETE FROM "{name}" WHERE "_row_key" IN ({",".join("?" for _ in part)})', part)
244
+ # One multi-row INSERT per batch: DuckDB's executemany is about 5 times slower at 24 months a pass.
245
+ con.execute(f'INSERT INTO "{name}" ({", ".join(chr(34) + c + chr(34) for c in cols)}) '
246
+ f'VALUES {", ".join(one for _ in part)}', [by_key[k].get(c) for k in part for c in cols])
 
247
  return len(by_key)
248
 
249
 
 
389
 
390
 
391
  def _run_report(key, table, start, end, rt, profile_id, connection_id, **extra):
392
+ """Run the group report that fills `table` under this connection's own credential. -> GroupReport."""
393
  options = {"rt": rt, "profile_id": profile_id, **extra}
394
  if connection_id:
395
  options["connection_id"] = connection_id
396
+ return client.run_group_report(key, schema.group_of(table), start, end, **options)
397
+
398
+
399
+ class _Pass:
400
+ """One connection's sync pass: its budget, clock, and what its group reports already covered.
401
+
402
+ `eligible` holds the tables a group report may land into this pass: requested, offered, due and
403
+ not yet finished. `covered` holds each table's spans landed this pass, `pulled` the rows a
404
+ sibling's report landed, `outcome` a group's failure, `notes` Amazon's text per table.
405
+ """
406
+
407
+ def __init__(self, con, key, rt, connection, report, *, today, used, deadline, stamp, checkpointed):
408
+ self.con, self.key, self.rt, self.report = con, key, rt, report
409
+ self.cid, self.label, self.profile = connection[:3]
410
+ self.today, self.used, self.deadline, self.stamp = today, used, deadline, stamp
411
+ self.checkpointed = checkpointed
412
+ self.eligible, self.covered, self.pulled, self.outcome, self.notes = set(), {}, {}, {}, {}
413
+ self.blocked = False
414
+
415
+ def wait(self, spec, reason):
416
+ self.report["waits"].append(f"{self.label}: {spec['label']}: {reason}")
417
+
418
+ def checkpoint(self, table):
419
+ if table_name(table) not in self.checkpointed:
420
+ self.checkpointed.append(table_name(table))
421
+
422
+ def covers(self, table, start, end):
423
+ """True when reports this pass already read every day from `start` to `end` for `table`."""
424
+ spans = self.covered.get(table, ())
425
+ return all(any(lo <= start + timedelta(days=i) <= hi for lo, hi in spans)
426
+ for i in range((end - start).days + 1))
427
+
428
+ def bounds(self):
429
+ """The poll limit for one new report: none, or the time left in this bounded pass."""
430
+ return {} if self.deadline is None else {"max_wait": max(0.0, self.deadline - _clock()), "now": _clock}
431
+
432
+ def fail(self, table, exc):
433
+ """Record a group report's failure for every member. -> (state, reason) for `table`."""
434
+ self.outcome[schema.group_of(table).name] = exc
435
+ return self.failure(table, exc)
436
+
437
+ def failure(self, table, exc):
438
+ spec = schema.TABLE_SPECS[table]
439
+ if isinstance(exc, client.AdsReportUnavailable):
440
+ return "not_offered", _not_offered_reason(spec, exc)
441
+ reason = str(exc)[:client.ERROR_TEXT_LIMIT]
442
+ self.report["problems"].append(f"{self.label}: {spec['label']}: {reason}")
443
+ return "failed", reason
444
+
445
+ def land(self, table, result, start, end):
446
+ """Land one group report into `table` and each eligible sibling, covering the span. -> rows into `table`."""
447
+ own = 0
448
+ for member in schema.group_of(table).members:
449
+ if member != table and member not in self.eligible:
450
+ continue
451
+ _ensure_table(self.con, member)
452
+ rows = _land(self.con, member, result.rows.get(member, []), result.report_id, start, end,
453
+ self.profile, self.cid, self.label)
454
+ self.covered.setdefault(member, []).append((start, end))
455
+ self.notes[member] = list(dict.fromkeys(self.notes.get(member, []) + list(result.notes)))
456
+ self.checkpoint(member)
457
+ if member == table:
458
+ own = rows
459
+ else:
460
+ self.pulled[member] = self.pulled.get(member, 0) + rows
461
+ for note in result.notes:
462
+ if f"{self.label}: {note}" not in self.report["problems"]:
463
+ self.report["problems"].append(f"{self.label}: {note}")
464
+ return own
465
+
466
+
467
+ def _resume(p):
468
+ """Read every report this connection left preparing before it creates another. -> {table: outcome}."""
469
+ resumed = {}
470
+ for table in TABLES:
471
+ pending = _read_pending(p.con, table, p.cid)
472
+ if pending is None:
473
+ continue
474
+ spec = schema.TABLE_SPECS[table]
475
+ _ensure_table(p.con, table)
476
+ p.checkpoint(table)
477
+ expired = f"Amazon Ads report {pending['report_id']} did not finish in 3 hours"
478
+ age = (p.stamp() - pending["created"]).total_seconds()
479
+ if age >= client.REPORT_MAX_WAIT_SECONDS:
480
+ _clear_pending(p.con, table, p.cid)
481
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
482
+ continue
483
+ limit = client.REPORT_MAX_WAIT_SECONDS - age
484
+ if p.deadline is not None:
485
+ limit = min(limit, max(0.0, p.deadline - _clock()))
486
+ try:
487
+ result = _run_report(p.key, table, pending["start"], pending["end"], p.rt, p.profile, p.cid,
488
+ max_wait=limit, report_id=pending["report_id"], now=_clock)
489
+ except client.AdsReportPending:
490
+ if (p.stamp() - pending["created"]).total_seconds() < client.REPORT_MAX_WAIT_SECONDS:
491
+ p.blocked = True
492
+ p.wait(spec, PREPARING_REASON)
493
+ resumed[table] = ("deferred", PREPARING_REASON, 0)
494
+ continue
495
+ _clear_pending(p.con, table, p.cid)
496
+ resumed[table] = (*p.failure(table, client.AdsError(expired)), 0)
497
+ continue
498
+ except client.AdsError as exc:
499
+ _clear_pending(p.con, table, p.cid)
500
+ resumed[table] = (*p.fail(table, exc), 0)
501
+ continue
502
+ _clear_pending(p.con, table, p.cid)
503
+ resumed[table] = ("done", "", p.land(table, result, pending["start"], pending["end"]))
504
+ return resumed
505
+
506
+
507
+ def _run_windows(p, table, connect_day):
508
+ """Run the group report for each window of `table` no report covered yet. -> (state, reason, rows)."""
509
+ spec = schema.TABLE_SPECS[table]
510
+ failed = p.outcome.get(schema.group_of(table).name)
511
+ if failed is not None:
512
+ return (*p.failure(table, failed), 0)
513
+ pulled = 0
514
+ for start, end in windows(p.con, table, connect_day, p.today, p.cid):
515
+ if p.covers(table, start, end):
516
+ continue
517
+ if p.used >= client.REPORTS_PER_DAY:
518
+ reason = (f"{connector_spec.AMAZON_ADS_THROUGHPUT} Today's {client.REPORTS_PER_DAY} "
519
+ f"reports are used, so days from {start.isoformat()} load on the next sync.")
520
+ p.report["problems"].append(f"{p.label}: {spec['label']}: waiting for tomorrow's report limit")
521
+ p.report["waits"].append(p.report["problems"][-1])
522
+ return "deferred", reason, pulled
523
+ if p.deadline is not None and _clock() >= p.deadline:
524
+ p.wait(spec, TIME_LIMIT_REASON)
525
+ return "deferred", TIME_LIMIT_REASON, pulled
526
+ p.used += 1
527
+ _spend_budget(p.con, p.today, p.used, p.cid)
528
+ created = p.stamp()
529
+ try:
530
+ result = _run_report(p.key, table, start, end, p.rt, p.profile, p.cid, **p.bounds())
531
+ except client.AdsReportPending as exc:
532
+ if (p.deadline is None or not exc.report_id
533
+ or (p.stamp() - created).total_seconds() >= client.REPORT_MAX_WAIT_SECONDS):
534
+ return (*p.failure(table, exc), pulled)
535
+ _write_pending(p.con, table, p.cid, exc.report_id, start, end, created)
536
+ p.checkpoint(table)
537
+ p.blocked = True
538
+ p.wait(spec, PREPARING_REASON)
539
+ return "deferred", PREPARING_REASON, pulled
540
+ except client.AdsError as exc:
541
+ return (*p.fail(table, exc), pulled)
542
+ pulled += p.land(table, result, start, end)
543
+ return "done", "", pulled
544
 
545
 
546
  def sync(tenant_key="royal-imports", *, rt=None, con=None, tables=None, now=None, log=print,
 
551
  through `amazon_store.persist_mirror`. A caller that lends `con` persists itself.
552
  `max_seconds` bounds the pass's wait on Amazon. A report still preparing then waits in the
553
  pending ledger, and the next pass reads it before it creates any other report.
554
+ One group report fills every requested member table, so a sibling's covered days spend no report.
555
  """
556
  key = amazon_store._tenant_key(tenant_key)
557
  report = {"tenant": key, "tables": {}, "problems": [], "limits": {}, "connections": [],
 
568
  own = con is None
569
  if own:
570
  con, _path = amazon_store._connect_tenant_mirror(key, rt)
 
571
  started = _clock()
572
+ deadline = None if max_seconds is None else started + float(max_seconds)
573
+ synced, checkpointed = [], []
574
  try:
575
  datastore._ensure_sync_state(con)
576
  now = _utc_now(now)
 
579
  def stamp():
580
  return now + timedelta(seconds=max(0.0, _clock() - started))
581
 
582
+ requested = [schema.short_name(t) for t in (tables or TABLES)]
583
+ for connection in connections:
584
+ cid, label, _profile, legacy = connection
 
 
 
585
  connect_day = _connect_day(con, today, cid, legacy=legacy)
586
  report.setdefault("connectDay", connect_day.isoformat())
587
  entry = {"id": cid, "label": label, "connectDay": connect_day.isoformat(), "tables": {}}
588
  report["connections"].append(entry)
589
+ p = _Pass(con, key, rt, connection, report, today=today, used=_budget(con, today, cid, legacy=legacy),
590
+ deadline=deadline, stamp=stamp, checkpointed=checkpointed)
591
+ p.eligible = {t for t in requested if schema.TABLE_SPECS[t]["offered"] and _cadence_due(con, t, now, cid)}
592
+ resumed = _resume(p)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
593
  for table in list(dict.fromkeys(requested + list(resumed))):
594
+ landed = _sync_table(p, table, entry, connect_day, now, legacy, table in requested,
595
+ resumed.get(table), log)
596
+ p.eligible.discard(table)
597
+ if landed and table_name(table) not in synced:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
598
  synced.append(table_name(table))
599
+ _record_tables(con, report, connections)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
600
  if own and rt is not None and (synced or checkpointed):
601
  _persist(key, rt, con, list(dict.fromkeys(synced + checkpointed)), report, log)
602
  finally:
 
606
  return report
607
 
608
 
609
+ _SEVERITY = {"skipped": -1, "done": 0, "not_offered": 1, "deferred": 2, "failed": 3}
610
+
611
+
612
+ def _sync_table(p, table, entry, connect_day, now, legacy, requested, prior, log):
613
+ """One table's turn in one connection's pass: run or skip it, record its state. -> True if it changed."""
614
+ con, cid, label, report = p.con, p.cid, p.label, p.report
615
+ spec = schema.TABLE_SPECS[table]
616
+ _ensure_table(con, table)
617
+ origin_changed = _refresh_origin(con, table, cid, label, legacy)
618
+ state, reason, pulled = prior or ("done", "", 0)
619
+ pulled += p.pulled.pop(table, 0)
620
+ if state != "done" or not requested:
621
+ pass
622
+ elif not spec["offered"]:
623
+ state, reason = "not_offered", spec["not_offered_reason"]
624
+ elif not _cadence_due(con, table, now, cid):
625
+ state = "skipped"
626
+ elif p.blocked:
627
+ state, reason = "deferred", PREPARING_REASON
628
+ p.wait(spec, reason)
629
+ else:
630
+ state, reason, rows = _run_windows(p, table, connect_day)
631
+ pulled += rows
632
+ if state == "done":
633
+ _record_cadence(con, table, now, cid)
634
+ if state == "done" and not reason and p.notes.get(table):
635
+ reason = " ".join(p.notes[table])
636
+ if state == "deferred" and reason in (PREPARING_REASON, TIME_LIMIT_REASON):
637
+ report["partial"] = True
638
+ if reason:
639
+ _record_limit(con, table, reason, cid, state=state)
640
+ shown = f"{label}: {reason}" if cid else reason
641
+ report["limits"][table] = " ".join(filter(None, (report["limits"].get(table), shown)))
642
+ elif state == "done":
643
+ _clear_limit(con, table, cid)
644
+ total = _count(con, table)
645
+ entry["tables"][table] = {"state": state, "pulled": pulled, "in_mirror": _count(con, table, cid)}
646
+ if state != "skipped":
647
+ amazon_store._record_state(con, _state_key(amazon_store.entity_key(_marker_table(table)), cid),
648
+ "done" if state == "done" else "partial", _count(con, table, cid), state)
649
+ previous = report["tables"].get(table, {})
650
+ report["tables"][table] = {"state": max((state, previous.get("state", state)), key=_SEVERITY.get),
651
+ "pulled": previous.get("pulled", 0) + pulled, "in_mirror": total}
652
+ log(f" {table_name(table):<36} pulled {pulled:>7} mirror total {total:>7} {state}")
653
+ return bool(pulled or origin_changed)
654
+
655
+
656
+ def _record_tables(con, report, connections):
657
+ """Write each table's aggregate state and limit across connections, and the pass's pending count."""
658
+ for table, state in report["tables"].items():
659
+ if state["state"] == "skipped":
660
+ continue
661
+ amazon_store._record_state(con, amazon_store.entity_key(_marker_table(table)),
662
+ "done" if state["state"] == "done" else "partial",
663
+ state["in_mirror"], state["state"])
664
+ if table in report["limits"]:
665
+ _record_limit(con, table, report["limits"][table], state=state["state"])
666
+ else:
667
+ _clear_limit(con, table)
668
+ report["pendingReports"] = sum(_read_pending(con, table, cid) is not None
669
+ for cid, *_ in connections for table in TABLES)
670
+ if report["partial"]:
671
+ report["continueAfterSeconds"] = CONTINUE_AFTER_SECONDS
672
+
673
+
674
  def _not_offered_reason(spec, exc):
675
  return (f"Amazon Reporting v1 does not offer {spec['label']} to this advertiser "
676
+ f"({str(exc)[:client.ERROR_TEXT_LIMIT]}). The table keeps the days it has. To ask for it, "
677
+ f"contact Amazon Ads support and quote Amazon's message.")
678
 
679
 
680
  def _persist(tenant_key, rt, con, synced, report, log):
platform/harness/amazon_client.py CHANGED
@@ -450,11 +450,8 @@ REPORT_TYPES = {
450
  "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
451
  "history": {"from": "2025-04-01",
452
  "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
453
- "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed (2026-11-14)",
454
- "periods": {"WEEK": {"from": "2025-07-13", "provisional": True,
455
- "evidence": "2026-11-14 probe refused 2025-07-06 and served 2025-10-05. "
456
- "The intervening twelve weeks are unasked. 2025-07-13 is the "
457
- "first unasked Sunday, not a proved earliest served floor"}}},
458
  "role": "Brand Analytics",
459
  "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
460
  "json_root": "dataByAsin", "monthly": True,
@@ -1127,7 +1124,7 @@ for _tables, _passes, _daily, _intraday in (
1127
  from harness import amazon_ads_schema as _ads_schema
1128
 
1129
  CADENCE.update({
1130
- "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"}
1131
  for table in _ads_schema.TABLE_SPECS
1132
  })
1133
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
 
450
  "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
451
  "history": {"from": "2025-04-01",
452
  "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
453
+ "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the "
454
+ "week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"},
 
 
 
455
  "role": "Brand Analytics",
456
  "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
457
  "json_root": "dataByAsin", "monthly": True,
 
1124
  from harness import amazon_ads_schema as _ads_schema
1125
 
1126
  CADENCE.update({
1127
+ "ads_" + table: {"perDay": 4, "lookbackDays": 60, "intradayLookbackDays": 5, "period": "day"}
1128
  for table in _ads_schema.TABLE_SPECS
1129
  })
1130
  del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
platform/harness/amazon_schema.py CHANGED
@@ -95,8 +95,6 @@ PROVENANCE = {
95
  "search_query_performance": "measured_at_sync",
96
  "sku_keywords": "derived",
97
  "keyword_ranks": "apify_dataset",
98
- "competitor_snapshots": "competitor_run",
99
- "competitor_images": "competitor_run",
100
  "fee_estimates_api": "measured_at_sync",
101
  }
102
 
@@ -107,13 +105,10 @@ PROVENANCE.update(dict.fromkeys((t + "_history" for t in connector_spec.AMAZON_H
107
  REVIEW_UNAVAILABLE_NOTE = (
108
  "Amazon's API does not supply written reviews, average star rating or review count. "
109
  "Read them on the product page or in Seller Central.")
110
- SQP_LIMIT_NOTE = "Amazon lists at most 100 searches a month for each product."
111
 
112
  #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
113
  #: screen and standing rule 2 governs it.
114
  PROVENANCE_COPY = {
115
- "competitor_run": ("Daily competitor facts saved by the competitor run. Every snapshot day is kept. "
116
- "Image rows keep each distinct variant and link, dated when first saved."),
117
  "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
118
  "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
119
  "Global position is separate. Unmatched observations remain in the raw mirror."),
@@ -308,8 +303,8 @@ def catalogue():
308
  descriptor = descriptors[key]
309
  label, native = descriptor.label, descriptor.native
310
  table[key] = {"label": label, "native": native}
311
- if provenance in {"derived", "apify_dataset", "competitor_run"}:
312
- table["columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
313
  if field["name"] != STORE]
314
  table["note"] = PROVENANCE_COPY[provenance]
315
  table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
@@ -319,7 +314,7 @@ def catalogue():
319
 
320
  def columns_for(table):
321
  """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
322
- if PROVENANCE.get(str(table)) in {"derived", "apify_dataset", "competitor_run"}:
323
  return []
324
  spec = (catalogue().get("tables") or {}).get(str(table)) or {}
325
  return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
@@ -382,17 +377,17 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
382
  fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
383
  spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
384
  spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
385
- local_sources = {
386
- "derived": ("listing_items", "search_query_performance"),
387
- "apify_dataset": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"),
388
- "competitor_run": ("amazon_rivals.run_quotes", "amazon_quotes_loop.run_tenant"),
389
- }
390
- provenance = PROVENANCE[table]
391
- if provenance in local_sources:
392
  spec["columns_source"] = {
393
  "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
394
- "provenance": provenance, "assumption": PROVENANCE_COPY[provenance],
395
- "sources": local_sources[provenance],
396
  }
397
  return spec
398
 
@@ -575,13 +570,9 @@ TABLE_SPECS.update({
575
  "browse_node_review_topics": _table_spec("browse_node_review_topics",
576
  ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
577
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
578
- (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_period_asin_query"),
579
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
580
  (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
581
- "competitor_snapshots": _table_spec("competitor_snapshots", ("asin", "snapshot_date"),
582
- grain="asin_day"),
583
- "competitor_images": _table_spec("competitor_images", ("asin", "variant", "link"),
584
- grain="asin_variant_link"),
585
  })
586
 
587
  # A history table is one row per version, identified by its version key and first valid date.
 
95
  "search_query_performance": "measured_at_sync",
96
  "sku_keywords": "derived",
97
  "keyword_ranks": "apify_dataset",
 
 
98
  "fee_estimates_api": "measured_at_sync",
99
  }
100
 
 
105
  REVIEW_UNAVAILABLE_NOTE = (
106
  "Amazon's API does not supply written reviews, average star rating or review count. "
107
  "Read them on the product page or in Seller Central.")
 
108
 
109
  #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
110
  #: screen and standing rule 2 governs it.
111
  PROVENANCE_COPY = {
 
 
112
  "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
113
  "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
114
  "Global position is separate. Unmatched observations remain in the raw mirror."),
 
303
  descriptor = descriptors[key]
304
  label, native = descriptor.label, descriptor.native
305
  table[key] = {"label": label, "native": native}
306
+ if provenance in {"derived", "apify_dataset"}:
307
+ table["columns"] = [dict(field) for field in AMAZON_FIELDS[key]
308
  if field["name"] != STORE]
309
  table["note"] = PROVENANCE_COPY[provenance]
310
  table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
 
314
 
315
  def columns_for(table):
316
  """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
317
+ if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"}:
318
  return []
319
  spec = (catalogue().get("tables") or {}).get(str(table)) or {}
320
  return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
 
377
  fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
378
  spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
379
  spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
380
+ if PROVENANCE[table] == "derived":
381
+ spec["columns_source"] = {
382
+ "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
383
+ "provenance": "derived", "assumption": PROVENANCE_COPY["derived"],
384
+ "sources": ("listing_items", "search_query_performance"),
385
+ }
386
+ if PROVENANCE[table] == "apify_dataset":
387
  spec["columns_source"] = {
388
  "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
389
+ "provenance": "apify_dataset", "assumption": PROVENANCE_COPY["apify_dataset"],
390
+ "sources": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"),
391
  }
392
  return spec
393
 
 
570
  "browse_node_review_topics": _table_spec("browse_node_review_topics",
571
  ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
572
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
573
+ (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
574
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
575
  (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
 
 
 
 
576
  })
577
 
578
  # A history table is one row per version, identified by its version key and first valid date.
platform/harness/amazon_store.py CHANGED
@@ -57,7 +57,6 @@ import threading
57
  import uuid
58
  from contextlib import closing, contextmanager
59
  from datetime import datetime, timedelta, timezone
60
- from decimal import Decimal, InvalidOperation
61
  from pathlib import Path
62
 
63
  _HERE = Path(__file__).resolve().parent
@@ -92,7 +91,7 @@ DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
92
  #: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
93
  HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
94
  HISTORY_TABLES = tuple(t + "_history" for t in HISTORY_SOURCES)
95
- DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES + ("competitor_snapshots", "competitor_images")
96
  PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES
97
  LEGACY_SNAPSHOT_TABLES = ("listings", "inventory")
98
 
@@ -2021,35 +2020,18 @@ def replace_keys_for(table):
2021
  if c not in _ORIGIN_TYPES)
2022
 
2023
 
2024
- @contextmanager
2025
- def _report_transaction(con):
2026
- con.execute("BEGIN TRANSACTION")
2027
- try:
2028
- yield
2029
- con.execute("COMMIT")
2030
- except Exception:
2031
- con.execute("ROLLBACK")
2032
- raise
2033
-
2034
-
2035
- def replace_window(con, table, cols, types, rows, keys, after=None, before=None):
2036
  """Replace incoming key groups atomically, or report why only insertion is possible.
2037
 
2038
  `after(con)` runs inside the same transaction, after the insert and before the commit.
2039
  """
2040
  missing = [c for c in keys if c not in cols]
2041
- if missing or not keys or not rows:
2042
  note = (f"{table}: window replacement unavailable; missing identity columns: "
2043
- f"{', '.join(missing)}; rows inserted; use a report carrying these columns") if missing else None
2044
- if before is None and after is None:
2045
- return _replace(con, table, cols, types, rows), note
2046
- with _report_transaction(con):
2047
- if before is not None:
2048
- before(con)
2049
- count = _replace(con, table, cols, types, rows)
2050
- if after is not None:
2051
- after(con)
2052
- return count, note
2053
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
2054
  target = quote(table)
2055
  # The window's rows are staged in batches inside the transaction, typed as the mirror and
@@ -2060,9 +2042,8 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None)
2060
  try:
2061
  predicate = " AND ".join(
2062
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
2063
- with _report_transaction(con):
2064
- if before is not None:
2065
- before(con)
2066
  count = _replace(con, temporary, cols, types, rows, source_table=table)
2067
  inserted = cols
2068
  if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys):
@@ -2074,6 +2055,10 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None)
2074
  con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}")
2075
  if after is not None:
2076
  after(con)
 
 
 
 
2077
  return count, None
2078
  finally:
2079
  con.execute(f"DROP TABLE IF EXISTS {temporary}")
@@ -2617,11 +2602,7 @@ def sqp_progress_sentence(detail):
2617
  if any(type(n) is not int or n < 0 for n in steps):
2618
  raise ValueError("Invalid saved Search Query Performance progress")
2619
  months_done, months, done, total, eta = steps
2620
- weeks_done, weeks = progress.get("weeksDone", 0), progress.get("weeksTotal", 0)
2621
- if any(type(n) is not int or n < 0 for n in (weeks_done, weeks)):
2622
- raise ValueError("Invalid saved Search Query Performance weekly progress")
2623
- weekly = f", {weeks_done:,} of {weeks:,} weeks complete" if weeks else ""
2624
- sentence = f"{months_done:,} of {months:,} months complete{weekly}, {done:,} of {total:,} batches."
2625
  if eta:
2626
  hours, minutes = round(eta / 3600), max(1, round(eta / 60))
2627
  left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes")
@@ -2649,7 +2630,7 @@ def _sqp_month(con, start, end):
2649
  "period": period,
2650
  "start": _utc(start).date().isoformat(),
2651
  "end": (_utc(end) - timedelta(days=1)).date().isoformat(),
2652
- "batches": {}, "asins": {}, "refusedAsins": {}, "retries": {}}
2653
 
2654
 
2655
  def _sqp_save_month(con, start, marker):
@@ -2658,97 +2639,17 @@ def _sqp_save_month(con, start, marker):
2658
 
2659
 
2660
  def _sqp_done(con, start, end, batch):
2661
- return not _sqp_missing_asins(_sqp_month(con, start, end), batch.split())
2662
-
2663
-
2664
- def _sqp_missing_asins(marker, asins):
2665
- """Successful ASIN summaries and durable named exclusions complete a period independently of batching."""
2666
- done = set(marker["asins"]) | set(marker.get("refusedAsins", {}))
2667
- for batch, retry in marker["retries"].items():
2668
- done.update(set(batch.split()) - set(retry["asins"]))
2669
- return [asin for asin in asins if asin not in done]
2670
-
2671
-
2672
- def _sqp_owed_tasks(con, tasks):
2673
- """Batch only each period's missing ASINs, retaining unfinished retry identities across catalogue changes."""
2674
- windows = {}
2675
- for task in tasks:
2676
- if task[0] == SQP_TABLE:
2677
- windows.setdefault(task[1:3], []).extend(task[3].split())
2678
- result = []
2679
- for task in tasks:
2680
- if task[0] != SQP_TABLE:
2681
- result.append(task)
2682
- continue
2683
- asins = windows.pop(task[1:3], None)
2684
- if asins is None:
2685
- continue
2686
- marker = _sqp_month(con, *task[1:3])
2687
- missing = _sqp_missing_asins(marker, list(dict.fromkeys(asins)))
2688
- remaining, batches = set(missing), []
2689
- for batch, retry in marker["retries"].items():
2690
- requested = set(retry["asins"])
2691
- if requested and requested <= remaining:
2692
- batches.append(batch)
2693
- remaining.difference_update(requested)
2694
- batches.extend(sqp_batches([asin for asin in missing if asin in remaining],
2695
- _client().REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"]))
2696
- result.extend((SQP_TABLE, *task[1:3], batch) for batch in batches)
2697
- return result
2698
 
2699
 
2700
  def _sqp_week_tasks(start, end, batches):
2701
- """Schedule every completed week newest first through the same ASIN batch queue."""
2702
  if "WEEK" not in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()):
2703
  return []
2704
- return [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_week_windows(start, end))
2705
  for batch in batches]
2706
 
2707
 
2708
- def _sqp_history_done(con, end):
2709
- """Every admitted period and current catalogue ASIN needs its own durable completion evidence."""
2710
- asins = _sqp_catalogue(con)
2711
- tasks = first_load_plan([SQP_TABLE], history_floor(con, SQP_TABLE, end), end,
2712
- sqp_asins=asins, starts=history_starts(con, [SQP_TABLE], end))
2713
- return bool(asins) and all(_sqp_done(con, task[1], task[2], task[3]) for task in tasks)
2714
-
2715
-
2716
- def _sqp_progress(con, months, weeks, asins, batches, end):
2717
- """Count durable per-ASIN completion separately for admitted MONTH and WEEK windows."""
2718
- from harness.amazon_client import REPORT_SUSTAINED_SECONDS
2719
-
2720
- progress = {}
2721
- for period, windows, unit, label in (("MONTH", months, "month", "%Y-%m"),
2722
- ("WEEK", weeks, "week", "%Y-%m-%d")):
2723
- floor = history_floor(con, SQP_TABLE, end, period=period)
2724
- rows = []
2725
- for w0, w1 in windows:
2726
- if w0 < floor:
2727
- continue
2728
- missing = set(_sqp_missing_asins(_sqp_month(con, w0, w1), asins))
2729
- rows.append({unit: w0.strftime(label), "total": len(batches),
2730
- "done": sum(not missing.intersection(batch.split()) for batch in batches)})
2731
- progress[unit + "s"] = rows
2732
- progress[unit + "sDone"] = sum(r["done"] >= r["total"] for r in rows)
2733
- progress[unit + "sTotal"] = len(rows)
2734
- rows = progress["months"] + progress["weeks"]
2735
- done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows)
2736
- progress.update(batchesDone=done, batchesTotal=total,
2737
- etaSeconds=int((total - done) * REPORT_SUSTAINED_SECONDS))
2738
- return progress if rows and batches else None
2739
-
2740
-
2741
- def _sqp_first_load_status(con, cohort):
2742
- """Project current period floors and per-ASIN evidence through the shared SQP progress shape."""
2743
- asins = _sqp_catalogue(con)
2744
- spec = _client().REPORT_TYPES[SQP_TABLE]
2745
- months = _month_windows(history_floor(con, SQP_TABLE, cohort), max(cohort, datetime.now(timezone.utc)))
2746
- weeks = (_week_windows(history_floor(con, SQP_TABLE, cohort, period="WEEK"), cohort)
2747
- if "WEEK" in spec.get("periods", ()) else [])
2748
- progress = _sqp_progress(con, months, weeks, asins, sqp_batches(asins, spec["asin_option_max_chars"]), cohort)
2749
- return {"progress": progress, "sentence": sqp_progress_sentence({"progress": progress})} if progress else None
2750
-
2751
-
2752
  class SQPBudgetDeferred(RuntimeError):
2753
  """A batch or its single retry remains durable for the next bounded pass."""
2754
 
@@ -2757,42 +2658,6 @@ class SQPPassDeferred(SQPBudgetDeferred):
2757
  """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running."""
2758
 
2759
 
2760
- def _sqp_document_rows(text, spec, requested, marker, meta):
2761
- """Validate the source document before flattening rows and computing per-ASIN totals."""
2762
- text = _document_text(text)
2763
- doc = json.loads(text)
2764
- if not isinstance(doc, dict):
2765
- raise ValueError("Search Query Performance document must be a JSON object")
2766
- raw = doc.get(spec["json_root"])
2767
- if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
2768
- raise ValueError("Search Query Performance document has no dataByAsin row array")
2769
- totals = {asin: {"count": 0, "purchases": 0} for asin in requested}
2770
- for row in raw:
2771
- asin = row.get("asin")
2772
- if not isinstance(asin, str):
2773
- raise ValueError("Search Query Performance ASIN must be a string")
2774
- if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]:
2775
- raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch")
2776
- purchase_data = row.get("purchaseData")
2777
- if not isinstance(purchase_data, dict):
2778
- raise ValueError("Search Query Performance purchaseData must be a JSON object")
2779
- purchase = purchase_data.get("asinPurchaseCount")
2780
- if type(purchase) is not int or purchase < 0:
2781
- raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count")
2782
- totals[asin]["count"] += 1
2783
- totals[asin]["purchases"] += purchase
2784
- columns, rows = parse_json_report(text, spec["json_root"],
2785
- document_id=meta.get("reportDocumentId"), report_type=spec["report"])
2786
- seen = set()
2787
- for row in rows:
2788
- key = amazon_schema.row_identity_for(SQP_TABLE, row)
2789
- if key in seen:
2790
- raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query")
2791
- seen.add(key)
2792
- row["_report_period"] = marker.get("period", "MONTH")
2793
- return columns, rows, totals, len(raw)
2794
-
2795
-
2796
  def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2797
  """One completed period and batch using the shared report transport and pass budget.
2798
 
@@ -2823,7 +2688,6 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2823
  reason = reason.replace("\u2014", "; ").replace("\u2013", " to ")
2824
  for asin in refused:
2825
  limit["refusedAsins"][asin] = reason
2826
- marker.setdefault("refusedAsins", {})[asin] = reason
2827
  problem(sqp_refusal_message(asin, reason))
2828
  if not refused:
2829
  limit["errors"] = [reason]
@@ -2843,15 +2707,39 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2843
  _sqp_save_month(con, start, marker)
2844
  return 0
2845
 
2846
- columns, rows, totals, source_count = _sqp_document_rows(text, spec, requested, marker, meta)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2847
  identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"]
 
 
 
 
 
 
 
2848
  columns = list(dict.fromkeys([*columns, *identity, "_report_period",
2849
  "purchase_data_asin_purchase_count", *_ORIGIN_TYPES]))
2850
  mirror = table_name(SQP_TABLE)
2851
  types = _ensure(con, mirror, columns, rows, preserve_rows=True,
2852
  declared={"purchase_data_asin_purchase_count": "BIGINT"})
2853
  marker["asins"].update(totals)
2854
- marker["batches"][batch] = {"asins": requested, "count": source_count,
2855
  "purchases": sum(t["purchases"] for t in totals.values()),
2856
  "reportDocumentId": meta["reportDocumentId"]}
2857
  con.execute("BEGIN TRANSACTION")
@@ -2895,24 +2783,6 @@ def _validate_sqp(con):
2895
  return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"}
2896
 
2897
 
2898
- def _sku_keyword_source_fields(columns):
2899
- from harness.amazon_sqp_reads import report_period_sql
2900
-
2901
- fields = {"start_date": "start_date", "end_date": "end_date", "asin": "asin",
2902
- "search_query": "search_query_data_search_query",
2903
- "impressions": "impression_data_asin_impression_count",
2904
- "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count",
2905
- "purchases": "purchase_data_asin_purchase_count"}
2906
- expressions = []
2907
- for name, source in fields.items():
2908
- numeric = name in ("impressions", "clicks", "cart_adds", "purchases")
2909
- kind, cast = ("BIGINT", "TRY_CAST") if numeric else ("VARCHAR", "CAST")
2910
- expressions.append(f'{cast}("{source}" AS {kind}) AS {name}' if source in columns else
2911
- f'NULL::{kind} AS {name}')
2912
- period = report_period_sql(columns) if {"start_date", "end_date"} <= columns else "NULL::VARCHAR"
2913
- return ", ".join([*expressions, f"{period} AS _report_period"])
2914
-
2915
-
2916
  def refresh_sku_keywords(con):
2917
  """Bind a local derived view to listing summaries and raw SQP without another source copy."""
2918
  listing_columns = _columns_of(con, "amzn_listing_items")
@@ -2940,7 +2810,19 @@ def refresh_sku_keywords(con):
2940
  output_meta = ', "_connection_id", store' if scoped else ""
2941
  backend_meta = ', l."_connection_id", l.store' if scoped else ""
2942
  backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2943
- select = _sku_keyword_source_fields(raw_columns)
 
 
 
 
 
 
 
 
 
 
 
 
2944
  raw_from = " FROM amzn_search_query_performance" if raw_columns else " WHERE FALSE"
2945
  con.execute(f"""
2946
  CREATE OR REPLACE VIEW amzn_sku_keywords AS
@@ -2956,25 +2838,23 @@ def refresh_sku_keywords(con):
2956
  SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group}
2957
  ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS (
2958
  SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
2959
- FROM raw WHERE _report_period IS NOT NULL
2960
  ), matched AS (
2961
  SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta}
2962
  FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
2963
  JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
2964
  )
2965
  SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id,
2966
- sku, asin, CASE WHEN _report_period='MONTH' THEN substr(start_date, 1, 7) || '-01' END AS month,
2967
- start_date, end_date, search_query,
2968
  impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count,
2969
  'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) ||
2970
- ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis,
2971
- _report_period{output_meta}
2972
  FROM matched
2973
  UNION ALL
2974
  SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)),
2975
  l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
2976
  l.backend_keywords, NULL, NULL,
2977
- 'Listing backend keywords only. No matching Search Query Performance rows.', NULL::VARCHAR{backend_meta}
2978
  FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match})
2979
  """)
2980
  refresh_keyword_ranks(con)
@@ -3071,56 +2951,43 @@ def _validate_keyword_ranks(con):
3071
 
3072
 
3073
  def _validate_sku_keywords(con):
3074
- """Tie distinct derived grains and purchases to raw rows separately for every MONTH and WEEK."""
3075
- from harness.amazon_sqp_reads import _independent_period
3076
-
3077
  basis = amazon_schema.PROVENANCE_COPY["derived"]
3078
- columns = _columns_of(con, table_name(SQP_TABLE))
3079
- if not columns:
3080
  return {"state": "unproven", "reason": "Search Query Performance source is absent", "source_basis": basis}
3081
  try:
3082
- declared = "_report_period" if "_report_period" in columns else "NULL::VARCHAR"
3083
- account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in columns else ""
3084
- derived_account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") else ""
3085
- raw = con.execute(f"""
3086
- SELECT CAST(start_date AS VARCHAR), CAST(end_date AS VARCHAR), list(DISTINCT {declared}),
3087
- COUNT(*), COUNT(DISTINCT (asin, search_query_data_search_query{account})),
3088
  SUM(purchase_data_asin_purchase_count), COUNT(purchase_data_asin_purchase_count),
3089
  COUNT(*) FILTER (WHERE purchase_data_asin_purchase_count < 0 OR start_date IS NULL
3090
  OR end_date IS NULL OR asin IS NULL OR search_query_data_search_query IS NULL)
3091
- FROM amzn_search_query_performance GROUP BY ALL
3092
  """).fetchall()
3093
- derived = con.execute(f"""
3094
- SELECT start_date, end_date, _report_period, COUNT(*), SUM(purchases), SUM(invalid)
3095
  FROM (
3096
- SELECT start_date, end_date, _report_period, asin, search_query{derived_account}, MIN(purchases) AS purchases,
3097
  CASE WHEN COUNT(purchases) <> COUNT(*) OR MIN(purchases) <> MAX(purchases)
3098
- OR COUNT(*) <> COUNT(DISTINCT sku)
3099
- OR COUNT(*) FILTER (WHERE month IS DISTINCT FROM CASE WHEN _report_period='MONTH'
3100
- THEN substr(start_date, 1, 7) || '-01' END) > 0 THEN 1 ELSE 0 END AS invalid
3101
  FROM amzn_sku_keywords WHERE source_grain IS NOT NULL
3102
- GROUP BY ALL
3103
- ) GROUP BY ALL
3104
  """).fetchall()
3105
- actual = {(start, end, period): {"grains": n, "purchases": purchases, "invalid": invalid}
3106
- for start, end, period, n, purchases, invalid in derived}
3107
- held, months = {}, {}
3108
- for start, end, declarations, count, distinct, purchases, present, invalid in raw:
3109
- labels = {_independent_period(start, end, declared) for declared in declarations}
3110
- period = labels.pop() if len(labels) == 1 else None
3111
- key = (start, end, period)
3112
- observed = actual.get(key, {"grains": 0, "purchases": 0, "invalid": 0})
3113
- good = (period is not None and count == distinct == present and not invalid and not observed["invalid"]
3114
  and observed["grains"] == distinct and observed["purchases"] == purchases)
3115
- held[key] = {"state": "ok" if good else "refused", "start": start, "end": end, "period": period,
3116
- "expected": {"grains": distinct, "purchases": purchases}, "actual": observed}
3117
- if period == "MONTH":
3118
- months[start[:7]] = held[key]
3119
- extra = set(actual) - set(held)
3120
- good = bool(held) and not extra and all(row["state"] == "ok" for row in held.values())
3121
- return {"state": "ok" if good else "refused" if held or extra else "unproven",
3122
- "months": months, "periods": list(held.values()), "source_basis": basis,
3123
- "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every period" if good else
3124
  "SQP coverage or purchases differ. Unmapped ASINs, duplicate grains and missing counts cannot reconcile")}
3125
  except Exception as exc:
3126
  return {"state": "refused", "source_basis": basis,
@@ -3156,12 +3023,8 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY
3156
  for table in tables:
3157
  spec = _client().REPORT_TYPES[table]
3158
  table_start = starts.get(table, start) if starts is not None else start
3159
- if table == SQP_TABLE:
3160
- rule = _history_rule(spec, "WEEK")
3161
- week_start = _date_utc(rule["from"]) if rule.get("from") else table_start
3162
- week_start = starts.get((table, "WEEK"), week_start) if starts is not None else week_start
3163
- weekly_history.extend(_sqp_week_tasks(week_start, end, sqp_batches(
3164
- sqp_asins, spec["asin_option_max_chars"])))
3165
  if table == "settlements":
3166
  continue
3167
  if spec.get("snapshot"):
@@ -4096,13 +3959,6 @@ def standing_refusals(report):
4096
  return sorted(holds["standing"]) if holds else []
4097
 
4098
 
4099
- def _deferred_table(reason, refusal, *, current=False):
4100
- """Keep a seller-wide refusal visible when a scheduling hold prevents another request."""
4101
- if refusal and refusal.get("state") == "not_offered":
4102
- return {"state": "refused", "amazonStatus": "NOT_OFFERED", "reason": refusal["reason"]}
4103
- return {"state": "deferred", "reason": reason, "current": current}
4104
-
4105
-
4106
  class _AmazonMirrorStore(_mirrors.MirrorStore):
4107
  def __init__(self):
4108
  self.connector, self.spec = "amazon", _mirror_spec("amazon")
@@ -4180,10 +4036,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
4180
  """
4181
  rows = report.get("tables") or {}
4182
  problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
4183
- # Landed SQP batches and dated periods publish even while older history stays owed.
4184
  if report.get("state") in {"partial", "failed"} and not any(
4185
  isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE
4186
- or bool(row.get("landedMonths") or row.get("landedWeeks")))
4187
  and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
4188
  return None
4189
  unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
@@ -4208,7 +4064,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
4208
  unfinished |= set(partner)
4209
  for names in scoped.values():
4210
  names |= {pairs[name] for name in list(names) if name in pairs and pairs[name] not in unfinished}
4211
- if not unfinished and not scoped and not problems and report.get("state") not in {"partial", "failed"}:
4212
  return None
4213
  owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))}
4214
  account_owned = {str(problem) for cid, result in accounts.items()
@@ -5255,10 +5111,8 @@ def _restore_first_load_wait(tenant_key, con):
5255
  return due
5256
 
5257
 
5258
- def _loaded_refresh_hold(con, table, now, *, slot_pass=False, history=False):
5259
  """Return a loaded table's cadence hold while its first load continues, or None when due."""
5260
- if history and table == "brand_analytics" and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") == "1":
5261
- return None
5262
  now = _utc(now)
5263
  tick = _read_state(con, entity_key("sync_tick_" + table))
5264
  # A queued resync is a repair: only Amazon's own floor holds it.
@@ -5550,8 +5404,6 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table
5550
  floor = read_backfill_floor(con, table)
5551
  if cohort is None:
5552
  return False
5553
- if table == SQP_TABLE:
5554
- return _sqp_history_done(con, cohort)
5555
  # The same per-table floor first_load_plan uses, so a load that lands every window completes.
5556
  return floor is not None and floor <= history_floor(con, table, cohort)
5557
  # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
@@ -5621,8 +5473,6 @@ def first_load_status(tenant_key, rt=None):
5621
  "nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None}
5622
  if job:
5623
  out["job"] = job
5624
- if SQP_TABLE in enabled_tables:
5625
- out["sqp"] = _sqp_first_load_status(con, cohort)
5626
  return out
5627
  finally:
5628
  con.close()
@@ -6444,22 +6294,9 @@ def first_load_job_summary(docs, *, table_status=None):
6444
  "remaining": record.get("remaining", last.get("remainingAfter")),
6445
  "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"),
6446
  "operation": last.get("operation"), "resume_at": last.get("resume_at"),
6447
- "sqp": last.get("sqp"),
6448
  "tables": {**(last.get("tables") or {}), **(table_status or {})}}
6449
 
6450
 
6451
- def _checkpoint_first_load_sqp(tenant_key, rt, cohort, record, save):
6452
- """Checkpoint derived SQP progress after publication without making it another completion authority."""
6453
- if cohort is None or not read_database_enablement(rt)["databases"].get(amazon_schema.DATABASE_PREFIX + SQP_TABLE):
6454
- return
6455
- con, _path = _connect_tenant_mirror(tenant_key, rt)
6456
- try:
6457
- record["sqp"] = _sqp_first_load_status(con, cohort)
6458
- finally:
6459
- con.close()
6460
- save(lambda doc: doc.update(sqp=record["sqp"]))
6461
-
6462
-
6463
  def _record_generation(docs, row):
6464
  def add(doc):
6465
  rows = [r for r in (doc.get("generations") or []) if r.get("generation") != row["generation"]]
@@ -6469,18 +6306,6 @@ def _record_generation(docs, row):
6469
  docs.update(FIRST_LOAD_GENERATIONS_KEY, add, flush="sync")
6470
 
6471
 
6472
- def _record_brand_week_write(record, task, wall_seconds, cpu_seconds, rows):
6473
- """Accumulate measured writes while retaining each week's own CPU and wall time."""
6474
- timing = record.setdefault("brandAnalytics", {"wallSeconds": 0, "cpuSeconds": 0, "periods": []})
6475
- week = {"account": task["account"], "_period_start": task["start"][:10],
6476
- "_period_end": task["end"][:10], "wallSeconds": round(wall_seconds, 3),
6477
- "cpuSeconds": round(cpu_seconds, 3)}
6478
- timing["periods"].append(week)
6479
- for field in ("wallSeconds", "cpuSeconds"):
6480
- timing[field] = round(sum(item[field] for item in timing["periods"]), 3)
6481
- timing["rows"] = rows
6482
-
6483
-
6484
  def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_seconds=5 * 3600, log=print,
6485
  docs=None, plan=None, client=None, write=None, delegate=None, counts=None,
6486
  holder=None, now=time.time, sleep=time.sleep, poll_seconds=None, job_id=None):
@@ -6546,13 +6371,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6546
  if cursor.get("end") and end and cursor["end"] != _iso(end):
6547
  # A new cohort end means a new first load; the old cursor's windows describe another plan.
6548
  cursor = {}
6549
- cohort = end or (_utc(cursor["end"]) if cursor.get("end") else None)
6550
  finished = dict(cursor.get("finished") or {})
6551
  opened = dict(cursor.get("open") or {})
6552
  throttled = dict(cursor.get("throttled") or {})
6553
  failed = dict(cursor.get("failed") or {})
6554
  downloads = dict(cursor.get("downloads") or {})
6555
- save(lambda doc: doc.update(tenant=key, end=_iso(cohort) if cohort else None, backfillDays=DEFAULT_BACKFILL_DAYS,
6556
  generation=int(generation), finished=finished, open=opened,
6557
  throttled=throttled, failed=failed, downloads=downloads))
6558
 
@@ -6561,7 +6385,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6561
 
6562
  record["remainingBefore"] = sum(owed(t) for t in tasks)
6563
  mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed)
6564
- # A write that finishes no window waits for the next generation, including unserved documents.
6565
  stalled = set()
6566
  own_tables = sorted({t["table"] for t in mine if owed(t)})
6567
  ready = {}
@@ -6678,8 +6502,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6678
  prefix.append(task)
6679
  if not prefix:
6680
  continue
6681
- if table == "brand_analytics":
6682
- prefix = prefix[:1] # Each week needs its own measured write interval.
6683
  sibling = SHARED_REPORT_SIBLINGS.get(table)
6684
  pair = [t for t in by_table.get(sibling, []) if owed(t)][:len(prefix)] if sibling else []
6685
  units = [table] + ([sibling] if pair and [t["key"] for t in pair] == [t["key"] for t in prefix]
@@ -6692,30 +6514,28 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6692
  log(f"{table}: the first-load write failed ({type(exc).__name__}); "
6693
  "its windows stay owed and the next generation downloads them again.")
6694
  for t in prefix:
6695
- stalled.add(t["id"])
6696
  ready.pop(t["key"], None)
6697
  continue
6698
- wall_seconds, cpu_seconds = time.time() - t0, time.process_time() - c0
 
 
 
6699
  stamp_done = _epoch_iso(now())
6700
  for unit in units:
6701
  row = result["tables"].get(unit) or {}
6702
- ok = result["published"] and (row.get("state", "done") not in {
6703
  "failed", "refused", "throttled", "pending", "held", "deferred", "disabled"}
6704
- or unit == "brand_analytics" and row.get("state") == "deferred"
6705
- and row.get("current") and row.get("landedWeeks", 0) > 0)
6706
  if not ok:
6707
  continue
6708
  for t in (prefix if unit == table else pair):
6709
  if t["key"] in result["served"]:
6710
- if unit == "brand_analytics":
6711
- _record_brand_week_write(record, t, wall_seconds, cpu_seconds, row.get("rows"))
6712
  finished[t["id"]] = {"at": stamp_done, "generation": int(generation),
6713
  "reportId": (opened.get(t["key"]) or {}).get("reportId")}
6714
  record["windowsWritten"] += 1
6715
  if unit not in record["tablesPublished"]:
6716
  record["tablesPublished"].append(unit)
6717
  for t in prefix + pair:
6718
- if t["id"] not in finished:
6719
  stalled.add(t["id"])
6720
  ready.pop(t["key"], None)
6721
  if t["id"] in finished or t["key"] not in result["served"]:
@@ -6755,8 +6575,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6755
  break
6756
  record["otherTablesPasses"] += 1
6757
  result = result if isinstance(result, dict) else {}
6758
- if plan is None:
6759
- _checkpoint_first_load_sqp(key, rt, cohort, record, save)
6760
  if result.get("state") != "partial" or not result.get("landedTasks"):
6761
  break
6762
  try:
@@ -6765,16 +6583,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
6765
  after = tasks
6766
  record["remainingAfter"] = sum(t["id"] not in finished and not t.get("landed") and t["id"] not in failed
6767
  for t in after)
6768
- record["brandAnalyticsPeriods"] = sorted({t["start"][:10] for t in tasks + after
6769
- if t["table"] == "brand_analytics" and t.get("start")
6770
- and (t.get("landed") or t["id"] in finished)})
6771
  try:
6772
  row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES)))
6773
  except Exception: # noqa: BLE001
6774
  row_counts = {}
6775
  table_status = None
6776
  if plan is None:
6777
- _checkpoint_first_load_sqp(key, rt, cohort, record, save)
6778
  con, _path = _connect_tenant_mirror(key, rt)
6779
  try:
6780
  table_status = {"inventory": inventory_table_status(con)}
@@ -6816,129 +6630,6 @@ _AMAZON_FLOOR_TEXT = re.compile(
6816
  _FLOOR_MIN_AGE = timedelta(days=45)
6817
 
6818
 
6819
- def _sqp_depth_start(value, period):
6820
- """Admit a complete MONTH or Sunday-based WEEK boundary."""
6821
- if period not in _client().SQP_PERIOD:
6822
- raise ValueError("SQP depth supports MONTH and WEEK only")
6823
- if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
6824
- value += "T00:00:00Z"
6825
- day = _utc(value)
6826
- boundary = _grain_floor(day, period.lower())
6827
- if day != boundary:
6828
- raise ValueError("SQP depth needs the first day of a complete period")
6829
- return day
6830
-
6831
-
6832
- def _sqp_depth_shift(start, period, count):
6833
- if period == "WEEK":
6834
- return start + timedelta(weeks=count)
6835
- return _months_before(start, -count)
6836
-
6837
-
6838
- def _sqp_depth_eligible(row, now):
6839
- """Only Amazon's old out-of-range refusal can bound served history."""
6840
- return (row["state"] == "out_of_range" and row.get("amazonStatus") in ("FATAL", 400)
6841
- and bool(_AMAZON_FLOOR_TEXT.search(row.get("amazon") or ""))
6842
- and _date_utc(row["end"]) + timedelta(days=1) <= now - _FLOOR_MIN_AGE)
6843
-
6844
-
6845
- def _sqp_depth_observation(request, asin, period, start, now):
6846
- cx = _client()
6847
- end = _sqp_depth_shift(start, period, 1)
6848
- row = {"period": period, "start": start.date().isoformat(),
6849
- "end": (end - timedelta(days=1)).date().isoformat(), "reportId": None,
6850
- "state": "empty", "hasRows": False, "floorEligible": False}
6851
- try:
6852
- text, meta = request(cx.REPORT_TYPES[SQP_TABLE]["report"], start=_iso(start),
6853
- end=_iso(end - timedelta(seconds=1)),
6854
- options={"reportPeriod": period, "asin": asin})
6855
- except cx.AmazonError as exc:
6856
- words, status = str(getattr(exc, "vendor_reason", "") or exc), getattr(exc, "status", None)
6857
- if status not in ("FATAL", 400) or not _AMAZON_FLOOR_TEXT.search(words):
6858
- raise
6859
- row.update(state="out_of_range", amazonStatus=status, amazon=words,
6860
- reportId=getattr(exc, "report_id", None))
6861
- row["floorEligible"] = _sqp_depth_eligible(row, now)
6862
- return row
6863
- _columns, _rows, _totals, count = _sqp_document_rows(
6864
- text, cx.REPORT_TYPES[SQP_TABLE], [asin], row, meta)
6865
- row.update(state="served" if count else "empty", hasRows=bool(count),
6866
- reportId=meta.get("reportId"), reportDocumentId=meta.get("reportDocumentId"), amazonStatus="DONE")
6867
- return row
6868
-
6869
-
6870
- def _sqp_depth_linear(rows, start, period):
6871
- refusals = 0
6872
- while start.date().isoformat() in rows:
6873
- row = rows[start.date().isoformat()]
6874
- refusals = refusals + 1 if row["floorEligible"] else 0
6875
- if refusals >= 2:
6876
- return None
6877
- start = _sqp_depth_shift(start, period, -1)
6878
- return start
6879
-
6880
-
6881
- def _sqp_depth_next(rows, newest, period):
6882
- served = sorted(day for day, row in rows.items() if row["state"] == "served")
6883
- if not served:
6884
- return _sqp_depth_linear(rows, newest, period)
6885
- first = _sqp_depth_start(served[0], period)
6886
- refused = sorted(day for day, row in rows.items() if row["floorEligible"] and day < served[0])
6887
- if refused:
6888
- low = _sqp_depth_start(refused[-1], period)
6889
- distance = ((first - low).days // 7 if period == "WEEK" else
6890
- (first.year - low.year) * 12 + first.month - low.month)
6891
- if distance > 1:
6892
- middle = _sqp_depth_shift(low, period, distance // 2)
6893
- if middle.date().isoformat() not in rows:
6894
- return middle
6895
- return _sqp_depth_linear(rows, _sqp_depth_shift(first, period, -1), period)
6896
-
6897
-
6898
- def _sqp_depth_result(rows, period, newest, reports, *, done=False):
6899
- served = sorted(day for day, row in rows.items() if row["state"] == "served")
6900
- first = _sqp_depth_start(served[0], period) if served else None
6901
- bounded = bool(first) and all(rows.get(_sqp_depth_shift(first, period, -n).date().isoformat(), {}).get(
6902
- "floorEligible", False) for n in (1, 2))
6903
- return {"period": period, "newest": newest.date().isoformat(),
6904
- "earliestServed": served[0] if served else None,
6905
- "floor": served[0] if done and bounded else None,
6906
- "state": ("measured" if bounded else "unproven") if done else "running",
6907
- "reportsRequested": reports, "observations": [rows[day] for day in sorted(rows, reverse=True)]}
6908
-
6909
-
6910
- def sqp_depth_walk(request, asin, period, newest, known=(), *, now=None, checkpoint=None):
6911
- """Find one ASIN's served boundary, retaining empty periods and requiring two adjacent old refusals."""
6912
- if not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin):
6913
- raise ValueError("SQP depth needs one ASIN")
6914
- newest, now = _sqp_depth_start(newest, period), _utc(now)
6915
- if _sqp_depth_shift(newest, period, 1) > now:
6916
- raise ValueError("SQP depth needs a completed newest period")
6917
- rows = {}
6918
- for saved in known:
6919
- row = dict(saved)
6920
- start = _sqp_depth_start(row["start"], period)
6921
- if row.get("period") != period or start > newest or row.get("state") not in (
6922
- "served", "empty", "out_of_range"):
6923
- raise ValueError("SQP depth known observations must belong to this period and window")
6924
- end = (_sqp_depth_shift(start, period, 1) - timedelta(days=1)).date().isoformat()
6925
- if row.get("end") != end or (row["state"] == "served" and row.get("hasRows") is not True):
6926
- raise ValueError("SQP depth known observations need matching windows and row evidence")
6927
- row["floorEligible"] = _sqp_depth_eligible(row, now)
6928
- rows[row["start"]] = row
6929
- reports = 0
6930
- while (start := _sqp_depth_next(rows, newest, period)) is not None:
6931
- row = _sqp_depth_observation(request, asin, period, start, now)
6932
- rows[row["start"]] = row
6933
- reports += 1
6934
- if checkpoint:
6935
- checkpoint(_sqp_depth_result(rows, period, newest, reports))
6936
- result = _sqp_depth_result(rows, period, newest, reports, done=True)
6937
- if checkpoint:
6938
- checkpoint(result)
6939
- return result
6940
-
6941
-
6942
  def _dated_table(table):
6943
  """A table whose first load plans dated history windows."""
6944
  spec = _client().REPORT_TYPES.get(table)
@@ -7046,23 +6737,20 @@ def _seller_floor(con, end):
7046
  f"{DEFAULT_BACKFILL_DAYS} day fallback.")
7047
 
7048
 
7049
- def _history_limit_key(table, period=None):
7050
- """Keep the legacy MONTH key and isolate every other period's observed history floor."""
7051
- suffix = "" if period in (None, "MONTH") else "_" + period
7052
- return entity_key(f"limit_{table}_history{suffix}")
7053
 
7054
 
7055
- def _observed_history_limit(con, table, period=None):
7056
  """-> (floor, reason) Amazon showed by refusing an older window, or None."""
7057
- state = _read_state(con, _history_limit_key(table, period))
7058
  try:
7059
  detail = json.loads(state["detail"]) if state and state.get("detail") else {}
7060
  if detail.get("floor"):
7061
  return _utc(detail["floor"]), str(detail.get("reason") or "")
7062
  except (ValueError, TypeError, AttributeError):
7063
  pass
7064
- if period not in (None, "MONTH"):
7065
- return None
7066
  legacy = _read_state(con, entity_key("limit_" + table))
7067
  try:
7068
  reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else ""
@@ -7074,13 +6762,7 @@ def _observed_history_limit(con, table, period=None):
7074
  return None
7075
 
7076
 
7077
- def _history_rule(spec, period=None):
7078
- """A period-specific rule replaces the table default without inheriting another period's floor."""
7079
- rule = spec.get("history") or {}
7080
- return rule.get("periods", {}).get(period, rule)
7081
-
7082
-
7083
- def history_bound(con, table, end, *, now=None, period=None):
7084
  """-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it.
7085
 
7086
  max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain.
@@ -7088,9 +6770,9 @@ def history_bound(con, table, end, *, now=None, period=None):
7088
  `end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with.
7089
  """
7090
  spec = _client().REPORT_TYPES.get(table) or {}
7091
- grain = "week" if period == "WEEK" else _history_grain(spec)
7092
  now = _utc(now) if now is not None else datetime.now(timezone.utc)
7093
- rule = _history_rule(spec, period)
7094
  if rule.get("from") or rule.get("seller_floor") is False:
7095
  # Brand-level data (Brand Analytics) predates the seller's first order, so the seller month
7096
  # does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses.
@@ -7112,36 +6794,29 @@ def history_bound(con, table, end, *, now=None, period=None):
7112
  elif rule.get("from"):
7113
  bound = _grain_ceil(_date_utc(rule["from"]), grain)
7114
  rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only."
7115
- if rule.get("provisional"):
7116
- rule_cause = (f"Weekly history starts at the provisional planning bound {bound.date().isoformat()}. "
7117
- "The earliest served week is unproved; an older range refusal sets the observed floor.")
7118
  if bound is not None and bound > floor:
7119
  floor, cause = bound, rule_cause
7120
- observed = _observed_history_limit(con, table, period)
7121
  if observed and observed[0] > floor:
7122
  floor, cause = observed[0], observed[1] or (
7123
  f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.")
7124
- if (spec.get("weekly") and not WEEKLY_HISTORY_PLANNED
7125
- and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") != "1"):
7126
  latest = _week_start(_utc(end)) - timedelta(days=7)
7127
  if latest > floor:
7128
  older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else ""
7129
  floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, "
7130
- "because weekly history requires AIOS_AMAZON_WEEKLY_HISTORY=1." + older)
7131
  return floor, cause
7132
 
7133
 
7134
- def history_floor(con, table, end, *, now=None, period=None):
7135
  """The oldest history start `first_load_plan` and `needs_first_load` use for `table`."""
7136
- return history_bound(con, table, end, now=now, period=period)[0]
7137
 
7138
 
7139
  def history_starts(con, tables, end, *, now=None):
7140
  """{table: history_floor} for each dated table; the one dict every first-load planner passes."""
7141
- starts = {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)}
7142
- if SQP_TABLE in starts and "WEEK" in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()):
7143
- starts[SQP_TABLE, "WEEK"] = history_floor(con, SQP_TABLE, end, now=now, period="WEEK")
7144
- return starts
7145
 
7146
 
7147
  def history_status(con, tables, end):
@@ -7167,7 +6842,8 @@ def _history_reopened(con, table, cohort):
7167
  return False
7168
  bound = history_floor(con, table, cohort)
7169
  if table == SQP_TABLE:
7170
- return bool(_sqp_catalogue(con)) and not _sqp_history_done(con, cohort)
 
7171
  floor = read_backfill_floor(con, table)
7172
  return floor is not None and floor > bound
7173
 
@@ -7468,12 +7144,11 @@ def _report_document(con, table, fields, report_type, **kwargs):
7468
  return result
7469
 
7470
 
7471
- def _newer_history_landed(con, table, end, period=None):
7472
  """A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage."""
7473
  if table == SQP_TABLE:
7474
- windows = _week_windows if period == "WEEK" else _month_windows
7475
  return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values())
7476
- for w0, w1 in windows(end, datetime.now(timezone.utc)))
7477
  floor = read_backfill_floor(con, table)
7478
  return floor is not None and end <= floor
7479
 
@@ -7488,11 +7163,10 @@ def _close_history_floor(con, table, exc, start, end):
7488
  return None
7489
  now = datetime.now(timezone.utc)
7490
  status = getattr(exc, "status", None)
7491
- period = _sqp_period(start, end) if table == SQP_TABLE else None
7492
  words = str(getattr(exc, "vendor_reason", "") or exc)
7493
  floor = read_backfill_floor(con, table)
7494
  said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc))
7495
- and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end, period))
7496
  unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True)
7497
  and floor is not None and end <= floor and start < now - timedelta(days=365)
7498
  and bool(_read_state(con, entity_key("reconcile_" + table))))
@@ -7502,25 +7176,14 @@ def _close_history_floor(con, table, exc, start, end):
7502
  reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. "
7503
  "Keep the connection enabled for newer data; ask Seller Support for older history.")
7504
  amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300]
7505
- _record_state(con, _history_limit_key(table, period), "done", detail=json.dumps({
7506
- "floor": _iso(stop), "period": period, "reason": reason, "amazon": amazon,
7507
- "window": [_iso(start), _iso(end)]}))
7508
  if table != SQP_TABLE:
7509
  # SQP's limit_ key is its progress document; its floor lives in the history marker only.
7510
  _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason}))
7511
  return reason
7512
 
7513
 
7514
- def _history_task_period(task):
7515
- return _sqp_period(task[1], task[2]) if task[0] == SQP_TABLE and task[1] is not None else None
7516
-
7517
-
7518
- def _history_cut_tasks(tasks, table, period, end):
7519
- """Only the refused period's windows at or below its floor are no longer owed."""
7520
- return (task for task in tasks if task[0] == table and task[2] is not None
7521
- and task[2] <= end and _history_task_period(task) == period)
7522
-
7523
-
7524
  def _record_reconciliation(con, table, marker):
7525
  """Keep the most recent landed period, even while backfill travels backwards."""
7526
  key = entity_key("reconcile_" + table)
@@ -9146,156 +8809,6 @@ def _order_metric_totals(records, *, flattened=False):
9146
  return totals
9147
 
9148
 
9149
- ORDER_LINE_PROOF = "reconcile_order_items_"
9150
-
9151
-
9152
- def _order_report_months(start, end):
9153
- first = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
9154
- while first < _utc(end):
9155
- following = (first.replace(day=28) + timedelta(days=4)).replace(day=1)
9156
- yield first, following
9157
- first = following
9158
-
9159
-
9160
- def _order_previous_months(con, orders):
9161
- if not {"amazon_order_id", "purchase_date"} <= _columns_of(con, "amzn_order_items"):
9162
- return set()
9163
- result = con.execute("SELECT DISTINCT strftime(TRY_CAST(purchase_date AS TIMESTAMPTZ), '%Y-%m-01') "
9164
- "FROM amzn_order_items WHERE amazon_order_id IN (SELECT unnest(?))", [list(orders)])
9165
- return {month for (month,) in result.fetchall()}
9166
-
9167
-
9168
- def _order_line_summary(summary, row):
9169
- status = str(row.get("order_status") or "").strip().lower()
9170
- if status in ("cancelled", "canceled"):
9171
- return
9172
- summary["count"] += 1
9173
- origin = f"{row['_report_document_id']}:{row['_report_row_number']}".encode()
9174
- summary["origin_xor"] = str(int(summary["origin_xor"]) ^ int.from_bytes(
9175
- hashlib.md5(origin, usedforsecurity=False).digest(), "little"))
9176
- currency = row.get("currency")
9177
- if not status or not currency or (summary["currency"] and summary["currency"] != currency):
9178
- summary["invalid"] = True
9179
- summary["currency"] = currency
9180
- try:
9181
- price = Decimal(str(row.get("item_price")))
9182
- if not price.is_finite() or price < 0:
9183
- raise ValueError("invalid order line price")
9184
- summary["sales"] = str(Decimal(summary["sales"]) + price)
9185
- except (InvalidOperation, ValueError):
9186
- summary["invalid"] = True
9187
-
9188
-
9189
- def _order_report_observations(con, cols, rows, meta, report_type, start, end):
9190
- """Measure raw report windows before order grouping, typing or insertion."""
9191
- spec = _client().REPORT_TYPES["order_items"]
9192
- required = {"amazon_order_id", "purchase_date", "asin", "order_status", "item_price", "currency"}
9193
- proof = {"reportType": report_type, "reportDocumentId": meta.get("reportDocumentId"),
9194
- "start": _iso(start), "end": _iso(end), "months": {}, "touched": set()}
9195
- if report_type == spec["report"]:
9196
- for first, following in _order_report_months(start, end):
9197
- proof["months"][first.date().isoformat()] = {
9198
- "start": _iso(max(first, start)), "end": _iso(min(following, end)), "asins": {},
9199
- "invalid": bool(cols and not required <= set(cols))}
9200
- orders = set()
9201
- for row in rows:
9202
- orders.add(str(row.get("amazon_order_id") or ""))
9203
- if len(orders) >= 1000:
9204
- proof["touched"].update(_order_previous_months(con, orders))
9205
- orders.clear()
9206
- try:
9207
- purchased = _utc(row["purchase_date"])
9208
- if not row["purchase_date"]:
9209
- raise ValueError("missing purchase date")
9210
- except (KeyError, TypeError, ValueError, AttributeError):
9211
- proof["touched"].add(None)
9212
- continue
9213
- month = purchased.strftime("%Y-%m-01")
9214
- proof["touched"].add(month)
9215
- if report_type != spec["report"] or not start <= purchased < end:
9216
- continue
9217
- window = proof["months"][month]
9218
- if not row.get("asin") or not row.get("amazon_order_id"):
9219
- window["invalid"] = True
9220
- summary = window["asins"].setdefault(str(row.get("asin") or ""),
9221
- {"count": 0, "sales": "0", "currency": None, "invalid": False, "origin_xor": "0"})
9222
- _order_line_summary(summary, row)
9223
- proof["touched"].update(_order_previous_months(con, orders))
9224
- return proof
9225
-
9226
-
9227
- def _order_report_totals(windows):
9228
- totals = {}
9229
- for window in windows:
9230
- for asin, raw in window["asins"].items():
9231
- total = totals.setdefault(asin, {"count": 0, "sales": "0", "currency": None,
9232
- "invalid": False, "origin_xor": "0"})
9233
- total["count"] += raw["count"]
9234
- total["sales"] = str(Decimal(total["sales"]) + Decimal(raw["sales"]))
9235
- total["origin_xor"] = str(int(total["origin_xor"]) ^ int(raw["origin_xor"]))
9236
- total["invalid"] |= raw["invalid"] or bool(
9237
- raw["count"] and total["currency"] and total["currency"] != raw["currency"])
9238
- if raw["count"]:
9239
- total["currency"] = raw["currency"]
9240
- return totals
9241
-
9242
-
9243
- def _record_order_report(con, proof):
9244
- """Publish raw line totals with disjoint purchase-date windows or an explicit invalidation."""
9245
- prefix = entity_key(ORDER_LINE_PROOF)
9246
- touched = proof["touched"] | proof["months"].keys()
9247
- unknown = None in touched
9248
- if unknown:
9249
- touched.update(row[0][len(prefix):].replace("_", "-") + "-01" for row in con.execute(
9250
- "SELECT entity FROM _sync_state WHERE starts_with(entity, ?)", [prefix]).fetchall())
9251
- for month in sorted(touched - {None}):
9252
- key = prefix + month[:7].replace("-", "_")
9253
- saved = _read_state(con, key) or {}
9254
- try:
9255
- previous = json.loads(saved.get("detail") or "{}")
9256
- windows = previous.get("windows", [])
9257
- if not isinstance(windows, list):
9258
- raise ValueError("invalid order source windows")
9259
- window = proof["months"].get(month)
9260
- windows = [old for old in windows if window and
9261
- (old["end"] <= window["start"] or old["start"] >= window["end"])]
9262
- except (ValueError, TypeError, KeyError, AttributeError):
9263
- windows, window = [], proof["months"].get(month)
9264
- if window and not unknown:
9265
- windows.append({**window, "reportDocumentId": proof["reportDocumentId"]})
9266
- first = _utc(month + "T00:00:00Z")
9267
- following = (first.replace(day=28) + timedelta(days=4)).replace(day=1)
9268
- marker = {"version": 1, "source": _client().REPORT_TYPES["order_items"]["validates_against"],
9269
- "reportType": _client().REPORT_TYPES["order_items"]["report"], "cancelled_excluded": True,
9270
- "start": month, "end": following.date().isoformat(), "windows": windows,
9271
- "asins": _order_report_totals(windows),
9272
- "reportDocumentIds": list(dict.fromkeys(w["reportDocumentId"] for w in windows))}
9273
- if not windows:
9274
- marker["reason"] = "Order updates require purchase-month report proof to be refreshed"
9275
- marker["invalidated_by"] = {key: proof[key] for key in
9276
- ("reportType", "reportDocumentId", "start", "end")}
9277
- _record_state(con, key, "done", detail=json.dumps(marker))
9278
-
9279
-
9280
- def _reset_order_report(con):
9281
- if _columns_of(con, "amzn_order_items"):
9282
- con.execute("DELETE FROM amzn_order_items")
9283
- con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [entity_key(ORDER_LINE_PROOF)])
9284
-
9285
-
9286
- def _write_order_report_window(con, cols, types, rows, proof, *, reset=False):
9287
- before = _reset_order_report if reset else None
9288
- after = lambda target: _record_order_report(target, proof)
9289
- if not cols:
9290
- with _report_transaction(con):
9291
- if before:
9292
- before(con)
9293
- after(con)
9294
- return 0, None
9295
- return replace_window(con, "amzn_order_items", cols, types, rows, replace_keys_for("order_items"),
9296
- before=before, after=after)
9297
-
9298
-
9299
  def _order_status_observations(con, rows, meta):
9300
  """Retain status bounds for the current Sales window in the existing account sync state."""
9301
  saved = _read_state(con, entity_key("reconcile_order_metrics"))
@@ -10249,18 +9762,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10249
  and (table in completed or (read_watermark(con, table) and table not in repairs))]
10250
  due_loaded = []
10251
  for table in loaded:
10252
- history = table == "brand_analytics" and table in reopened
10253
- hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass, history=history)
10254
  if hold is None:
10255
- if not history:
10256
- due_loaded.append(table)
10257
  continue
10258
  next_due, reason = hold
10259
  report["tables"][table] = {"state": "deferred", "reason": reason, "nextDue": _iso(next_due),
10260
  "current": current_table(table)}
10261
  log(f"{table}: DEFERRED: {reason}")
10262
- if history:
10263
- want.remove(table)
10264
  if due_loaded:
10265
  # Refreshes use their own floors; the history partition retains its bounded budget.
10266
  con.close()
@@ -10292,13 +9801,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10292
  or table in completed or table in FEEDBACK_TABLES
10293
  or (table == "settlements" and not refresh_loaded)):
10294
  continue
10295
- tick = _read_state(con, entity_key("sync_tick_" + table))
10296
- reason = ("This table has no recorded sync yet. Its first load is still running." if not tick else
10297
- "This table's first load is still running. Its previous rows were kept.")
10298
- report["tables"][table] = _deferred_table(reason, prior_refusals.get(table))
10299
  # The planner waits for the next phase on this hold. The card's tick stays as it was.
10300
  _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred")
10301
- if table == SQP_TABLE and tick and tick.get("state") == "throttled":
 
10302
  # The first load owns SQP now; a passed SQP deadline must not make it due every minute.
10303
  _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
10304
  want = [table for table in want if (read_watermark(con, table)
@@ -10327,7 +9835,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10327
  plan = sync_plan(ticks, want, now=end, slot_pass=slot_pass)
10328
  want = plan["tables"]
10329
  for table, reason in plan["deferred"].items():
10330
- report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table))
 
10331
  log(f"{table}: DEFERRED: {reason}")
10332
  elif not first_load:
10333
  # The vendor's report floor also applies to an operator's Sync now click.
@@ -10339,7 +9848,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10339
  reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
10340
  datetime.now(timezone.utc))
10341
  if reason:
10342
- report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table))
 
10343
  want.remove(table)
10344
 
10345
  # Heads go first even when the caller listed lines first. Scheduled plans still select one.
@@ -10367,10 +9877,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10367
  if prepare_brand_analytics(con):
10368
  changed_tables.add(table_name("brand_analytics"))
10369
 
10370
- def write_window(table, cols, types, rows, after=None, order_proof=None, reset=False):
10371
- if table == "order_items":
10372
- count, note = _write_order_report_window(con, cols, types, rows, order_proof, reset=reset)
10373
- elif cx.REPORT_TYPES[table].get("weekly"):
10374
  count, note = write_week(con, table, cols, types, rows, tenant_key, report, log)
10375
  else:
10376
  count, note = replace_window(con, table_name(table), cols, types, rows,
@@ -10441,7 +9949,20 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10441
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS)
10442
 
10443
  def sqp_progress():
10444
- sqp_limit["progress"] = _sqp_progress(con, sqp_months, sqp_weeks, sqp_asins, batches, end)
 
 
 
 
 
 
 
 
 
 
 
 
 
10445
 
10446
  try:
10447
  measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end
@@ -10458,15 +9979,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10458
  plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending}
10459
  undocumented = repairs.intersection(pending)
10460
  floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
10461
- wanted = [table for table in pending if (table not in completed or table in exhausted) and (table == SQP_TABLE or not read_watermark(con, table)
10462
  or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])]
10463
  planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins,
10464
  starts=starts)
10465
- for entry in _sqp_owed_tasks(con, _sqp_after_cohort(planned, end, sqp_asins)):
10466
  task = entry if len(entry) == 4 else (*entry, None)
10467
  table, w0, w1, batch = task
10468
  if table == SQP_TABLE:
10469
- work.append(task)
 
10470
  elif _after_cohort(table, w0, end):
10471
  # A month after the frozen end has its own marker: the backfill floor only moves backwards.
10472
  if not _cohort_month_done(con, table, w0):
@@ -10511,22 +10033,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10511
  if not recent <= mark <= end:
10512
  mark = recent
10513
  plan = plans[table] = sync_window(table, mark, now=end, since=since,
10514
- floor=history_floor(con, table, end),
10515
  intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
10516
  if table == SQP_TABLE:
10517
- # Monthly history precedes every completed week's ASIN batches.
10518
- start = plan["start"] if since is not None else history_floor(con, table, end)
10519
- months = list(reversed(_month_windows(start, plan["end"])))
10520
- tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches]
10521
- weekly_tasks = _sqp_week_tasks(
10522
- _utc(since) if since is not None else history_floor(con, table, end, period="WEEK"),
10523
- end, batches)
10524
- owed = [*tasks, *weekly_tasks]
10525
- if since is None:
10526
- owed = _sqp_owed_tasks(con, owed)
10527
- work.extend(owed)
10528
- if not any(_sqp_period(w0, w1) == "MONTH" for _table, w0, w1, _batch in owed):
10529
- work.append((table, None, None, None))
10530
  else:
10531
  work.append((table, None, None, None))
10532
 
@@ -10562,15 +10078,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10562
  sqp_limit["weeklyStart"] = (_week_start(end) - timedelta(days=7)).date().isoformat()
10563
  if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
10564
  sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
10565
- plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since,
10566
- floor=history_floor(con, SQP_TABLE, end))
10567
  # A first load also owes the months that closed after its frozen cohort end.
10568
  sqp_months = sorted({*_month_windows(plan["start"], plan["end"]),
10569
  *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()),
10570
  *((w0, w1) for name, w0, w1, _batch in work
10571
  if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")})
10572
- sqp_weeks = (_week_windows(history_floor(con, SQP_TABLE, end, period="WEEK"), end)
10573
- if "WEEK" in cx.REPORT_TYPES[SQP_TABLE].get("periods", ()) else [])
10574
  sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months]
10575
  sqp_progress()
10576
  sqp_limit["deferredReason"] = (
@@ -10578,12 +10091,11 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10578
  if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
10579
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10580
 
10581
- history_cut = {} # Each table and period owns its observed floor.
10582
  for index, (table, task_start, task_end, task_batch) in enumerate(work):
10583
  if (first_load or table == SQP_TABLE) and table in failed:
10584
  continue
10585
- history_key = (table, _history_task_period((table, task_start, task_end, task_batch)))
10586
- if history_cut.get(history_key) is not None and task_end is not None and task_end <= history_cut[history_key]:
10587
  continue
10588
  if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget)
10589
  and (task_end is not None or table == "settlements"
@@ -10621,7 +10133,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10621
  pending_reports.add(table)
10622
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
10623
  _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"])
10624
- except cx.AmazonError as exc:
10625
  if isinstance(exc, cx.AmazonThrottled):
10626
  problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log)
10627
  report["tables"][table] = problem
@@ -10632,34 +10144,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10632
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
10633
  _record_state(con, entity_key(table), "throttled", detail=problem["reason"])
10634
  continue
10635
- limit_reason = _close_history_floor(con, table, exc, task_start, task_end)
10636
- if limit_reason:
10637
- history_cut[history_key] = task_end
10638
- landed.update(_history_cut_tasks(work, table, "WEEK", task_end))
10639
- sqp_limit["weeklyCause"] = limit_reason
10640
- sqp_limit["errors"] = []
10641
- sqp_progress()
10642
- _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10643
- log(limit_reason)
10644
- continue
10645
  causes.append(str(exc))
10646
- if not _AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc)):
10647
  marker = _sqp_month(con, task_start, task_end)
10648
  marker["batches"][task_batch] = {"refused": True, "reason": str(exc), "asins": []}
10649
- marker.setdefault("refusedAsins", {}).update({a: str(exc) for a in task_batch.split()})
10650
  _record_state(con, _sqp_month_key(task_start, "WEEK"), "refused", detail=json.dumps(marker))
10651
  landed.add((table, task_start, task_end, task_batch))
10652
- else:
10653
- failed.add(table)
10654
- report["tables"][table] = {"state": "deferred", "reason": str(exc)}
10655
- except ValueError as exc:
10656
- # Unreadable source content remains owed without creating an ASIN refusal or history floor.
10657
- causes.append(str(exc))
10658
  if causes and not weekly_cause_logged:
10659
  sqp_limit["weeklyCause"] = "Amazon weekly Search Query Performance: " + "; ".join(dict.fromkeys(causes))
10660
  sqp_problem(sqp_limit["weeklyCause"])
10661
  weekly_cause_logged = True
10662
- sqp_progress()
10663
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10664
  if weekly_deferred:
10665
  break
@@ -10898,7 +10392,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10898
  floors[table] = w0
10899
  stop = _note_history_window(con, table, w0, count)
10900
  if stop:
10901
- history_cut[history_key] = w0
10902
  landed.update(task for task in work if task[0] == table and task[2] is not None
10903
  and task[2] <= w0)
10904
  log(stop)
@@ -10984,32 +10478,28 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
10984
  declared = {}
10985
  if cols and (weekly or "_period_start" in replace_keys_for(table)):
10986
  cols, rows = stamp_week(cols, rows, w0, w1)
10987
- order_proof = (_order_report_observations(con, cols, rows, meta, rtype, w0, w1)
10988
- if "order_items" in (table, sibling) else None)
10989
  if not cols:
10990
  if first_load:
10991
  # Even an empty recent window clears old rows exactly once. A weekly
10992
  # table never clears: R18's retention keeps watched history.
10993
  for empty_table in (table, sibling):
10994
- if empty_table and empty_table != "order_items" and floors[empty_table] is None and not weekly:
10995
  if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?",
10996
  [table_name(empty_table)]).fetchone():
10997
  con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
10998
- cleared = table != "order_items"
10999
- elif order_proof is None:
11000
  completed_windows.append(window)
11001
  continue
11002
  if sibling:
11003
  sibling_mirror = table_name(sibling)
11004
  sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared,
11005
  preserve_rows=True) if cols else {}
11006
- sibling_reset = not sibling_cleared and not sibling_plan["incremental"]
11007
- if cols and sibling_reset and sibling != "order_items":
11008
  con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1")
11009
- if cols or order_proof is not None:
11010
- sibling_written += write_window(sibling, cols, sibling_types, rows,
11011
- order_proof=order_proof, reset=sibling_reset)
11012
  sibling_cleared = True
 
 
11013
  order_observations = None
11014
  if table == "orders":
11015
  rows = _dedupe_orders(rows)
@@ -11026,17 +10516,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
11026
  preserve_rows=True) if cols else {}
11027
  # Explicit windows and first loads replace the mirror; incremental passes keep history.
11028
  # A weekly table replaces by its period identity instead (D-2675).
11029
- order_reset = not cleared and not plan["incremental"]
11030
- if order_reset and not weekly and table != "order_items":
11031
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
11032
  cleared = True
11033
- if cols or table == "order_items":
11034
  after = (lambda target: _record_state(target, entity_key("reconcile_orders"), "done",
11035
  detail=json.dumps(order_observations))) if order_observations else None
11036
- written += write_window(table, cols, types, rows, after=after,
11037
- order_proof=order_proof, reset=order_reset)
11038
- if table == "order_items":
11039
- cleared = True
11040
  if _periodic(cx.REPORT_TYPES[table]):
11041
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
11042
  "count": len(rows)})
@@ -11052,7 +10538,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
11052
  floors[reached] = w0
11053
  stop = _note_history_window(con, table, w0, len(rows) if cols else 0)
11054
  if stop:
11055
- history_cut[history_key] = w0
11056
  landed.update(task for task in work if task[0] == table and task[2] is not None
11057
  and task[2] <= w0)
11058
  log(stop)
@@ -11217,14 +10703,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
11217
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
11218
  break
11219
  except Exception as e: # noqa: BLE001
11220
- limit_reason = (_close_history_floor(con, table, e, task_start, task_end)
11221
- if first_load or table == SQP_TABLE else None)
11222
  if limit_reason and table == SQP_TABLE:
11223
- history_cut[history_key] = task_end
11224
- landed.update(_history_cut_tasks(work, table, "MONTH", task_end))
 
 
11225
  if sqp_limit is not None:
11226
  sqp_limit["errors"] = []
11227
- sqp_progress()
11228
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
11229
  report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason}
11230
  log(limit_reason)
@@ -11318,20 +10804,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
11318
  sqp_progress()
11319
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
11320
  for table in pending if first_load else ():
11321
- # Landed periods publish with their markers while the remaining history stays owed.
11322
  months = sum(task[0] == table and task[2] is not None for task in landed)
11323
  row = report["tables"].get(table)
11324
- if (table == SQP_TABLE or not _periodic(cx.REPORT_TYPES[table]) or not months
11325
  or (row is not None and not (isinstance(row, dict)
11326
  and row.get("state") in {"deferred", "throttled", "pending"}))):
11327
  continue
11328
- grain = "week" if cx.REPORT_TYPES[table].get("weekly") else "month"
11329
- row = row or {"reason": f"{months} {grain}{'' if months == 1 else 's'} landed. The remaining {grain}s continue next pass."}
11330
- count_key = "landedWeeks" if grain == "week" else "landedMonths"
11331
- row_count = (con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
11332
- if _columns_of(con, table_name(table)) else 0)
11333
- report["tables"][table] = {**row, "state": "deferred", "current": True,
11334
- count_key: months, "rows": row_count}
11335
  if first_load:
11336
  refusals ={row["table"]: row for row in _refused_reports(con)}
11337
  not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"}
@@ -13453,11 +12934,6 @@ def _sqp_resume_legs(ck, runtime, asins):
13453
  con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins])
13454
  if watermark:
13455
  record_watermark(con, SQP_TABLE, _utc(watermark))
13456
- # A fixture watermark's earlier months need the same per-ASIN evidence as a real completed load.
13457
- for w0, w1 in _month_windows(history_floor(con, SQP_TABLE, _utc(watermark)), _utc(watermark)):
13458
- marker = _sqp_month(con, w0, w1)
13459
- marker["asins"] = {a: {"count": 0, "purchases": 0} for a in asins}
13460
- _sqp_save_month(con, w0, marker)
13461
  calls.clear()
13462
  return con
13463
 
 
57
  import uuid
58
  from contextlib import closing, contextmanager
59
  from datetime import datetime, timedelta, timezone
 
60
  from pathlib import Path
61
 
62
  _HERE = Path(__file__).resolve().parent
 
91
  #: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
92
  HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
93
  HISTORY_TABLES = tuple(t + "_history" for t in HISTORY_SOURCES)
94
+ DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES
95
  PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES
96
  LEGACY_SNAPSHOT_TABLES = ("listings", "inventory")
97
 
 
2020
  if c not in _ORIGIN_TYPES)
2021
 
2022
 
2023
+ def replace_window(con, table, cols, types, rows, keys, after=None):
 
 
 
 
 
 
 
 
 
 
 
2024
  """Replace incoming key groups atomically, or report why only insertion is possible.
2025
 
2026
  `after(con)` runs inside the same transaction, after the insert and before the commit.
2027
  """
2028
  missing = [c for c in keys if c not in cols]
2029
+ if missing:
2030
  note = (f"{table}: window replacement unavailable; missing identity columns: "
2031
+ f"{', '.join(missing)}; rows inserted; use a report carrying these columns")
2032
+ return _replace(con, table, cols, types, rows), note
2033
+ if not keys or not rows:
2034
+ return _replace(con, table, cols, types, rows), None
 
 
 
 
 
 
2035
  quote = lambda name: '"' + str(name).replace('"', '""') + '"'
2036
  target = quote(table)
2037
  # The window's rows are staged in batches inside the transaction, typed as the mirror and
 
2042
  try:
2043
  predicate = " AND ".join(
2044
  f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
2045
+ con.execute("BEGIN TRANSACTION")
2046
+ try:
 
2047
  count = _replace(con, temporary, cols, types, rows, source_table=table)
2048
  inserted = cols
2049
  if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys):
 
2055
  con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}")
2056
  if after is not None:
2057
  after(con)
2058
+ con.execute("COMMIT")
2059
+ except Exception:
2060
+ con.execute("ROLLBACK")
2061
+ raise
2062
  return count, None
2063
  finally:
2064
  con.execute(f"DROP TABLE IF EXISTS {temporary}")
 
2602
  if any(type(n) is not int or n < 0 for n in steps):
2603
  raise ValueError("Invalid saved Search Query Performance progress")
2604
  months_done, months, done, total, eta = steps
2605
+ sentence = f"{months_done:,} of {months:,} months complete, {done:,} of {total:,} batches."
 
 
 
 
2606
  if eta:
2607
  hours, minutes = round(eta / 3600), max(1, round(eta / 60))
2608
  left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes")
 
2630
  "period": period,
2631
  "start": _utc(start).date().isoformat(),
2632
  "end": (_utc(end) - timedelta(days=1)).date().isoformat(),
2633
+ "batches": {}, "asins": {}, "retries": {}}
2634
 
2635
 
2636
  def _sqp_save_month(con, start, marker):
 
2639
 
2640
 
2641
  def _sqp_done(con, start, end, batch):
2642
+ return batch in _sqp_month(con, start, end)["batches"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2643
 
2644
 
2645
  def _sqp_week_tasks(start, end, batches):
2646
+ """Schedule the latest completed week through the same ASIN batch queue."""
2647
  if "WEEK" not in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()):
2648
  return []
2649
+ return [(SQP_TABLE, w0, w1, batch) for w0, w1 in _week_windows(start, end)[-1:]
2650
  for batch in batches]
2651
 
2652
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2653
  class SQPBudgetDeferred(RuntimeError):
2654
  """A batch or its single retry remains durable for the next bounded pass."""
2655
 
 
2658
  """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running."""
2659
 
2660
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2661
  def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
2662
  """One completed period and batch using the shared report transport and pass budget.
2663
 
 
2688
  reason = reason.replace("\u2014", "; ").replace("\u2013", " to ")
2689
  for asin in refused:
2690
  limit["refusedAsins"][asin] = reason
 
2691
  problem(sqp_refusal_message(asin, reason))
2692
  if not refused:
2693
  limit["errors"] = [reason]
 
2707
  _sqp_save_month(con, start, marker)
2708
  return 0
2709
 
2710
+ # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
2711
+ text = _document_text(text)
2712
+ doc = json.loads(text)
2713
+ raw = doc.get(spec["json_root"])
2714
+ if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
2715
+ raise ValueError("Search Query Performance document has no dataByAsin row array")
2716
+ totals = {asin: {"count": 0, "purchases": 0} for asin in requested}
2717
+ for row in raw:
2718
+ asin = row.get("asin")
2719
+ if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]:
2720
+ raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch")
2721
+ purchase = (row.get("purchaseData") or {}).get("asinPurchaseCount")
2722
+ if type(purchase) is not int or purchase < 0:
2723
+ raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count")
2724
+ totals[asin]["count"] += 1
2725
+ totals[asin]["purchases"] += purchase
2726
+ columns, rows = parse_json_report(text, spec["json_root"],
2727
+ document_id=meta.get("reportDocumentId"), report_type=spec["report"])
2728
  identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"]
2729
+ seen = set()
2730
+ for row in rows:
2731
+ key = amazon_schema.row_identity_for(SQP_TABLE, row)
2732
+ if key in seen:
2733
+ raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query")
2734
+ seen.add(key)
2735
+ row["_report_period"] = marker.get("period", "MONTH")
2736
  columns = list(dict.fromkeys([*columns, *identity, "_report_period",
2737
  "purchase_data_asin_purchase_count", *_ORIGIN_TYPES]))
2738
  mirror = table_name(SQP_TABLE)
2739
  types = _ensure(con, mirror, columns, rows, preserve_rows=True,
2740
  declared={"purchase_data_asin_purchase_count": "BIGINT"})
2741
  marker["asins"].update(totals)
2742
+ marker["batches"][batch] = {"asins": requested, "count": len(raw),
2743
  "purchases": sum(t["purchases"] for t in totals.values()),
2744
  "reportDocumentId": meta["reportDocumentId"]}
2745
  con.execute("BEGIN TRANSACTION")
 
2783
  return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"}
2784
 
2785
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2786
  def refresh_sku_keywords(con):
2787
  """Bind a local derived view to listing summaries and raw SQP without another source copy."""
2788
  listing_columns = _columns_of(con, "amzn_listing_items")
 
2810
  output_meta = ', "_connection_id", store' if scoped else ""
2811
  backend_meta = ', l."_connection_id", l.store' if scoped else ""
2812
  backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else ""
2813
+ raw_fields = {
2814
+ "start_date": "start_date", "end_date": "end_date", "asin": "asin",
2815
+ "search_query": "search_query_data_search_query",
2816
+ "impressions": "impression_data_asin_impression_count",
2817
+ "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count",
2818
+ "purchases": "purchase_data_asin_purchase_count",
2819
+ }
2820
+ select = ", ".join(
2821
+ f'{"TRY_CAST" if name in ("impressions", "clicks", "cart_adds", "purchases") else "CAST"}'
2822
+ f'("{source}" AS {"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"}) AS {name}'
2823
+ if source in raw_columns else
2824
+ f'NULL::{"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"} AS {name}'
2825
+ for name, source in raw_fields.items())
2826
  raw_from = " FROM amzn_search_query_performance" if raw_columns else " WHERE FALSE"
2827
  con.execute(f"""
2828
  CREATE OR REPLACE VIEW amzn_sku_keywords AS
 
2838
  SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group}
2839
  ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS (
2840
  SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain
2841
+ FROM raw
2842
  ), matched AS (
2843
  SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta}
2844
  FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join}
2845
  JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join}
2846
  )
2847
  SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id,
2848
+ sku, asin, substr(start_date, 1, 7) || '-01' AS month, start_date, end_date, search_query,
 
2849
  impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count,
2850
  'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) ||
2851
+ ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis{output_meta}
 
2852
  FROM matched
2853
  UNION ALL
2854
  SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)),
2855
  l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
2856
  l.backend_keywords, NULL, NULL,
2857
+ 'Listing backend keywords only. No matching Search Query Performance rows.'{backend_meta}
2858
  FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match})
2859
  """)
2860
  refresh_keyword_ranks(con)
 
2951
 
2952
 
2953
  def _validate_sku_keywords(con):
2954
+ """Compare each month's distinct derived SQP grains with independently grouped raw source rows."""
 
 
2955
  basis = amazon_schema.PROVENANCE_COPY["derived"]
2956
+ if not _columns_of(con, table_name(SQP_TABLE)):
 
2957
  return {"state": "unproven", "reason": "Search Query Performance source is absent", "source_basis": basis}
2958
  try:
2959
+ raw = con.execute("""
2960
+ SELECT substr(CAST(start_date AS VARCHAR), 1, 7),
2961
+ COUNT(*), COUNT(DISTINCT (start_date, end_date, asin, search_query_data_search_query)),
 
 
 
2962
  SUM(purchase_data_asin_purchase_count), COUNT(purchase_data_asin_purchase_count),
2963
  COUNT(*) FILTER (WHERE purchase_data_asin_purchase_count < 0 OR start_date IS NULL
2964
  OR end_date IS NULL OR asin IS NULL OR search_query_data_search_query IS NULL)
2965
+ FROM amzn_search_query_performance GROUP BY 1
2966
  """).fetchall()
2967
+ derived = con.execute("""
2968
+ SELECT substr(month, 1, 7), COUNT(*), SUM(purchases), SUM(invalid)
2969
  FROM (
2970
+ SELECT month, start_date, end_date, asin, search_query, MIN(purchases) AS purchases,
2971
  CASE WHEN COUNT(purchases) <> COUNT(*) OR MIN(purchases) <> MAX(purchases)
2972
+ OR COUNT(*) <> COUNT(DISTINCT sku) THEN 1 ELSE 0 END AS invalid
 
 
2973
  FROM amzn_sku_keywords WHERE source_grain IS NOT NULL
2974
+ GROUP BY month, start_date, end_date, asin, search_query
2975
+ ) GROUP BY 1
2976
  """).fetchall()
2977
+ actual = {month: {"grains": n, "purchases": purchases, "invalid": invalid}
2978
+ for month, n, purchases, invalid in derived}
2979
+ months = {}
2980
+ for month, count, distinct, purchases, present, invalid in raw:
2981
+ observed = actual.get(month, {"grains": 0, "purchases": 0, "invalid": 0})
2982
+ good = (count == distinct == present and not invalid and not observed["invalid"]
 
 
 
2983
  and observed["grains"] == distinct and observed["purchases"] == purchases)
2984
+ months[month] = {"state": "ok" if good else "refused",
2985
+ "expected": {"grains": distinct, "purchases": purchases}, "actual": observed}
2986
+ extra = set(actual) - set(months)
2987
+ good = bool(months) and not extra and all(row["state"] == "ok" for row in months.values())
2988
+ return {"state": "ok" if good else "refused" if months or extra else "unproven",
2989
+ "months": months, "source_basis": basis,
2990
+ "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every month" if good else
 
 
2991
  "SQP coverage or purchases differ. Unmapped ASINs, duplicate grains and missing counts cannot reconcile")}
2992
  except Exception as exc:
2993
  return {"state": "refused", "source_basis": basis,
 
3023
  for table in tables:
3024
  spec = _client().REPORT_TYPES[table]
3025
  table_start = starts.get(table, start) if starts is not None else start
3026
+ if table == SQP_TABLE and table_start < end:
3027
+ recent.extend(_sqp_week_tasks(table_start, end, sqp_batches(sqp_asins, spec["asin_option_max_chars"])))
 
 
 
 
3028
  if table == "settlements":
3029
  continue
3030
  if spec.get("snapshot"):
 
3959
  return sorted(holds["standing"]) if holds else []
3960
 
3961
 
 
 
 
 
 
 
 
3962
  class _AmazonMirrorStore(_mirrors.MirrorStore):
3963
  def __init__(self):
3964
  self.connector, self.spec = "amazon", _mirror_spec("amazon")
 
4036
  """
4037
  rows = report.get("tables") or {}
4038
  problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []]
4039
+ # Landed SQP batches and landed months count as finished work, so a chunk of only that progress still publishes.
4040
  if report.get("state") in {"partial", "failed"} and not any(
4041
  isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE
4042
+ or bool(row.get("landedMonths")))
4043
  and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()):
4044
  return None
4045
  unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting}
 
4064
  unfinished |= set(partner)
4065
  for names in scoped.values():
4066
  names |= {pairs[name] for name in list(names) if name in pairs and pairs[name] not in unfinished}
4067
+ if not unfinished and not scoped and not problems:
4068
  return None
4069
  owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))}
4070
  account_owned = {str(problem) for cid, result in accounts.items()
 
5111
  return due
5112
 
5113
 
5114
+ def _loaded_refresh_hold(con, table, now, *, slot_pass=False):
5115
  """Return a loaded table's cadence hold while its first load continues, or None when due."""
 
 
5116
  now = _utc(now)
5117
  tick = _read_state(con, entity_key("sync_tick_" + table))
5118
  # A queued resync is a repair: only Amazon's own floor holds it.
 
5404
  floor = read_backfill_floor(con, table)
5405
  if cohort is None:
5406
  return False
 
 
5407
  # The same per-table floor first_load_plan uses, so a load that lands every window completes.
5408
  return floor is not None and floor <= history_floor(con, table, cohort)
5409
  # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left
 
5473
  "nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None}
5474
  if job:
5475
  out["job"] = job
 
 
5476
  return out
5477
  finally:
5478
  con.close()
 
6294
  "remaining": record.get("remaining", last.get("remainingAfter")),
6295
  "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"),
6296
  "operation": last.get("operation"), "resume_at": last.get("resume_at"),
 
6297
  "tables": {**(last.get("tables") or {}), **(table_status or {})}}
6298
 
6299
 
 
 
 
 
 
 
 
 
 
 
 
 
6300
  def _record_generation(docs, row):
6301
  def add(doc):
6302
  rows = [r for r in (doc.get("generations") or []) if r.get("generation") != row["generation"]]
 
6306
  docs.update(FIRST_LOAD_GENERATIONS_KEY, add, flush="sync")
6307
 
6308
 
 
 
 
 
 
 
 
 
 
 
 
 
6309
  def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_seconds=5 * 3600, log=print,
6310
  docs=None, plan=None, client=None, write=None, delegate=None, counts=None,
6311
  holder=None, now=time.time, sleep=time.sleep, poll_seconds=None, job_id=None):
 
6371
  if cursor.get("end") and end and cursor["end"] != _iso(end):
6372
  # A new cohort end means a new first load; the old cursor's windows describe another plan.
6373
  cursor = {}
 
6374
  finished = dict(cursor.get("finished") or {})
6375
  opened = dict(cursor.get("open") or {})
6376
  throttled = dict(cursor.get("throttled") or {})
6377
  failed = dict(cursor.get("failed") or {})
6378
  downloads = dict(cursor.get("downloads") or {})
6379
+ save(lambda doc: doc.update(tenant=key, end=_iso(end) if end else None, backfillDays=DEFAULT_BACKFILL_DAYS,
6380
  generation=int(generation), finished=finished, open=opened,
6381
  throttled=throttled, failed=failed, downloads=downloads))
6382
 
 
6385
 
6386
  record["remainingBefore"] = sum(owed(t) for t in tasks)
6387
  mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed)
6388
+ # A window the writer served but did not finish (deferred, held) waits for the next generation.
6389
  stalled = set()
6390
  own_tables = sorted({t["table"] for t in mine if owed(t)})
6391
  ready = {}
 
6502
  prefix.append(task)
6503
  if not prefix:
6504
  continue
 
 
6505
  sibling = SHARED_REPORT_SIBLINGS.get(table)
6506
  pair = [t for t in by_table.get(sibling, []) if owed(t)][:len(prefix)] if sibling else []
6507
  units = [table] + ([sibling] if pair and [t["key"] for t in pair] == [t["key"] for t in prefix]
 
6514
  log(f"{table}: the first-load write failed ({type(exc).__name__}); "
6515
  "its windows stay owed and the next generation downloads them again.")
6516
  for t in prefix:
 
6517
  ready.pop(t["key"], None)
6518
  continue
6519
+ if table == "brand_analytics":
6520
+ record["brandAnalytics"] = {"wallSeconds": round(time.time() - t0, 1),
6521
+ "cpuSeconds": round(time.process_time() - c0, 1),
6522
+ "rows": (result["tables"].get(table) or {}).get("rows")}
6523
  stamp_done = _epoch_iso(now())
6524
  for unit in units:
6525
  row = result["tables"].get(unit) or {}
6526
+ ok = result["published"] and row.get("state", "done") not in {
6527
  "failed", "refused", "throttled", "pending", "held", "deferred", "disabled"}
 
 
6528
  if not ok:
6529
  continue
6530
  for t in (prefix if unit == table else pair):
6531
  if t["key"] in result["served"]:
 
 
6532
  finished[t["id"]] = {"at": stamp_done, "generation": int(generation),
6533
  "reportId": (opened.get(t["key"]) or {}).get("reportId")}
6534
  record["windowsWritten"] += 1
6535
  if unit not in record["tablesPublished"]:
6536
  record["tablesPublished"].append(unit)
6537
  for t in prefix + pair:
6538
+ if t["id"] not in finished and t["key"] in result["served"]:
6539
  stalled.add(t["id"])
6540
  ready.pop(t["key"], None)
6541
  if t["id"] in finished or t["key"] not in result["served"]:
 
6575
  break
6576
  record["otherTablesPasses"] += 1
6577
  result = result if isinstance(result, dict) else {}
 
 
6578
  if result.get("state") != "partial" or not result.get("landedTasks"):
6579
  break
6580
  try:
 
6583
  after = tasks
6584
  record["remainingAfter"] = sum(t["id"] not in finished and not t.get("landed") and t["id"] not in failed
6585
  for t in after)
 
 
 
6586
  try:
6587
  row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES)))
6588
  except Exception: # noqa: BLE001
6589
  row_counts = {}
6590
  table_status = None
6591
  if plan is None:
 
6592
  con, _path = _connect_tenant_mirror(key, rt)
6593
  try:
6594
  table_status = {"inventory": inventory_table_status(con)}
 
6630
  _FLOOR_MIN_AGE = timedelta(days=45)
6631
 
6632
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6633
  def _dated_table(table):
6634
  """A table whose first load plans dated history windows."""
6635
  spec = _client().REPORT_TYPES.get(table)
 
6737
  f"{DEFAULT_BACKFILL_DAYS} day fallback.")
6738
 
6739
 
6740
+ def _history_limit_key(table):
6741
+ """Amazon's observed floor for one table; `limit_` family, so a table reset clears it too."""
6742
+ return entity_key(f"limit_{table}_history")
 
6743
 
6744
 
6745
+ def _observed_history_limit(con, table):
6746
  """-> (floor, reason) Amazon showed by refusing an older window, or None."""
6747
+ state = _read_state(con, _history_limit_key(table))
6748
  try:
6749
  detail = json.loads(state["detail"]) if state and state.get("detail") else {}
6750
  if detail.get("floor"):
6751
  return _utc(detail["floor"]), str(detail.get("reason") or "")
6752
  except (ValueError, TypeError, AttributeError):
6753
  pass
 
 
6754
  legacy = _read_state(con, entity_key("limit_" + table))
6755
  try:
6756
  reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else ""
 
6762
  return None
6763
 
6764
 
6765
+ def history_bound(con, table, end, *, now=None):
 
 
 
 
 
 
6766
  """-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it.
6767
 
6768
  max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain.
 
6770
  `end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with.
6771
  """
6772
  spec = _client().REPORT_TYPES.get(table) or {}
6773
+ grain = _history_grain(spec)
6774
  now = _utc(now) if now is not None else datetime.now(timezone.utc)
6775
+ rule = spec.get("history") or {}
6776
  if rule.get("from") or rule.get("seller_floor") is False:
6777
  # Brand-level data (Brand Analytics) predates the seller's first order, so the seller month
6778
  # does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses.
 
6794
  elif rule.get("from"):
6795
  bound = _grain_ceil(_date_utc(rule["from"]), grain)
6796
  rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only."
 
 
 
6797
  if bound is not None and bound > floor:
6798
  floor, cause = bound, rule_cause
6799
+ observed = _observed_history_limit(con, table)
6800
  if observed and observed[0] > floor:
6801
  floor, cause = observed[0], observed[1] or (
6802
  f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.")
6803
+ if spec.get("weekly") and not WEEKLY_HISTORY_PLANNED:
 
6804
  latest = _week_start(_utc(end)) - timedelta(days=7)
6805
  if latest > floor:
6806
  older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else ""
6807
  floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, "
6808
+ "because the weekly sync defers a past week until its next weekly turn." + older)
6809
  return floor, cause
6810
 
6811
 
6812
+ def history_floor(con, table, end, *, now=None):
6813
  """The oldest history start `first_load_plan` and `needs_first_load` use for `table`."""
6814
+ return history_bound(con, table, end, now=now)[0]
6815
 
6816
 
6817
  def history_starts(con, tables, end, *, now=None):
6818
  """{table: history_floor} for each dated table; the one dict every first-load planner passes."""
6819
+ return {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)}
 
 
 
6820
 
6821
 
6822
  def history_status(con, tables, end):
 
6842
  return False
6843
  bound = history_floor(con, table, cohort)
6844
  if table == SQP_TABLE:
6845
+ return bool(_sqp_catalogue(con)) and any(
6846
+ not _sqp_month(con, w0, w1)["batches"] for w0, w1 in _month_windows(bound, cohort))
6847
  floor = read_backfill_floor(con, table)
6848
  return floor is not None and floor > bound
6849
 
 
7144
  return result
7145
 
7146
 
7147
+ def _newer_history_landed(con, table, end):
7148
  """A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage."""
7149
  if table == SQP_TABLE:
 
7150
  return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values())
7151
+ for w0, w1 in _month_windows(end, datetime.now(timezone.utc)))
7152
  floor = read_backfill_floor(con, table)
7153
  return floor is not None and end <= floor
7154
 
 
7163
  return None
7164
  now = datetime.now(timezone.utc)
7165
  status = getattr(exc, "status", None)
 
7166
  words = str(getattr(exc, "vendor_reason", "") or exc)
7167
  floor = read_backfill_floor(con, table)
7168
  said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc))
7169
+ and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end))
7170
  unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True)
7171
  and floor is not None and end <= floor and start < now - timedelta(days=365)
7172
  and bool(_read_state(con, entity_key("reconcile_" + table))))
 
7176
  reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. "
7177
  "Keep the connection enabled for newer data; ask Seller Support for older history.")
7178
  amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300]
7179
+ _record_state(con, _history_limit_key(table), "done", detail=json.dumps({
7180
+ "floor": _iso(stop), "reason": reason, "amazon": amazon, "window": [_iso(start), _iso(end)]}))
 
7181
  if table != SQP_TABLE:
7182
  # SQP's limit_ key is its progress document; its floor lives in the history marker only.
7183
  _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason}))
7184
  return reason
7185
 
7186
 
 
 
 
 
 
 
 
 
 
 
7187
  def _record_reconciliation(con, table, marker):
7188
  """Keep the most recent landed period, even while backfill travels backwards."""
7189
  key = entity_key("reconcile_" + table)
 
8809
  return totals
8810
 
8811
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8812
  def _order_status_observations(con, rows, meta):
8813
  """Retain status bounds for the current Sales window in the existing account sync state."""
8814
  saved = _read_state(con, entity_key("reconcile_order_metrics"))
 
9762
  and (table in completed or (read_watermark(con, table) and table not in repairs))]
9763
  due_loaded = []
9764
  for table in loaded:
9765
+ hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass)
 
9766
  if hold is None:
9767
+ due_loaded.append(table)
 
9768
  continue
9769
  next_due, reason = hold
9770
  report["tables"][table] = {"state": "deferred", "reason": reason, "nextDue": _iso(next_due),
9771
  "current": current_table(table)}
9772
  log(f"{table}: DEFERRED: {reason}")
 
 
9773
  if due_loaded:
9774
  # Refreshes use their own floors; the history partition retains its bounded budget.
9775
  con.close()
 
9801
  or table in completed or table in FEEDBACK_TABLES
9802
  or (table == "settlements" and not refresh_loaded)):
9803
  continue
9804
+ report["tables"][table] = {"state": "deferred", "reason":
9805
+ "The first load is continuing in bounded passes on the resident sync thread."}
 
 
9806
  # The planner waits for the next phase on this hold. The card's tick stays as it was.
9807
  _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred")
9808
+ tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None
9809
+ if tick and tick.get("state") == "throttled":
9810
  # The first load owns SQP now; a passed SQP deadline must not make it due every minute.
9811
  _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
9812
  want = [table for table in want if (read_watermark(con, table)
 
9835
  plan = sync_plan(ticks, want, now=end, slot_pass=slot_pass)
9836
  want = plan["tables"]
9837
  for table, reason in plan["deferred"].items():
9838
+ report["tables"][table] = {"state": "deferred", "reason": reason,
9839
+ "current": current_table(table)}
9840
  log(f"{table}: DEFERRED: {reason}")
9841
  elif not first_load:
9842
  # The vendor's report floor also applies to an operator's Sync now click.
 
9848
  reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
9849
  datetime.now(timezone.utc))
9850
  if reason:
9851
+ report["tables"][table] = {"state": "deferred", "reason": reason,
9852
+ "current": current_table(table)}
9853
  want.remove(table)
9854
 
9855
  # Heads go first even when the caller listed lines first. Scheduled plans still select one.
 
9877
  if prepare_brand_analytics(con):
9878
  changed_tables.add(table_name("brand_analytics"))
9879
 
9880
+ def write_window(table, cols, types, rows, after=None):
9881
+ if cx.REPORT_TYPES[table].get("weekly"):
 
 
9882
  count, note = write_week(con, table, cols, types, rows, tenant_key, report, log)
9883
  else:
9884
  count, note = replace_window(con, table_name(table), cols, types, rows,
 
9949
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS)
9950
 
9951
  def sqp_progress():
9952
+ """Batches done of total per planned month, from the durable month markers."""
9953
+ rows = []
9954
+ for w0, w1 in sqp_months:
9955
+ marker = _sqp_month(con, w0, w1)
9956
+ rows.append({"month": w0.strftime("%Y-%m"), "total": len(batches),
9957
+ "done": sum(batch in marker["batches"] for batch in batches)})
9958
+ done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows)
9959
+ weekly = sum(not _sqp_done(con, w0, w1, batch) for table, w0, w1, batch in work
9960
+ if table == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "WEEK")
9961
+ from harness.amazon_client import REPORT_SUSTAINED_SECONDS
9962
+ sqp_limit["progress"] = {
9963
+ "months": rows, "batchesDone": done, "batchesTotal": total,
9964
+ "monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows),
9965
+ "etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None
9966
 
9967
  try:
9968
  measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end
 
9979
  plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending}
9980
  undocumented = repairs.intersection(pending)
9981
  floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending}
9982
+ wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table)
9983
  or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])]
9984
  planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins,
9985
  starts=starts)
9986
+ for entry in _sqp_after_cohort(planned, end, sqp_asins):
9987
  task = entry if len(entry) == 4 else (*entry, None)
9988
  table, w0, w1, batch = task
9989
  if table == SQP_TABLE:
9990
+ if not _sqp_done(con, w0, w1, batch):
9991
+ work.append(task)
9992
  elif _after_cohort(table, w0, end):
9993
  # A month after the frozen end has its own marker: the backfill floor only moves backwards.
9994
  if not _cohort_month_done(con, table, w0):
 
10033
  if not recent <= mark <= end:
10034
  mark = recent
10035
  plan = plans[table] = sync_window(table, mark, now=end, since=since,
 
10036
  intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
10037
  if table == SQP_TABLE:
10038
+ # Closed months land newest first, before the latest week's batches.
10039
+ months = list(reversed(_month_windows(plan["start"], plan["end"])))
10040
+ tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
10041
+ if since is not None or not _sqp_done(con, w0, w1, batch)]
10042
+ weekly_tasks = [task for task in _sqp_week_tasks(
10043
+ _utc(since) if since is not None else end - timedelta(days=7), end, batches)
10044
+ if since is not None or not _sqp_done(con, task[1], task[2], task[3])]
10045
+ work.extend([*tasks, *weekly_tasks] if tasks else [*weekly_tasks, (table, None, None, None)])
 
 
 
 
 
10046
  else:
10047
  work.append((table, None, None, None))
10048
 
 
10078
  sqp_limit["weeklyStart"] = (_week_start(end) - timedelta(days=7)).date().isoformat()
10079
  if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]:
10080
  sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "")
10081
+ plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
 
10082
  # A first load also owes the months that closed after its frozen cohort end.
10083
  sqp_months = sorted({*_month_windows(plan["start"], plan["end"]),
10084
  *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()),
10085
  *((w0, w1) for name, w0, w1, _batch in work
10086
  if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")})
 
 
10087
  sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months]
10088
  sqp_progress()
10089
  sqp_limit["deferredReason"] = (
 
10091
  if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
10092
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10093
 
10094
+ history_cut = {} # table -> Amazon's floor observed this pass; older tasks are already counted landed.
10095
  for index, (table, task_start, task_end, task_batch) in enumerate(work):
10096
  if (first_load or table == SQP_TABLE) and table in failed:
10097
  continue
10098
+ if history_cut.get(table) is not None and task_end is not None and task_end <= history_cut[table]:
 
10099
  continue
10100
  if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget)
10101
  and (task_end is not None or table == "settlements"
 
10133
  pending_reports.add(table)
10134
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
10135
  _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"])
10136
+ except Exception as exc:
10137
  if isinstance(exc, cx.AmazonThrottled):
10138
  problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log)
10139
  report["tables"][table] = problem
 
10144
  report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
10145
  _record_state(con, entity_key(table), "throttled", detail=problem["reason"])
10146
  continue
 
 
 
 
 
 
 
 
 
 
10147
  causes.append(str(exc))
10148
+ if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)):
10149
  marker = _sqp_month(con, task_start, task_end)
10150
  marker["batches"][task_batch] = {"refused": True, "reason": str(exc), "asins": []}
 
10151
  _record_state(con, _sqp_month_key(task_start, "WEEK"), "refused", detail=json.dumps(marker))
10152
  landed.add((table, task_start, task_end, task_batch))
 
 
 
 
 
 
10153
  if causes and not weekly_cause_logged:
10154
  sqp_limit["weeklyCause"] = "Amazon weekly Search Query Performance: " + "; ".join(dict.fromkeys(causes))
10155
  sqp_problem(sqp_limit["weeklyCause"])
10156
  weekly_cause_logged = True
 
10157
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10158
  if weekly_deferred:
10159
  break
 
10392
  floors[table] = w0
10393
  stop = _note_history_window(con, table, w0, count)
10394
  if stop:
10395
+ history_cut[table] = w0
10396
  landed.update(task for task in work if task[0] == table and task[2] is not None
10397
  and task[2] <= w0)
10398
  log(stop)
 
10478
  declared = {}
10479
  if cols and (weekly or "_period_start" in replace_keys_for(table)):
10480
  cols, rows = stamp_week(cols, rows, w0, w1)
 
 
10481
  if not cols:
10482
  if first_load:
10483
  # Even an empty recent window clears old rows exactly once. A weekly
10484
  # table never clears: R18's retention keeps watched history.
10485
  for empty_table in (table, sibling):
10486
+ if empty_table and floors[empty_table] is None and not weekly:
10487
  if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?",
10488
  [table_name(empty_table)]).fetchone():
10489
  con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
10490
+ cleared = True
10491
+ else:
10492
  completed_windows.append(window)
10493
  continue
10494
  if sibling:
10495
  sibling_mirror = table_name(sibling)
10496
  sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared,
10497
  preserve_rows=True) if cols else {}
10498
+ if cols and not sibling_cleared and not sibling_plan["incremental"]:
 
10499
  con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1")
 
 
 
10500
  sibling_cleared = True
10501
+ if cols:
10502
+ sibling_written += write_window(sibling, cols, sibling_types, rows)
10503
  order_observations = None
10504
  if table == "orders":
10505
  rows = _dedupe_orders(rows)
 
10516
  preserve_rows=True) if cols else {}
10517
  # Explicit windows and first loads replace the mirror; incremental passes keep history.
10518
  # A weekly table replaces by its period identity instead (D-2675).
10519
+ if not cleared and not plan["incremental"] and not weekly:
 
10520
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
10521
  cleared = True
10522
+ if cols:
10523
  after = (lambda target: _record_state(target, entity_key("reconcile_orders"), "done",
10524
  detail=json.dumps(order_observations))) if order_observations else None
10525
+ written += write_window(table, cols, types, rows, after=after)
 
 
 
10526
  if _periodic(cx.REPORT_TYPES[table]):
10527
  _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1),
10528
  "count": len(rows)})
 
10538
  floors[reached] = w0
10539
  stop = _note_history_window(con, table, w0, len(rows) if cols else 0)
10540
  if stop:
10541
+ history_cut[table] = w0
10542
  landed.update(task for task in work if task[0] == table and task[2] is not None
10543
  and task[2] <= w0)
10544
  log(stop)
 
10703
  report["tables"][table] = {"state": "deferred", "reason": str(exc)}
10704
  break
10705
  except Exception as e: # noqa: BLE001
10706
+ limit_reason = _close_history_floor(con, table, e, task_start, task_end) if first_load else None
 
10707
  if limit_reason and table == SQP_TABLE:
10708
+ # Only months at or below Amazon's floor stop; newer months and the weekly batches continue.
10709
+ history_cut[table] = task_end
10710
+ landed.update(task for task in work if task[0] == table and task[2] is not None
10711
+ and task[2] <= task_end)
10712
  if sqp_limit is not None:
10713
  sqp_limit["errors"] = []
 
10714
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10715
  report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason}
10716
  log(limit_reason)
 
10804
  sqp_progress()
10805
  _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
10806
  for table in pending if first_load else ():
10807
+ # Landed months publish with their markers, as SQP batches do; the months still owed run next pass.
10808
  months = sum(task[0] == table and task[2] is not None for task in landed)
10809
  row = report["tables"].get(table)
10810
+ if (table == SQP_TABLE or not cx.REPORT_TYPES[table].get("monthly") or not months
10811
  or (row is not None and not (isinstance(row, dict)
10812
  and row.get("state") in {"deferred", "throttled", "pending"}))):
10813
  continue
10814
+ row = row or {"reason": f"{months} month{'' if months == 1 else 's'} landed. The remaining months continue next pass."}
10815
+ report["tables"][table] = {**row, "state": "deferred", "current": True, "landedMonths": months}
 
 
 
 
 
10816
  if first_load:
10817
  refusals ={row["table"]: row for row in _refused_reports(con)}
10818
  not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"}
 
12934
  con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins])
12935
  if watermark:
12936
  record_watermark(con, SQP_TABLE, _utc(watermark))
 
 
 
 
 
12937
  calls.clear()
12938
  return con
12939
 
platform/harness/datastore.py CHANGED
@@ -3732,7 +3732,6 @@ REPLACE_EACH_PASS = {
3732
  "product_tag": {"domain": []},
3733
  }
3734
  REPLACE_PAGE = 5000
3735
- REPLACE_READ_ATTEMPTS = 3
3736
 
3737
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
3738
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
@@ -4019,13 +4018,13 @@ def _base_domain(spec):
4019
  return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
4020
 
4021
 
4022
- def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None):
4023
  """Publish a single requested entity through the same source boundary as a full pass."""
4024
  if source_paused():
4025
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4026
  candidate = getattr(_SYNC_TLS, "candidate", None)
4027
  if candidate is not None:
4028
- phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report)
4029
  # A recurring refresh may report changed pages to a bespoke cursor, but a
4030
  # tenant pass cannot claim completion until those private pages finish too.
4031
  pending = _connection_pending(candidate["con"], key)
@@ -4034,7 +4033,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None):
4034
  return phase, pulled
4035
  rt = _resident_odoo_runtime()
4036
  if rt is None:
4037
- return _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report)
4038
  report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
4039
  return report[key]["phase"], report[key]["pulled"]
4040
 
@@ -4184,17 +4183,16 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print):
4184
  con.close()
4185
 
4186
 
4187
- def _sync_one(key, max_batches=200, batch=BATCH, log=print, report=None):
4188
  """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
4189
  _connection_model(ENTITIES[key])
4190
  plan = REPLACE_EACH_PASS.get(key)
4191
  if plan is None or source_paused():
4192
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4193
  if plan.get("scope") != "internal":
4194
- return _replace_entity(key, log=log, report=report)
4195
  phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4196
- replaced, changed = _replace_entity(key, log=log, report=report)
4197
- return (phase if replaced == "live" else replaced), pulled + changed
4198
 
4199
 
4200
  def _replace_scope(con, key, plan):
@@ -4208,44 +4206,37 @@ def _replace_scope(con, key, plan):
4208
  return "(" + " OR ".join(terms) + ") AND " + _connection_where()
4209
 
4210
 
4211
- def _read_replace_rows(key, spec, domain):
4212
- """Read uncapped pages again if neither surrounding count matches, with a fixed attempt bound."""
4213
- fields = spec["fields"] + list(spec.get("m2m") or [])
4214
- for _ in range(REPLACE_READ_ATTEMPTS):
4215
- expected = int(O.get_odoo().search_count(spec["model"], domain))
4216
- rows, last = [], 0
4217
- while True:
4218
- recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
4219
- limit=REPLACE_PAGE, order="id asc")
4220
- if any(field not in rec for rec in recs for field in fields):
4221
- raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
4222
- rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
4223
- named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
4224
- # Only an empty page ends a read, since Odoo may return short pages before the end.
4225
- if not recs:
4226
- break
4227
- last = recs[-1]["id"]
4228
- if len(rows) == expected:
4229
- return rows
4230
- after = int(O.get_odoo().search_count(spec["model"], domain))
4231
- if len(rows) == after:
4232
- return rows
4233
- return None
4234
 
4235
-
4236
- def _replace_entity(key, log=print, report=None):
4237
- """Replace a complete read atomically, or hold a moving entity before changing rows or state."""
4238
  if source_paused():
4239
  _paused_notice(f"full read of {key}", log)
4240
  return PAUSED_PHASE, 0
4241
  spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
4242
- rows = _read_replace_rows(key, spec, list(plan["domain"]))
4243
- if rows is None:
4244
- if report is not None:
4245
- report["cause"] = {"classifier": "odoo_entity_moving", "entity": key,
4246
- "attempts": REPLACE_READ_ATTEMPTS}
4247
- log(f" {key}: Odoo changed during all {REPLACE_READ_ATTEMPTS} full reads. Previous rows are kept.")
4248
- return "backfill", 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4249
  con = connect()
4250
  try:
4251
  _ensure_table(con, key, spec)
@@ -4337,7 +4328,6 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
4337
  with binding:
4338
  for key in selected:
4339
  t0 = time.time()
4340
- row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
4341
  marker = _connection_state_key(key + ".pass")
4342
  previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
4343
  if entry and candidate.get("resuming") and previous == ("done",):
@@ -4346,11 +4336,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
4346
  if entry:
4347
  candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
4348
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
4349
- **({"batch": batch} if batch != BATCH else {}),
4350
- **({"report": row} if candidate is not None else {}))
4351
  if entry and phase == "live":
4352
  candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
4353
  [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
 
4354
  if phase != "live":
4355
  row["phase"] = phase
4356
  row["pulled"] += pulled
 
3732
  "product_tag": {"domain": []},
3733
  }
3734
  REPLACE_PAGE = 5000
 
3735
 
3736
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
3737
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
 
4018
  return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
4019
 
4020
 
4021
+ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
4022
  """Publish a single requested entity through the same source boundary as a full pass."""
4023
  if source_paused():
4024
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4025
  candidate = getattr(_SYNC_TLS, "candidate", None)
4026
  if candidate is not None:
4027
+ phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log)
4028
  # A recurring refresh may report changed pages to a bespoke cursor, but a
4029
  # tenant pass cannot claim completion until those private pages finish too.
4030
  pending = _connection_pending(candidate["con"], key)
 
4033
  return phase, pulled
4034
  rt = _resident_odoo_runtime()
4035
  if rt is None:
4036
+ return _sync_one(key, max_batches=max_batches, batch=batch, log=log)
4037
  report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
4038
  return report[key]["phase"], report[key]["pulled"]
4039
 
 
4183
  con.close()
4184
 
4185
 
4186
+ def _sync_one(key, max_batches=200, batch=BATCH, log=print):
4187
  """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
4188
  _connection_model(ENTITIES[key])
4189
  plan = REPLACE_EACH_PASS.get(key)
4190
  if plan is None or source_paused():
4191
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4192
  if plan.get("scope") != "internal":
4193
+ return _replace_entity(key, log=log)
4194
  phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4195
+ return phase, pulled + _replace_entity(key, log=log)[1]
 
4196
 
4197
 
4198
  def _replace_scope(con, key, plan):
 
4206
  return "(" + " OR ".join(terms) + ") AND " + _connection_where()
4207
 
4208
 
4209
+ def _replace_entity(key, log=print):
4210
+ """Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4211
 
4212
+ Returns (phase, changed_rows). A failed or short read raises before any delete.
4213
+ """
 
4214
  if source_paused():
4215
  _paused_notice(f"full read of {key}", log)
4216
  return PAUSED_PHASE, 0
4217
  spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
4218
+ domain = list(plan["domain"])
4219
+ fields = spec["fields"] + list(spec.get("m2m") or [])
4220
+ expected = int(O.get_odoo().search_count(spec["model"], domain))
4221
+ rows, last = [], 0
4222
+ while True:
4223
+ recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
4224
+ limit=REPLACE_PAGE, order="id asc")
4225
+ if any(field not in rec for rec in recs for field in fields):
4226
+ raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
4227
+ rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
4228
+ named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
4229
+ # Stop on an EMPTY page: a server-side page cap returns short pages before the end.
4230
+ if not recs:
4231
+ break
4232
+ last = recs[-1]["id"]
4233
+ # Rows moved during the read when the read matches a second count. Refuse only a read
4234
+ # that disagrees with both counts, before any delete.
4235
+ if len(rows) != expected:
4236
+ after = int(O.get_odoo().search_count(spec["model"], domain))
4237
+ if len(rows) != after:
4238
+ raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were "
4239
+ f"{expected} and {after}. The mirror keeps its previous rows.")
4240
  con = connect()
4241
  try:
4242
  _ensure_table(con, key, spec)
 
4328
  with binding:
4329
  for key in selected:
4330
  t0 = time.time()
 
4331
  marker = _connection_state_key(key + ".pass")
4332
  previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
4333
  if entry and candidate.get("resuming") and previous == ("done",):
 
4336
  if entry:
4337
  candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
4338
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
4339
+ **({"batch": batch} if batch != BATCH else {}))
 
4340
  if entry and phase == "live":
4341
  candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
4342
  [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
4343
+ row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
4344
  if phase != "live":
4345
  row["phase"] = phase
4346
  row["pulled"] += pulled
platform/harness/mirror_store.py CHANGED
@@ -355,8 +355,7 @@ def _sync_result(connector, report):
355
  completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
356
  for row in report.values())
357
  return completed, completed and all(row["phase"] == "live" for row in report.values())
358
- if (report.get("problems") or report.get("audit_problems") or (connector == "amazon"
359
- and any(row.get("state") == "refused" for row in (report.get("tables") or {}).values()))):
360
  # A problem owned by one held table publishes the rest; it never makes the pass current.
361
  holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
362
  if holds is None or holds["blocked"]:
@@ -381,34 +380,16 @@ def _sync_result(connector, report):
381
  return completed, current
382
 
383
 
384
- def odoo_moving_entities(report):
385
- """Return only typed, exhausted moving-entity holds whose cause matches their entity."""
386
- if not isinstance(report, dict):
387
- return {}
388
- return {name: row["cause"] for name, row in (report or {}).items()
389
- if re.fullmatch(r"[a-z][a-z0-9_]{0,79}", str(name)) and isinstance(row, dict)
390
- and row.get("phase") == "backfill" and isinstance(row.get("cause"), dict)
391
- and row["cause"].get("classifier") == "odoo_entity_moving"
392
- and row["cause"].get("entity") == name
393
- and type(row["cause"].get("attempts")) is int and row["cause"]["attempts"] == 3}
394
-
395
-
396
- def record_sync_attempt(connector, rt):
397
- """Record writer activity without claiming a completed pass or replacing a previous refusal."""
398
- with _CATCH_UP_LOCK:
399
- state = _catch_up_source(rt, connector)
400
- if not state["current"]:
401
- state.update(pass_pending=True, attempt_at=time.time())
402
-
403
-
404
  def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
405
  """Count completed passes and require a clean result before clearing startup catch-up."""
406
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
407
  rt = runtime
408
  if rt is None:
409
  return pull(*args, **kwargs)
410
- record_sync_attempt(connector, rt)
411
  with _CATCH_UP_LOCK:
 
 
 
412
  active = (connector, _tenant_key(rt.key))
413
  _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
414
  try:
@@ -540,99 +521,6 @@ def _sql_literal(value):
540
  return "'" + str(value).replace("'", "''") + "'"
541
 
542
 
543
- def _merge_publication_entries(base, local, stored):
544
- """Merge keyed observations against their common base, naming conflicting local changes."""
545
- missing = object()
546
- merged, conflicts = {}, []
547
- for name in sorted(set(base) | set(local) | set(stored)):
548
- before, ours, theirs = (entries.get(name, missing) for entries in (base, local, stored))
549
- if _same_publication_entry(ours, before):
550
- value = theirs
551
- elif _same_publication_entry(ours, theirs):
552
- value = theirs
553
- elif _same_publication_entry(theirs, before):
554
- value = ours
555
- else:
556
- conflicts.append(name)
557
- continue
558
- if value is not missing:
559
- merged[name] = value
560
- return merged, conflicts
561
-
562
-
563
- def _same_publication_entry(left, right):
564
- """A content-addressed table retains its identity when only its write timestamp differs."""
565
- if isinstance(left, dict) and isinstance(right, dict):
566
- return {k: v for k, v in left.items() if k != "written_at"} == {
567
- k: v for k, v in right.items() if k != "written_at"}
568
- return left == right
569
-
570
-
571
- def _marker_freshness(row):
572
- """An invalid observation time cannot outrank a valid source-state observation."""
573
- try:
574
- return _utc(row[5])
575
- except (ValueError, OverflowError):
576
- return datetime.min.replace(tzinfo=timezone.utc)
577
-
578
-
579
- def _merge_publication_markers(base, local, stored):
580
- """Merge source state independently of observation time, retaining its newest valid freshness."""
581
- versions = [{row[0]: tuple(row) for row in rows} for rows in (base, local, stored)]
582
- states = [{name: row[:5] for name, row in version.items()} for version in versions]
583
- merged, conflicts = _merge_publication_entries(*states)
584
- for name, state in merged.items():
585
- matching = [version[name] for version in versions
586
- if name in version and version[name][:5] == state]
587
- merged[name] = max(matching, key=_marker_freshness)
588
- return merged, conflicts
589
-
590
-
591
- def _candidate_checkpoint(con):
592
- """Read the private candidate's common-base manifest and authoritative local markers."""
593
- if not con.execute("SELECT 1 FROM information_schema.tables "
594
- "WHERE table_name='_candidate_checkpoint'").fetchone():
595
- return None
596
- row = con.execute("SELECT document FROM _candidate_checkpoint").fetchone()
597
- return json.loads(row[0]) if row else None
598
-
599
-
600
- def _candidate_base_tables(checkpoint):
601
- """Keep initial local content provenance distinct from an actual saved manifest."""
602
- return {**checkpoint.get("local_tables", {}), **checkpoint["base"].get("tables", {})}
603
-
604
-
605
- def _save_candidate_checkpoint(con, base, markers, *, local_tables=None):
606
- """Keep per-table base provenance separate from the installed serving publication."""
607
- con.execute("CREATE TABLE IF NOT EXISTS _candidate_checkpoint (document VARCHAR)")
608
- con.execute("DELETE FROM _candidate_checkpoint")
609
- con.execute("INSERT INTO _candidate_checkpoint VALUES (?)", [json.dumps({
610
- "base": base, "markers": markers, "local_tables": local_tables or {}})])
611
-
612
-
613
- def _save_initial_candidate_checkpoint(store, rt, candidate, published):
614
- """Capture pre-manifest local table content before private source work changes it."""
615
- base = datastore._publication(published, store.connector)
616
- local_tables = {}
617
- missing = store._tables(published) - set(base.get("tables", {}))
618
- if missing:
619
- sealed = store._sealed_mode(rt)
620
- part_limit = store.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
621
- with tempfile.TemporaryDirectory(prefix="mirror-local-base-") as tmp:
622
- for name in sorted(missing):
623
- entry, _fresh = store._checkpoint_table(published, name, {}, set(), tmp,
624
- sealed=sealed, part_limit=part_limit)
625
- local_tables[name] = entry
626
- _save_candidate_checkpoint(candidate, base, store._state_rows(published), local_tables=local_tables)
627
-
628
-
629
- def _table_marker_rows(store, rows, tables):
630
- """Pair held data markers and direct table status with their table's rows."""
631
- names = {row[0] for row in store._held_state_rows(rows, tables)}
632
- names.update(row[0] for row in rows if row[0].split("__", 1)[-1] in tables)
633
- return [row for row in rows if row[0] in names]
634
-
635
-
636
  class MirrorStore:
637
  """One connector policy, with state cached separately for each tenant mirror."""
638
 
@@ -1410,232 +1298,162 @@ class MirrorStore:
1410
  return entry, fresh
1411
 
1412
 
1413
- def _rebase_publication(self, tenant_key, rt, con, base, manifest, rows, *, allowed, report):
1414
- """Keep stored changes and their markers, holding overlapping private changes for retry."""
1415
- raw = self._download(rt, _MANIFEST_KEY)
1416
- if raw is None:
1417
- # A local snapshot identity alone does not prove a saved manifest existed.
1418
- if "tables" in base:
1419
- raise PublicationHeld(f"{self.label} checkpoint held: the saved base publication is missing.")
1420
- return rows, False
1421
- stored = json.loads(raw)
1422
- if stored.get("tenant") != _tenant_key(tenant_key):
1423
- raise ValueError("Mirror manifest belongs to another tenant")
1424
- has_base = con.execute("SELECT 1 FROM information_schema.tables "
1425
- "WHERE table_name='_candidate_base'").fetchone()
1426
- candidate_base = con.execute("SELECT identity FROM _candidate_base").fetchone() if has_base else None
1427
- base_id = candidate_base[0] if candidate_base else base.get("snapshot_id", "unknown")
1428
- if base_id != base.get("snapshot_id", "unknown"):
1429
- raise PublicationHeld(f"{self.label} checkpoint held: private rows have a different base publication.")
1430
- if (stored.get("snapshot_id", "unknown") == base_id
1431
- and stored.get("tables") == base.get("tables")
1432
- and stored.get("sync_state") == base.get("sync_state")):
1433
- return rows, False
1434
- checkpoint = _candidate_checkpoint(con)
1435
- tables, conflicts = _merge_publication_entries(
1436
- _candidate_base_tables(checkpoint) if checkpoint else base.get("tables", {}),
1437
- manifest["tables"], stored.get("tables", {}))
1438
- if not allowed:
1439
- names = ", ".join(conflicts) or "the source's held tables"
1440
- raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed {names}. "
1441
- "Private rows keep their previous base for the next pass.")
1442
- with tempfile.TemporaryDirectory(prefix="mirror-rebase-") as tmp:
1443
- self._before_restore(tenant_key, rt)
1444
- stored, stored_rows, _paths, _total, _refused = self._validated_snapshot(
1445
- tenant_key, rt, con, raw, tmp)
1446
- if checkpoint is None:
1447
- raise PublicationHeld(f"{self.label} checkpoint held: common-base markers are unavailable.")
1448
- base_rows = checkpoint["markers"]
1449
- markers, marker_conflicts = _merge_publication_markers(base_rows, rows, stored_rows)
1450
- versions = base_rows + rows + stored_rows
1451
- conflicts = sorted(set(conflicts) | {
1452
- name for name in report.get("held_tables", ())
1453
- if set(marker_conflicts) & {row[0] for row in _table_marker_rows(self, versions, {name})}})
1454
- held_markers = {row[0] for row in _table_marker_rows(self, versions, set(conflicts))}
1455
- if set(marker_conflicts) - held_markers:
1456
- raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed markers "
1457
- + ", ".join(marker_conflicts) + ". Private rows stay for the next pass.")
1458
- if conflicts:
1459
- report["conflicts"] = conflicts
1460
- report["problems"].append(f"{self.label} checkpoint held tables: a newer stored publication changed "
1461
- + ", ".join(conflicts) + ". Private rows keep their previous base for the next pass.")
1462
- tables.update({name: stored["tables"][name] for name in conflicts if name in stored["tables"]})
1463
- markers = {name: row for name, row in markers.items() if name not in held_markers}
1464
- markers.update({row[0]: tuple(row) for row in stored_rows if row[0] in held_markers})
1465
- manifest["tables"] = tables
1466
- return list(markers.values()), True
1467
-
1468
-
1469
- def _checkpoint_table(self, con, name, prior, named, tmp, *, sealed, part_limit):
1470
- """Prepare changed immutable content, reusing every part the base already names."""
1471
- path = Path(tmp) / (name + ".parquet")
1472
- size = None
1473
- if "parts" not in prior:
1474
- con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
1475
- "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1476
- size = path.stat().st_size
1477
- if size is None or size > part_limit:
1478
- return self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
1479
- metadata = _blob_metadata(path.read_bytes())
1480
- if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
1481
- and (".sealed-v1." in prior["key"]) == sealed):
1482
- return None, []
1483
- suffix = ".sealed-v1.parquet" if sealed else ".parquet"
1484
- key = _content_key(name, suffix, metadata)
1485
- metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
1486
- written_at=_iso(datetime.now(timezone.utc)), key=key)
1487
- if key in named and named[key] != metadata["sha256"]:
1488
- raise ValueError(f"{self.label} mirror content key collides with a published blob")
1489
- return metadata, [] if key in named else [(key, path)]
1490
-
1491
-
1492
- def _checkpoint_tables(self, rt, con, manifest, dirty, named, tmp, *, sealed, part_limit):
1493
- """Prepare local changes and migrate legacy remote entries without dropping refused tables."""
1494
- tables, changed, pending = self._tables(con), [], []
1495
- legacy = {name for name, meta in manifest["tables"].items()
1496
- if (not meta.get("key") and "parts" not in meta)
1497
- or any((".sealed-v1." in key) != sealed for key, _ in self._entry_blob_keys(name, meta))}
1498
- for name in sorted(legacy - tables):
1499
- prior, migrated = manifest["tables"][name], []
1500
- for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
1501
- if (".sealed-v1." in key) == sealed:
1502
- migrated.append(dict(metadata))
1503
- continue
1504
- path = Path(tmp) / f"{name}.migrate{index}.parquet"
1505
- path.write_bytes(self._checked_mirror_blob(rt, key, metadata))
1506
- new_key = key.replace(".parquet", ".sealed-v1.parquet")
1507
- migrated.append({**metadata, "key": new_key})
1508
- pending.append((new_key, path))
1509
- manifest["tables"][name] = ({**prior, "parts": migrated} if "parts" in prior
1510
- else {**prior, **migrated[0]})
1511
- changed.append(name)
1512
- for name in sorted((dirty | legacy | (tables - set(manifest["tables"]))) & tables):
1513
- entry, fresh = self._checkpoint_table(con, name, manifest["tables"].get(name, {}), named, tmp,
1514
- sealed=sealed, part_limit=part_limit)
1515
- if entry is not None:
1516
- manifest["tables"][name] = entry
1517
- changed.append(name)
1518
- pending.extend(fresh)
1519
- return changed, pending
1520
-
1521
-
1522
- def _upload_checkpoint_parts(self, tenant_key, rt, pending, part_limit, progress):
1523
- """Ledger immutable keys before uploading bounded parts, retaining byte progress on failure."""
1524
- if any(path.stat().st_size > part_limit for _key, path in pending):
1525
- raise ValueError("mirror part exceeds the byte limit")
1526
- if pending:
1527
- self._ledger_add(rt, tenant_key, [key for key, _path in pending])
1528
- for key, path in pending:
1529
- data = path.read_bytes()
1530
- self._upload(rt, key, data)
1531
- progress["uploaded"] += len(data)
1532
-
1533
-
1534
- def _publish_checkpoint(self, tenant_key, rt, con, cache, manifest, changed, pending, tmp, *,
1535
- publication, hold, rebase_allowed, report, log, progress):
1536
- """Publish one complete manifest after content upload and a fresh stored-base comparison."""
1537
- checkpoint = _candidate_checkpoint(con)
1538
- old = checkpoint["base"] if checkpoint else cache["manifest"]
1539
- sealed = bool(manifest.get("blob_envelope_version"))
1540
- part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1541
- rows = self._state_rows(con)
1542
- if not self._state_complete(rows):
1543
- return None
1544
- suffix = ".sealed-v1" if sealed else ""
1545
- state_meta, _blobs = self._state_entry(rows, part_limit, suffix)
1546
- if not (publication or changed or state_meta != old.get("sync_state")
1547
- or manifest["tables"] != old.get("tables", {}) or old.get("version") != MANIFEST_VERSION):
1548
- return None
1549
- self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
1550
- rows, report["rebased"] = self._rebase_publication(
1551
- tenant_key, rt, con, old, manifest, rows,
1552
- allowed=rebase_allowed and publication is not None and hold is None, report=report)
1553
- state_meta, blobs = self._state_entry(rows, part_limit, suffix)
1554
- pending, named = [], self._manifest_keys(old)
1555
- for index, (key, state) in enumerate(blobs):
1556
- if key not in named:
1557
- path = Path(tmp) / f"sync_state.{index}.json"
1558
- path.write_bytes(state)
1559
- pending.append((key, path))
1560
- self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
1561
- stamp = _iso(datetime.now(timezone.utc))
1562
- manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, **(publication or {}))
1563
- if publication is not None:
1564
- manifest["completed_at"] = stamp
1565
- if hold is not None:
1566
- manifest["hold"] = hold
1567
- self._upload(rt, _HELD_KEY, _mirror_json(manifest))
1568
- return manifest
1569
- superseded = _superseded_hold(self, tenant_key, rt)
1570
- if superseded:
1571
- manifest["superseded_hold"] = superseded
1572
- published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
1573
- progress["uploaded"] += len(published)
1574
- if superseded is not None:
1575
- try:
1576
- rt.delete_path(self.prefix + _HELD_KEY)
1577
- except Exception as exc:
1578
- log(f"{self.label} superseded hold cleanup deferred ({type(exc).__name__}); "
1579
- "the next publication retries")
1580
- if publication is None:
1581
- datastore._write_publication(con, self.connector, manifest)
1582
- cache["manifest"] = manifest
1583
- cache["problems"].clear()
1584
- self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
1585
- return manifest
1586
-
1587
-
1588
  def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
1589
- _publication=None, _hold=None, _rebase_allowed=True):
1590
- """Checkpoint immutable parts and markers before publishing one manifest, retaining private work on failure."""
 
 
 
 
 
 
 
 
 
 
 
1591
  candidate = getattr(datastore._SYNC_TLS, "candidate", None)
1592
  if candidate is not None and _publication is None:
1593
  if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
1594
  raise ValueError("Checkpoint belongs to another source candidate")
1595
- return
1596
- progress = {"uploaded": 0}
1597
  with self.lock:
1598
  cache = self.restore_mirror(tenant_key, rt, log=log)
1599
  cache["dirty"].update(changed_tables)
1600
  try:
1601
  if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
1602
- raise PublicationHeld(
1603
- f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot "
1604
- "at start. The saved copy is kept; restore tenant-store access and restart to "
1605
- "restore it before checkpointing.")
1606
- checkpoint = _candidate_checkpoint(con)
1607
- old = checkpoint["base"] if checkpoint else cache["manifest"]
 
1608
  sealed = self._sealed_mode(rt)
1609
- if old.get("blob_envelope_version") and not sealed:
1610
- raise tenant_keys.TenantKeyLocked("A sealed mirror cannot publish without its tenant key.")
 
 
1611
  if old.get("snapshot_id") and _publication is None:
1612
  raise ValueError("An identified source requires the complete sync publication boundary")
1613
- tables = self._tables(con) | set(cache.get("refused_tables", ()))
 
 
1614
  manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
1615
- "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}}
 
1616
  if sealed:
1617
- manifest["blob_envelope_version"] = be.VERSION
1618
- part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1619
  with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
1620
- changed, pending = self._checkpoint_tables(
1621
- rt, con, manifest, cache["dirty"], self._manifest_keys(old), tmp,
1622
- sealed=sealed, part_limit=part_limit)
1623
- saved = self._publish_checkpoint(tenant_key, rt, con, cache, manifest, changed, pending, tmp,
1624
- publication=_publication, hold=_hold, rebase_allowed=_rebase_allowed,
1625
- report=report, log=log, progress=progress)
1626
- if _hold is not None:
1627
- return saved
1628
- if saved or not changed:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1629
  cache["dirty"].clear()
1630
- log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, "
1631
- f"{progress['uploaded']:,} bytes uploaded")
1632
- return saved
1633
- except PublicationHeld as exc:
1634
- report["problems"].append(str(exc))
1635
- log(str(exc))
1636
  except Exception as exc:
1637
- message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); "
1638
- f"{progress['uploaded']:,} bytes uploaded. "
1639
  "Local rows were kept; restore tenant-store access and resync before restarting.")
1640
  report["problems"].append(message)
1641
  log(message)
@@ -1820,20 +1638,8 @@ def _export_tables(store, con, directory):
1820
  return paths
1821
 
1822
 
1823
- def _candidate_matches(store, candidate, published, previous):
1824
- """Resume a known same-tenant common base even after another publication was installed."""
1825
- if not previous:
1826
- return False
1827
- installed = datastore._publication(published, store.connector)
1828
- checkpoint = _candidate_checkpoint(candidate)
1829
- if checkpoint is None:
1830
- return previous[0] == installed.get("snapshot_id", "unknown")
1831
- base = checkpoint["base"]
1832
- return previous[0] == base.get("snapshot_id", "unknown") and base.get("tenant") == installed.get("tenant")
1833
-
1834
-
1835
  def _open_candidate(store, tenant_key, rt, published):
1836
- """Resume known common-base provenance or seed private rows from the serving copy."""
1837
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1838
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1839
  datastore._ensure_sync_state(candidate)
@@ -1843,9 +1649,7 @@ def _open_candidate(store, tenant_key, rt, published):
1843
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
1844
  previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
1845
  base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1846
- if _candidate_matches(store, candidate, published, previous):
1847
- if _candidate_checkpoint(candidate) is None:
1848
- _save_initial_candidate_checkpoint(store, rt, candidate, published)
1849
  return candidate, previous[1]
1850
  started = _iso(datetime.now(timezone.utc))
1851
  try:
@@ -1860,7 +1664,6 @@ def _open_candidate(store, tenant_key, rt, published):
1860
  candidate.execute("DELETE FROM _candidate_pending")
1861
  candidate.execute("DELETE FROM _candidate_held")
1862
  candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
1863
- _save_initial_candidate_checkpoint(store, rt, candidate, published)
1864
  return candidate, started
1865
  except BaseException:
1866
  candidate.close()
@@ -1880,7 +1683,7 @@ def pending_sync_entities(connector, tenant_key, rt):
1880
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1881
  try:
1882
  base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
1883
- if not _candidate_matches(store, candidate, published, base):
1884
  return None
1885
  pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
1886
  if store.connector == "odoo":
@@ -1898,13 +1701,6 @@ def _holds_rows(holds):
1898
  return holds is not None and bool(holds["tables"] or holds.get("accounts"))
1899
 
1900
 
1901
- def _held_tables(holds):
1902
- """Retain whole private tables and their original bases even when only an account is held."""
1903
- if holds is None:
1904
- return set()
1905
- return set(holds["tables"]).union(*(hold["tables"] for hold in (holds.get("accounts") or {}).values()))
1906
-
1907
-
1908
  def _hold_previous(store, candidate, published, tables):
1909
  """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1910
  prior = store._tables(published)
@@ -1925,36 +1721,6 @@ def _hold_previous(store, candidate, published, tables):
1925
  candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1926
 
1927
 
1928
- def _retain_conflicting_candidate(store, candidate, published, manifest, conflicts):
1929
- """Refresh independent tables while retaining held private rows, markers and their original common bases."""
1930
- checkpoint = _candidate_checkpoint(candidate)
1931
- tables = set(conflicts)
1932
- private = _table_marker_rows(store, store._state_rows(candidate), tables)
1933
- saved_rows = store._state_rows(published)
1934
- held_names = {row[0] for row in _table_marker_rows(store, checkpoint["markers"] + saved_rows, tables)}
1935
- base_rows = [row for row in saved_rows if row[0] not in held_names]
1936
- base_rows += [row for row in checkpoint["markers"] if row[0] in held_names]
1937
- base = {**manifest, "snapshot_id": checkpoint["base"].get("snapshot_id", "unknown"),
1938
- "tables": {name: entry for name, entry in manifest["tables"].items() if name not in tables}}
1939
- base["tables"].update({name: entry for name, entry in _candidate_base_tables(checkpoint).items()
1940
- if name in tables})
1941
- candidate.execute("BEGIN TRANSACTION")
1942
- try:
1943
- _hold_previous(store, candidate, published, (store._tables(candidate) | store._tables(published)) - tables)
1944
- owned = store._state_rows(candidate)
1945
- if owned:
1946
- candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in owned])
1947
- rows = [row for row in saved_rows if row[0] not in held_names] + private
1948
- if rows:
1949
- candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows)
1950
- _save_candidate_checkpoint(candidate, base, base_rows)
1951
- candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", [[name] for name in conflicts])
1952
- candidate.execute("COMMIT")
1953
- except BaseException:
1954
- candidate.execute("ROLLBACK")
1955
- raise
1956
-
1957
-
1958
  def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1959
  """One log line: what a pass published and what it held, each with its reason."""
1960
  rows = report.get("tables") or {}
@@ -2124,7 +1890,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2124
  prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
2125
  metadata.pop("rows")
2126
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
2127
- result = {"problems": [], "held_tables": _held_tables(holds)}
2128
  save_started = time.monotonic()
2129
  with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
2130
  exporting = time.monotonic()
@@ -2142,18 +1908,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2142
  report["problems"].extend(problems)
2143
  return
2144
  raise RuntimeError("; ".join(problems))
2145
- if report is not None and "problems" in report:
2146
- report["problems"].extend(result["problems"])
2147
- retained = set(result.get("conflicts", ())) | result["held_tables"]
2148
- kept = bool(retained)
2149
  try:
2150
  installing = time.monotonic()
2151
- rows = store._state_rows(candidate)
2152
- if result.get("rebased"):
2153
- store._before_restore(tenant_key, rt)
2154
- _saved, rows, paths, _total, _refused = store._validated_snapshot(
2155
- tenant_key, rt, published, _mirror_json(manifest), tmp)
2156
- store._install_snapshot(published, manifest, rows, paths)
2157
  installed = True
2158
  store._after_publish(tenant_key, manifest)
2159
  except BaseException:
@@ -2163,19 +1920,20 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2163
  note("install", installing)
2164
  for listener in list(_PUBLISH_LISTENERS):
2165
  try:
2166
- listener(store.connector, tenant_key, manifest, rows, paths)
2167
  except Exception as exc:
2168
  # The snapshot is saved and installed here. A failed hand-off only delays readers.
2169
  log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
2170
  if _holds_rows(holds):
2171
  candidate.execute("ROLLBACK")
2172
- if retained:
2173
- _retain_conflicting_candidate(store, candidate, published, manifest, retained)
2174
- if odoo and holds is not None:
2175
  # D-11329: a held entity keeps its private progress but never narrows the next pass.
2176
  names = [[name] for name in sorted(holds["held"])]
2177
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
2178
  candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
 
2179
  if holds is not None:
2180
  log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
2181
  if odoo:
 
355
  completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
356
  for row in report.values())
357
  return completed, completed and all(row["phase"] == "live" for row in report.values())
358
+ if report.get("problems") or report.get("audit_problems"):
 
359
  # A problem owned by one held table publishes the rest; it never makes the pass current.
360
  holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
361
  if holds is None or holds["blocked"]:
 
380
  return completed, current
381
 
382
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
383
  def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
384
  """Count completed passes and require a clean result before clearing startup catch-up."""
385
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
386
  rt = runtime
387
  if rt is None:
388
  return pull(*args, **kwargs)
 
389
  with _CATCH_UP_LOCK:
390
+ state = _catch_up_source(rt, connector)
391
+ if not state["current"]:
392
+ state.update(pass_pending=True, attempt_at=time.time())
393
  active = (connector, _tenant_key(rt.key))
394
  _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
395
  try:
 
521
  return "'" + str(value).replace("'", "''") + "'"
522
 
523
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
524
  class MirrorStore:
525
  """One connector policy, with state cached separately for each tenant mirror."""
526
 
 
1298
  return entry, fresh
1299
 
1300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1301
  def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
1302
+ _publication=None, _hold=None):
1303
+ """Checkpoint changed tables and the connector's markers, publishing the manifest last.
1304
+
1305
+ Candidates include partially written tables. Hashing their parquet locally avoids reuploading
1306
+ identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain
1307
+ dirty for retry on the next pass, while landed local rows and cursors stay usable.
1308
+
1309
+ S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a
1310
+ CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists
1311
+ every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row
1312
+ upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4)
1313
+ leaves the previous manifest naming only complete, untouched blobs.
1314
+ """
1315
  candidate = getattr(datastore._SYNC_TLS, "candidate", None)
1316
  if candidate is not None and _publication is None:
1317
  if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
1318
  raise ValueError("Checkpoint belongs to another source candidate")
1319
+ return # The enclosing source pass owns publication after every table completes.
1320
+ uploaded = 0
1321
  with self.lock:
1322
  cache = self.restore_mirror(tenant_key, rt, log=log)
1323
  cache["dirty"].update(changed_tables)
1324
  try:
1325
  if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
1326
+ message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot "
1327
+ "at start. The saved copy is kept; restore tenant-store access and restart to "
1328
+ "restore it before checkpointing.")
1329
+ report["problems"].append(message)
1330
+ log(message)
1331
+ return
1332
+ old = cache["manifest"]
1333
  sealed = self._sealed_mode(rt)
1334
+ if old.get('blob_envelope_version') and not sealed:
1335
+ raise tenant_keys.TenantKeyLocked('A sealed mirror cannot publish without its tenant key.')
1336
+ suffix = '.sealed-v1' if sealed else ''
1337
+ part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1338
  if old.get("snapshot_id") and _publication is None:
1339
  raise ValueError("An identified source requires the complete sync publication boundary")
1340
+ named = self._manifest_keys(old)
1341
+ tables = self._tables(con)
1342
+ refused_tables = set(cache.get("refused_tables", ()))
1343
  manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
1344
+ "tables": {name: meta for name, meta in old.get("tables", {}).items()
1345
+ if name in tables or name in refused_tables}}
1346
  if sealed:
1347
+ manifest['blob_envelope_version'] = be.VERSION
1348
+ changed, pending = [], []
1349
  with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
1350
+ # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
1351
+ legacy = {name for name, meta in manifest["tables"].items()
1352
+ if (not meta.get("key") and "parts" not in meta)
1353
+ or any((".sealed-v1." in key) != sealed
1354
+ for key, _ in self._entry_blob_keys(name, meta))}
1355
+ # Refused local tables still name good remote parts: migrate those bytes too.
1356
+ for name in sorted(legacy - tables):
1357
+ prior = manifest['tables'][name]
1358
+ migrated = []
1359
+ for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
1360
+ if ('.sealed-v1.' in key) == sealed:
1361
+ migrated.append(dict(metadata))
1362
+ continue
1363
+ data = self._checked_mirror_blob(rt, key, metadata)
1364
+ path = Path(tmp) / f'{name}.migrate{index}.parquet'
1365
+ path.write_bytes(data)
1366
+ new_key = key.replace('.parquet', '.sealed-v1.parquet')
1367
+ migrated.append({**metadata, 'key': new_key})
1368
+ pending.append((new_key, path))
1369
+ manifest['tables'][name] = ({**prior, 'parts': migrated} if 'parts' in prior
1370
+ else {**prior, **migrated[0]})
1371
+ changed.append(name)
1372
+ for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
1373
+ if not self._valid_table(name):
1374
+ raise ValueError(f"{self.label} mirror table name is invalid")
1375
+ prior = manifest["tables"].get(name, {})
1376
+ path = Path(tmp) / (name + ".parquet")
1377
+ size = None
1378
+ if "parts" not in prior:
1379
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
1380
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1381
+ size = path.stat().st_size
1382
+ if size is None or size > part_limit:
1383
+ entry, fresh = self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
1384
+ if entry is None:
1385
+ continue
1386
+ manifest["tables"][name] = entry
1387
+ changed.append(name)
1388
+ pending.extend(fresh)
1389
+ continue
1390
+ data = path.read_bytes()
1391
+ metadata = _blob_metadata(data)
1392
+ if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
1393
+ and (".sealed-v1." in prior["key"]) == sealed):
1394
+ continue
1395
+ key = _content_key(name, suffix + ".parquet", metadata)
1396
+ metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
1397
+ written_at=_iso(datetime.now(timezone.utc)), key=key)
1398
+ manifest["tables"][name] = metadata
1399
+ changed.append(name)
1400
+ if key in named:
1401
+ # The same content under its own name is already stored and published.
1402
+ if named[key] != metadata["sha256"]:
1403
+ raise ValueError(f"{self.label} mirror content key collides with a published blob")
1404
+ continue
1405
+ pending.append((key, path))
1406
+ rows = self._state_rows(con)
1407
+ state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
1408
+ published = None
1409
+ if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1410
+ or manifest["tables"] != old.get("tables", {})
1411
+ or old.get("version") != MANIFEST_VERSION):
1412
+ for index, (state_key, state) in enumerate(state_blobs):
1413
+ if state_key not in named:
1414
+ state_path = Path(tmp) / f"sync_state.{index}.json"
1415
+ state_path.write_bytes(state)
1416
+ pending.append((state_key, state_path))
1417
+ if any(path.stat().st_size > part_limit for _key, path in pending):
1418
+ raise ValueError("mirror part exceeds the byte limit")
1419
+ if pending:
1420
+ self._ledger_add(rt, tenant_key, [key for key, _path in pending])
1421
+ for key, path in pending:
1422
+ data = path.read_bytes()
1423
+ self._upload(rt, key, data)
1424
+ uploaded += len(data)
1425
+ stamp = _iso(datetime.now(timezone.utc))
1426
+ manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp,
1427
+ **(_publication or {}))
1428
+ if _publication is not None:
1429
+ manifest["completed_at"] = stamp
1430
+ if _hold is not None:
1431
+ manifest["hold"] = _hold
1432
+ self._upload(rt, _HELD_KEY, _mirror_json(manifest))
1433
+ return manifest
1434
+ superseded = _superseded_hold(self, tenant_key, rt)
1435
+ if superseded:
1436
+ manifest["superseded_hold"] = superseded
1437
+ published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
1438
+ uploaded += len(published)
1439
+ if superseded is not None:
1440
+ # A newer publication replaces a stored hold. Nobody has to release it.
1441
+ try:
1442
+ rt.delete_path(self.prefix + _HELD_KEY)
1443
+ except Exception: # noqa: BLE001
1444
+ log(f"{self.label} superseded hold pointer stays until the next publication")
1445
+ if _publication is None:
1446
+ datastore._write_publication(con, self.connector, manifest)
1447
+ cache["manifest"] = manifest
1448
+ cache["problems"].clear()
1449
+ if published:
1450
+ self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
1451
+ if published or not changed:
1452
  cache["dirty"].clear()
1453
+ log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded")
1454
+ return manifest if published else None
 
 
 
 
1455
  except Exception as exc:
1456
+ message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. "
 
1457
  "Local rows were kept; restore tenant-store access and resync before restarting.")
1458
  report["problems"].append(message)
1459
  log(message)
 
1638
  return paths
1639
 
1640
 
 
 
 
 
 
 
 
 
 
 
 
 
1641
  def _open_candidate(store, tenant_key, rt, published):
1642
+ """Resume private rows only while their base publication still matches the serving copy."""
1643
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1644
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1645
  datastore._ensure_sync_state(candidate)
 
1649
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
1650
  previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
1651
  base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1652
+ if previous and previous[0] == base:
 
 
1653
  return candidate, previous[1]
1654
  started = _iso(datetime.now(timezone.utc))
1655
  try:
 
1664
  candidate.execute("DELETE FROM _candidate_pending")
1665
  candidate.execute("DELETE FROM _candidate_held")
1666
  candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
 
1667
  return candidate, started
1668
  except BaseException:
1669
  candidate.close()
 
1683
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1684
  try:
1685
  base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
1686
+ if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"):
1687
  return None
1688
  pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
1689
  if store.connector == "odoo":
 
1701
  return holds is not None and bool(holds["tables"] or holds.get("accounts"))
1702
 
1703
 
 
 
 
 
 
 
 
1704
  def _hold_previous(store, candidate, published, tables):
1705
  """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1706
  prior = store._tables(published)
 
1721
  candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1722
 
1723
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1724
  def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1725
  """One log line: what a pass published and what it held, each with its reason."""
1726
  rows = report.get("tables") or {}
 
1890
  prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1891
  metadata.pop("rows")
1892
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
1893
+ result = {"problems": []}
1894
  save_started = time.monotonic()
1895
  with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
1896
  exporting = time.monotonic()
 
1908
  report["problems"].extend(problems)
1909
  return
1910
  raise RuntimeError("; ".join(problems))
 
 
 
 
1911
  try:
1912
  installing = time.monotonic()
1913
+ store._install_snapshot(published, manifest, store._state_rows(candidate), paths)
 
 
 
 
 
1914
  installed = True
1915
  store._after_publish(tenant_key, manifest)
1916
  except BaseException:
 
1920
  note("install", installing)
1921
  for listener in list(_PUBLISH_LISTENERS):
1922
  try:
1923
+ listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths)
1924
  except Exception as exc:
1925
  # The snapshot is saved and installed here. A failed hand-off only delays readers.
1926
  log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
1927
  if _holds_rows(holds):
1928
  candidate.execute("ROLLBACK")
1929
+ candidate.execute("DELETE FROM _candidate_base")
1930
+ candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
1931
+ if odoo:
1932
  # D-11329: a held entity keeps its private progress but never narrows the next pass.
1933
  names = [[name] for name in sorted(holds["held"])]
1934
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
1935
  candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
1936
+ kept = True
1937
  if holds is not None:
1938
  log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
1939
  if odoo:
platform/modules/amazon_profit.py CHANGED
@@ -31,11 +31,9 @@ import re
31
  import shutil
32
  import tempfile
33
  import threading
34
- from decimal import Decimal, ROUND_HALF_UP
35
  from pathlib import Path
36
 
37
  from modules import amazon_sku_metrics as M
38
- from modules import amazon_part_dates as _part_dates
39
  import core.user_tables as _registry
40
  from core import module_tables
41
  from core import amazon_rivals as _rivals
@@ -121,58 +119,14 @@ def _field(key, label, ftype, description, *, computed, **extra):
121
  return out
122
 
123
 
124
- def _progress_fields(description, *, key="projects", label="Projects", empty_label="No projects"):
125
  """Project progress and its completed-count label, written by the Projects doors."""
126
- return (_field(key, label, "pct", description, computed=True,
127
- format={"display": "progress", "labelKey": key + "_label", "emptyLabel": empty_label}),
128
- _field(key + "_label", label + (" completed" if key == "projects" else " label"), "text",
129
- "Completed projects of the counted ones." if key == "projects" else "Open PDP workflow stage and progress.",
130
  computed=True))
131
 
132
 
133
- HISTORY_FIELD = _field("history", "History", "text", "Open the row's recorded changes.", computed=True)
134
- ASIN_APP_KEYS = ("asin", "projects", "pdp_update", "trend_score", "search_trend", "market_share_trend",
135
- "click_share_trend", "exposure", "funnel_score", "pdp_last_updated", "title_last_updated",
136
- "bullets_last_updated", "images_last_updated", "aplus_last_updated", "history")
137
- SEARCH_TRENDS = (("search", "search_trend", "Search trend", "int"),
138
- ("market_share", "market_share_trend", "Market share trend", "pct"),
139
- ("click_share", "click_share_trend", "Click share trend", "pct"))
140
- EXPOSURES = {"under": "Under-exposed", "match": "Matched", "over": "Over-exposed",
141
- "few": "Few", "none": "Not seen"}
142
- PDP_PART_DATE_KEYS = tuple(part + "_last_updated" for part in ("title", "bullets", "images", "aplus"))
143
- PDP_CELL_KEYS = ("pdp_update", "pdp_update_label", "pdp_update_more") + PDP_PART_DATE_KEYS
144
- _PDP_PROGRESS_FIELDS = _progress_fields("Progress of the earliest open PDP workflow.", key="pdp_update",
145
- label="PDP update", empty_label="No open workflow")
146
- _SIGNAL_FIELDS = (
147
- _field("trend_score", "Trend score", "int", "Search trend score from 0 to 100.", computed=True),
148
- *(_field(key, label, "pct", "Change against the same reported months last year.", computed=True)
149
- for _, key, label, _ in SEARCH_TRENDS),
150
- _field("exposure", "Exposure", "select", "Sales or click share against impression share.",
151
- computed=True, options=list(EXPOSURES.values())),
152
- _field("funnel_score", "Funnel score", "int", "Search-weighted funnel score, with 5 at the market.",
153
- computed=True, format={"decimals": 1}),
154
- )
155
- _ASIN_DATE_FIELDS = tuple(_field(key, label, "date", "Latest recorded listing or part change.", computed=True)
156
- for key, label in zip(("pdp_last_updated",) + PDP_PART_DATE_KEYS,
157
- ("PDP last updated", "Title last updated", "Bullets last updated",
158
- "Images last updated", "A+ last updated")))
159
- _SIGNAL_COMPANIONS = tuple(field for kind, key, label, now_type in SEARCH_TRENDS for field in (
160
- _field(key + "_series", label + " series", "text", "Every reported historical month, with raw values, ASIN gaps and reasons.", computed=True),
161
- _field(key + "_now", label + " now", now_type, "Searches in the newest complete store month." if kind == "search"
162
- else "Share aggregated over the twelve-month calculation window.", computed=True),
163
- _field(key + "_reason", label + " reason", "text", "Why the year comparison is unavailable.", computed=True),
164
- _field(key + "_now_reason", label + " now reason", "text", "Why the current value is unavailable.", computed=True),
165
- )) + tuple(_field(key + "_reason", label + " reason", "text", "Why this signal is unavailable.", computed=True)
166
- for key, label in (("trend_score", "Trend score"), ("funnel_score", "Funnel score"), ("exposure", "Exposure")))
167
- _ASIN_COMPANIONS = (_PDP_PROGRESS_FIELDS[1:]
168
- + (_field("pdp_update_more", "More PDP workflows", "int", "Other open PDP workflows.", computed=True),)
169
- + _SIGNAL_COMPANIONS
170
- + (_field("exposure_label", "Exposure detail", "text", "Exposure multiple and comparison base.", computed=True),
171
- _field("pdp_last_updated_reason", "PDP date reason", "text", "Why Amazon's listing date is unavailable.", computed=True)))
172
- _ASIN_SOURCE_KEYS = tuple(f["key"] for f in _SIGNAL_FIELDS + _SIGNAL_COMPANIONS) + (
173
- "exposure_label", "pdp_last_updated", "pdp_last_updated_reason")
174
-
175
-
176
  def _metric_fields(skip=()):
177
  """The sibling metric module's columns, as computed fields of this module."""
178
  return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
@@ -226,7 +180,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x)
226
  for k, label, t, d, x in _rivals.PARENT_COLUMNS)
227
  COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
228
  _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
229
- COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (HISTORY_FIELD, PICTURE_FIELD) + _ECON_FIELDS[1:] + _metric_fields()
230
  + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
231
  + COMPETITOR_FIELDS)
232
  #: Owned columns: people edit them and a refresh never writes them.
@@ -378,7 +332,7 @@ def cost_note(row, *, code=None, found=True):
378
  if source and row.get("cogs_per_unit") is None:
379
  return source if source.endswith(".") else source + "."
380
  return NO_COST_NOTE
381
- _IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label", "history")
382
  _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
383
 
384
  #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
@@ -397,12 +351,9 @@ def _grain_fields(key_column, grain):
397
  else (metric["parent_asin"], PICTURE_FIELD))
398
  where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
399
  progress = _progress_fields(f"Average progress of the projects on {where}.")
400
- app = (_PDP_PROGRESS_FIELDS[:1] + _SIGNAL_FIELDS + _ASIN_DATE_FIELDS + (HISTORY_FIELD,)
401
- if key_column == "asin" else (HISTORY_FIELD,))
402
- companions = _ASIN_COMPANIONS if key_column == "asin" else ()
403
- return (ids[:1] + progress[:1] + app + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
404
  + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
405
- + progress[1:] + companions)
406
 
407
 
408
  ASIN_FIELDS = _grain_fields("asin", "asin")
@@ -448,7 +399,7 @@ PROJECT_COMPUTED_KEYS = tuple(f["key"] for f in PROJECT_COMPUTED_FIELDS)
448
 
449
  def table_specs():
450
  """Every module table in creation order."""
451
- specs = (
452
  {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
453
  "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
454
  {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
@@ -456,10 +407,6 @@ def table_specs():
456
  {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
457
  "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
458
  )
459
- for spec in specs:
460
- fields = {f["key"]: f for f in spec["fields"]}
461
- spec["fields"] = tuple(fields[key] for key in _project_field_order(fields))
462
- return specs
463
 
464
 
465
  # ---------------------------------------------------------------- period and threshold
@@ -1760,7 +1707,7 @@ def _sync_cursor(tenant_key, rt):
1760
  else:
1761
  import duckdb
1762
  cur = duckdb.connect(str(path), read_only=True)
1763
- if _mirror_has_economics(cur) or {'sku', 'attributes', 'summaries'} <= _columns(cur, 'amzn_listing_items'):
1764
  return cur, cur.close
1765
  cur.close()
1766
  for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
@@ -1773,7 +1720,7 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
1773
  """Refresh the current period, apply decisions, then suggest for every unlinked SKU.
1774
  Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
1775
  synced = {str(t or "").strip().lower() for t in synced_tables or ()}
1776
- if not synced & {"economics", "listings", "listing_items"}:
1777
  return {"state": "skipped", "synced": sorted(synced)}
1778
  if st is None:
1779
  from harness import amazon_store
@@ -1785,17 +1732,6 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
1785
  con, close = _sync_cursor(tenant_key, st)
1786
  try:
1787
  report = {"state": "done"}
1788
- if synced & {"listings", "listing_items"}:
1789
- try:
1790
- report["partDates"] = _part_dates.observe(st, con, day=today or _today())
1791
- report["partDateCells"] = paint_asin_part_dates(st)
1792
- except Exception as exc: # quality-ok: Date storage failure must not block profit matching.
1793
- report['partDatesError'] = f'{type(exc).__name__}: {exc}'
1794
- _log(f"{tenant_key}: listing part dates failed ({report['partDatesError']}).")
1795
- if not synced & {'economics', 'listings'}:
1796
- return dict(report, state='failed', reason=report['partDatesError'])
1797
- if not synced & {"economics", "listings"}:
1798
- return report
1799
  try:
1800
  refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
1801
  report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
@@ -1871,16 +1807,12 @@ def _definition_fields(fields=None, owned_keys=None):
1871
 
1872
 
1873
  def _project_field_order(keys):
1874
- """Pin App fields in schema defaults and keep companions last, without changing saved layouts."""
1875
  keys = list(keys)
1876
- if not keys or keys[0] not in ("sku", "asin", "parent_asin"):
1877
- return keys
1878
- pinned = ASIN_APP_KEYS if keys[0] == "asin" else (keys[0], "projects", "history")
1879
- companions = (("projects_label",) + tuple(f["key"] for f in _ASIN_COMPANIONS)
1880
- if keys[0] == "asin" else ())
1881
- return ([key for key in pinned if key in keys]
1882
- + [key for key in keys if key not in pinned + companions]
1883
- + [key for key in companions if key in keys])
1884
 
1885
 
1886
  def _format_repairs(fields, wanted):
@@ -1900,9 +1832,9 @@ def _grain_word_repairs(fields, wanted):
1900
  continue
1901
  if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
1902
  out.setdefault("grain", {})["options"] = list(target["options"])
1903
- if field["key"] in ("projects", "pdp_update") and (field.get("type") != target["type"] or "link" in field
1904
  or (field.get("format") or {}) != (target.get("format") or {})):
1905
- out[field["key"]] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None}
1906
  return out
1907
 
1908
 
@@ -2093,9 +2025,6 @@ def write_rows(st, computed, *, table_key=TABLE_KEY, key_column="sku", label=TAB
2093
  for rid, row in table_rows(st, table_key).items()
2094
  if str(row.get(key_column) or "").strip() not in computed}
2095
  wanted = {rid: values for rid, values in wanted.items() if values}
2096
- for rid, row in table_rows(st, table_key).items():
2097
- if row.get("history") != "View":
2098
- wanted.setdefault(rid, {})["history"] = "View"
2099
  if wanted:
2100
  import core.user_tables as ut
2101
 
@@ -2639,7 +2568,7 @@ def grain_rows(rows, key, *, thin=THIN_DEFAULT, period_label="", fields=()):
2639
  flat["cost_note"] = _grain_cost_note(flat, children)
2640
  flat["period"] = period_label
2641
  out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
2642
- if k not in ("projects", "projects_label") + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS}
2643
  return out
2644
 
2645
 
@@ -2871,117 +2800,8 @@ def _withhold_untied(by_grain, checks):
2871
 
2872
  def _grain_period_keys(fields, key_column):
2873
  return tuple(f["key"] for f in fields
2874
- if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label", "history")
2875
- + COMPETITOR_KEYS + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS)
2876
-
2877
-
2878
- def _listing_dates(con):
2879
- """Read the newest Amazon lastUpdatedDate per ASIN from raw listing summaries."""
2880
- if "summaries" not in _columns(con, "amzn_listing_items"):
2881
- return {}
2882
- rows = con.execute("""SELECT json_extract_string(s.value, '$.asin'),
2883
- CAST(MAX(TRY_CAST(json_extract_string(s.value, '$.lastUpdatedDate') AS TIMESTAMPTZ))
2884
- AT TIME ZONE 'UTC' AS DATE)
2885
- FROM amzn_listing_items l, json_each(TRY_CAST(l.summaries AS JSON)) s GROUP BY 1""").fetchall()
2886
- return {asin: date.isoformat() for asin, date in rows if asin and date is not None}
2887
-
2888
-
2889
- def _exposure_detail(exposure):
2890
- """Format the recorded exposure multiple with the prototype's one-decimal rounding."""
2891
- if not exposure or exposure["multiple"] is None:
2892
- return None
2893
- multiple = Decimal.from_float(float(exposure["multiple"])).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)
2894
- return f"{multiple}x {exposure['base']} share"
2895
-
2896
-
2897
- def _asin_signal_cells(signal, history):
2898
- """Store scalar cells and raw monthly bars for every reported historical month."""
2899
- out = {key: stored_cell(_FIELD_BY_KEY[key], signal[key]) for key in ("trend_score", "funnel_score")}
2900
- out.update({key + "_reason": value for key, value in signal["reasons"].items()
2901
- if key in ("trend_score", "funnel_score", "exposure")})
2902
- exposure = signal["exposure"]
2903
- out["exposure"] = EXPOSURES[exposure["kind"]] if exposure else None
2904
- out["exposure_label"] = _exposure_detail(exposure)
2905
- for kind, key, _label, _now_type in SEARCH_TRENDS:
2906
- trend = signal["trends"][kind]
2907
- points = {point["month"]: point for point in trend["series"]}
2908
- series = [points[month] for month in history]
2909
- latest = series[-1] if series else {"value": None, "reason": signal["note"] or "No search report"}
2910
- now = latest["value"] if kind == "search" else signal[kind]
2911
- out.update({key: None if trend["change"] is None else str(trend["change"] * 100),
2912
- key + "_series": json.dumps(series, separators=(",", ":")),
2913
- key + "_now": (stored_cell(_FIELD_BY_KEY[key + "_now"], now) if kind == "search"
2914
- else None if now is None else str(now * 100)),
2915
- key + "_reason": trend["reason"],
2916
- key + "_now_reason": (signal["note"] or latest["reason"] if kind == "search"
2917
- else signal["reasons"][kind])})
2918
- return out
2919
-
2920
-
2921
- def paint_asin_signals(st, con, *, today):
2922
- """Refresh source signals and Amazon's date for every stored ASIN, independent of the profit period."""
2923
- from modules import amazon_search_signals as signals
2924
- asins = [str(row["asin"]).strip() for row in table_rows(st, ASIN_KEY).values() if row.get("asin")]
2925
- result = signals.signals(con, asins, today=today)
2926
- dates = _listing_dates(con)
2927
- parts = _part_dates.cells(st, asins)
2928
- computed = {asin: dict(_asin_signal_cells(signal, result["history"]), **parts[asin], pdp_last_updated=dates.get(asin),
2929
- pdp_last_updated_reason=None if asin in dates else "No listing lastUpdatedDate reported")
2930
- for asin, signal in result["asins"].items()}
2931
- return write_rows(st, computed, table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=())
2932
-
2933
-
2934
- def paint_asin_part_dates(st):
2935
- """Refresh recorded part dates without profit validation, search reports or Odoo matching."""
2936
- asins = [str(row['asin']).strip() for row in table_rows(st, ASIN_KEY).values() if row.get('asin')]
2937
- if not asins:
2938
- return {'rows': 0}
2939
- return write_rows(st, _part_dates.cells(st, asins), table_key=ASIN_KEY,
2940
- key_column='asin', label=ASIN_LABEL, period_keys=())
2941
-
2942
-
2943
- def asin_signal_checks(st, con, *, today=None):
2944
- """Tie stored signal cells to the signal reader's independently validated raw monthly inputs."""
2945
- from modules import amazon_search_signals as signals
2946
- rows = {str(row["asin"]).strip(): row for row in table_rows(st, ASIN_KEY).values() if row.get("asin")}
2947
- result = signals.signals(con, rows, today=today)
2948
- verified = signals.validate(con, result, today=today)
2949
- differences = sum(any(rows[asin].get(key) != value for key, value in _asin_signal_cells(signal, result["history"]).items())
2950
- for asin, signal in result["asins"].items())
2951
- return [{"check": "amazon_profit: stored ASIN search signals tie to raw monthly reports",
2952
- "ok": verified["ok"] and not differences, "rows": len(rows), "differences": differences,
2953
- "reason": verified["reason"]}]
2954
-
2955
-
2956
- def listing_date_checks(st, con):
2957
- """Independently tie stored PDP dates to the newest raw summary timestamp per ASIN."""
2958
- newest = {}
2959
- available = "summaries" in _columns(con, "amzn_listing_items")
2960
- if available:
2961
- cursor = con.execute("SELECT summaries FROM amzn_listing_items")
2962
- for batch in iter(lambda: cursor.fetchmany(2048), []):
2963
- for (raw,) in batch:
2964
- try:
2965
- summaries = json.loads(raw) if isinstance(raw, str) else raw
2966
- except (TypeError, ValueError):
2967
- continue
2968
- for summary in summaries if isinstance(summaries, list) else ():
2969
- if not isinstance(summary, dict) or not summary.get("asin"):
2970
- continue
2971
- try:
2972
- stamp = dt.datetime.fromisoformat(str(summary.get("lastUpdatedDate")).replace("Z", "+00:00"))
2973
- except ValueError:
2974
- continue
2975
- stamp = stamp.replace(tzinfo=stamp.tzinfo or dt.timezone.utc).astimezone(dt.timezone.utc)
2976
- asin = summary["asin"]
2977
- if asin not in newest or stamp > newest[asin]:
2978
- newest[asin] = stamp
2979
- rows = list(table_rows(st, ASIN_KEY).values())
2980
- differences = sum(row.get("pdp_last_updated") != (newest[row["asin"]].date().isoformat()
2981
- if row.get("asin") in newest else None) for row in rows)
2982
- return [{"check": "amazon_profit: PDP last updated ties to newest listing lastUpdatedDate per ASIN",
2983
- "ok": available and not differences, "rows": len(rows), "differences": differences,
2984
- "reason": None if available else "Listing summaries are unavailable"}]
2985
 
2986
 
2987
  def _date(value):
@@ -3322,7 +3142,6 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N
3322
  report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
3323
  key_column="parent_asin", label=PARENT_LABEL,
3324
  period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
3325
- report["asinSignals"] = paint_asin_signals(st, con, today=day)
3326
  report["blankReasons"] = blank_check(stored_tables(st))
3327
  bands = {sku: values.get("band") for sku, values in computed.items()}
3328
  report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
@@ -3468,7 +3287,6 @@ def _mirror_has_economics(con):
3468
 
3469
 
3470
  def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
3471
- "amzn_listing_items", "amzn_search_query_performance",
3472
  "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
3473
  "amzn_ads_sd_advertised_products")):
3474
  """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
@@ -3630,10 +3448,10 @@ def stored_grain_checks(st, con):
3630
 
3631
 
3632
  def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
3633
- typed_values=None, typed_period=None, st=None, today=None):
3634
  """Tie net proceeds components to independent Amazon sources within 2% over settled days.
3635
  Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
3636
- With `st`, stored grain figures, search signals and listing dates tie to their sources."""
3637
  month = month or last_complete_month()
3638
  bounds = period_bounds(month)
3639
  start, end = bounds["start"], bounds["end"]
@@ -3693,8 +3511,6 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No
3693
  checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
3694
  if st is not None:
3695
  checks.extend(stored_grain_checks(st, con))
3696
- checks.extend(listing_date_checks(st, con))
3697
- checks.extend(asin_signal_checks(st, con, today=today))
3698
  return checks
3699
 
3700
 
 
31
  import shutil
32
  import tempfile
33
  import threading
 
34
  from pathlib import Path
35
 
36
  from modules import amazon_sku_metrics as M
 
37
  import core.user_tables as _registry
38
  from core import module_tables
39
  from core import amazon_rivals as _rivals
 
119
  return out
120
 
121
 
122
+ def _progress_fields(description):
123
  """Project progress and its completed-count label, written by the Projects doors."""
124
+ return (_field("projects", "Projects", "pct", description, computed=True,
125
+ format={"display": "progress", "labelKey": "projects_label", "emptyLabel": "No projects"}),
126
+ _field("projects_label", "Projects completed", "text", "Completed projects of the counted ones.",
 
127
  computed=True))
128
 
129
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  def _metric_fields(skip=()):
131
  """The sibling metric module's columns, as computed fields of this module."""
132
  return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
 
180
  for k, label, t, d, x in _rivals.PARENT_COLUMNS)
181
  COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
182
  _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
183
+ COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields()
184
  + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
185
  + COMPETITOR_FIELDS)
186
  #: Owned columns: people edit them and a refresh never writes them.
 
332
  if source and row.get("cogs_per_unit") is None:
333
  return source if source.endswith(".") else source + "."
334
  return NO_COST_NOTE
335
+ _IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label")
336
  _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
337
 
338
  #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
 
351
  else (metric["parent_asin"], PICTURE_FIELD))
352
  where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
353
  progress = _progress_fields(f"Average progress of the projects on {where}.")
354
+ return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
 
 
 
355
  + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
356
+ + progress[1:])
357
 
358
 
359
  ASIN_FIELDS = _grain_fields("asin", "asin")
 
399
 
400
  def table_specs():
401
  """Every module table in creation order."""
402
+ return (
403
  {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
404
  "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
405
  {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
 
407
  {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
408
  "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
409
  )
 
 
 
 
410
 
411
 
412
  # ---------------------------------------------------------------- period and threshold
 
1707
  else:
1708
  import duckdb
1709
  cur = duckdb.connect(str(path), read_only=True)
1710
+ if _mirror_has_economics(cur):
1711
  return cur, cur.close
1712
  cur.close()
1713
  for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
 
1720
  """Refresh the current period, apply decisions, then suggest for every unlinked SKU.
1721
  Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
1722
  synced = {str(t or "").strip().lower() for t in synced_tables or ()}
1723
+ if not synced & {"economics", "listings"}:
1724
  return {"state": "skipped", "synced": sorted(synced)}
1725
  if st is None:
1726
  from harness import amazon_store
 
1732
  con, close = _sync_cursor(tenant_key, st)
1733
  try:
1734
  report = {"state": "done"}
 
 
 
 
 
 
 
 
 
 
 
1735
  try:
1736
  refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
1737
  report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
 
1807
 
1808
 
1809
  def _project_field_order(keys):
1810
+ """Place Projects beside a product identity in schema defaults, never saved user layouts."""
1811
  keys = list(keys)
1812
+ if keys and keys[0] in ("sku", "asin", "parent_asin") and "projects" in keys:
1813
+ keys.remove("projects")
1814
+ keys.insert(1, "projects")
1815
+ return keys
 
 
 
 
1816
 
1817
 
1818
  def _format_repairs(fields, wanted):
 
1832
  continue
1833
  if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
1834
  out.setdefault("grain", {})["options"] = list(target["options"])
1835
+ if field["key"] == "projects" and (field.get("type") != target["type"] or "link" in field
1836
  or (field.get("format") or {}) != (target.get("format") or {})):
1837
+ out["projects"] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None}
1838
  return out
1839
 
1840
 
 
2025
  for rid, row in table_rows(st, table_key).items()
2026
  if str(row.get(key_column) or "").strip() not in computed}
2027
  wanted = {rid: values for rid, values in wanted.items() if values}
 
 
 
2028
  if wanted:
2029
  import core.user_tables as ut
2030
 
 
2568
  flat["cost_note"] = _grain_cost_note(flat, children)
2569
  flat["period"] = period_label
2570
  out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
2571
+ if k not in ("projects", "projects_label")}
2572
  return out
2573
 
2574
 
 
2800
 
2801
  def _grain_period_keys(fields, key_column):
2802
  return tuple(f["key"] for f in fields
2803
+ if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label")
2804
+ + COMPETITOR_KEYS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2805
 
2806
 
2807
  def _date(value):
 
3142
  report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
3143
  key_column="parent_asin", label=PARENT_LABEL,
3144
  period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
 
3145
  report["blankReasons"] = blank_check(stored_tables(st))
3146
  bands = {sku: values.get("band") for sku, values in computed.items()}
3147
  report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
 
3287
 
3288
 
3289
  def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
 
3290
  "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
3291
  "amzn_ads_sd_advertised_products")):
3292
  """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
 
3448
 
3449
 
3450
  def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
3451
+ typed_values=None, typed_period=None, st=None):
3452
  """Tie net proceeds components to independent Amazon sources within 2% over settled days.
3453
  Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
3454
+ With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why."""
3455
  month = month or last_complete_month()
3456
  bounds = period_bounds(month)
3457
  start, end = bounds["start"], bounds["end"]
 
3511
  checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
3512
  if st is not None:
3513
  checks.extend(stored_grain_checks(st, con))
 
 
3514
  return checks
3515
 
3516
 
platform/modules/amazon_profit_account.py CHANGED
@@ -555,124 +555,6 @@ def validate_pace(con, bounds, pace, elapsed, month_days, reason):
555
  "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
556
 
557
 
558
- class _SourceRows:
559
- def __init__(self, rows):
560
- self.rows = rows
561
-
562
- def fetchone(self):
563
- return self.rows[0] if self.rows else None
564
-
565
- def fetchall(self):
566
- return list(self.rows)
567
-
568
-
569
- def _settlement_source(sql):
570
- """Parse distinct settlement labels and referenced order dates without losing source lines or null keys."""
571
- expressions = {"posted": ("posted_date", f"CAST({profit._moment_sql('s.posted_date')} AS DATE)")}
572
- for name, field in (("description", "amount_description"), ("kind", "amount_type"),
573
- ("transaction", "transaction_type")):
574
- expressions[name] = field, f"lower(regexp_replace(COALESCE({field}, ''), '[^a-zA-Z0-9]', '', 'g'))"
575
- fields = [field for field, _expression in expressions.values()]
576
- projection = ", ".join(expression.replace("s.posted_date", "posted_date") + " AS " + name
577
- for name, (_field, expression) in expressions.items())
578
- keys = ", ".join(f"{field} AS source_{field}" for field in fields)
579
- labels = (f"labels AS MATERIALIZED (SELECT {keys}, {projection} "
580
- f"FROM (SELECT DISTINCT {', '.join(fields)} FROM {profit.SETTLEMENTS})), ")
581
- for name, (_field, expression) in expressions.items():
582
- sql = sql.replace(expression + " AS " + name, "label." + name + " AS " + name)
583
- join = " AND ".join(f"s.{field} IS NOT DISTINCT FROM label.source_{field}" for field in fields)
584
- sql = sql.replace(f"FROM {profit.SETTLEMENTS} s JOIN streams",
585
- f"FROM {profit.SETTLEMENTS} s JOIN labels label ON {join} JOIN streams")
586
- sql = sql.replace("FROM amzn_orders GROUP BY amazon_order_id",
587
- f"FROM amzn_orders WHERE amazon_order_id IN (SELECT trim(order_id) "
588
- f"FROM {profit.SETTLEMENTS}) GROUP BY amazon_order_id")
589
- return sql.replace("WITH orders AS (", "WITH " + labels + "orders AS (", 1)
590
-
591
-
592
- class _ValidationSources:
593
- """Share settlement source preparation across this request's independent window sums."""
594
-
595
- def __init__(self, con, bounds):
596
- self.con, self.results, self.spans = con, {}, {}
597
- self.bounds = [span for bound in bounds for span in (bound, *_comparisons(bound).values())]
598
-
599
- def _spans(self, currency=None):
600
- if currency in self.spans:
601
- return self.spans[currency]
602
- spans = {(bound["start"], bound["end"]) for bound in self.bounds}
603
- if currency is not None:
604
- for start, end in tuple(spans):
605
- covered = profit.settlement_coverage(self, start, end, currency)
606
- spans.add((covered["start"] or start, covered["end"] or end))
607
- self.spans[currency] = sorted(spans)
608
- return self.spans[currency]
609
-
610
- def _components(self, sql, params):
611
- marker = "SELECT component, COALESCE(SUM(currency_amount), 0)"
612
- prefix, select = sql.split(marker, 1)
613
- aggregates, select = (marker + select).split("FROM dated WHERE", 1)
614
- aggregates = aggregates.removeprefix("SELECT component, ")
615
- # The grouping keys retain every input to the original window predicate and its bad-row counts.
616
- prefix += (" , prepared(component, basis, posted, dates, bad_dates, amount, source_rows, "
617
- "bad_amounts, unmatched_orders) AS (SELECT component, basis, posted, dates, bad_dates, "
618
- + aggregates + " FROM dated GROUP BY component, basis, posted, dates, bad_dates) ")
619
- spans = self._spans(params[0])
620
- for bound in ("lo", "hi", "lo", "hi"):
621
- select = select.replace("CAST(? AS DATE)", "span." + bound, 1)
622
- ranges = ", ".join(f"({index}, CAST(? AS DATE), CAST(? AS DATE))" for index in range(len(spans)))
623
- select = f"FROM prepared JOIN (VALUES {ranges}) AS span(id, lo, hi) ON" + select
624
- select = select.replace("GROUP BY component ORDER BY component",
625
- "GROUP BY span.id, component ORDER BY span.id, component")
626
- bindings = list(params[:2]) + [value for span in spans for value in span]
627
- rows = self.con.execute(_settlement_source(prefix) + "SELECT span.id, component, "
628
- "COALESCE(SUM(amount), 0), SUM(source_rows), SUM(bad_amounts), SUM(unmatched_orders) "
629
- + select, bindings).fetchall()
630
- for index, span in enumerate(spans):
631
- self.results[(sql, tuple(params[:2]) + span * 2)] = [row[1:] for row in rows if row[0] == index]
632
-
633
- def _unscoped(self, sql):
634
- marker = "AND (CAST("
635
- source, _date_filter = sql.split(marker, 1)
636
- source = source.replace("SELECT count(*)", f"SELECT CAST({profit._moment_sql('posted_date')} AS DATE) AS posted", 1)
637
- spans = self._spans(profit.economics_currency(self.con))
638
- sums = ["count(*) FILTER (WHERE posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)"
639
- for _span in spans]
640
- counts = self.con.execute("SELECT " + ", ".join(sums) + " FROM (" + source + ")",
641
- [value for span in spans for value in span]).fetchone()
642
- for index, count in enumerate(counts):
643
- self.results[(sql, spans[index])] = [(count,)]
644
-
645
- def execute(self, sql, params=()):
646
- params = tuple(params or ())
647
- key = sql, params
648
- if key in self.results:
649
- return _SourceRows(self.results[key])
650
- if sql == "SELECT column_name FROM information_schema.columns WHERE table_name = ?":
651
- self.results[key] = self.con.execute(sql, params).fetchall()
652
- return _SourceRows(self.results[key])
653
- components = (sql.startswith("WITH orders AS (") and len(params) == 6
654
- and sql.endswith(" GROUP BY component ORDER BY component"))
655
- unscoped = (sql.startswith(f"SELECT count(*) FROM {profit.SETTLEMENTS} s\n")
656
- and "WHERE h.settlement_id IS NULL" in sql and len(params) == 2)
657
- if components:
658
- self._components(sql, params)
659
- elif unscoped:
660
- self._unscoped(sql)
661
- if key in self.results:
662
- return _SourceRows(self.results[key])
663
- return self.con.execute(sql, params)
664
-
665
-
666
- def _account_bounds(chosen, today):
667
- bounds = [chosen, _bounds(today.replace(day=1), today - dt.timedelta(days=1), "mtd")]
668
- current = today.replace(day=1)
669
- for _ in range(2):
670
- last = current - dt.timedelta(days=1)
671
- current = last.replace(day=1)
672
- bounds.append(_bounds(current, last, current.strftime("%Y-%m")))
673
- return bounds
674
-
675
-
676
  def account_view(con, period=None, *, stored=None, settings=None, today=None):
677
  """Return selected, comparison and monthly account windows without refreshing stored inputs."""
678
  today = today or profit._today()
@@ -692,8 +574,6 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
692
  "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
693
  raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
694
  stored, settings = stored or {}, settings or {}
695
- periods = _account_bounds(chosen, today)
696
- con = _ValidationSources(con, periods)
697
  cache = {}
698
  def window(bounds):
699
  key = bounds["start"], bounds["end"]
@@ -702,8 +582,13 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
702
  return {**cache[key], "period": bounds}
703
  def compared(bounds):
704
  return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
705
- mtd = compared(periods[1])
706
- months = [compared(bounds) for bounds in periods[2:]]
 
 
 
 
 
707
  pace = {}
708
  elapsed = mtd["period"]["days"]
709
  month_days = calendar.monthrange(today.year, today.month)[1]
 
555
  "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
556
 
557
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
558
  def account_view(con, period=None, *, stored=None, settings=None, today=None):
559
  """Return selected, comparison and monthly account windows without refreshing stored inputs."""
560
  today = today or profit._today()
 
574
  "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
575
  raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
576
  stored, settings = stored or {}, settings or {}
 
 
577
  cache = {}
578
  def window(bounds):
579
  key = bounds["start"], bounds["end"]
 
582
  return {**cache[key], "period": bounds}
583
  def compared(bounds):
584
  return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
585
+ current = today.replace(day=1)
586
+ mtd = compared(_bounds(current, today - dt.timedelta(days=1), "mtd"))
587
+ months = []
588
+ for _ in range(2):
589
+ last = current - dt.timedelta(days=1)
590
+ current = last.replace(day=1)
591
+ months.append(compared(_bounds(current, last, current.strftime("%Y-%m"))))
592
  pace = {}
593
  elapsed = mtd["period"]["days"]
594
  month_days = calendar.monthrange(today.year, today.month)[1]
platform/modules/competitor_watch.py CHANGED
@@ -300,7 +300,7 @@ class _TermIndex(_DocumentIndex):
300
  self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
301
  total_params = ([self.slug, *([display_term(search)] if search else [])]
302
  if search or tracked else [self.key(KEY_TERM_INDEX)])
303
- tail = ("), "
304
  "bounds AS (SELECT total, CASE WHEN %s >= total THEN "
305
  "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
306
  "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
@@ -308,12 +308,6 @@ class _TermIndex(_DocumentIndex):
308
  + ("AND strpos(lower(label), %s) > 0 " if search else "") +
309
  'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
310
  "FROM bounds")
311
- if isinstance(total, str):
312
- statement = self.term_sql("WITH totals(total) AS (" + total + tail)
313
- else:
314
- # PostgreSQL's `self.sql` returns a psycopg Composed: compose it, never concatenate it as text.
315
- from psycopg import sql as psql
316
- statement = psql.SQL("WITH totals(total) AS (") + total + psql.SQL(self.term_sql(tail))
317
  params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
318
  *([display_term(search)] if search else []), limit]
319
  with self.connection() as con:
@@ -469,25 +463,15 @@ def _ba_columns(cur):
469
 
470
 
471
  def _sqp(cur, term=None):
472
- """Monthly SQP rows as (term, asin, impressions, end_date) for our ASINs."""
473
  need = {"search_query_data_search_query", "asin"}
474
  cols = _columns(cur, SQP_TABLE) if cur is not None else set()
475
  if not need <= cols:
476
  return []
477
- if "_report_period" in cols:
478
- period = '"_report_period" = \'MONTH\''
479
- elif {"start_date", "end_date"} <= cols:
480
- start = 'TRY_CAST("start_date" AS DATE)'
481
- end_date = 'TRY_CAST("end_date" AS DATE)'
482
- period = f'day({start}) = 1 AND {end_date} = last_day({start})'
483
- else:
484
- return []
485
  imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
486
  if "impression_data_asin_impression_count" in cols else "0")
487
  end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
488
- where = f" WHERE {period}"
489
- if term is not None:
490
- where += f' AND {_term_sql("search_query_data_search_query")} = ?'
491
  rows = cur.execute(
492
  f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
493
  [term] if term is not None else []).fetchall()
 
300
  self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
301
  total_params = ([self.slug, *([display_term(search)] if search else [])]
302
  if search or tracked else [self.key(KEY_TERM_INDEX)])
303
+ statement = self.term_sql("WITH totals(total) AS (" + total + "), "
304
  "bounds AS (SELECT total, CASE WHEN %s >= total THEN "
305
  "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
306
  "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
 
308
  + ("AND strpos(lower(label), %s) > 0 " if search else "") +
309
  'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
310
  "FROM bounds")
 
 
 
 
 
 
311
  params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
312
  *([display_term(search)] if search else []), limit]
313
  with self.connection() as con:
 
463
 
464
 
465
  def _sqp(cur, term=None):
466
+ """SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only."""
467
  need = {"search_query_data_search_query", "asin"}
468
  cols = _columns(cur, SQP_TABLE) if cur is not None else set()
469
  if not need <= cols:
470
  return []
 
 
 
 
 
 
 
 
471
  imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
472
  if "impression_data_asin_impression_count" in cols else "0")
473
  end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
474
+ where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else ""
 
 
475
  rows = cur.execute(
476
  f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
477
  [term] if term is not None else []).fetchall()
requirements.txt CHANGED
@@ -25,7 +25,6 @@ duckdb==1.5.4
25
  # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
26
  pytz>=2024.1
27
  pyyaml>=6.0
28
- jsonschema==4.26.0
29
  pillow>=10.0
30
  beautifulsoup4>=4.12
31
  lxml>=5.0
 
25
  # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
26
  pytz>=2024.1
27
  pyyaml>=6.0
 
28
  pillow>=10.0
29
  beautifulsoup4>=4.12
30
  lxml>=5.0
web/package-lock.json CHANGED
@@ -25,7 +25,6 @@
25
  "@types/react": "^18.3.12",
26
  "@types/react-dom": "^18.3.1",
27
  "@vitejs/plugin-react": "^6.0.3",
28
- "esbuild": "^0.27.0",
29
  "openapi-typescript": "^7.13.0",
30
  "oxlint": "^1.71.0",
31
  "typescript": "~6.0.2",
@@ -390,448 +389,6 @@
390
  "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
391
  "license": "MIT"
392
  },
393
- "node_modules/@esbuild/aix-ppc64": {
394
- "version": "0.27.7",
395
- "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz",
396
- "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==",
397
- "cpu": [
398
- "ppc64"
399
- ],
400
- "dev": true,
401
- "license": "MIT",
402
- "optional": true,
403
- "os": [
404
- "aix"
405
- ],
406
- "engines": {
407
- "node": ">=18"
408
- }
409
- },
410
- "node_modules/@esbuild/android-arm": {
411
- "version": "0.27.7",
412
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz",
413
- "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==",
414
- "cpu": [
415
- "arm"
416
- ],
417
- "dev": true,
418
- "license": "MIT",
419
- "optional": true,
420
- "os": [
421
- "android"
422
- ],
423
- "engines": {
424
- "node": ">=18"
425
- }
426
- },
427
- "node_modules/@esbuild/android-arm64": {
428
- "version": "0.27.7",
429
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz",
430
- "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==",
431
- "cpu": [
432
- "arm64"
433
- ],
434
- "dev": true,
435
- "license": "MIT",
436
- "optional": true,
437
- "os": [
438
- "android"
439
- ],
440
- "engines": {
441
- "node": ">=18"
442
- }
443
- },
444
- "node_modules/@esbuild/android-x64": {
445
- "version": "0.27.7",
446
- "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz",
447
- "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==",
448
- "cpu": [
449
- "x64"
450
- ],
451
- "dev": true,
452
- "license": "MIT",
453
- "optional": true,
454
- "os": [
455
- "android"
456
- ],
457
- "engines": {
458
- "node": ">=18"
459
- }
460
- },
461
- "node_modules/@esbuild/darwin-arm64": {
462
- "version": "0.27.7",
463
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz",
464
- "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==",
465
- "cpu": [
466
- "arm64"
467
- ],
468
- "dev": true,
469
- "license": "MIT",
470
- "optional": true,
471
- "os": [
472
- "darwin"
473
- ],
474
- "engines": {
475
- "node": ">=18"
476
- }
477
- },
478
- "node_modules/@esbuild/darwin-x64": {
479
- "version": "0.27.7",
480
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz",
481
- "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==",
482
- "cpu": [
483
- "x64"
484
- ],
485
- "dev": true,
486
- "license": "MIT",
487
- "optional": true,
488
- "os": [
489
- "darwin"
490
- ],
491
- "engines": {
492
- "node": ">=18"
493
- }
494
- },
495
- "node_modules/@esbuild/freebsd-arm64": {
496
- "version": "0.27.7",
497
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz",
498
- "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==",
499
- "cpu": [
500
- "arm64"
501
- ],
502
- "dev": true,
503
- "license": "MIT",
504
- "optional": true,
505
- "os": [
506
- "freebsd"
507
- ],
508
- "engines": {
509
- "node": ">=18"
510
- }
511
- },
512
- "node_modules/@esbuild/freebsd-x64": {
513
- "version": "0.27.7",
514
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz",
515
- "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==",
516
- "cpu": [
517
- "x64"
518
- ],
519
- "dev": true,
520
- "license": "MIT",
521
- "optional": true,
522
- "os": [
523
- "freebsd"
524
- ],
525
- "engines": {
526
- "node": ">=18"
527
- }
528
- },
529
- "node_modules/@esbuild/linux-arm": {
530
- "version": "0.27.7",
531
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz",
532
- "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==",
533
- "cpu": [
534
- "arm"
535
- ],
536
- "dev": true,
537
- "license": "MIT",
538
- "optional": true,
539
- "os": [
540
- "linux"
541
- ],
542
- "engines": {
543
- "node": ">=18"
544
- }
545
- },
546
- "node_modules/@esbuild/linux-arm64": {
547
- "version": "0.27.7",
548
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz",
549
- "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==",
550
- "cpu": [
551
- "arm64"
552
- ],
553
- "dev": true,
554
- "license": "MIT",
555
- "optional": true,
556
- "os": [
557
- "linux"
558
- ],
559
- "engines": {
560
- "node": ">=18"
561
- }
562
- },
563
- "node_modules/@esbuild/linux-ia32": {
564
- "version": "0.27.7",
565
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz",
566
- "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==",
567
- "cpu": [
568
- "ia32"
569
- ],
570
- "dev": true,
571
- "license": "MIT",
572
- "optional": true,
573
- "os": [
574
- "linux"
575
- ],
576
- "engines": {
577
- "node": ">=18"
578
- }
579
- },
580
- "node_modules/@esbuild/linux-loong64": {
581
- "version": "0.27.7",
582
- "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz",
583
- "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==",
584
- "cpu": [
585
- "loong64"
586
- ],
587
- "dev": true,
588
- "license": "MIT",
589
- "optional": true,
590
- "os": [
591
- "linux"
592
- ],
593
- "engines": {
594
- "node": ">=18"
595
- }
596
- },
597
- "node_modules/@esbuild/linux-mips64el": {
598
- "version": "0.27.7",
599
- "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz",
600
- "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==",
601
- "cpu": [
602
- "mips64el"
603
- ],
604
- "dev": true,
605
- "license": "MIT",
606
- "optional": true,
607
- "os": [
608
- "linux"
609
- ],
610
- "engines": {
611
- "node": ">=18"
612
- }
613
- },
614
- "node_modules/@esbuild/linux-ppc64": {
615
- "version": "0.27.7",
616
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz",
617
- "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==",
618
- "cpu": [
619
- "ppc64"
620
- ],
621
- "dev": true,
622
- "license": "MIT",
623
- "optional": true,
624
- "os": [
625
- "linux"
626
- ],
627
- "engines": {
628
- "node": ">=18"
629
- }
630
- },
631
- "node_modules/@esbuild/linux-riscv64": {
632
- "version": "0.27.7",
633
- "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz",
634
- "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==",
635
- "cpu": [
636
- "riscv64"
637
- ],
638
- "dev": true,
639
- "license": "MIT",
640
- "optional": true,
641
- "os": [
642
- "linux"
643
- ],
644
- "engines": {
645
- "node": ">=18"
646
- }
647
- },
648
- "node_modules/@esbuild/linux-s390x": {
649
- "version": "0.27.7",
650
- "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz",
651
- "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==",
652
- "cpu": [
653
- "s390x"
654
- ],
655
- "dev": true,
656
- "license": "MIT",
657
- "optional": true,
658
- "os": [
659
- "linux"
660
- ],
661
- "engines": {
662
- "node": ">=18"
663
- }
664
- },
665
- "node_modules/@esbuild/linux-x64": {
666
- "version": "0.27.7",
667
- "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz",
668
- "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==",
669
- "cpu": [
670
- "x64"
671
- ],
672
- "dev": true,
673
- "license": "MIT",
674
- "optional": true,
675
- "os": [
676
- "linux"
677
- ],
678
- "engines": {
679
- "node": ">=18"
680
- }
681
- },
682
- "node_modules/@esbuild/netbsd-arm64": {
683
- "version": "0.27.7",
684
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz",
685
- "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==",
686
- "cpu": [
687
- "arm64"
688
- ],
689
- "dev": true,
690
- "license": "MIT",
691
- "optional": true,
692
- "os": [
693
- "netbsd"
694
- ],
695
- "engines": {
696
- "node": ">=18"
697
- }
698
- },
699
- "node_modules/@esbuild/netbsd-x64": {
700
- "version": "0.27.7",
701
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz",
702
- "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==",
703
- "cpu": [
704
- "x64"
705
- ],
706
- "dev": true,
707
- "license": "MIT",
708
- "optional": true,
709
- "os": [
710
- "netbsd"
711
- ],
712
- "engines": {
713
- "node": ">=18"
714
- }
715
- },
716
- "node_modules/@esbuild/openbsd-arm64": {
717
- "version": "0.27.7",
718
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz",
719
- "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==",
720
- "cpu": [
721
- "arm64"
722
- ],
723
- "dev": true,
724
- "license": "MIT",
725
- "optional": true,
726
- "os": [
727
- "openbsd"
728
- ],
729
- "engines": {
730
- "node": ">=18"
731
- }
732
- },
733
- "node_modules/@esbuild/openbsd-x64": {
734
- "version": "0.27.7",
735
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz",
736
- "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==",
737
- "cpu": [
738
- "x64"
739
- ],
740
- "dev": true,
741
- "license": "MIT",
742
- "optional": true,
743
- "os": [
744
- "openbsd"
745
- ],
746
- "engines": {
747
- "node": ">=18"
748
- }
749
- },
750
- "node_modules/@esbuild/openharmony-arm64": {
751
- "version": "0.27.7",
752
- "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz",
753
- "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==",
754
- "cpu": [
755
- "arm64"
756
- ],
757
- "dev": true,
758
- "license": "MIT",
759
- "optional": true,
760
- "os": [
761
- "openharmony"
762
- ],
763
- "engines": {
764
- "node": ">=18"
765
- }
766
- },
767
- "node_modules/@esbuild/sunos-x64": {
768
- "version": "0.27.7",
769
- "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz",
770
- "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==",
771
- "cpu": [
772
- "x64"
773
- ],
774
- "dev": true,
775
- "license": "MIT",
776
- "optional": true,
777
- "os": [
778
- "sunos"
779
- ],
780
- "engines": {
781
- "node": ">=18"
782
- }
783
- },
784
- "node_modules/@esbuild/win32-arm64": {
785
- "version": "0.27.7",
786
- "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz",
787
- "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==",
788
- "cpu": [
789
- "arm64"
790
- ],
791
- "dev": true,
792
- "license": "MIT",
793
- "optional": true,
794
- "os": [
795
- "win32"
796
- ],
797
- "engines": {
798
- "node": ">=18"
799
- }
800
- },
801
- "node_modules/@esbuild/win32-ia32": {
802
- "version": "0.27.7",
803
- "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz",
804
- "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==",
805
- "cpu": [
806
- "ia32"
807
- ],
808
- "dev": true,
809
- "license": "MIT",
810
- "optional": true,
811
- "os": [
812
- "win32"
813
- ],
814
- "engines": {
815
- "node": ">=18"
816
- }
817
- },
818
- "node_modules/@esbuild/win32-x64": {
819
- "version": "0.27.7",
820
- "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz",
821
- "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==",
822
- "cpu": [
823
- "x64"
824
- ],
825
- "dev": true,
826
- "license": "MIT",
827
- "optional": true,
828
- "os": [
829
- "win32"
830
- ],
831
- "engines": {
832
- "node": ">=18"
833
- }
834
- },
835
  "node_modules/@glideapps/glide-data-grid": {
836
  "version": "6.0.3",
837
  "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
@@ -2292,48 +1849,6 @@
2292
  "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
2293
  "license": "MIT"
2294
  },
2295
- "node_modules/esbuild": {
2296
- "version": "0.27.7",
2297
- "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz",
2298
- "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==",
2299
- "dev": true,
2300
- "hasInstallScript": true,
2301
- "license": "MIT",
2302
- "bin": {
2303
- "esbuild": "bin/esbuild"
2304
- },
2305
- "engines": {
2306
- "node": ">=18"
2307
- },
2308
- "optionalDependencies": {
2309
- "@esbuild/aix-ppc64": "0.27.7",
2310
- "@esbuild/android-arm": "0.27.7",
2311
- "@esbuild/android-arm64": "0.27.7",
2312
- "@esbuild/android-x64": "0.27.7",
2313
- "@esbuild/darwin-arm64": "0.27.7",
2314
- "@esbuild/darwin-x64": "0.27.7",
2315
- "@esbuild/freebsd-arm64": "0.27.7",
2316
- "@esbuild/freebsd-x64": "0.27.7",
2317
- "@esbuild/linux-arm": "0.27.7",
2318
- "@esbuild/linux-arm64": "0.27.7",
2319
- "@esbuild/linux-ia32": "0.27.7",
2320
- "@esbuild/linux-loong64": "0.27.7",
2321
- "@esbuild/linux-mips64el": "0.27.7",
2322
- "@esbuild/linux-ppc64": "0.27.7",
2323
- "@esbuild/linux-riscv64": "0.27.7",
2324
- "@esbuild/linux-s390x": "0.27.7",
2325
- "@esbuild/linux-x64": "0.27.7",
2326
- "@esbuild/netbsd-arm64": "0.27.7",
2327
- "@esbuild/netbsd-x64": "0.27.7",
2328
- "@esbuild/openbsd-arm64": "0.27.7",
2329
- "@esbuild/openbsd-x64": "0.27.7",
2330
- "@esbuild/openharmony-arm64": "0.27.7",
2331
- "@esbuild/sunos-x64": "0.27.7",
2332
- "@esbuild/win32-arm64": "0.27.7",
2333
- "@esbuild/win32-ia32": "0.27.7",
2334
- "@esbuild/win32-x64": "0.27.7"
2335
- }
2336
- },
2337
  "node_modules/escalade": {
2338
  "version": "3.2.0",
2339
  "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
 
25
  "@types/react": "^18.3.12",
26
  "@types/react-dom": "^18.3.1",
27
  "@vitejs/plugin-react": "^6.0.3",
 
28
  "openapi-typescript": "^7.13.0",
29
  "oxlint": "^1.71.0",
30
  "typescript": "~6.0.2",
 
389
  "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
390
  "license": "MIT"
391
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
  "node_modules/@glideapps/glide-data-grid": {
393
  "version": "6.0.3",
394
  "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
 
1849
  "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
1850
  "license": "MIT"
1851
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1852
  "node_modules/escalade": {
1853
  "version": "3.2.0",
1854
  "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
web/package-lock.linux.sha256 CHANGED
@@ -1,2 +1,2 @@
1
- 9cd826154f5fecd9784f6632ad62e9da94db63b1ed343ab137fbd29ce5690c07 package-lock.json
2
- # resolved on linux/x64 by HF job 6afd11b54643bac7ac631093 on 2026-11-18, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170).
 
1
+ 31cb494538043db42bdf698846c2262f5eb234fb686aa03d9cf3ab1cbfdbe2b8 package-lock.json
2
+ # resolved on linux/x64 in the SDLC 72 lane A HF container on 2026-10-09 (S72-T05, npm install --save-exact @phosphor-icons/react@2.1.10); npm ci --ignore-scripts passed there (228 packages). Regenerate ONLY with lockfile_linux.py (W-170).
web/package.json CHANGED
@@ -39,7 +39,6 @@
39
  "@types/react": "^18.3.12",
40
  "@types/react-dom": "^18.3.1",
41
  "@vitejs/plugin-react": "^6.0.3",
42
- "esbuild": "^0.27.0",
43
  "openapi-typescript": "^7.13.0",
44
  "oxlint": "^1.71.0",
45
  "typescript": "~6.0.2",
 
39
  "@types/react": "^18.3.12",
40
  "@types/react-dom": "^18.3.1",
41
  "@vitejs/plugin-react": "^6.0.3",
 
42
  "openapi-typescript": "^7.13.0",
43
  "oxlint": "^1.71.0",
44
  "typescript": "~6.0.2",
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 4f29d08d01160aa6ccd46623bfc075cb2ca87d826499ba6a6cc4196a66b79c1d
 
1
+ 55137f24b4cb4023d1609a3ec3179bd65aba9f362ca4a4a7403d2839fd270873
web/src/apps/amazon-profit/AppView.tsx CHANGED
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
2
  import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
3
  import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
4
  OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
5
- import type { AppCellDraw, AppTableHost, AppViewProps, Row } from "../../customer-grid/seam";
6
  import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
7
  CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
8
  import { LogoLoader } from "../../ui/LogoLoader";
@@ -11,36 +11,12 @@ import PicturePopup from "./PicturePopup";
11
  import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
12
  import type { ProjectsTarget } from "./Projects";
13
  import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
14
- import GridFilter, { SIGNAL_SORTS } from "./pdp/GridFilter";
15
- import HistoryPopup from "./pdp/HistoryPopup";
16
- import PdpUpdatePopup, { type PdpListing, type PdpNavigation } from "./pdp/PdpUpdatePopup";
17
- import WorkflowWindow from "./pdp/WorkflowWindow";
18
- import type { PdpProjectNavigation } from "./pdp/ContinueWorkflow";
19
- import type { PdpProjectPane } from "./pdp/contract";
20
  import "../../styles/sdlc81/command-center-app.css";
21
- import "../../styles/sdlc86/project-manager-pdp.css";
22
 
23
  const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
24
  const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
25
  /** The App's 56 px rows (R5). Table keeps its own default. */
26
  const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
27
- const ASIN_COLUMNS = { rowHeight: 56, headerTone: { projects: "primary", pdp_update: "primary", history: "primary" },
28
- widths: { asin: 150, projects: 210, pdp_update: 170, trend_score: 150, search_trend: 210,
29
- market_share_trend: 240, click_share_trend: 240, exposure: 190, funnel_score: 150, history: 150 } } as const;
30
- const noReport = (reasonKey: string) => ({ reasonKey, equals: "No search report", text: "No search report" });
31
- const signalTrend = (key: string, share: boolean): AppCellDraw => ({ kind: "trend", seriesKey: key + "_series",
32
- valueKey: key + "_now", changeKey: key, format: share ? "percent" : "number", showValue: share,
33
- omitMissingValue: true, valuePrecision: "adaptive", changeDecimals: 0, changeNeutralBand: .5, fitLabels: true, barTone: "primary",
34
- unavailable: noReport(key + "_reason") });
35
- const SIGNAL_CELLS: AppTableHost["cellRenderers"] = {
36
- trend_score: { draw: { kind: "ring", labelPosition: "after", unavailable: noReport("trend_score_reason") } },
37
- search_trend: { draw: signalTrend("search_trend", false) },
38
- market_share_trend: { draw: signalTrend("market_share_trend", true) },
39
- click_share_trend: { draw: signalTrend("click_share_trend", true) },
40
- exposure: { draw: { kind: "badge", subKey: "exposure_label", emptyText: "None", unavailable: noReport("exposure_reason") } },
41
- funnel_score: { draw: { kind: "score", neutralRange: [4.5, 5.5], outerRange: [3, 7], emptyText: "Few",
42
- unavailable: noReport("funnel_score_reason") } },
43
- };
44
 
45
  /** Admission belongs to the selected tab's fresh probe. */
46
  type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
@@ -48,17 +24,12 @@ type TableProbe = { table: string; status: "ready"; tables: Record<string, boole
48
 
49
  // The Command Center App body: the Table module's own Grid, hosted under the App frame.
50
  // The tab strip switches `table` through the context, so the URL stays on the App.
51
- export default function ProjectManagementView(props: AppViewProps & PdpNavigation) {
52
  const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
53
  const [probe, setProbe] = useState<TableProbe>(null);
54
  const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
55
- const [projects, setProjects] = useState<(ProjectsTarget & {
56
- selectedProjectId?: string; originListing?: PdpListing; initialPane?: PdpProjectPane;
57
- }) | null>(null);
58
  const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
59
- const [history, setHistory] = useState<Row | null>(null);
60
- const [pdp, setPdp] = useState<PdpListing | null>(null);
61
- const [workflow, setWorkflow] = useState<{ id: string; listing: PdpListing; resumeRevision?: number } | null>(null);
62
  const [gridRevision, setGridRevision] = useState(0);
63
  const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
64
  const openTable = useCallback((next: string) => {
@@ -69,9 +40,6 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
69
  setPicture(null);
70
  setProjects(null);
71
  setCompetitors(null);
72
- setHistory(null);
73
- setPdp(null);
74
- setWorkflow(null);
75
  setTable(next);
76
  }, [table]);
77
  const reloadGrid = useCallback(() => {
@@ -120,37 +88,12 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
120
  }, [dataEntry, admitted, table]);
121
  const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
122
  const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
123
- const openProject = (id: string, asin: string, listing: PdpListing, initialPane?: PdpProjectPane) => {
124
- if (props.onOpenProject) { props.onOpenProject(id, asin); return; }
125
- setPdp(null);
126
- setProjects({ grain: "asin", rowKey: asin, title: listing.title, selectedProjectId: id, originListing: listing, initialPane });
127
- };
128
- const resumeWorkflow = () => {
129
- setProjects(null);
130
- setWorkflow(current => current ? { ...current, resumeRevision: (current.resumeRevision ?? 0) + 1 } : current);
131
- };
132
- const projectNavigation: PdpProjectNavigation = { onOpenWorkflow: (next, item, target) => {
133
- if (workflow?.id === next.id && workflow.listing.asin === item.asin) {
134
- resumeWorkflow();
135
- return;
136
- }
137
- setProjects(null);
138
- if (props.onOpenWorkflow) { props.onOpenWorkflow(next.id, item.asin); return; }
139
- setWorkflow({ id: next.id, listing: { asin: item.asin, title: target?.title ?? "", sku: target?.sku ?? null, brand: null } });
140
- } };
141
  // C2: progress colours, button doors and the Picture popup, declared once per table.
142
  // Only the Projects table keeps New record (`append`).
143
  const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
144
- ...(table === "ut_amazon_asin_profit" ? SIGNAL_CELLS : {}),
145
  "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
146
  projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
147
  competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
148
- history: { button: true, activate: row => setHistory(row) },
149
- pdp_update: { button: true, activate: row => {
150
- if (!row.asin) return false;
151
- setPdp({ asin: String(row.asin), title: String(row.title || ""),
152
- sku: row.sku ? String(row.sku) : null, brand: row.brand ? String(row.brand) : null });
153
- } },
154
  "type:image": { activate: (record, field) => {
155
  const ref = String(record[field.key] ?? "").trim();
156
  if (!ref) return false;
@@ -172,35 +115,17 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
172
  </div>
173
  : table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
174
  <CommandCenterTabStrip database={table} onOpenTable={openTable} />
175
- <ProjectsPage pdp={projectNavigation} />
176
  </div>
177
  : !entryChecked ? <LogoLoader data-app-reason="loading" /> :
178
  <AppTableGrid key={`${table}:${gridRevision}`}
179
  data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
180
- columns={table === "ut_amazon_asin_profit" ? ASIN_COLUMNS : COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers}
181
- history={false} filterPanel={table === "ut_amazon_asin_profit" ? props => <GridFilter {...props} /> : undefined}
182
- sortChoices={table === "ut_amazon_asin_profit" ? SIGNAL_SORTS : undefined}
183
  toolbar={<AmazonToolbar database={table} />} />}
184
  {picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
185
  onClose={() => setPicture(null)} />}
186
- {projects && <ProjectsPopup {...projects} pdp={projectNavigation}
187
- onClose={projects.initialPane ? resumeWorkflow : () => setProjects(null)} onChanged={reloadGrid}
188
- backLabel={projects.initialPane ? "Back to workflow" : projects.originListing ? "Back to projects" : undefined}
189
- onBack={projects.initialPane ? resumeWorkflow : projects.originListing
190
- ? () => { setPdp(projects.originListing ?? null); setProjects(null); setWorkflow(null); } : undefined} />}
191
  {competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
192
- {history && <HistoryPopup table={table} row={history} onClose={() => setHistory(null)} onChanged={reloadGrid} />}
193
- {pdp && <PdpUpdatePopup listing={pdp} onClose={() => setPdp(null)} onChanged={reloadGrid}
194
- onOpenWorkflow={props.onOpenWorkflow ?? ((id, asin) => {
195
- setWorkflow({ id, listing: { ...pdp, asin } }); setPdp(null);
196
- })} onAddWorkflow={props.onAddWorkflow}
197
- onOpenProject={(id, asin) => openProject(id, asin, pdp)} />}
198
- {workflow && <WorkflowWindow key={`${workflow.id}:${workflow.listing.asin}`} workflowId={workflow.id} listing={workflow.listing}
199
- resumeRevision={workflow.resumeRevision}
200
- onClose={() => setWorkflow(null)} onChanged={reloadGrid}
201
- onBack={() => { setPdp(workflow.listing); setWorkflow(null); }}
202
- onOpenProject={() => openProject(workflow.id, workflow.listing.asin, workflow.listing)}
203
- onOpenProjectPane={pane => openProject(workflow.id, workflow.listing.asin, workflow.listing, pane)} />}
204
  </OverlayProvider>
205
  </CommandCenterTableContext.Provider>;
206
  }
 
2
  import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
3
  import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
4
  OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
5
+ import type { AppTableHost, AppViewProps } from "../../customer-grid/seam";
6
  import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
7
  CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
8
  import { LogoLoader } from "../../ui/LogoLoader";
 
11
  import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
12
  import type { ProjectsTarget } from "./Projects";
13
  import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
 
 
 
 
 
 
14
  import "../../styles/sdlc81/command-center-app.css";
 
15
 
16
  const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
17
  const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
18
  /** The App's 56 px rows (R5). Table keeps its own default. */
19
  const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  /** Admission belongs to the selected tab's fresh probe. */
22
  type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
 
24
 
25
  // The Command Center App body: the Table module's own Grid, hosted under the App frame.
26
  // The tab strip switches `table` through the context, so the URL stays on the App.
27
+ export default function ProjectManagementView(_props: AppViewProps) {
28
  const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
29
  const [probe, setProbe] = useState<TableProbe>(null);
30
  const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
31
+ const [projects, setProjects] = useState<ProjectsTarget | null>(null);
 
 
32
  const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
 
 
 
33
  const [gridRevision, setGridRevision] = useState(0);
34
  const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
35
  const openTable = useCallback((next: string) => {
 
40
  setPicture(null);
41
  setProjects(null);
42
  setCompetitors(null);
 
 
 
43
  setTable(next);
44
  }, [table]);
45
  const reloadGrid = useCallback(() => {
 
88
  }, [dataEntry, admitted, table]);
89
  const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
90
  const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  // C2: progress colours, button doors and the Picture popup, declared once per table.
92
  // Only the Projects table keeps New record (`append`).
93
  const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
 
94
  "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
95
  projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
96
  competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
 
 
 
 
 
 
97
  "type:image": { activate: (record, field) => {
98
  const ref = String(record[field.key] ?? "").trim();
99
  if (!ref) return false;
 
115
  </div>
116
  : table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
117
  <CommandCenterTabStrip database={table} onOpenTable={openTable} />
118
+ <ProjectsPage />
119
  </div>
120
  : !entryChecked ? <LogoLoader data-app-reason="loading" /> :
121
  <AppTableGrid key={`${table}:${gridRevision}`}
122
  data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
123
+ columns={COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers}
 
 
124
  toolbar={<AmazonToolbar database={table} />} />}
125
  {picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
126
  onClose={() => setPicture(null)} />}
127
+ {projects && <ProjectsPopup {...projects} onClose={() => setProjects(null)} onChanged={reloadGrid} />}
 
 
 
 
128
  {competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
 
 
 
 
 
 
 
 
 
 
 
 
129
  </OverlayProvider>
130
  </CommandCenterTableContext.Provider>;
131
  }
web/src/apps/amazon-profit/ProjectOverview.tsx CHANGED
@@ -11,8 +11,6 @@ import { X } from "@phosphor-icons/react/dist/csr/X";
11
  import { useEscapeClose } from "../../ui/useEscapeClose";
12
  import ProjectTasks from "./ProjectTasks";
13
  import ProjectSkuPicker from "./ProjectSkuPicker";
14
- import { PdpApprovalRow, PdpOverviewRows, PdpStep, type ProjectPdp } from "./pdp/ContinueWorkflow";
15
- import { removeItems } from "./pdp/api";
16
  import {
17
  EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
18
  type Field, type FilterTree, type Row, type Sorts,
@@ -238,12 +236,11 @@ function ProjectDeleteDialog({ name, busy, error, onCancel, onConfirm }: {
238
  }
239
 
240
  export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
241
- error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope, pdp }: {
242
  id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
243
  error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
244
  setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
245
  rowScope?: RowScope | null; onClearRowScope?: () => void;
246
- pdp?: ProjectPdp;
247
  }) {
248
  const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
249
  const [people, setPeople] = useState<ProjectPerson[]>([]);
@@ -321,12 +318,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
321
  setBusy(true);
322
  const fallback = "Products were not removed.";
323
  try {
324
- if (pdp) {
325
- if (!pdp.detail?.canEdit) throw new Error("The workflow is unavailable. Reload the project.");
326
- pdp.accept((await removeItems(id, pairs.map(pair => pair.key))).workflow);
327
- setTick(value => value + 1);
328
- return { refused: [] };
329
- }
330
  const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
331
  { method: "POST", body: { targets: pairs } }, fallback);
332
  if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
@@ -352,10 +343,10 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
352
  writing.current = false;
353
  setBusy(false);
354
  }
355
- }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick, pdp]);
356
 
357
  const { project, targets } = detail;
358
- const canEdit = mayEdit && detail.canEdit && (!pdp || Boolean(pdp.detail?.canEdit));
359
  const deleteOpen = confirming && canEdit && active;
360
  useEffect(() => {
361
  if (!canEdit || !active) { setConfirming(false); setAdding(false); }
@@ -469,7 +460,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
469
  </section>
470
 
471
  <div className="lp-proj-setup-fields">
472
- {project.type === "pdp" && pdp ? <PdpOverviewRows pdp={pdp} /> : null}
473
  <SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
474
  {canEdit
475
  ? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
@@ -492,7 +482,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
492
  onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
493
  ? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
494
  </SetupRow>
495
- {project.type === "pdp" && pdp ? <PdpApprovalRow pdp={pdp} canEdit={canEdit} /> : null}
496
  <SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
497
  <div className="lp-proj-contributors">
498
  {contributors.map(person => <span key={person.username} className="lp-proj-contributor">
@@ -566,7 +555,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
566
  : null}
567
  </div>
568
  {canEdit && active && adding
569
- ? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)} pdp={pdp}
570
  onAdded={answer => {
571
  setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
572
  cause: answer.cause } : current);
@@ -577,7 +566,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
577
  : null}
578
  {targets.length
579
  ? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
580
- onRemove={removeTargets} pdp={pdp} />
581
  : null}
582
  </section>
583
  </SetupRow>
@@ -665,10 +654,9 @@ async function readLinked(table: string, grain: ProjectGrain, tree: FilterTree,
665
  }
666
  }
667
 
668
- function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove, pdp }: {
669
  targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
670
  onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
671
- pdp?: ProjectPdp;
672
  }) {
673
  const id = useId();
674
  const [chosen, setChosen] = useState<ProjectGrain>("sku");
@@ -698,14 +686,13 @@ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemov
698
  </div> : null}
699
  <SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
700
  targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
701
- onRemove={onRemove} pdp={pdp} />
702
  </div>;
703
  }
704
 
705
- function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove, pdp }: {
706
  grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
707
  busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
708
- pdp?: ProjectPdp;
709
  }) {
710
  const table = TABLES[grain];
711
  const label = GRAIN_LABEL[grain];
@@ -799,7 +786,7 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
799
  : new Set(Array.from(current).filter(key => shown.has(key))));
800
  }, [term, blocked, shownSignature]);
801
  const columns = LIST_COLUMNS[grain];
802
- const span = (canEdit ? 1 : 0) + 1 + columns.length + (pdp ? 2 : 0);
803
  const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
804
  const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
805
  const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
@@ -864,7 +851,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
864
  {canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
865
  <th scope="col">Product</th>
866
  {columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
867
- {pdp ? <><th scope="col">Step</th><th scope="col"><span className="lp-proj-sku-sr">Workflow</span></th></> : null}
868
  </tr></thead>
869
  <tbody>{groups.map(([value, members]) => <Fragment key={value}>
870
  {group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
@@ -887,7 +873,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
887
  return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
888
  {shown || <span className="lp-proj-muted">Not set</span>}</td>;
889
  })}
890
- {pdp ? <PdpSkuCells pdp={pdp} target={row.target} /> : null}
891
  </tr>;
892
  })}
893
  </Fragment>)}</tbody>
@@ -913,14 +898,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
913
  </div>;
914
  }
915
 
916
- function PdpSkuCells({ pdp, target }: { pdp: ProjectPdp; target: Target }) {
917
- const asin = target.grain === "asin" ? target.key : target.asin;
918
- const item = pdp.detail?.workflow.items.find(item => item.asin === asin);
919
- return <><td>{item ? <PdpStep item={item} /> : <span className="lp-proj-muted">Unavailable</span>}</td>
920
- <td><button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={!item || !pdp.open}
921
- onClick={() => { if (item) pdp.open?.(item); }}>Open</button></td></>;
922
- }
923
-
924
  export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
925
  targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
926
  onCancel: () => void; onForbidden: () => void;
 
11
  import { useEscapeClose } from "../../ui/useEscapeClose";
12
  import ProjectTasks from "./ProjectTasks";
13
  import ProjectSkuPicker from "./ProjectSkuPicker";
 
 
14
  import {
15
  EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
16
  type Field, type FilterTree, type Row, type Sorts,
 
236
  }
237
 
238
  export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
239
+ error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope }: {
240
  id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
241
  error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
242
  setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
243
  rowScope?: RowScope | null; onClearRowScope?: () => void;
 
244
  }) {
245
  const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
246
  const [people, setPeople] = useState<ProjectPerson[]>([]);
 
318
  setBusy(true);
319
  const fallback = "Products were not removed.";
320
  try {
 
 
 
 
 
 
321
  const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
322
  { method: "POST", body: { targets: pairs } }, fallback);
323
  if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
 
343
  writing.current = false;
344
  setBusy(false);
345
  }
346
+ }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick]);
347
 
348
  const { project, targets } = detail;
349
+ const canEdit = mayEdit && detail.canEdit;
350
  const deleteOpen = confirming && canEdit && active;
351
  useEffect(() => {
352
  if (!canEdit || !active) { setConfirming(false); setAdding(false); }
 
460
  </section>
461
 
462
  <div className="lp-proj-setup-fields">
 
463
  <SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
464
  {canEdit
465
  ? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
 
482
  onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
483
  ? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
484
  </SetupRow>
 
485
  <SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
486
  <div className="lp-proj-contributors">
487
  {contributors.map(person => <span key={person.username} className="lp-proj-contributor">
 
555
  : null}
556
  </div>
557
  {canEdit && active && adding
558
+ ? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)}
559
  onAdded={answer => {
560
  setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
561
  cause: answer.cause } : current);
 
566
  : null}
567
  {targets.length
568
  ? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
569
+ onRemove={removeTargets} />
570
  : null}
571
  </section>
572
  </SetupRow>
 
654
  }
655
  }
656
 
657
+ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove }: {
658
  targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
659
  onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
 
660
  }) {
661
  const id = useId();
662
  const [chosen, setChosen] = useState<ProjectGrain>("sku");
 
686
  </div> : null}
687
  <SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
688
  targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
689
+ onRemove={onRemove} />
690
  </div>;
691
  }
692
 
693
+ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove }: {
694
  grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
695
  busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
 
696
  }) {
697
  const table = TABLES[grain];
698
  const label = GRAIN_LABEL[grain];
 
786
  : new Set(Array.from(current).filter(key => shown.has(key))));
787
  }, [term, blocked, shownSignature]);
788
  const columns = LIST_COLUMNS[grain];
789
+ const span = (canEdit ? 1 : 0) + 1 + columns.length;
790
  const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
791
  const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
792
  const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
 
851
  {canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
852
  <th scope="col">Product</th>
853
  {columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
 
854
  </tr></thead>
855
  <tbody>{groups.map(([value, members]) => <Fragment key={value}>
856
  {group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
 
873
  return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
874
  {shown || <span className="lp-proj-muted">Not set</span>}</td>;
875
  })}
 
876
  </tr>;
877
  })}
878
  </Fragment>)}</tbody>
 
898
  </div>;
899
  }
900
 
 
 
 
 
 
 
 
 
901
  export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
902
  targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
903
  onCancel: () => void; onForbidden: () => void;
web/src/apps/amazon-profit/ProjectSkuPicker.tsx CHANGED
@@ -11,9 +11,6 @@ import { FilterBuilderPanel, Popover, SortBuilder, countConditions,
11
  import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
12
  type Project, type ProjectGrain, type Target } from "./ProjectsShared";
13
  import "../../styles/sdlc85/project-sku-picker.css";
14
- import { projectClaimNames, readProjectClaims, type ProjectPdp } from "./pdp/ContinueWorkflow";
15
- import { addItems, PdpRequestError } from "./pdp/api";
16
- import type { PdpClaim } from "./pdp/contract";
17
 
18
  export type FilterTree = FilterBuilderPanelProps["filters"];
19
  type FilterNode = FilterTree["nodes"][number];
@@ -175,7 +172,7 @@ export type TaskTargetSource = {
175
  name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
176
  targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
177
  };
178
- type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void; pdp?: ProjectPdp };
179
  export type SkuPickerProps = PickerBase & (
180
  | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
181
  | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
@@ -204,12 +201,11 @@ function taskStart(task: TaskTargetSource | null) {
204
  done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
205
  }
206
 
207
- export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose, pdp }: {
208
  projectId: string; existing: Target[]; canEdit: boolean;
209
  onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
210
- pdp?: ProjectPdp;
211
  }) {
212
- return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose} pdp={pdp} />;
213
  }
214
 
215
  /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
@@ -217,15 +213,11 @@ export function SkuPicker(props: SkuPickerProps) {
217
  const { projectId, canEdit, onClose } = props;
218
  const task = props.mode === "task" ? props : null;
219
  const taskMode = task !== null;
220
- const pdp = props.mode === "add" ? props.pdp : undefined;
221
- const pdpMode = Boolean(pdp);
222
- const workflow = pdp?.detail?.workflow;
223
- const [taken, setTaken] = useState<PdpClaim[]>([]);
224
  const existing = props.mode === "add" ? props.existing : NO_TARGETS;
225
  const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
226
  const id = useId();
227
  const [start] = useState(() => taskStart(task?.task ?? null));
228
- const [grain, setGrain] = useState<ProjectGrain>(pdp ? "asin" : start.grain);
229
  const [query, setQuery] = useState("");
230
  const [search, setSearch] = useState("");
231
  const [filters, setFilters] = useState<FilterTree>(start.filter);
@@ -269,17 +261,15 @@ export function SkuPicker(props: SkuPickerProps) {
269
  const order = useMemo<Sorts>(() => group
270
  ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
271
  const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
272
- const holder = (key: string) => workflow ? projectClaimNames(taken, workflow, key) : "";
273
- const selectedCount = allMatching ? Math.max(0, available - excluded.size)
274
- : workflow ? Array.from(selected).filter(key => !holder(key)).length : selected.size;
275
- const ready = canEdit && (!pdp || Boolean(pdp.detail?.canEdit)) && !forbidden && !loading && !busy && query === search && match !== null && !error;
276
  const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
277
- const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key) && !holder(key));
278
  const checkedOnPage = pageKeys.filter(isSelected).length;
279
  const preview = taskMode && kind === "filter";
280
  const picks = selected.size + Object.entries(held)
281
  .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
282
- const grains = pdp ? ["asin"] as const : taskMode ? TASK_GRAINS : GRAINS;
283
 
284
  const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
285
  const resetRead = () => {
@@ -335,11 +325,6 @@ export function SkuPicker(props: SkuPickerProps) {
335
  if (counted && counted.total > 0 && !products.length) {
336
  throw new Error("Matching products did not load. Retry to refresh the matches.");
337
  }
338
- if (pdpMode) {
339
- const claims = await readProjectClaims(products.map(row => text(row.asin)), controller.signal);
340
- if (controller.signal.aborted) return;
341
- setTaken(current => cursor === null ? claims : [...current, ...claims]);
342
- }
343
  setFields(page.fields);
344
  setRows(current => {
345
  if (counted?.total === 0) return [];
@@ -370,7 +355,7 @@ export function SkuPicker(props: SkuPickerProps) {
370
  })
371
  .finally(() => { if (!controller.signal.aborted) setLoading(false); });
372
  return () => controller.abort();
373
- }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode, pdpMode]);
374
 
375
  const toggleKeys = (keys: string[], checked: boolean) => {
376
  if (allMatching) setExcluded(current => {
@@ -389,11 +374,6 @@ export function SkuPicker(props: SkuPickerProps) {
389
  if (!ready || !selectedCount || submitting.current) return;
390
  submitting.current = true; setBusy(true); setRefused([]);
391
  try {
392
- if (pdp) {
393
- pdp.accept((await addItems(projectId, Array.from(selected).filter(key => !holder(key)))).workflow);
394
- onClose();
395
- return;
396
- }
397
  // Exclusions apply after server ranking, so they cannot promote a different top-N row.
398
  const complete = allMatching && excluded.size
399
  ? await matchingKeys(projectId, grain, tree, match!.total) : null;
@@ -413,11 +393,7 @@ export function SkuPicker(props: SkuPickerProps) {
413
  } else onClose();
414
  } catch (reason) {
415
  setError(messageOf(reason, "Products were not added."));
416
- if (reason instanceof PdpRequestError && reason.status === 409) {
417
- try { setTaken(await readProjectClaims(rows.map(row => text(row.asin)))); }
418
- catch (refreshReason) { setError(messageOf(refreshReason, "Workflow claims did not load. Retry.")); }
419
- }
420
- if ((reason instanceof ProjectRequestError || reason instanceof PdpRequestError) && [401, 403].includes(reason.status)) {
421
  setForbidden(true); setRows([]); setFields([]); setMatch(null);
422
  }
423
  } finally { submitting.current = false; setBusy(false); }
@@ -438,10 +414,7 @@ export function SkuPicker(props: SkuPickerProps) {
438
  try {
439
  const every = await matchingKeys(projectId, grain, tree, current.total);
440
  if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
441
- const claims = workflow ? await readProjectClaims(every.keys) : [];
442
- if (workflow) setTaken(claims);
443
- const eligible = workflow ? every.keys.filter(key => !linked.has(key) && !projectClaimNames(claims, workflow, key)) : every.keys;
444
- setSelected(previous => new Set([...previous, ...eligible]));
445
  } catch (reason) { refuse(reason, "Matches did not load."); }
446
  finally { submitting.current = false; setBusy(false); }
447
  };
@@ -535,7 +508,7 @@ export function SkuPicker(props: SkuPickerProps) {
535
  {match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
536
  {match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
537
  onClick={() => {
538
- if (taskMode || pdp) { void selectMatching(); return; }
539
  setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
540
  }}>
541
  Select all {number(match.total)} matching
@@ -570,15 +543,13 @@ export function SkuPicker(props: SkuPickerProps) {
570
  {members.map(row => {
571
  const key = text(row[grain]);
572
  const alreadyLinked = linked.has(key);
573
- const heldBy = holder(key);
574
  const picture = pictureSrc(text(row.picture));
575
  return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
576
- {preview ? null : <td><SelectBox checked={isSelected(key) && !heldBy} disabled={!ready || alreadyLinked || Boolean(heldBy)}
577
  label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
578
  <td><div className="lp-proj-sku-product">
579
  {picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
580
- <span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}
581
- {heldBy ? <small>{heldBy} already changes the same parts</small> : null}</span>
582
  </div></td>
583
  <td className="lp-proj-sku-code" title={key}>{key}</td>
584
  <td className="lp-proj-sku-code">{grain === "parent_asin"
 
11
  import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
12
  type Project, type ProjectGrain, type Target } from "./ProjectsShared";
13
  import "../../styles/sdlc85/project-sku-picker.css";
 
 
 
14
 
15
  export type FilterTree = FilterBuilderPanelProps["filters"];
16
  type FilterNode = FilterTree["nodes"][number];
 
172
  name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
173
  targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
174
  };
175
+ type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void };
176
  export type SkuPickerProps = PickerBase & (
177
  | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
178
  | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
 
201
  done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
202
  }
203
 
204
+ export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose }: {
205
  projectId: string; existing: Target[]; canEdit: boolean;
206
  onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
 
207
  }) {
208
+ return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose} />;
209
  }
210
 
211
  /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
 
213
  const { projectId, canEdit, onClose } = props;
214
  const task = props.mode === "task" ? props : null;
215
  const taskMode = task !== null;
 
 
 
 
216
  const existing = props.mode === "add" ? props.existing : NO_TARGETS;
217
  const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
218
  const id = useId();
219
  const [start] = useState(() => taskStart(task?.task ?? null));
220
+ const [grain, setGrain] = useState<ProjectGrain>(start.grain);
221
  const [query, setQuery] = useState("");
222
  const [search, setSearch] = useState("");
223
  const [filters, setFilters] = useState<FilterTree>(start.filter);
 
261
  const order = useMemo<Sorts>(() => group
262
  ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
263
  const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
264
+ const selectedCount = allMatching ? Math.max(0, available - excluded.size) : selected.size;
265
+ const ready = canEdit && !forbidden && !loading && !busy && query === search && match !== null && !error;
 
 
266
  const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
267
+ const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key));
268
  const checkedOnPage = pageKeys.filter(isSelected).length;
269
  const preview = taskMode && kind === "filter";
270
  const picks = selected.size + Object.entries(held)
271
  .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
272
+ const grains = taskMode ? TASK_GRAINS : GRAINS;
273
 
274
  const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
275
  const resetRead = () => {
 
325
  if (counted && counted.total > 0 && !products.length) {
326
  throw new Error("Matching products did not load. Retry to refresh the matches.");
327
  }
 
 
 
 
 
328
  setFields(page.fields);
329
  setRows(current => {
330
  if (counted?.total === 0) return [];
 
355
  })
356
  .finally(() => { if (!controller.signal.aborted) setLoading(false); });
357
  return () => controller.abort();
358
+ }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode]);
359
 
360
  const toggleKeys = (keys: string[], checked: boolean) => {
361
  if (allMatching) setExcluded(current => {
 
374
  if (!ready || !selectedCount || submitting.current) return;
375
  submitting.current = true; setBusy(true); setRefused([]);
376
  try {
 
 
 
 
 
377
  // Exclusions apply after server ranking, so they cannot promote a different top-N row.
378
  const complete = allMatching && excluded.size
379
  ? await matchingKeys(projectId, grain, tree, match!.total) : null;
 
393
  } else onClose();
394
  } catch (reason) {
395
  setError(messageOf(reason, "Products were not added."));
396
+ if (reason instanceof ProjectRequestError && [401, 403].includes(reason.status)) {
 
 
 
 
397
  setForbidden(true); setRows([]); setFields([]); setMatch(null);
398
  }
399
  } finally { submitting.current = false; setBusy(false); }
 
414
  try {
415
  const every = await matchingKeys(projectId, grain, tree, current.total);
416
  if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
417
+ setSelected(previous => new Set([...previous, ...every.keys]));
 
 
 
418
  } catch (reason) { refuse(reason, "Matches did not load."); }
419
  finally { submitting.current = false; setBusy(false); }
420
  };
 
508
  {match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
509
  {match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
510
  onClick={() => {
511
+ if (taskMode) { void selectMatching(); return; }
512
  setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
513
  }}>
514
  Select all {number(match.total)} matching
 
543
  {members.map(row => {
544
  const key = text(row[grain]);
545
  const alreadyLinked = linked.has(key);
 
546
  const picture = pictureSrc(text(row.picture));
547
  return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
548
+ {preview ? null : <td><SelectBox checked={isSelected(key)} disabled={!ready || alreadyLinked}
549
  label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
550
  <td><div className="lp-proj-sku-product">
551
  {picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
552
+ <span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}</span>
 
553
  </div></td>
554
  <td className="lp-proj-sku-code" title={key}>{key}</td>
555
  <td className="lp-proj-sku-code">{grain === "parent_asin"
web/src/apps/amazon-profit/Projects.tsx CHANGED
@@ -1,16 +1,12 @@
1
- import { useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react";
2
  import {
3
- CenteredPopup, GRAIN_LABEL, LoadingRows, ProjectHeader, call, enc, messageOf,
4
  type Detail, type ProjectGrain, type ProjectList,
5
  } from "./ProjectsShared";
6
  import ProjectResults from "./ProjectResults";
7
  import { AddExisting, ProjectBrowser } from "./ProjectsList";
8
- import ProjectOverview, { ProjectForm, RowScopeChip, inRowScope, type RowScope } from "./ProjectOverview";
9
  import ProjectActivities from "./ProjectActivities";
10
- import PdpActivities from "./pdp/PdpActivities";
11
- import BrandRules from "./pdp/BrandRules";
12
- import ContinueWorkflow, { useProjectPdp, type PdpProjectNavigation } from "./pdp/ContinueWorkflow";
13
- import type { PdpProjectPane } from "./pdp/contract";
14
 
15
  export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
16
  export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
@@ -23,29 +19,16 @@ const PROJECT_PANES = [
23
  { key: "results", label: "Dashboard" },
24
  { key: "log", label: "Activities" },
25
  ] as const;
26
- type ExtraPane = { key: string; label: string; content: ReactNode };
27
- type ProjectExtras = {
28
- pdp?: PdpProjectNavigation; backLabel?: string; onBack?: () => void;
29
- initialPane?: PdpProjectPane;
30
- headerExtra?: ReactNode; extraPanes?: ExtraPane[];
31
- };
32
  const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
33
 
34
- function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel,
35
- pdp: navigation, headerExtra, extraPanes, initialPane }: {
36
  id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
37
- } & ProjectExtras) {
38
  const [detail, setDetail] = useState<Detail | null>(null);
39
  const [error, setError] = useState("");
40
  const [tick, setTick] = useState(0);
41
- const [pane, setPane] = useState<string>(initialPane ?? "setup");
42
- const projectChanged = () => { setTick(value => value + 1); onChanged(); };
43
- const pdp = useProjectPdp(detail?.project, detail?.targets ?? [], tick, mayEdit && Boolean(detail?.canEdit), projectChanged, navigation);
44
- const extras = detail?.project.type === "pdp" ? [
45
- { key: "brand", label: "Brand and rules", content: <BrandRules key={id} workflowId={id} projectName={detail.project.name}
46
- canEdit={mayEdit && detail.canEdit} active={pane === "brand"} /> }, ...(extraPanes ?? []),
47
- ] : [];
48
- const panes = extras.length ? [...PROJECT_PANES, ...extras] : PROJECT_PANES;
49
  // The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
50
  const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
51
  ? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
@@ -81,20 +64,20 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
81
 
82
  const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
83
  <div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
84
- {panes.map(({ key, label }, index) => <button key={key} type="button" role="tab"
85
  className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
86
  id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
87
  aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
88
  onKeyDown={event => {
89
  let next: number;
90
- if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % panes.length;
91
- else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + panes.length) % panes.length;
92
  else if (event.key === "Home") next = 0;
93
- else if (event.key === "End") next = panes.length - 1;
94
  else return;
95
  event.preventDefault();
96
  event.stopPropagation();
97
- setPane(panes[next].key);
98
  railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
99
  }}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
100
  </div>
@@ -114,19 +97,15 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
114
  </div>;
115
  const { project } = detail;
116
  const canEdit = mayEdit && detail.canEdit;
117
- const scopedAsins = rowScope ? [...new Set(detail.targets.filter(target => inRowScope(target, rowScope))
118
- .flatMap(target => target.grain === "asin" ? [target.key] : target.asin ? [target.asin] : []))] : [];
119
 
120
  return <div className="lp-proj-detail-layout">
121
- <ProjectHeader>{pdp ? <ContinueWorkflow pdp={pdp} targets={detail.targets}
122
- asin={scopedAsins.length === 1 ? scopedAsins[0] : undefined} /> : null}{headerExtra}</ProjectHeader>
123
  {rail}
124
  <div className="lp-proj-panes" ref={panesRef}>
125
  <section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
126
  aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
127
  <ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
128
  onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
129
- formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope} pdp={pdp} />
130
  </section>
131
  <section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
132
  aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
@@ -135,24 +114,17 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
135
  <ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
136
  updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
137
  </section>
138
- {project.type === "pdp"
139
- ? <PdpActivities project={project} formId={formId} active={pane === "log"} />
140
- : <ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />}
141
- {extras.map(extra => <section key={extra.key} className="lp-proj-pane" role="tabpanel" tabIndex={0}
142
- id={`${formId}-pane-${extra.key}`} aria-labelledby={`${formId}-tab-${extra.key}`} hidden={pane !== extra.key}>
143
- {extra.content}
144
- </section>)}
145
  </div>
146
  </div>;
147
  }
148
 
149
  // ---- The popup a Projects cell opens ----
150
 
151
- export interface ProjectsPopupProps extends ProjectExtras {
152
  grain: ProjectGrain;
153
  rowKey: string;
154
  title?: string;
155
- selectedProjectId?: string;
156
  onClose: () => void;
157
  onChanged?: () => void;
158
  }
@@ -160,13 +132,11 @@ export interface ProjectsPopupProps extends ProjectExtras {
160
  type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
161
 
162
  export function ProjectsPopup(props: ProjectsPopupProps) {
163
- return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}:${props.selectedProjectId ?? ""}:${props.initialPane ?? ""}`} {...props} />;
164
  }
165
 
166
- function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose, onChanged,
167
- pdp, backLabel, onBack, headerExtra, extraPanes, initialPane }: ProjectsPopupProps) {
168
- const [view, setView] = useState<PopupView>(() => selectedProjectId
169
- ? { kind: "detail", id: selectedProjectId } : { kind: "list" });
170
  const [read, setRead] = useState<ProjectList | null>(null);
171
  const [error, setError] = useState("");
172
  const [tick, setTick] = useState(0);
@@ -186,7 +156,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
186
  const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
187
 
188
  const actions = view.kind === "detail"
189
- ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={onBack ?? (() => setView({ kind: "list" }))}>{backLabel ?? "All projects"}</button>
190
  : view.kind === "list" && canEdit
191
  ? <>
192
  <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
@@ -213,8 +183,6 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
213
  : null}
214
  {view.kind === "detail"
215
  ? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
216
- initialPane={view.id === selectedProjectId ? initialPane : undefined}
217
- pdp={pdp} backLabel={backLabel} onBack={onBack} headerExtra={headerExtra} extraPanes={extraPanes}
218
  onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
219
  : null}
220
  {view.kind === "new" && canEdit
@@ -236,7 +204,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
236
 
237
  // ---- The Projects tab: every project, the same rows ----
238
 
239
- export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {}) {
240
  const [read, setRead] = useState<ProjectList | null>(null);
241
  const [error, setError] = useState("");
242
  const [tick, setTick] = useState(0);
@@ -273,7 +241,6 @@ export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {
273
  {open
274
  ? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
275
  <ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
276
- pdp={pdp} headerExtra={headerExtra} extraPanes={extraPanes}
277
  onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
278
  </CenteredPopup>
279
  : null}
 
1
+ import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
2
  import {
3
+ CenteredPopup, GRAIN_LABEL, LoadingRows, call, enc, messageOf,
4
  type Detail, type ProjectGrain, type ProjectList,
5
  } from "./ProjectsShared";
6
  import ProjectResults from "./ProjectResults";
7
  import { AddExisting, ProjectBrowser } from "./ProjectsList";
8
+ import ProjectOverview, { ProjectForm, RowScopeChip, type RowScope } from "./ProjectOverview";
9
  import ProjectActivities from "./ProjectActivities";
 
 
 
 
10
 
11
  export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
12
  export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
 
19
  { key: "results", label: "Dashboard" },
20
  { key: "log", label: "Activities" },
21
  ] as const;
22
+ type ProjectPane = (typeof PROJECT_PANES)[number]["key"];
 
 
 
 
 
23
  const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
24
 
25
+ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel }: {
 
26
  id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
27
+ }) {
28
  const [detail, setDetail] = useState<Detail | null>(null);
29
  const [error, setError] = useState("");
30
  const [tick, setTick] = useState(0);
31
+ const [pane, setPane] = useState<ProjectPane>("setup");
 
 
 
 
 
 
 
32
  // The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
33
  const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
34
  ? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
 
64
 
65
  const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
66
  <div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
67
+ {PROJECT_PANES.map(({ key, label }, index) => <button key={key} type="button" role="tab"
68
  className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
69
  id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
70
  aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
71
  onKeyDown={event => {
72
  let next: number;
73
+ if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % PROJECT_PANES.length;
74
+ else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + PROJECT_PANES.length) % PROJECT_PANES.length;
75
  else if (event.key === "Home") next = 0;
76
+ else if (event.key === "End") next = PROJECT_PANES.length - 1;
77
  else return;
78
  event.preventDefault();
79
  event.stopPropagation();
80
+ setPane(PROJECT_PANES[next].key);
81
  railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
82
  }}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
83
  </div>
 
97
  </div>;
98
  const { project } = detail;
99
  const canEdit = mayEdit && detail.canEdit;
 
 
100
 
101
  return <div className="lp-proj-detail-layout">
 
 
102
  {rail}
103
  <div className="lp-proj-panes" ref={panesRef}>
104
  <section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
105
  aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
106
  <ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
107
  onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
108
+ formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope} />
109
  </section>
110
  <section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
111
  aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
 
114
  <ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
115
  updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
116
  </section>
117
+ <ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />
 
 
 
 
 
 
118
  </div>
119
  </div>;
120
  }
121
 
122
  // ---- The popup a Projects cell opens ----
123
 
124
+ export interface ProjectsPopupProps {
125
  grain: ProjectGrain;
126
  rowKey: string;
127
  title?: string;
 
128
  onClose: () => void;
129
  onChanged?: () => void;
130
  }
 
132
  type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
133
 
134
  export function ProjectsPopup(props: ProjectsPopupProps) {
135
+ return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}`} {...props} />;
136
  }
137
 
138
+ function ProjectsPopupContent({ grain, rowKey, title, onClose, onChanged }: ProjectsPopupProps) {
139
+ const [view, setView] = useState<PopupView>({ kind: "list" });
 
 
140
  const [read, setRead] = useState<ProjectList | null>(null);
141
  const [error, setError] = useState("");
142
  const [tick, setTick] = useState(0);
 
156
  const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
157
 
158
  const actions = view.kind === "detail"
159
+ ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "list" })}>All projects</button>
160
  : view.kind === "list" && canEdit
161
  ? <>
162
  <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
 
183
  : null}
184
  {view.kind === "detail"
185
  ? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
 
 
186
  onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
187
  : null}
188
  {view.kind === "new" && canEdit
 
204
 
205
  // ---- The Projects tab: every project, the same rows ----
206
 
207
+ export function ProjectsPage() {
208
  const [read, setRead] = useState<ProjectList | null>(null);
209
  const [error, setError] = useState("");
210
  const [tick, setTick] = useState(0);
 
241
  {open
242
  ? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
243
  <ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
 
244
  onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
245
  </CenteredPopup>
246
  : null}
web/src/apps/amazon-profit/ProjectsShared.tsx CHANGED
@@ -1,4 +1,4 @@
1
- import { createContext, useContext, useEffect, useId, useRef, useState, type KeyboardEvent, type ReactNode } from "react";
2
  import { createPortal } from "react-dom";
3
  import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
4
  import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
@@ -35,7 +35,6 @@ const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [
35
 
36
  export interface Project {
37
  id: string;
38
- type?: "project" | "pdp";
39
  name: string;
40
  status: Status;
41
  priority: Priority;
@@ -338,20 +337,6 @@ export function Owner({ name, compact = false }: { name: string | null; compact?
338
  // ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
339
 
340
  const ProjectFooterHost = createContext<HTMLElement | null>(null);
341
- export function projectMenuKeys(event: KeyboardEvent<HTMLElement>) {
342
- const items = Array.from(event.currentTarget.querySelectorAll<HTMLElement>('[role="menuitem"]:not(:disabled)'));
343
- const index = items.indexOf(document.activeElement as HTMLElement);
344
- const next = event.key === "ArrowDown" ? items[(index + 1) % items.length]
345
- : event.key === "ArrowUp" ? items[(index - 1 + items.length) % items.length]
346
- : event.key === "Home" ? items[0] : event.key === "End" ? items.at(-1) : undefined;
347
- if (next) { event.preventDefault(); next.focus(); }
348
- }
349
-
350
- const ProjectHeaderHost = createContext<HTMLElement | null>(null);
351
- export function ProjectHeader({ children }: { children: ReactNode }) {
352
- const host = useContext(ProjectHeaderHost);
353
- return host ? createPortal(children, host) : null;
354
- }
355
  export function ProjectFooter({ children }: { children: ReactNode }) {
356
  const host = useContext(ProjectFooterHost);
357
  return host ? createPortal(children, host) : <>{children}</>;
@@ -372,7 +357,6 @@ export interface CenteredPopupProps {
372
  export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
373
  const panelRef = useRef<HTMLDivElement>(null);
374
  const [footer, setFooter] = useState<HTMLDivElement | null>(null);
375
- const [header, setHeader] = useState<HTMLSpanElement | null>(null);
376
  const titleId = useId();
377
  useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
378
  return <BodyPortal>
@@ -388,14 +372,13 @@ export function CenteredPopup({ title, subtitle, actions, onClose, children, cla
388
  </div>
389
  <div className="lp-cpop-actions">
390
  {actions}
391
- {className?.includes("lp-proj-popup") ? <span className="pdp-project-header" ref={setHeader} /> : null}
392
  <button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
393
  </div>
394
  </header>
395
- <ProjectHeaderHost.Provider value={header}><ProjectFooterHost.Provider value={className?.includes("lp-proj-popup") ? footer : null}>
396
  <div className="lp-cpop-body">{children}</div>
397
  {className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
398
- </ProjectFooterHost.Provider></ProjectHeaderHost.Provider>
399
  </div>
400
  </div>
401
  </BodyPortal>;
 
1
+ import { createContext, useContext, useEffect, useId, useRef, useState, type ReactNode } from "react";
2
  import { createPortal } from "react-dom";
3
  import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
4
  import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
 
35
 
36
  export interface Project {
37
  id: string;
 
38
  name: string;
39
  status: Status;
40
  priority: Priority;
 
337
  // ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
338
 
339
  const ProjectFooterHost = createContext<HTMLElement | null>(null);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
340
  export function ProjectFooter({ children }: { children: ReactNode }) {
341
  const host = useContext(ProjectFooterHost);
342
  return host ? createPortal(children, host) : <>{children}</>;
 
357
  export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
358
  const panelRef = useRef<HTMLDivElement>(null);
359
  const [footer, setFooter] = useState<HTMLDivElement | null>(null);
 
360
  const titleId = useId();
361
  useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
362
  return <BodyPortal>
 
372
  </div>
373
  <div className="lp-cpop-actions">
374
  {actions}
 
375
  <button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
376
  </div>
377
  </header>
378
+ <ProjectFooterHost.Provider value={className?.includes("lp-proj-popup") ? footer : null}>
379
  <div className="lp-cpop-body">{children}</div>
380
  {className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
381
+ </ProjectFooterHost.Provider>
382
  </div>
383
  </div>
384
  </BodyPortal>;
web/src/apps/templateMarks.tsx CHANGED
@@ -85,10 +85,6 @@ export const TEMPLATE_MARKS: Readonly<Record<string, ReactElement>> = {
85
  alerts: <>
86
  <path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
87
  </>,
88
- reports: <>
89
- <path className={ACCENT} d="M14 3v5h5z" />
90
- <path className={INK} d="M14 3H5v18h14V8zM14 3v5h5M8 12h8M8 16h8" />
91
- </>,
92
  };
93
 
94
  const AMAZON_MARKS = new Set([
 
85
  alerts: <>
86
  <path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
87
  </>,
 
 
 
 
88
  };
89
 
90
  const AMAZON_MARKS = new Set([
web/src/automation/AutomationDetail.tsx CHANGED
@@ -22,7 +22,7 @@ import PanelToggleIcon from "../shell/PanelToggleIcon";
22
  // what this codebase treats as a defect rather than a summary.
23
  // ---------------------------------------------------------------------------
24
  import { useCallback, useEffect, useMemo, useRef, useState } from "react";
25
- import { useRefusalToast } from "../shell/ToastStack";
26
 
27
  import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
28
  import HarnessPanel from "./HarnessPanel";
@@ -335,6 +335,28 @@ function RetiredAutomationResult({ automation }: { automation: Automation }) {
335
  );
336
  }
337
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  export default function AutomationDetailView(props: Props) {
339
  return props.automation.retired
340
  ? <RetiredAutomationResult automation={props.automation} />
 
22
  // what this codebase treats as a defect rather than a summary.
23
  // ---------------------------------------------------------------------------
24
  import { useCallback, useEffect, useMemo, useRef, useState } from "react";
25
+ import { TOAST_EVENT } from "../apiContract";
26
 
27
  import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
28
  import HarnessPanel from "./HarnessPanel";
 
335
  );
336
  }
337
 
338
+ function useRefusalToast() {
339
+ const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
340
+ const sequence = useRef(0);
341
+ const delivered = useRef(0);
342
+ useEffect(() => {
343
+ if (!messages.length) return;
344
+ const through = messages[messages.length - 1].id;
345
+ for (const message of messages) {
346
+ if (message.id <= delivered.current) continue;
347
+ delivered.current = message.id;
348
+ const text = message.text;
349
+ window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
350
+ }
351
+ setMessages((current) => current.filter((message) => message.id > through));
352
+ }, [messages]);
353
+ return useCallback((text: string) => {
354
+ if (!text) return;
355
+ const message = { id: ++sequence.current, text };
356
+ setMessages((current) => [...current, message]);
357
+ }, []);
358
+ }
359
+
360
  export default function AutomationDetailView(props: Props) {
361
  return props.automation.retired
362
  ? <RetiredAutomationResult automation={props.automation} />
web/src/automation/AutomationSurface.tsx CHANGED
@@ -18,8 +18,7 @@ import type { KeyboardEvent as ReactKeyboardEvent } from "react";
18
  import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
19
 
20
  import type { AutomationOpenDetail } from "../apiContract";
21
- import { AUTOMATION_OPEN_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
22
- import { useRefusalToast } from "../shell/ToastStack";
23
  // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
24
  // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
25
  // C7 says consume the existing icon component and do not inline a new SVG path, and
@@ -212,6 +211,27 @@ if (typeof window !== "undefined") {
212
  * that a database can be created BY asking for an automation.
213
  */
214
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
215
  export default function AutomationSurface() {
216
  /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
217
  through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
 
18
  import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
19
 
20
  import type { AutomationOpenDetail } from "../apiContract";
21
+ import { AUTOMATION_OPEN_EVENT, TOAST_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
 
22
  // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
23
  // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
24
  // C7 says consume the existing icon component and do not inline a new SVG path, and
 
211
  * that a database can be created BY asking for an automation.
212
  */
213
 
214
+ function useRefusalToast() {
215
+ const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
216
+ const sequence = useRef(0);
217
+ const delivered = useRef(0);
218
+ useEffect(() => {
219
+ if (!messages.length) return;
220
+ const through = messages[messages.length - 1].id;
221
+ for (const { id, text } of messages) {
222
+ if (id <= delivered.current) continue;
223
+ delivered.current = id;
224
+ window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
225
+ }
226
+ setMessages((current) => current.filter((message) => message.id > through));
227
+ }, [messages]);
228
+ return (text: string) => {
229
+ if (!text) return;
230
+ const message = { id: ++sequence.current, text };
231
+ setMessages((current) => [...current, message]);
232
+ };
233
+ }
234
+
235
  export default function AutomationSurface() {
236
  /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
237
  through `data === null` and therefore never paints the skeleton. The lazy initialiser runs