fsanyoto commited on
Commit
1abb0a4
·
verified ·
1 Parent(s): 8a705fa

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/amazon_quotes_loop.py +5 -93
  4. api/amazon_relational.py +1 -6
  5. api/app_templates.py +1 -11
  6. api/application_registry.py +6 -28
  7. api/connectors_amazon.py +10 -5
  8. api/external_freshness.py +55 -150
  9. api/main.py +26 -52
  10. api/odoo_relational.py +7 -1
  11. api/process_role.py +35 -101
  12. api/routes_amazon_projects.py +3 -9
  13. api/routes_brand.py +2 -52
  14. api/routes_changes.py +0 -3
  15. api/routes_competitors.py +10 -32
  16. api/routes_connectors.py +6 -1
  17. api/routes_grid.py +0 -7
  18. api/routes_odoo_tables.py +2 -10
  19. api/routes_shares.py +32 -27
  20. api/routes_starred.py +0 -5
  21. api/routes_tables.py +3 -25
  22. api/sync_runtime.py +10 -55
  23. api/test_odoo_rank_refresh.py +70 -1
  24. api/usage_ledger.py +53 -59
  25. platform/core/amazon_projects.py +21 -90
  26. platform/core/amazon_rivals.py +117 -575
  27. platform/core/connector_spec.py +1 -46
  28. platform/core/external_tools.py +1 -1
  29. platform/core/shares.py +2 -24
  30. platform/harness/amazon_client.py +2 -5
  31. platform/harness/amazon_schema.py +13 -22
  32. platform/harness/amazon_store.py +0 -0
  33. platform/harness/datastore.py +34 -44
  34. platform/harness/mirror_store.py +152 -394
  35. platform/modules/amazon_profit.py +23 -207
  36. platform/modules/amazon_profit_account.py +7 -122
  37. platform/modules/competitor_watch.py +3 -19
  38. requirements.txt +0 -1
  39. web/package-lock.json +0 -485
  40. web/package-lock.linux.sha256 +2 -2
  41. web/package.json +0 -1
  42. web/public/AIOS_DEPLOY_RECEIPT +1 -1
  43. web/src/apps/amazon-profit/AppView.tsx +6 -81
  44. web/src/apps/amazon-profit/ProjectOverview.tsx +9 -32
  45. web/src/apps/amazon-profit/ProjectSkuPicker.tsx +14 -43
  46. web/src/apps/amazon-profit/Projects.tsx +20 -53
  47. web/src/apps/amazon-profit/ProjectsShared.tsx +3 -20
  48. web/src/apps/templateMarks.tsx +0 -4
  49. web/src/automation/AutomationDetail.tsx +23 -1
  50. web/src/automation/AutomationSurface.tsx +22 -2
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "2532efa50c",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "b1a532ae668",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 2532efa50c
 
1
+ b1a532ae668
api/amazon_quotes_loop.py CHANGED
@@ -9,12 +9,10 @@ import logging
9
  from pathlib import Path
10
  import threading
11
  import time
12
- import uuid
13
 
14
  JOB = "amazon-rival-quotes"
15
  #: The loop asks each tenant whether its daily run is due this often.
16
  POLL_SECONDS = 600
17
- SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")
18
 
19
  #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
20
  #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
@@ -28,29 +26,17 @@ _thread = None
28
  _stop = None
29
 
30
 
31
- class SnapshotBlocked(RuntimeError):
32
- """A safe admission cause that can be recorded without exposing provider details."""
33
-
34
-
35
  def _say(message):
36
  print(f"[{JOB}] {message}", flush=True)
37
 
38
 
39
  def readers(fields):
40
- """Read prices and complete facts with the tenant credential and the shared Amazon pacers."""
41
  from harness import amazon_client as cx
42
- from harness import amazon_competitor_facts as facts
43
-
44
- def complete(asins):
45
- catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
46
- return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
47
- "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
48
- for asin in asins}
49
-
50
  return {"offers": lambda asins: cx.item_offers(fields, asins),
51
  # The pricing sync draws this same `competitive_summary` pacer in this worker process.
52
  "summaries": lambda asins: cx.competitive_summaries(fields, asins),
53
- "facts": complete}
54
 
55
 
56
  @contextmanager
@@ -78,69 +64,9 @@ def _after_quotes(tenant, changed):
78
  "the run result is unchanged.")
79
 
80
 
81
- def _snapshot_cause(runtime, *, check_lease=True):
82
- from harness import amazon_store
83
-
84
- enabled = amazon_store.read_database_enablement(runtime)
85
- if not enabled["available"]:
86
- return "Competitor database settings could not be read. Restore settings access and retry the run."
87
- if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
88
- return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
89
- if check_lease and amazon_store.first_load_job_live(runtime):
90
- return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
91
- return None
92
-
93
-
94
- def _publish_snapshots(tenant, runtime, result):
95
- from harness import amazon_competitor_facts as facts, amazon_store, mirror_store
96
-
97
- lock = mirror_store.for_connector("amazon").lock
98
- if not lock.acquire(blocking=False):
99
- raise SnapshotBlocked("The Amazon mirror writer is busy.")
100
- holder = f"{JOB}:{result['id']}"
101
- leased = False
102
- try:
103
- leased = amazon_store.acquire_amazon_lease(runtime, holder)
104
- if not leased:
105
- raise SnapshotBlocked("The Amazon mirror writer lease is held.")
106
- cause = _snapshot_cause(runtime, check_lease=False)
107
- if cause:
108
- raise SnapshotBlocked(cause)
109
- report = {"tables": {}, "problems": []}
110
- with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
111
- con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
112
- try:
113
- stored = facts.import_snapshots(con, result["snapshots"])
114
- for table in SNAPSHOT_TABLES:
115
- total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
116
- report["tables"][table] = {"state": "done", "rows": total}
117
- finally:
118
- con.close()
119
- if report["problems"]:
120
- raise RuntimeError("Competitor mirror publication failed")
121
- result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
122
- finally:
123
- try:
124
- if leased:
125
- amazon_store.release_amazon_lease(runtime, holder)
126
- finally:
127
- lock.release()
128
-
129
-
130
- def _run_blocked(runtime, cause):
131
- from core import amazon_rivals as rivals
132
-
133
- run_id = uuid.uuid4().hex
134
- rivals.start_run(runtime, run_id, asin_total=0)
135
- return {"id": run_id, "progress": 0, "cause": cause,
136
- "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}
137
-
138
-
139
  def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
140
  """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
141
  from core import amazon_rivals as rivals
142
- if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
143
- raise ValueError("Competitor quotes require the matching tenant runtime")
144
  try:
145
  if not rivals.due(runtime, now=now):
146
  return None
@@ -154,23 +80,9 @@ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
154
  fields = amazon_store.credential(tenant, runtime)
155
  if not fields:
156
  return None
157
- started = time.monotonic()
158
- cause = _snapshot_cause(runtime)
159
- result = (_run_blocked(runtime, cause) if cause else
160
- rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
161
- seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
162
- log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
163
- if result["snapshots"]:
164
- try:
165
- _publish_snapshots(tenant, runtime, result)
166
- except Exception as exc:
167
- cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
168
- f"Competitor snapshots could not be published ({type(exc).__name__}).")
169
- result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
170
- f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
171
- result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
172
- rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
173
- counts=result["counts"], cause=result["cause"])
174
  if result["cause"] is None:
175
  _after_quotes(tenant, result["changed"])
176
  return result
 
9
  from pathlib import Path
10
  import threading
11
  import time
 
12
 
13
  JOB = "amazon-rival-quotes"
14
  #: The loop asks each tenant whether its daily run is due this often.
15
  POLL_SECONDS = 600
 
16
 
17
  #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
18
  #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
 
26
  _stop = None
27
 
28
 
 
 
 
 
29
  def _say(message):
30
  print(f"[{JOB}] {message}", flush=True)
31
 
32
 
33
  def readers(fields):
34
+ """The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
35
  from harness import amazon_client as cx
 
 
 
 
 
 
 
 
36
  return {"offers": lambda asins: cx.item_offers(fields, asins),
37
  # The pricing sync draws this same `competitive_summary` pacer in this worker process.
38
  "summaries": lambda asins: cx.competitive_summaries(fields, asins),
39
+ "ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
40
 
41
 
42
  @contextmanager
 
64
  "the run result is unchanged.")
65
 
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
68
  """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
69
  from core import amazon_rivals as rivals
 
 
70
  try:
71
  if not rivals.due(runtime, now=now):
72
  return None
 
80
  fields = amazon_store.credential(tenant, runtime)
81
  if not fields:
82
  return None
83
+ result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
84
+ seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
85
+ log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  if result["cause"] is None:
87
  _after_quotes(tenant, result["changed"])
88
  return result
api/amazon_relational.py CHANGED
@@ -650,8 +650,6 @@ def _fields_for(cur, key, *, read_through=False):
650
  "pinned": name == pinned,
651
  **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
652
  if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
653
- **({"description": amazon_schema.SQP_LIMIT_NOTE}
654
- if name == "asin" and key == PREFIX + "search_query_performance" else {}),
655
  **({"format": fmt} if fmt else {}),
656
  }, flow="amazon_relational"))
657
  for lookup in derived.get(name, ()):
@@ -725,8 +723,7 @@ READ_THROUGH = frozenset(
725
  and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
726
  PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
727
  # Change history grows without a cap (standing rule 1), so it is never stored.
728
- PREFIX + table for table in amazon_store.HISTORY_TABLES) | frozenset((
729
- PREFIX + "competitor_snapshots", PREFIX + "competitor_images"))
730
 
731
 
732
  def _quoted(name):
@@ -1601,8 +1598,6 @@ def apply_plan(rt, built, username="amazon", today=None):
1601
  doc[key]["description"] = ads_descriptions[key]
1602
  if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
1603
  doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
1604
- if key == PREFIX + "search_query_performance":
1605
- doc[key]["description"] = amazon_schema.SQP_LIMIT_NOTE
1606
  if key == PREFIX + "listing_items":
1607
  doc[key]["amazonListingIdentity"] = listing_version
1608
  return doc
 
650
  "pinned": name == pinned,
651
  **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
652
  if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
 
 
653
  **({"format": fmt} if fmt else {}),
654
  }, flow="amazon_relational"))
655
  for lookup in derived.get(name, ()):
 
723
  and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
724
  PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
725
  # Change history grows without a cap (standing rule 1), so it is never stored.
726
+ PREFIX + table for table in amazon_store.HISTORY_TABLES)
 
727
 
728
 
729
  def _quoted(name):
 
1598
  doc[key]["description"] = ads_descriptions[key]
1599
  if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
1600
  doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
 
 
1601
  if key == PREFIX + "listing_items":
1602
  doc[key]["amazonListingIdentity"] = listing_version
1603
  return doc
api/app_templates.py CHANGED
@@ -127,22 +127,12 @@ TEMPLATES = (
127
  # Project management reads its own module tables (ut_amazon_*), so it declares no
128
  # source: a cold mirror must not refuse an App whose rows live in the store.
129
  _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
130
- "Create value", (),
131
- optional_sources=("ut_amzn_search_query_performance", "ut_amzn_order_items",
132
- "ut_amzn_brand_analytics", "ut_amzn_listings", "ut_amzn_catalog_items",
133
- "ut_amzn_listing_items", "ut_amzn_competitor_snapshots",
134
- "ut_amzn_competitor_images"),
135
- optional_connectors=("odoo",), proposes=False),
136
  {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
137
  "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
138
  "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
139
  "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
140
  "tools": (), "proposes": False},
141
- {"kind": "reports", "label": "Reports", "purpose": "Download reports shared with you.",
142
- "connector": "odoo", "connectors": ("odoo",), "tags": ["Odoo"], "sources": (),
143
- "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "reports",
144
- "module": "modules.reports", "router": "routes_reports", "view": "reports",
145
- "tools": (), "proposes": False},
146
  )
147
  if not all(validate_connectors(row) for row in TEMPLATES):
148
  raise ValueError("An App template omits a source connector.")
 
127
  # Project management reads its own module tables (ut_amazon_*), so it declares no
128
  # source: a cold mirror must not refuse an App whose rows live in the store.
129
  _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
130
+ "Create value", (), optional_connectors=("odoo",), proposes=False),
 
 
 
 
 
131
  {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
132
  "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
133
  "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
134
  "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
135
  "tools": (), "proposes": False},
 
 
 
 
 
136
  )
137
  if not all(validate_connectors(row) for row in TEMPLATES):
138
  raise ValueError("An App template omits a source connector.")
api/application_registry.py CHANGED
@@ -624,16 +624,6 @@ def _source_rows_exist(session, sources):
624
  return False
625
 
626
 
627
- def _reports_readable(runtime, user, is_admin):
628
- """Report grants admit the Reports App independently of its App sharing record."""
629
- try:
630
- from modules import reports
631
- except Exception as exc: # A failed Reports import withholds its App.
632
- raise_if_resource_error(exc)
633
- return False
634
- return bool(is_admin or reports.granted_keys(runtime, user))
635
-
636
-
637
  def _readable_template_app(session, app_id):
638
  """Resolve App identity before reading any source or exposing a refusal."""
639
  from deps import err
@@ -645,10 +635,7 @@ def _readable_template_app(session, app_id):
645
  except Exception:
646
  raise err(503, "app_unavailable", "This App cannot be read right now.") from None
647
  if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
648
- or app.get("retired") is True
649
- or not (_reports_readable(session.runtime, session.uname, session.admin)
650
- if app.get("templateKind") == "reports"
651
- else may_read(app, session.uname, session.admin))):
652
  raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
653
  kind = app.get("templateKind")
654
  catalogue = _app_templates.BY_KIND.get(str(kind or ""))
@@ -1318,7 +1305,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
1318
 
1319
 
1320
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
1321
- is_default=False, readable=None):
1322
  row=copy.deepcopy(_app_projection(app))
1323
  row.setdefault("origin", None)
1324
  if row.get("type") == "code":
@@ -1327,10 +1314,9 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
1327
  compatible = [database for database in row.get("compatibleDatabases") or ()
1328
  if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
1329
  row["compatibleDatabases"] = compatible
1330
- app_readable = may_read(app, user, is_admin)
1331
- row["capabilities"]={"read":app_readable if readable is None else readable,"edit":may_edit(app,user,is_admin),
1332
  "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
1333
- "duplicate":app_readable,"star":app_readable,
1334
  "default":bool(is_admin)}
1335
  row["starred"] = user in (app.get("starredBy") or ())
1336
  row["isDefault"] = bool(is_default)
@@ -1391,7 +1377,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1391
  """List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
1392
  doc = load(runtime)
1393
  rows=[]
1394
- reports_reader = None
1395
  for app in doc["applications"].values():
1396
  if (isinstance(app, dict) and getattr(runtime, "key", None)
1397
  and app.get("tenant") != runtime.key):
@@ -1401,19 +1386,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1401
  if (isinstance(app, dict) and app.get("templateKind") == "alerts"
1402
  and not (may_open or _user_wall(runtime, user))("automation")):
1403
  continue
1404
- report_app = isinstance(app, dict) and app.get("templateKind") == "reports"
1405
- if report_app:
1406
- if reports_reader is None:
1407
- reports_reader = _reports_readable(runtime, user, is_admin)
1408
- if not reports_reader or app.get("retired") is True:
1409
- continue
1410
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
1411
  continue
1412
  if database:
1413
  # A View resolves its App only through this read, so a retired seed stays listed here.
1414
- if not report_app and not may_read(app,user,is_admin): continue
1415
  if database not in (app.get("compatibleDatabases") or ()): continue
1416
- elif not report_app and not _listed(app, user): continue
1417
  if (not _database_free(app)
1418
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
1419
  catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
@@ -1425,7 +1404,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
1425
  default_key = f"{database}::{app.get('type')}" if database else ""
1426
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
1427
  target_database=database,
1428
- readable=True if report_app else None,
1429
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
1430
  # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
1431
  # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
 
624
  return False
625
 
626
 
 
 
 
 
 
 
 
 
 
 
627
  def _readable_template_app(session, app_id):
628
  """Resolve App identity before reading any source or exposing a refusal."""
629
  from deps import err
 
635
  except Exception:
636
  raise err(503, "app_unavailable", "This App cannot be read right now.") from None
637
  if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
638
+ or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
 
 
 
639
  raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
640
  kind = app.get("templateKind")
641
  catalogue = _app_templates.BY_KIND.get(str(kind or ""))
 
1305
 
1306
 
1307
  def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
1308
+ is_default=False):
1309
  row=copy.deepcopy(_app_projection(app))
1310
  row.setdefault("origin", None)
1311
  if row.get("type") == "code":
 
1314
  compatible = [database for database in row.get("compatibleDatabases") or ()
1315
  if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
1316
  row["compatibleDatabases"] = compatible
1317
+ row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin),
 
1318
  "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
1319
+ "duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin),
1320
  "default":bool(is_admin)}
1321
  row["starred"] = user in (app.get("starredBy") or ())
1322
  row["isDefault"] = bool(is_default)
 
1377
  """List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
1378
  doc = load(runtime)
1379
  rows=[]
 
1380
  for app in doc["applications"].values():
1381
  if (isinstance(app, dict) and getattr(runtime, "key", None)
1382
  and app.get("tenant") != runtime.key):
 
1386
  if (isinstance(app, dict) and app.get("templateKind") == "alerts"
1387
  and not (may_open or _user_wall(runtime, user))("automation")):
1388
  continue
 
 
 
 
 
 
1389
  if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
1390
  continue
1391
  if database:
1392
  # A View resolves its App only through this read, so a retired seed stays listed here.
1393
+ if not may_read(app,user,is_admin): continue
1394
  if database not in (app.get("compatibleDatabases") or ()): continue
1395
+ elif not _listed(app, user): continue
1396
  if (not _database_free(app)
1397
  and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
1398
  catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
 
1404
  default_key = f"{database}::{app.get('type')}" if database else ""
1405
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
1406
  target_database=database,
 
1407
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
1408
  # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
1409
  # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
api/connectors_amazon.py CHANGED
@@ -611,11 +611,16 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
611
  answer[table]["warning"] = warning
612
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
613
  answer[table]["limit"] = limit
614
- for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"):
615
- answer[table] = {
616
- "rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False,
617
- "unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]],
618
- }
 
 
 
 
 
619
  # A history table is written in its source's transaction, so it shares the source's last success.
620
  for table in az.HISTORY_TABLES:
621
  rows = mirrored.get(table)
 
611
  answer[table]["warning"] = warning
612
  if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
613
  answer[table]["limit"] = limit
614
+ answer["sku_keywords"] = {
615
+ "rowsLoaded": mirrored.get("sku_keywords"),
616
+ "lastSuccess": None,
617
+ "inProgress": False,
618
+ "unknownCause": az.amazon_schema.PROVENANCE_COPY["derived"],
619
+ }
620
+ answer["keyword_ranks"] = {
621
+ "rowsLoaded": mirrored.get("keyword_ranks"), "lastSuccess": None, "inProgress": False,
622
+ "unknownCause": az.amazon_schema.PROVENANCE_COPY["apify_dataset"],
623
+ }
624
  # A history table is written in its source's transaction, so it shares the source's last success.
625
  for table in az.HISTORY_TABLES:
626
  rows = mirrored.get(table)
api/external_freshness.py CHANGED
@@ -82,24 +82,18 @@ def _amazon_raw(table_key, source=None):
82
  return None
83
 
84
 
85
- def amazon_sync_state(cur, rt=None, *, entry=None):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
- if rt is not None or entry is not None:
88
  from core import connector_sync, connector_spec
89
- if entry is None:
90
- entry = connector_sync.status(rt, "amazon_sp", fresh=True)
91
  if entry is not None:
92
  last_pass = entry.get("pass") or {}
93
  held = last_pass.get("held") or {}
94
  stamps = last_pass.get("tableSync") or {}
95
  tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
96
- state = {"amzn_sync_mark_" + name: stamp for name in tables
97
- if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
98
- if cur is not None and any(name not in stamps for name in held):
99
- previous = amazon_sync_state(cur)
100
- state.update({"amzn_sync_mark_" + name: stamp for name in held if name not in stamps
101
- and (stamp := _amazon_table_stamp(previous, name)) is not None})
102
- return state
103
  return {str(entity): at for entity, at in cur.execute(
104
  "SELECT entity, updated_at FROM _sync_state "
105
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
@@ -141,8 +135,8 @@ def amazon_sync_stamp(cur, table_key, state=None):
141
  return _amazon_table_stamp(state, table)
142
 
143
 
144
- def _amazon_refreshes(session, keys, *, entry, served=False):
145
- """Use the supplied canonical entry or served receipts for authorized Amazon databases."""
146
  import assistant_mirror
147
 
148
  cur, state, refreshed = None, None, {}
@@ -154,7 +148,7 @@ def _amazon_refreshes(session, keys, *, entry, served=False):
154
  cur = None
155
  continue
156
  if state is None:
157
- state = amazon_sync_state(cur, entry=None if served else entry)
158
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
159
  except Exception:
160
  continue
@@ -244,7 +238,7 @@ def _mirror_refreshes(session, definitions, keys):
244
  return refreshed
245
 
246
 
247
- def _served_behind(session, key, *, worker=None):
248
  """process_role's install lag for the connector behind one mirror database, or None."""
249
  key = str(key or "")
250
  connector = ("amazon" if key.startswith("ut_amzn_") else
@@ -253,82 +247,49 @@ def _served_behind(session, key, *, worker=None):
253
  return None
254
  try:
255
  import process_role
256
- if worker is None:
257
- worker = process_role.stored_worker(session.runtime)
258
- return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector,
259
- worker=worker)
260
  except Exception:
261
  return None
262
 
263
 
264
- def _retained_fact(served, state, reason):
265
- """Keep the successful source receipt while naming why its rows have not advanced."""
266
- fact = {"source_synced_at": utc_stamp(served), "source_sync": state, "source_sync_reason": reason}
267
- if (day := _sync_date(served)) is not None:
268
- fact["source_synced_on"] = day
269
- return fact
270
-
271
-
272
- def _stale_fact(served, lag=None):
273
  """A fact for a mirror whose newest published sync this deployment could not install."""
274
  day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
275
  reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
276
  if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
277
- if lag and lag.get("cause"):
278
- reason += " " + lag["cause"]
279
- if lag and (retry := utc_stamp(lag.get("nextRetryAt"))):
280
- reason += f" Next retry at {retry}."
281
- return _retained_fact(served, "stale", reason)
282
-
283
-
284
- def _amazon_holds(entry, keys):
285
- """Map each authorized table to its own canonical hold or its held input tables."""
286
- held = (entry.get("pass") or {}).get("held") or {}
287
- result = {}
288
- for key in keys:
289
- raw = _amazon_raw(key)
290
- table = raw.removeprefix("amzn_") if raw else ""
291
- dependencies = (table, *_AMAZON_DERIVED_INPUTS.get(table, ()))
292
- rows = [held[name] for name in dependencies if isinstance(held.get(name), dict)]
293
- if rows:
294
- result[key] = rows
295
- return result
296
-
297
-
298
- def _held_fact(served, entries):
299
- """Name the kept rows and each stored hold cause without changing their source date."""
300
- import dash_norm
301
 
302
- day = _sync_date(served) or (utc_stamp(served) or "")[:10]
303
- reason = f"Held at the rows synced on {day}." if day else "Held at the previously synced rows."
304
- causes = dict.fromkeys(dash_norm.plain(entry.get("cause") or "The last sync did not finish this table.")
305
- for entry in entries)
306
- reason += " " + " ".join(causes)
307
- retries = sorted({stamp for entry in entries if (stamp := utc_stamp(entry.get("nextTryAt")))})
308
- if retries:
309
- reason += f" Next retry at {retries[0]}."
310
- return _retained_fact(served, "stale", reason)
311
 
 
 
 
312
 
313
- def _edit_receipts(rt, edited):
314
- """Read the latest edit for all personal databases in one history aggregate."""
 
 
 
 
 
 
 
 
 
 
 
315
  edits = {}
316
  if edited:
 
 
 
317
  try:
318
  from core import history_rows
319
  latest = history_rows.backend(rt).latest(sorted(edited))
320
- except Exception: # Unavailable history keeps the definition's recorded edit time.
321
  latest = {}
322
  for key, at in latest.items() if isinstance(latest, dict) else ():
323
  if key in edited:
324
  edits[key] = _latest(edits.get(key), at)
325
- return edits
326
-
327
-
328
- def _connector_receipts(rt, connected, definitions):
329
- """Read matching jobs without borrowing another connection's successful sync."""
330
- from core import connector_sync
331
-
332
  receipts = {}
333
  if connected:
334
  try:
@@ -344,84 +305,27 @@ def _connector_receipts(rt, connected, definitions):
344
  or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
345
  continue
346
  receipts[key] = job
347
- return receipts
348
-
349
-
350
- class _StatusSnapshot:
351
- """Keep the request's fresh canonical status available to readers that request freshness again."""
352
-
353
- def __init__(self, runtime, statuses):
354
- self._runtime, self._statuses = runtime, statuses
355
-
356
- def get(self, name, fresh=False):
357
- from core import connector_sync
358
-
359
- if name == connector_sync.STATUS_KEY:
360
- return self._statuses
361
- return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)
362
-
363
- def __getattr__(self, name):
364
- return getattr(self._runtime, name)
365
-
366
-
367
- def _odoo_freshness(rt):
368
- """Read Odoo's metadata without restoring or querying its mirror."""
369
- import odoo_relational
370
- from core import user_tables
371
-
372
- docs = {}
373
- for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
374
- try:
375
- docs[name] = rt.get(name) or {}
376
- except Exception: # A missing status must not turn a discovery read into an error.
377
- docs[name] = {}
378
- return odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
379
-
380
-
381
- def _synced_fact(values):
382
- """Use a successful source timestamp or preserve an explicitly date-only receipt."""
383
- stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
384
- if stamp is not None:
385
- return {"source_synced_at": stamp, "source_sync": "synced"}
386
- day = next((day for value in values if (day := _sync_date(value)) is not None), None)
387
- if day is not None:
388
- return {"source_synced_on": day, "source_sync": "synced_date_only",
389
- "source_sync_reason": "The source reports only a sync date, without a time."}
390
- return None
391
-
392
-
393
- def snapshot(session, databases):
394
- """Index source receipts and edit times once for this authorized discovery call."""
395
- from core import connector_sync, user_tables
396
-
397
- wanted, rt = set(databases), session.runtime
398
- try:
399
- definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
400
- if not isinstance(definitions, dict):
401
- raise ValueError("source definitions unavailable")
402
- except Exception:
403
- return {key: {"source_synced_at": None, "source_sync": "never_synced",
404
- "source_sync_reason": "The source's connector status could not be verified."}
405
- for key in wanted}
406
- lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
407
- connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
408
- edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
409
- edits, receipts = _edit_receipts(rt, edited), _connector_receipts(rt, connected, definitions)
410
  odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
 
 
 
 
 
 
 
 
 
 
 
411
  amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
412
- statuses = connector_sync.status_all(rt, fresh=True) if amazon_keys or odoo_keys else {}
413
- odoo = _odoo_freshness(_StatusSnapshot(rt, statuses)) if odoo_keys else {}
414
- worker = (statuses.get("sync_worker") or {}).get("default") or {}
415
- amazon_entry = (statuses.get("amazon_sp") or {}).get("default")
416
- behind = {group: _served_behind(session, next(iter(keys)), worker=worker)
417
- for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
418
- amazon = _amazon_refreshes(session, amazon_keys, entry=amazon_entry,
419
- served=behind.get("amazon") is not None) if amazon_keys else {}
420
- held = _amazon_holds(amazon_entry or {}, amazon_keys)
421
  mirror_keys = {key for key in connected - amazon_keys
422
  if not user_tables.materialises(key, st=lent, defn=definitions[key])}
423
  mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
424
  facts = {}
 
 
 
425
  odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
426
  if isinstance(item, dict) and item.get("key")}
427
  for key in wanted:
@@ -432,10 +336,7 @@ def snapshot(session, databases):
432
  elif key == "ut_activity":
433
  fact["source_sync"] = "not_applicable"
434
  elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
435
- fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key),
436
- behind["amazon" if key in amazon_keys else "odoo"])
437
- elif key in held:
438
- fact = _held_fact(amazon.get(key), held[key])
439
  else:
440
  health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
441
  job = receipts.get(key) or {}
@@ -447,9 +348,13 @@ def snapshot(session, databases):
447
  if job.get("state") == "ready":
448
  successes.append(job.get("at"))
449
  values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
450
- synced = _synced_fact(values)
451
- if synced is not None:
452
- fact.update(synced)
 
 
 
 
453
  elif (own_failed or job.get("state") == "failed"
454
  or key in odoo_keys and odoo.get("phase") == "error"):
455
  fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
 
82
  return None
83
 
84
 
85
+ def amazon_sync_state(cur, rt=None):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
+ if rt is not None:
88
  from core import connector_sync, connector_spec
89
+ entry = connector_sync.status(rt, "amazon_sp", fresh=True)
 
90
  if entry is not None:
91
  last_pass = entry.get("pass") or {}
92
  held = last_pass.get("held") or {}
93
  stamps = last_pass.get("tableSync") or {}
94
  tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
95
+ return {"amzn_sync_mark_" + name: stamp for name in tables
96
+ if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
 
 
 
 
 
97
  return {str(entity): at for entity, at in cur.execute(
98
  "SELECT entity, updated_at FROM _sync_state "
99
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
 
135
  return _amazon_table_stamp(state, table)
136
 
137
 
138
+ def _amazon_refreshes(session, keys):
139
+ """Read only sync metadata for the authorized Amazon databases on this tenant's cursor."""
140
  import assistant_mirror
141
 
142
  cur, state, refreshed = None, None, {}
 
148
  cur = None
149
  continue
150
  if state is None:
151
+ state = amazon_sync_state(cur, session.runtime)
152
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
153
  except Exception:
154
  continue
 
238
  return refreshed
239
 
240
 
241
+ def _served_behind(session, key):
242
  """process_role's install lag for the connector behind one mirror database, or None."""
243
  key = str(key or "")
244
  connector = ("amazon" if key.startswith("ut_amzn_") else
 
247
  return None
248
  try:
249
  import process_role
250
+ return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector)
 
 
 
251
  except Exception:
252
  return None
253
 
254
 
255
+ def _stale_fact(served):
 
 
 
 
 
 
 
 
256
  """A fact for a mirror whose newest published sync this deployment could not install."""
257
  day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
258
  reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
259
  if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
260
+ return {"source_synced_at": None, "source_sync": "stale", "source_sync_reason": reason}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
 
 
 
 
 
 
 
 
 
 
262
 
263
+ def snapshot(session, databases):
264
+ """Index source receipts and edit times once for this authorized discovery call."""
265
+ from core import connector_sync, user_tables
266
 
267
+ wanted = set(databases)
268
+ rt = session.runtime
269
+ try:
270
+ definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
271
+ if not isinstance(definitions, dict):
272
+ raise ValueError("source definitions unavailable")
273
+ except Exception:
274
+ return {key: {"source_synced_at": None, "source_sync": "never_synced",
275
+ "source_sync_reason": "The source's connector status could not be verified."}
276
+ for key in wanted}
277
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
278
+ connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
279
+ edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
280
  edits = {}
281
  if edited:
282
+ # D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
283
+ # newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
284
+ # definition's `updated` stamp below and never fails discovery.
285
  try:
286
  from core import history_rows
287
  latest = history_rows.backend(rt).latest(sorted(edited))
288
+ except Exception:
289
  latest = {}
290
  for key, at in latest.items() if isinstance(latest, dict) else ():
291
  if key in edited:
292
  edits[key] = _latest(edits.get(key), at)
 
 
 
 
 
 
 
293
  receipts = {}
294
  if connected:
295
  try:
 
305
  or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
306
  continue
307
  receipts[key] = job
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
309
+ odoo = {}
310
+ if odoo_keys:
311
+ import odoo_relational
312
+
313
+ docs = {}
314
+ for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
315
+ try:
316
+ docs[name] = rt.get(name) or {}
317
+ except Exception:
318
+ docs[name] = {}
319
+ odoo = odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
320
  amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
321
+ amazon = _amazon_refreshes(session, amazon_keys) if amazon_keys else {}
 
 
 
 
 
 
 
 
322
  mirror_keys = {key for key in connected - amazon_keys
323
  if not user_tables.materialises(key, st=lent, defn=definitions[key])}
324
  mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
325
  facts = {}
326
+ # One install-lag read per connector: a secondary web worker reads it from the primary's file.
327
+ behind = {group: _served_behind(session, next(iter(keys)))
328
+ for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
329
  odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
330
  if isinstance(item, dict) and item.get("key")}
331
  for key in wanted:
 
336
  elif key == "ut_activity":
337
  fact["source_sync"] = "not_applicable"
338
  elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
339
+ fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key))
 
 
 
340
  else:
341
  health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
342
  job = receipts.get(key) or {}
 
348
  if job.get("state") == "ready":
349
  successes.append(job.get("at"))
350
  values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
351
+ stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
352
+ synced_on = next((day for value in values if (day := _sync_date(value)) is not None), None)
353
+ if stamp is not None:
354
+ fact.update(source_synced_at=stamp, source_sync="synced")
355
+ elif synced_on is not None:
356
+ fact.update(source_synced_on=synced_on, source_sync="synced_date_only",
357
+ source_sync_reason="The source reports only a sync date, without a time.")
358
  elif (own_failed or job.get("state") == "failed"
359
  or key in odoo_keys and odoo.get("phase") == "error"):
360
  fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
api/main.py CHANGED
@@ -758,13 +758,6 @@ import app_routes
758
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
759
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
760
  _mount_router("routes_amazon_projects")
761
- _mount_router("routes_amazon_pdp")
762
- _mount_router("routes_pdp_text")
763
- _mount_router("routes_pdp_images")
764
- if _mount_router("routes_pdp_publish"):
765
- # Amazon fetches published image URLs without a session, so the media router carries no auth.
766
- from routes_pdp_publish import media_router as _pdp_media_router
767
- app.include_router(_pdp_media_router)
768
  _mount_router("routes_competitors")
769
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
770
  _mount_router("routes_sku_actions")
@@ -1350,8 +1343,6 @@ def _pull_amazon(why, vendor=True):
1350
  # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
1351
  _install_amazon_publication(tenant_key, rt, why)
1352
  if _amazon.first_load_job_live(rt):
1353
- from harness import mirror_store
1354
- mirror_store.record_sync_attempt("amazon", rt)
1355
  print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
1356
  return False
1357
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
@@ -1387,7 +1378,6 @@ def _pull_amazon(why, vendor=True):
1387
  except Exception as exc:
1388
  # A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
1389
  if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
1390
- mirror_store.record_sync_attempt("amazon", rt)
1391
  print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
1392
  flush=True)
1393
  return False
@@ -1759,16 +1749,8 @@ def _amazon_sync_receipt(report, rt=None):
1759
 
1760
 
1761
  def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1762
- """Name a classified failure or a completed publication's typed entity hold."""
1763
  if ok:
1764
- if provider == "odoo":
1765
- from harness import mirror_store
1766
- held = mirror_store.odoo_moving_entities(cause)
1767
- if held:
1768
- entity = sorted(held)[0]
1769
- others = f" {len(held) - 1} other entities were held." if len(held) > 1 else ""
1770
- return (f"Odoo {entity} changed during all 3 full reads. Its previous rows were kept."
1771
- + others + " Other completed entities published.")
1772
  return detail
1773
  classifier, sentence = _sync_now_cause(cause)
1774
  if classifier == "nothing_due":
@@ -1969,35 +1951,9 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
1969
  **({"after_entity": _after_entity} if _after_entity else {}))
1970
  if not report and selected_entities != []:
1971
  raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
1972
- held = mirror_store.odoo_moving_entities(report)
1973
- if any(row.get("phase") != "live" and name not in held for name, row in report.items()):
1974
  raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
1975
- waiting = mirror_store.pending_sync_entities("odoo", tenant, rt)
1976
- if held and waiting == []:
1977
- import json
1978
- store = mirror_store.for_connector("odoo")
1979
- with store.lock:
1980
- published, _ = store._connect(tenant, rt)
1981
- candidate = None
1982
- try:
1983
- path = rt.datastore_path.with_suffix(".odoo.candidate.duckdb")
1984
- candidate = datastore.apply_limits(datastore.duckdb.connect(str(path), read_only=True))
1985
- retained = {row[0] for row in candidate.execute("SELECT name FROM _candidate_held").fetchall()}
1986
- publication = json.loads(store._download(rt, mirror_store._PASS_KEY) or b"{}")
1987
- installed = datastore._publication(published, "odoo")
1988
- if (retained == set(held) and publication.get("state") == "published"
1989
- and publication.get("tenant") == tenant and publication.get("connector") == "odoo"
1990
- and publication.get("completed_at") and installed.get("state") == "complete"
1991
- and publication.get("snapshot_id")
1992
- and publication.get("snapshot_id") == installed.get("snapshot_id")):
1993
- waiting = retained
1994
- finally:
1995
- if candidate is not None:
1996
- candidate.close()
1997
- published.close()
1998
- if ((held and (set(waiting or ()) != set(held)
1999
- or not any(row.get("phase") == "live" for row in report.values())))
2000
- or (not held and waiting is not None)):
2001
  raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
2002
  problems = []
2003
  elif provider == "amazon_sp":
@@ -2090,15 +2046,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
2090
  detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
2091
  cause=amazon_failure or problems[0])
2092
  else:
2093
- ok, detail = True, _sync_now_detail(provider, tenant, ok=True, detail="Sync finished.",
2094
- cause=report if provider == "odoo" else None)
2095
  if amazon_detail and not problems:
2096
  detail += " " + amazon_detail
2097
  except _SyncNowRefused as exc:
2098
  detail = str(exc)
2099
  except Exception as exc: # noqa: BLE001
2100
  detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
2101
- if progress and not amazon_detail and not (provider == "odoo" and ok and held):
2102
  import re
2103
  suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
2104
  cause_code = suffix[0] if suffix else ""
@@ -3565,8 +3520,6 @@ def _amazon_desk_pass(tenant, *, project=True):
3565
  job.join()
3566
  state["job"] = None
3567
  _project_desk(state["outcome"], tenant)
3568
- import sync_runtime
3569
- sync_runtime.relay_at_boundary()
3570
  receipt = connector_sync.sync_request("amazon_sp", tenant)
3571
  if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
3572
  or max(receipt.get("notBefore") or 0,
@@ -3884,6 +3837,7 @@ def _store_resync_loop():
3884
  _resync_status = globals().get("_publish_resync_state")
3885
  if _resync_status:
3886
  _resync_status(_cad["tenant"], started=True)
 
3887
  try:
3888
  selected_entities = _configured_mirror_entities(_cad["tenant"])
3889
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
@@ -3914,6 +3868,26 @@ def _store_resync_loop():
3914
  "source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
3915
  except Exception: # noqa: BLE001
3916
  pass # The resident mirror state above still reports this pass.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3917
  else:
3918
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
3919
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
 
758
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
759
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
760
  _mount_router("routes_amazon_projects")
 
 
 
 
 
 
 
761
  _mount_router("routes_competitors")
762
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
763
  _mount_router("routes_sku_actions")
 
1343
  # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
1344
  _install_amazon_publication(tenant_key, rt, why)
1345
  if _amazon.first_load_job_live(rt):
 
 
1346
  print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
1347
  return False
1348
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
 
1378
  except Exception as exc:
1379
  # A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
1380
  if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
 
1381
  print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
1382
  flush=True)
1383
  return False
 
1749
 
1750
 
1751
  def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1752
+ """Record one classified failure; leave successful receipt text untouched."""
1753
  if ok:
 
 
 
 
 
 
 
 
1754
  return detail
1755
  classifier, sentence = _sync_now_cause(cause)
1756
  if classifier == "nothing_due":
 
1951
  **({"after_entity": _after_entity} if _after_entity else {}))
1952
  if not report and selected_entities != []:
1953
  raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
1954
+ if any(row.get("phase") != "live" for row in report.values()):
 
1955
  raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
1956
+ if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1957
  raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
1958
  problems = []
1959
  elif provider == "amazon_sp":
 
2046
  detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
2047
  cause=amazon_failure or problems[0])
2048
  else:
2049
+ ok, detail = True, "Sync finished."
 
2050
  if amazon_detail and not problems:
2051
  detail += " " + amazon_detail
2052
  except _SyncNowRefused as exc:
2053
  detail = str(exc)
2054
  except Exception as exc: # noqa: BLE001
2055
  detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
2056
+ if progress and not amazon_detail:
2057
  import re
2058
  suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
2059
  cause_code = suffix[0] if suffix else ""
 
3520
  job.join()
3521
  state["job"] = None
3522
  _project_desk(state["outcome"], tenant)
 
 
3523
  receipt = connector_sync.sync_request("amazon_sp", tenant)
3524
  if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
3525
  or max(receipt.get("notBefore") or 0,
 
3837
  _resync_status = globals().get("_publish_resync_state")
3838
  if _resync_status:
3839
  _resync_status(_cad["tenant"], started=True)
3840
+ _pass_started_epoch = _t.time()
3841
  try:
3842
  selected_entities = _configured_mirror_entities(_cad["tenant"])
3843
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
 
3868
  "source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
3869
  except Exception: # noqa: BLE001
3870
  pass # The resident mirror state above still reports this pass.
3871
+ elif not _manual:
3872
+ # A held pass that published this pass's snapshot is a success. A refused publication writes nothing.
3873
+ try:
3874
+ from datetime import datetime as _held_dt
3875
+ from harness import runtime as _held_runtime
3876
+ import routes_connectors as _held_receipts
3877
+ _held_rt = _held_runtime.get_runtime(_cad["tenant"])
3878
+ _held_pass = (connector_sync.status(
3879
+ _held_rt, "odoo", RESYNC_STATE.get("connectionId") or "default",
3880
+ fresh=True) or {}).get("pass") or {}
3881
+ _published = (connector_sync._utc_stamp(_held_pass["publishedAt"])
3882
+ if _held_pass.get("publishedAt") else None)
3883
+ if _published and (_held_dt.fromisoformat(_published).timestamp()
3884
+ >= int(_pass_started_epoch)):
3885
+ RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
3886
+ _held_receipts.record_odoo_stage(
3887
+ _held_rt, RESYNC_STATE.get("connectionId") or "default",
3888
+ "source", "ready", at=_published, cadence_seconds=_every)
3889
+ except Exception: # noqa: BLE001
3890
+ pass # The pass row in the status document still reports this publication.
3891
  else:
3892
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
3893
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
api/odoo_relational.py CHANGED
@@ -6483,7 +6483,13 @@ def sync_freshness(rt, now=None):
6483
  status = status if isinstance(status, dict) else {}
6484
  from core import connector_sync
6485
  reader = getattr(connector_sync, "status", None)
6486
- canonical = reader(rt, "odoo", fresh=True) if reader is not None and rt is not None else None
 
 
 
 
 
 
6487
  canonical = canonical if isinstance(canonical, dict) else None
6488
  pass_status = _sync_pass_freshness(canonical, cfg, now)
6489
  phase = str(status.get("phase") or "unknown")
 
6483
  status = status if isinstance(status, dict) else {}
6484
  from core import connector_sync
6485
  reader = getattr(connector_sync, "status", None)
6486
+ canonical = None
6487
+ if reader is not None and rt is not None:
6488
+ # The resync loop writes under the keychain default id. Read that slot first, else "default".
6489
+ default_id = default_connection_id(rt)
6490
+ canonical = reader(rt, "odoo", default_id, fresh=True) if default_id else None
6491
+ if not isinstance(canonical, dict):
6492
+ canonical = reader(rt, "odoo", fresh=True)
6493
  canonical = canonical if isinstance(canonical, dict) else None
6494
  pass_status = _sync_pass_freshness(canonical, cfg, now)
6495
  phase = str(status.get("phase") or "unknown")
api/process_role.py CHANGED
@@ -56,20 +56,6 @@ READY_FILE = "READY.json"
56
  PRIMARY_LOCK = "web-primary.lock"
57
  PRIMARY_FILE = "web-primary.json"
58
  INSTALL_ATTEMPTS = 3
59
- INSTALL_RETRY_SECONDS = 3600
60
- _INSTALL_CAUSES = (
61
- (TimeoutError, "The install timed out."),
62
- (ConnectionError, "The saved sync could not be downloaded."),
63
- (PermissionError, "The install could not access the saved sync."),
64
- (FileNotFoundError, "The saved sync is missing a required file."),
65
- (OSError, "The install could not read or write its files."),
66
- (MemoryError, "The install ran out of memory."),
67
- (ValueError, "The saved sync failed validation."),
68
- (TypeError, "The saved sync has an invalid structure."),
69
- (LookupError, "The saved sync is missing required data."),
70
- (RuntimeError, "The install stopped with an internal error."),
71
- (Exception, "The install failed unexpectedly."),
72
- )
73
  INCOMPLETE_GRACE_SECONDS = 3600
74
  #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
75
  WORKER_STATUS = "sync_worker"
@@ -550,34 +536,37 @@ def pending_handoffs(root=None):
550
 
551
 
552
  def _failure_map(root=None):
553
- """Return each published snapshot's own failed install receipt."""
554
  if not is_primary():
555
  stored = primary_state(root).get("installFailures") or {}
556
  return {tuple(key.split("/", 1)): row for key, row in stored.items()
557
  if "/" in key and isinstance(row, dict)}
558
- return {key: dict(row) for key, snapshot in dict(_STORED["mirrors"]).items()
559
- if (row := _INSTALL_FAILURES.get((key, snapshot)))}
560
-
561
-
562
- def served_behind(tenant, connector, root=None, *, worker=None):
 
 
 
 
 
 
 
563
  """`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
564
 
565
  None when nothing is published, the published snapshot is served, or no install of it failed yet.
566
  """
567
  key = (str(tenant or "").strip().lower(), str(connector))
568
- worker = worker or {}
569
- published = ((worker.get("mirrors") or {}).get(key[1]) or {}).get("snapshotId") or _known_mirrors(root).get(key)
570
  if not published:
571
  return None
572
  served = _installed_map(root).get(key)
573
- failure = _failure_map(root).get(key) or (worker.get("installs") or {}).get(_install_key(key, published)) or {}
574
- if (failure.get("tenant"), failure.get("connector"), failure.get("snapshotId")) != (*key, published):
575
- return None
576
  attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
577
  if served == published or attempts < 1:
578
  return None
579
- return {"published": published, "served": served or failure.get("served"), "attempts": attempts,
580
- "error": failure.get("type"), "cause": failure.get("cause"), "nextRetryAt": failure.get("nextRetryAt")}
581
 
582
 
583
  # ----------------------------------------------------------------------------- several web workers
@@ -785,44 +774,6 @@ def _remove_dir(path):
785
  shutil.rmtree(path, ignore_errors=True)
786
 
787
 
788
- def _install_key(key, snapshot):
789
- return "/".join((*key, snapshot))
790
-
791
-
792
- def _install_due(key, snapshot, stamp):
793
- failure = _INSTALL_FAILURES.get((key, snapshot)) or {}
794
- return failure.get("attempts", 0) < INSTALL_ATTEMPTS or stamp >= failure["nextRetryAt"]
795
-
796
-
797
- def _record_install(key, snapshot, stamp, *, exc=None, rt=None, changed=True):
798
- """Keep one scoped install receipt and persist its safe cause and retry deadline."""
799
- from core import connector_sync
800
-
801
- identity = (key, snapshot)
802
- was_failed = identity in _INSTALL_FAILURES
803
- row = {"tenant": key[0], "connector": key[1], "snapshotId": snapshot, "at": stamp,
804
- "state": "failed" if exc is not None else "installed"}
805
- if exc is not None:
806
- kind, cause = next((kind, cause) for kind, cause in _INSTALL_CAUSES if isinstance(exc, kind))
807
- attempts = (_INSTALL_FAILURES.get(identity) or {}).get("attempts", 0) + 1
808
- row.update(attempts=attempts, type=kind.__name__, cause=cause, served=_INSTALLED.get(key),
809
- nextRetryAt=stamp + (INSTALL_RETRY_SECONDS if attempts >= INSTALL_ATTEMPTS else POLL_SECONDS))
810
- _INSTALL_FAILURES[identity] = row
811
- LAST_INSTALL_ERROR.clear()
812
- LAST_INSTALL_ERROR.update(row)
813
- else:
814
- row["changed"] = changed is not False
815
- _INSTALL_FAILURES.pop(identity, None)
816
- if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector"),
817
- LAST_INSTALL_ERROR.get("snapshotId")) == (*key, snapshot):
818
- LAST_INSTALL_ERROR.clear()
819
- if rt is not None and (exc is not None or was_failed):
820
- written = connector_sync.update_status(rt, WORKER_STATUS, "installs", {_install_key(key, snapshot): row})
821
- if not written:
822
- print(f"[process-role] install status not stored for {key[0]}/{key[1]}", flush=True)
823
- return row
824
-
825
-
826
  def install_ready(root=None, installer=None, *, now=None):
827
  """Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
828
 
@@ -835,22 +786,25 @@ def install_ready(root=None, installer=None, *, now=None):
835
  for key, rows in ready_handoffs(root).items():
836
  completed_at, newest_dir, ready = rows[-1]
837
  snapshot = ready.get("snapshotId") or newest_dir.name
838
- if _INSTALLED.get(key) != snapshot and _install_due(key, snapshot, stamp):
839
  try:
840
  changed = installer(key[1], key[0], ready)
841
- except Exception as exc: # Retry failed installs without exposing their diagnostic text.
842
- failure = _record_install(key, snapshot, stamp, exc=exc)
 
 
843
  print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
844
- f"({failure['type']})", flush=True)
845
  continue
846
  _INSTALLED[key] = snapshot
847
- _record_install(key, snapshot, stamp, changed=changed)
848
  if changed is False:
849
  # Already resident: nothing was rebuilt, so this is not an install.
850
  for _at, older, _ready in rows[:-1]:
851
  _remove_dir(older)
852
  continue
853
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
 
 
854
  installed.append(key)
855
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
856
  for _at, older, _ready in rows[:-1]:
@@ -908,31 +862,29 @@ def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
908
  installer = installer or _install_from_store
909
  stamp = time.time() if now is None else now
910
  installed = []
911
- stored = (stored_worker(rt).get("installs") or {}) if rt is not None else {}
912
  for connector, row in sorted((mirrors or {}).items()):
913
  snapshot = row.get("snapshotId") if isinstance(row, dict) else None
914
  if not snapshot:
915
  continue
916
- key = (str(tenant).strip().lower(), str(connector))
917
  _STORED["mirrors"][key] = snapshot
918
- previous = stored.get(_install_key(key, snapshot)) or {}
919
- if (previous.get("state") == "failed" and (key, snapshot) not in _INSTALL_FAILURES
920
- and (previous.get("tenant"), previous.get("connector"), previous.get("snapshotId")) == (*key, snapshot)):
921
- _INSTALL_FAILURES[(key, snapshot)] = previous
922
- if _INSTALLED.get(key) == snapshot or not _install_due(key, snapshot, stamp):
923
  continue
924
  try:
925
  changed = installer(key[1], key[0], rt, snapshot)
926
- except Exception as exc: # Preserve the serving copy and retry this snapshot on its own deadline.
927
- failure = _record_install(key, snapshot, stamp, exc=exc, rt=rt)
 
 
928
  print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
929
- f"({failure['type']})", flush=True)
930
  continue
931
  _INSTALLED[key] = snapshot
932
- _record_install(key, snapshot, stamp, rt=rt, changed=changed)
933
  if changed is False:
934
  continue
935
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
 
 
936
  installed.append(key)
937
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
938
  return installed
@@ -1283,23 +1235,6 @@ def start_web_services():
1283
  return True
1284
 
1285
 
1286
- def _install_diagnostics(stored):
1287
- """Read this tenant's current failed snapshot and last changed install across web processes."""
1288
- tenant = writer_tenant()
1289
- rows = {**(stored.get("installs") or {}), **{
1290
- _install_key(key, snapshot): row for (key, snapshot), row in dict(_INSTALL_FAILURES).items()}}
1291
- rows = sorted((row for row in rows.values() if isinstance(row, dict) and row.get("tenant") == tenant),
1292
- key=lambda row: row.get("at", 0), reverse=True)
1293
- mirrors = {key[1]: snapshot for key, snapshot in _known_mirrors().items() if key[0] == tenant}
1294
- mirrors.update({connector: row.get("snapshotId") for connector, row in (stored.get("mirrors") or {}).items()})
1295
- failure = next((row for row in rows if row.get("state") == "failed"
1296
- and mirrors.get(row["connector"]) == row["snapshotId"]
1297
- and _INSTALLED.get((tenant, row["connector"])) != row["snapshotId"]), None)
1298
- installed = next((row for row in rows if row.get("state") == "installed" and row.get("changed")), None)
1299
- local = dict(LAST_INSTALL) if LAST_INSTALL.get("tenant") == tenant else None
1300
- return local or installed, failure
1301
-
1302
-
1303
  def status():
1304
  """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
1305
 
@@ -1324,13 +1259,12 @@ def status():
1324
  fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
1325
  "lastExit": None, "jobId": row.get("jobId")}
1326
  import sync_runtime
1327
- last_install, last_error = _install_diagnostics(stored)
1328
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
1329
  "syncRuntime": sync_runtime.runtime_name()[0],
1330
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
1331
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1332
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1333
- "restarts": _WORKER["restarts"], "lastInstall": last_install,
1334
- "lastInstallError": last_error,
1335
  "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
1336
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
 
56
  PRIMARY_LOCK = "web-primary.lock"
57
  PRIMARY_FILE = "web-primary.json"
58
  INSTALL_ATTEMPTS = 3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  INCOMPLETE_GRACE_SECONDS = 3600
60
  #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
61
  WORKER_STATUS = "sync_worker"
 
536
 
537
 
538
  def _failure_map(root=None):
539
+ """{(tenant, connector): {snapshotId, attempts, type, at}} for each published snapshot that failed to install."""
540
  if not is_primary():
541
  stored = primary_state(root).get("installFailures") or {}
542
  return {tuple(key.split("/", 1)): row for key, row in stored.items()
543
  if "/" in key and isinstance(row, dict)}
544
+ last, failures = dict(LAST_INSTALL_ERROR), {}
545
+ for key, snapshot in dict(_STORED["mirrors"]).items():
546
+ attempts = _INSTALL_FAILURES.get((key, snapshot), 0)
547
+ if attempts:
548
+ # LAST_INSTALL_ERROR is one slot: its type and time belong only to the snapshot it names.
549
+ same = (last.get("tenant"), last.get("connector"), last.get("snapshotId")) == (*key, snapshot)
550
+ failures[key] = {"snapshotId": snapshot, "attempts": attempts,
551
+ "type": last.get("type") if same else None, "at": last.get("at") if same else None}
552
+ return failures
553
+
554
+
555
+ def served_behind(tenant, connector, root=None):
556
  """`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
557
 
558
  None when nothing is published, the published snapshot is served, or no install of it failed yet.
559
  """
560
  key = (str(tenant or "").strip().lower(), str(connector))
561
+ published = _known_mirrors(root).get(key)
 
562
  if not published:
563
  return None
564
  served = _installed_map(root).get(key)
565
+ failure = _failure_map(root).get(key) or {}
 
 
566
  attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
567
  if served == published or attempts < 1:
568
  return None
569
+ return {"published": published, "served": served, "attempts": attempts, "error": failure.get("type")}
 
570
 
571
 
572
  # ----------------------------------------------------------------------------- several web workers
 
774
  shutil.rmtree(path, ignore_errors=True)
775
 
776
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
777
  def install_ready(root=None, installer=None, *, now=None):
778
  """Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
779
 
 
786
  for key, rows in ready_handoffs(root).items():
787
  completed_at, newest_dir, ready = rows[-1]
788
  snapshot = ready.get("snapshotId") or newest_dir.name
789
+ if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
790
  try:
791
  changed = installer(key[1], key[0], ready)
792
+ except Exception as exc: # noqa: BLE001
793
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
794
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
795
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
796
  print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
797
+ f"({type(exc).__name__}: {exc})", flush=True)
798
  continue
799
  _INSTALLED[key] = snapshot
 
800
  if changed is False:
801
  # Already resident: nothing was rebuilt, so this is not an install.
802
  for _at, older, _ready in rows[:-1]:
803
  _remove_dir(older)
804
  continue
805
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
806
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
807
+ LAST_INSTALL_ERROR.clear()
808
  installed.append(key)
809
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
810
  for _at, older, _ready in rows[:-1]:
 
862
  installer = installer or _install_from_store
863
  stamp = time.time() if now is None else now
864
  installed = []
 
865
  for connector, row in sorted((mirrors or {}).items()):
866
  snapshot = row.get("snapshotId") if isinstance(row, dict) else None
867
  if not snapshot:
868
  continue
869
+ key = (str(tenant), str(connector))
870
  _STORED["mirrors"][key] = snapshot
871
+ if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
 
 
 
 
872
  continue
873
  try:
874
  changed = installer(key[1], key[0], rt, snapshot)
875
+ except Exception as exc: # noqa: BLE001
876
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
877
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
878
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
879
  print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
880
+ f"({type(exc).__name__}: {exc})", flush=True)
881
  continue
882
  _INSTALLED[key] = snapshot
 
883
  if changed is False:
884
  continue
885
  LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
886
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
887
+ LAST_INSTALL_ERROR.clear()
888
  installed.append(key)
889
  print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
890
  return installed
 
1235
  return True
1236
 
1237
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1238
  def status():
1239
  """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
1240
 
 
1259
  fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
1260
  "lastExit": None, "jobId": row.get("jobId")}
1261
  import sync_runtime
 
1262
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
1263
  "syncRuntime": sync_runtime.runtime_name()[0],
1264
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
1265
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1266
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
1267
+ "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
1268
+ "lastInstallError": dict(LAST_INSTALL_ERROR) or None,
1269
  "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
1270
  "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
api/routes_amazon_projects.py CHANGED
@@ -38,7 +38,7 @@ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "
38
  "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
39
  #: Project JSON keys a client may echo back. They are ignored, never written.
40
  READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
41
- "rollup", "impact", "via", "figures", "type", "listed"))
42
 
43
 
44
  def _call(fn, *args, **kwargs):
@@ -116,7 +116,6 @@ def _project(record, today):
116
  impact = dict(record.get("impact") or {})
117
  impact.pop(projects.RESULT_CONFIGS_KEY, None)
118
  return {"id": record["id"], "name": record["name"], "status": record["status"],
119
- "type": record.get("type", "project"), "listed": record.get("listed", True),
120
  "priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
121
  "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
122
  "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
@@ -268,7 +267,7 @@ def _row_ids(cells):
268
  return sorted(cells)
269
 
270
 
271
- def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_profit.VIA):
272
  import core.user_tables as ut
273
 
274
  def _set(cur):
@@ -284,7 +283,7 @@ def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_pr
284
  row[key] = value
285
  return cur
286
 
287
- ut.update_with_activity(table_key, _set, st=st, by=by, via=via,
288
  row_ids=_row_ids(cells))
289
 
290
 
@@ -875,11 +874,6 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi
875
  def delete_project(project_id: str, session: Session = Depends(require_session)):
876
  """Delete one project and its targets."""
877
  runtime = _writer(session)
878
- record = _call(projects.get, runtime, project_id)
879
- if record.get("type") == "pdp":
880
- from routes_amazon_pdp import delete_workflow
881
- delete_workflow(project_id, session=session)
882
- return {"ok": True, "id": project_id}
883
  pairs = _target_pairs(runtime, project_id)
884
  _call(projects.delete, runtime, project_id, by=_by(session))
885
  _repaint(runtime, pairs)
 
38
  "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
39
  #: Project JSON keys a client may echo back. They are ignored, never written.
40
  READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
41
+ "rollup", "impact", "via", "figures"))
42
 
43
 
44
  def _call(fn, *args, **kwargs):
 
116
  impact = dict(record.get("impact") or {})
117
  impact.pop(projects.RESULT_CONFIGS_KEY, None)
118
  return {"id": record["id"], "name": record["name"], "status": record["status"],
 
119
  "priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
120
  "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
121
  "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
 
267
  return sorted(cells)
268
 
269
 
270
+ def _write_cells(st, table_key, cells):
271
  import core.user_tables as ut
272
 
273
  def _set(cur):
 
283
  row[key] = value
284
  return cur
285
 
286
+ ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
287
  row_ids=_row_ids(cells))
288
 
289
 
 
874
  def delete_project(project_id: str, session: Session = Depends(require_session)):
875
  """Delete one project and its targets."""
876
  runtime = _writer(session)
 
 
 
 
 
877
  pairs = _target_pairs(runtime, project_id)
878
  _call(projects.delete, runtime, project_id, by=_by(session))
879
  _repaint(runtime, pairs)
api/routes_brand.py CHANGED
@@ -35,16 +35,13 @@ from __future__ import annotations
35
 
36
  import os
37
  import re
38
- from hashlib import sha256
39
  from xml.etree import ElementTree
40
  from pathlib import Path
41
 
42
- from fastapi import APIRouter, Depends, Request
43
- from fastapi.concurrency import run_in_threadpool
44
- from fastapi.responses import FileResponse, Response
45
 
46
  from deps import Session, err, require_session
47
- from core import tenant_brand
48
 
49
  router = APIRouter(prefix="/api/v1")
50
 
@@ -80,53 +77,6 @@ _MEDIA = {".png": "image/png", ".svg": "image/svg+xml"}
80
  _HERE = Path(__file__).resolve().parent
81
 
82
 
83
- def _logo_call(fn, *args):
84
- try:
85
- return fn(*args)
86
- except tenant_brand.LogoStorageError as error:
87
- raise err(503, "logo_storage_unavailable", str(error)) from None
88
- except ValueError as error:
89
- raise err(422, "invalid_logo", str(error)) from None
90
-
91
-
92
- def _logo_admin(session):
93
- if not session.admin:
94
- raise err(403, "admin_required", "Only an administrator can change the tenant logo.")
95
-
96
-
97
- @router.get("/tenant-brand/logo")
98
- def tenant_logo(session: Session = Depends(require_session)):
99
- """Serve this tenant's logo to any signed-in user without changing state."""
100
- result = _logo_call(tenant_brand.logo, session.runtime)
101
- if result is None:
102
- raise err(404, "no_logo", "This workspace has no logo.")
103
- data, mime = result
104
- return Response(data, media_type=mime, headers={
105
- "ETag": f'"{sha256(data).hexdigest()}"', "Cache-Control": "private, no-cache",
106
- "X-Content-Type-Options": "nosniff"})
107
-
108
-
109
- @router.put("/tenant-brand/logo")
110
- async def put_tenant_logo(request: Request, session: Session = Depends(require_session)):
111
- """Accept at most 2 MiB of raw PNG or JPEG bytes from an administrator."""
112
- _logo_admin(session)
113
- data = bytearray()
114
- async for chunk in request.stream():
115
- if len(data) + len(chunk) > tenant_brand.MAX_BYTES:
116
- raise err(413, "logo_too_large", tenant_brand.SIZE_LIMIT)
117
- data.extend(chunk)
118
- return await run_in_threadpool(
119
- _logo_call, tenant_brand.put_logo, session.runtime, bytes(data), session.uname)
120
-
121
-
122
- @router.delete("/tenant-brand/logo", status_code=204)
123
- def delete_tenant_logo(session: Session = Depends(require_session)):
124
- """Remove this tenant's logo for an administrator."""
125
- _logo_admin(session)
126
- _logo_call(tenant_brand.delete_logo, session.runtime)
127
- return Response(status_code=204)
128
-
129
-
130
  def _roots():
131
  env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
132
  out = [Path(env)] if env else []
 
35
 
36
  import os
37
  import re
 
38
  from xml.etree import ElementTree
39
  from pathlib import Path
40
 
41
+ from fastapi import APIRouter, Depends
42
+ from fastapi.responses import FileResponse
 
43
 
44
  from deps import Session, err, require_session
 
45
 
46
  router = APIRouter(prefix="/api/v1")
47
 
 
77
  _HERE = Path(__file__).resolve().parent
78
 
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  def _roots():
81
  env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
82
  out = [Path(env)] if env else []
api/routes_changes.py CHANGED
@@ -62,7 +62,6 @@ import core.store as store
62
  import core.user_tables as user_tables
63
  import modules.customer_data as customer_data
64
  import modules.product_data as product_data
65
- import virtual_sources
66
 
67
  from deps import Session, err, require_changes_session, require_session
68
 
@@ -201,8 +200,6 @@ def _may_watch(session, scope):
201
  @router.get("/changes")
202
  def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
203
  """Serve legacy change checks without making Pg a resident client."""
204
- if virtual_sources.get(_clean_scope(scope)) is not None:
205
- return {"scope": _clean_scope(scope), "tokens": {}, "backend": store.backend()}
206
  names = buckets_for(scope)
207
  if names is None:
208
  raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
 
62
  import core.user_tables as user_tables
63
  import modules.customer_data as customer_data
64
  import modules.product_data as product_data
 
65
 
66
  from deps import Session, err, require_changes_session, require_session
67
 
 
200
  @router.get("/changes")
201
  def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
202
  """Serve legacy change checks without making Pg a resident client."""
 
 
203
  names = buckets_for(scope)
204
  if names is None:
205
  raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
api/routes_competitors.py CHANGED
@@ -39,19 +39,6 @@ def _open_mirror(runtime):
39
  cur.close()
40
 
41
 
42
- @contextmanager
43
- def _boundary_errors():
44
- try:
45
- yield
46
- except rivals.MirrorUnreadable as exc:
47
- raise err(503, "amazon_mirror_unreadable",
48
- f"The Amazon data could not be read ({exc}). Try again shortly.") from None
49
- except rivals.BackendUnavailable:
50
- raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
51
- except RuntimeError:
52
- raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
53
-
54
-
55
  def _rt(session):
56
  session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
57
  try:
@@ -164,23 +151,6 @@ def competitors_list(grain: str, key: str, session: Session = Depends(require_se
164
  return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
165
 
166
 
167
- @router.get("/competitors/pool")
168
- def competitors_pool(grain: str, key: str, session: Session = Depends(require_session)):
169
- """Read the eligible Setup pool and its stored daily facts through the caller's tenant session."""
170
- runtime = _rt(session)
171
- key = _grain(grain, key)
172
- with _boundary_errors(), _open_mirror(runtime) as cur:
173
- target, parent = _target(cur, grain, key)
174
- if parent is not None:
175
- raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
176
- pool = rivals.pool(runtime, cur, target)
177
- payload = {field: pool[field] for field in ('asin', 'productType', 'dataAsOf', 'week', 'month',
178
- 'keywordRanksOn', 'cause', 'held', 'removed', 'competitors')}
179
- if not payload['competitors']:
180
- payload['cause'] = rivals.NO_POOL
181
- return {**payload, 'canEdit': routes_amazon_profit._may_edit(session)}
182
-
183
-
184
  def _writable(session, body):
185
  runtime = _rt(session)
186
  if not routes_amazon_profit._may_edit(session):
@@ -188,8 +158,16 @@ def _writable(session, body):
188
  body = body if isinstance(body, dict) else {}
189
  grain = body.get("grain")
190
  key = _grain(grain, body.get("key"))
191
- with _boundary_errors(), _open_mirror(runtime) as cur:
192
- target, parent = _target(cur, grain, key)
 
 
 
 
 
 
 
 
193
  if parent is not None:
194
  raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
195
  try:
 
39
  cur.close()
40
 
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  def _rt(session):
43
  session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
44
  try:
 
151
  return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
152
 
153
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
  def _writable(session, body):
155
  runtime = _rt(session)
156
  if not routes_amazon_profit._may_edit(session):
 
158
  body = body if isinstance(body, dict) else {}
159
  grain = body.get("grain")
160
  key = _grain(grain, body.get("key"))
161
+ try:
162
+ with _open_mirror(runtime) as cur:
163
+ target, parent = _target(cur, grain, key)
164
+ except rivals.MirrorUnreadable as exc:
165
+ raise err(503, "amazon_mirror_unreadable",
166
+ f"The Amazon data could not be read ({exc}). Try again shortly.") from None
167
+ except rivals.BackendUnavailable:
168
+ raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
169
+ except RuntimeError:
170
+ raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
171
  if parent is not None:
172
  raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
173
  try:
api/routes_connectors.py CHANGED
@@ -3315,6 +3315,9 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3315
  state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
3316
  else tick.get("state") or "never_run")
3317
  reason = row_reason(table, state)
 
 
 
3318
  # A source-internal mismatch published the table. Its warning stays until a clean reconcile.
3319
  warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
3320
  schedule.append({**row, "period": _amazon_table_period(table),
@@ -3354,7 +3357,9 @@ def _amazon_sync_staleness(completed, schedule, now=None):
3354
  fastest = max(amazon_client.CADENCE[table]["perDay"]
3355
  for table in amazon_client.GROUP_A)
3356
  allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
3357
- stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest]
 
 
3358
  if not stamps:
3359
  stamps = [completed]
3360
  now = now or datetime.now(timezone.utc)
 
3315
  state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
3316
  else tick.get("state") or "never_run")
3317
  reason = row_reason(table, state)
3318
+ if state == "refused" and (reason or "").startswith("Amazon does not offer this report"):
3319
+ # A report Amazon does not offer this seller is not available, never a failed sync.
3320
+ state = "not_offered"
3321
  # A source-internal mismatch published the table. Its warning stays until a clean reconcile.
3322
  warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
3323
  schedule.append({**row, "period": _amazon_table_period(table),
 
3357
  fastest = max(amazon_client.CADENCE[table]["perDay"]
3358
  for table in amazon_client.GROUP_A)
3359
  allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
3360
+ # A not_offered row can carry an old watermark stamp. Amazon never sends that report, so it never ages the card.
3361
+ stamps = [row.get("lastSyncAt") for row in schedule
3362
+ if row["perDay"] == fastest and row.get("state") != "not_offered"]
3363
  if not stamps:
3364
  stamps = [completed]
3365
  now = now or datetime.now(timezone.utc)
api/routes_grid.py CHANGED
@@ -20,7 +20,6 @@ from inspect import signature
20
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
21
 
22
  from deps import Session, err, module_gate, perms, require_session
23
- import virtual_sources
24
 
25
  router = APIRouter(prefix="/api/v1")
26
 
@@ -50,7 +49,6 @@ def row_patch_receipt(scope):
50
  raise err(400, 'bad_op_id', str(exc)) from exc
51
  session = bound.arguments['session']
52
  scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
53
- virtual_sources.refuse_write(scope_key)
54
  bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
55
 
56
  def apply():
@@ -235,7 +233,6 @@ def grid_choice_remove(scope: str = "customer", body: dict = Body(default=None),
235
 
236
  body = body if isinstance(body, dict) else {}
237
  scope = _scope_or_400(scope)
238
- virtual_sources.refuse_write(scope)
239
  key = str(body.get("key") or "").strip()[:80]
240
  options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
241
  confirm = body.get("confirm") is True
@@ -478,8 +475,6 @@ def workspace(scope: str = "customer",
478
  from routes_customers import grid_assembly
479
 
480
  scope = _scope_or_400(scope)
481
- if virtual_sources.get(scope) is not None:
482
- return {"workspace": virtual_sources.assembly(session, scope, with_rows=False)["workspace"]}
483
  # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
484
  # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
485
  # session (its wall is the table's own, inside the assembly).
@@ -1529,7 +1524,6 @@ def bulk_cells(body: dict = Body(default=None),
1529
  from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
1530
 
1531
  scope = _scope_or_400((body or {}).get("scopeKey"))
1532
- virtual_sources.refuse_write(scope)
1533
  if scope != "product":
1534
  # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
1535
  raise err(400, "scope_not_bulk_writable",
@@ -2132,7 +2126,6 @@ def grid_events_route(body: dict = Body(default=None),
2132
  # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
2133
  # what a scope is, or the surface you read is not the surface you wrote.
2134
  scope = _scope_or_400((body or {}).get("scopeKey"))
2135
- virtual_sources.refuse_write(scope)
2136
  if scope.startswith("ut_") and any(
2137
  isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
2138
  # Managed records require Route/Zone tools, including keys without stored definitions.
 
20
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
21
 
22
  from deps import Session, err, module_gate, perms, require_session
 
23
 
24
  router = APIRouter(prefix="/api/v1")
25
 
 
49
  raise err(400, 'bad_op_id', str(exc)) from exc
50
  session = bound.arguments['session']
51
  scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
 
52
  bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
53
 
54
  def apply():
 
233
 
234
  body = body if isinstance(body, dict) else {}
235
  scope = _scope_or_400(scope)
 
236
  key = str(body.get("key") or "").strip()[:80]
237
  options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
238
  confirm = body.get("confirm") is True
 
475
  from routes_customers import grid_assembly
476
 
477
  scope = _scope_or_400(scope)
 
 
478
  # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
479
  # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
480
  # session (its wall is the table's own, inside the assembly).
 
1524
  from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
1525
 
1526
  scope = _scope_or_400((body or {}).get("scopeKey"))
 
1527
  if scope != "product":
1528
  # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
1529
  raise err(400, "scope_not_bulk_writable",
 
2126
  # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
2127
  # what a scope is, or the surface you read is not the surface you wrote.
2128
  scope = _scope_or_400((body or {}).get("scopeKey"))
 
2129
  if scope.startswith("ut_") and any(
2130
  isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
2131
  # Managed records require Route/Zone tools, including keys without stored definitions.
api/routes_odoo_tables.py CHANGED
@@ -5096,12 +5096,10 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
5096
  def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
5097
  groupBy=None, groupKey=None, ids=None, cursor=None):
5098
  from harness import datastore
5099
- import virtual_sources
5100
 
5101
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
5102
  "sorts": _json_arg(sorts, "sorts"), "search": search})
5103
- virtual = virtual_sources.get(table_key) is not None
5104
- payload = virtual_sources.rows_payload(session, table_key, related=related) if virtual else None
5105
  window_offset = offset
5106
  if table_key == 'ut_activity':
5107
  if cursor is not None:
@@ -5140,9 +5138,7 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
5140
  options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
5141
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
5142
  cur=read['cur'], **read['frm'])
5143
- if table_key == 'ut_activity' or virtual:
5144
- if virtual and cursor is not None:
5145
- raise err(400, 'window_cursor_unsupported', 'Continue this source with an offset.')
5146
  win = datastore.window(**options)
5147
  win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
5148
  else:
@@ -5289,10 +5285,6 @@ def odoo_table_rows(table_key: str,
5289
  "limit, filters, sorts and search in the query string.")
5290
  ids = _record_ids(body)
5291
 
5292
- import virtual_sources
5293
- if virtual_sources.get(table_key) is not None:
5294
- return _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
5295
- sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor)
5296
  # Admission resolves this Amazon database without publishing a process-wide binding.
5297
  page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
5298
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
 
5096
  def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
5097
  groupBy=None, groupKey=None, ids=None, cursor=None):
5098
  from harness import datastore
 
5099
 
5100
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
5101
  "sorts": _json_arg(sorts, "sorts"), "search": search})
5102
+ payload = None
 
5103
  window_offset = offset
5104
  if table_key == 'ut_activity':
5105
  if cursor is not None:
 
5138
  options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
5139
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
5140
  cur=read['cur'], **read['frm'])
5141
+ if table_key == 'ut_activity':
 
 
5142
  win = datastore.window(**options)
5143
  win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
5144
  else:
 
5285
  "limit, filters, sorts and search in the query string.")
5286
  ids = _record_ids(body)
5287
 
 
 
 
 
5288
  # Admission resolves this Amazon database without publishing a process-wide binding.
5289
  page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
5290
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
api/routes_shares.py CHANGED
@@ -1221,8 +1221,6 @@ def get_cohort(scope: str, cohort_id: str, session: Session = Depends(require_se
1221
  @router.get("/share/{kind}/{oid}")
1222
  def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1223
  kind = _kind_or_400(kind)
1224
- if kind == "report" and not session.admin:
1225
- raise err(404, "no_object", "no such item, or it is not shared with this account")
1226
  if kind == "field":
1227
  table_key, field_key = shares.split_field_oid(oid)
1228
  if table_key and field_key:
@@ -1281,9 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1281
  **rec,
1282
  "role": role,
1283
  "mayAdminister": may_admin,
1284
- "maxGrantableRole": (
1285
- "view" if kind == "report" else "edit" if claiming else shares.max_grantable_role(
1286
- kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)),
1287
  "ownerLabel": owner_label,
1288
  # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
1289
  # them. `assignable_people` serves bare display names because `user`-kind CELLS store
@@ -1305,19 +1302,35 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1305
 
1306
 
1307
  def _people(tenant):
1308
- """Active tenant identities, excluding test accounts, ordered by display name."""
1309
- return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower())
1310
-
1311
-
1312
- def _report_entries_or_400(entries):
1313
- """A report grants download to named people alone."""
1314
- for entry in entries:
1315
- if (not isinstance(entry, dict) or not str(entry.get("user") or "").strip()
1316
- or str(entry.get("user")).strip() == shares.EVERYONE):
1317
- raise err(400, "bad_entries",
1318
- "Add people by name. A report is shared one person at a time.")
1319
- if str(entry.get("role") or "view").strip().lower() != "view":
1320
- raise err(400, "bad_role", "A report grant can only allow download.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1321
 
1322
 
1323
  def _stranger(session, kind, oid):
@@ -1342,11 +1355,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1342
  _notify=True):
1343
  """Apply the shared share admission, ceiling and atomic writer for both doors."""
1344
  kind = _kind_or_400(kind)
1345
- if kind == "report" and not session.admin:
1346
- if _stranger(session, kind, oid):
1347
- raise err(404, "no_object", "no such item, or it is not shared with this account")
1348
- raise err(403, "not_owner",
1349
- "Only an administrator can change who can download this report.")
1350
  if kind == "field":
1351
  table_key, field_key = shares.split_field_oid(oid)
1352
  if table_key and field_key:
@@ -1420,8 +1428,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1420
  raise err(400, "bad_entries",
1421
  "entries must be a list of {user, role}. Send [] to un-share, which is how "
1422
  "revoking is expressed")
1423
- if kind == "report":
1424
- _report_entries_or_400(entries)
1425
  _entries_or_400(session, entries)
1426
  # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
1427
  # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
@@ -1617,8 +1623,7 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
1617
 
1618
  try:
1619
  out = shares.set_grants(kind, oid, entries,
1620
- owner=(None if kind == "report" else
1621
- session.uname if claiming else rec["owner"]),
1622
  granter=session.uname,
1623
  st=session.runtime,
1624
  expect=rec["entries"])
 
1221
  @router.get("/share/{kind}/{oid}")
1222
  def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1223
  kind = _kind_or_400(kind)
 
 
1224
  if kind == "field":
1225
  table_key, field_key = shares.split_field_oid(oid)
1226
  if table_key and field_key:
 
1279
  **rec,
1280
  "role": role,
1281
  "mayAdminister": may_admin,
1282
+ "maxGrantableRole": "edit" if claiming else shares.max_grantable_role(
1283
+ kind, oid, session.uname, is_admin=_admin_share, st=session.runtime),
 
1284
  "ownerLabel": owner_label,
1285
  # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
1286
  # them. `assignable_people` serves bare display names because `user`-kind CELLS store
 
1302
 
1303
 
1304
  def _people(tenant):
1305
+ """[{username, name}] for this tenant — same population as `assignable_people`, with the
1306
+ BINDING identity alongside the display one.
1307
+
1308
+ ⛔⛔ W49-T23 / D-235 — A BLANK TENANT RETURNS NOBODY, NOT EVERYBODY. This read
1309
+ `if want and str(u.get('tenant') …) != want`, so a blank or `None` slug skipped the filter
1310
+ entirely and handed one workspace every tenant's roster: the exact `if want and …` fail-OPEN
1311
+ that W33-T37 closed in `core.users.assignable_people` / `avatar_map` one layer down, left
1312
+ behind in the copy that lives here. The three callers all pass `session.tenant`, which is
1313
+ `claims["t"]` and is already proven non-blank by `require_session` (an unresolvable slug is a
1314
+ 401 before any route runs), so this changes no answer any live caller gets today — it removes
1315
+ the shape, before a fourth caller arrives that does not carry that proof. Fail-closed, because
1316
+ from this function a blank slug and "every tenant at once" are the same observation, and one
1317
+ of the two callers below turns this roster into who may be GRANTED access.
1318
+ """
1319
+ try:
1320
+ reg = users.registry() or {}
1321
+ except Exception:
1322
+ return []
1323
+ want = str(tenant or '').strip().lower()
1324
+ if not want:
1325
+ return []
1326
+ out = []
1327
+ for uname, u in reg.items():
1328
+ if not isinstance(u, dict) or u.get('active') is False:
1329
+ continue
1330
+ if str(u.get('tenant') or 'royal-imports').strip().lower() != want:
1331
+ continue
1332
+ out.append({"username": str(uname), "name": str(u.get('name') or uname)})
1333
+ return sorted(out, key=lambda p: p["name"].lower())
1334
 
1335
 
1336
  def _stranger(session, kind, oid):
 
1355
  _notify=True):
1356
  """Apply the shared share admission, ceiling and atomic writer for both doors."""
1357
  kind = _kind_or_400(kind)
 
 
 
 
 
1358
  if kind == "field":
1359
  table_key, field_key = shares.split_field_oid(oid)
1360
  if table_key and field_key:
 
1428
  raise err(400, "bad_entries",
1429
  "entries must be a list of {user, role}. Send [] to un-share, which is how "
1430
  "revoking is expressed")
 
 
1431
  _entries_or_400(session, entries)
1432
  # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
1433
  # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
 
1623
 
1624
  try:
1625
  out = shares.set_grants(kind, oid, entries,
1626
+ owner=session.uname if claiming else rec["owner"],
 
1627
  granter=session.uname,
1628
  st=session.runtime,
1629
  expect=rec["entries"])
api/routes_starred.py CHANGED
@@ -1231,9 +1231,6 @@ def starred_records(database: str = "", session: Session = Depends(require_sessi
1231
  key = _clean_id(database)
1232
  if not key:
1233
  raise err(400, "bad_request", "no database was named")
1234
- import virtual_sources
1235
- if virtual_sources.get(key) is not None:
1236
- return {"database": key, "ids": [], "count": 0, "view": None}
1237
  return _records_payload(session, key, _database_or_refuse(session, key))
1238
 
1239
 
@@ -1252,8 +1249,6 @@ def set_starred_record(body: dict = Body(default=None),
1252
  raw_id = _clean_id(body.get("id"))
1253
  if not key:
1254
  raise err(400, "bad_request", "no database was named")
1255
- import virtual_sources
1256
- virtual_sources.refuse_write(key)
1257
  if not raw_id.isdigit():
1258
  # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
1259
  raise err(400, "bad_request", "a record id is a number")
 
1231
  key = _clean_id(database)
1232
  if not key:
1233
  raise err(400, "bad_request", "no database was named")
 
 
 
1234
  return _records_payload(session, key, _database_or_refuse(session, key))
1235
 
1236
 
 
1249
  raw_id = _clean_id(body.get("id"))
1250
  if not key:
1251
  raise err(400, "bad_request", "no database was named")
 
 
1252
  if not raw_id.isdigit():
1253
  # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
1254
  raise err(400, "bad_request", "a record id is a number")
api/routes_tables.py CHANGED
@@ -56,7 +56,6 @@ from fastapi import HTTPException, Request, Response
56
  from deps import Session, err, require_session
57
  from harness.runtime import DatastoreMismatch
58
  from routes_grid import row_patch_receipt
59
- import virtual_sources
60
 
61
  router = APIRouter(prefix="/api/v1")
62
 
@@ -286,8 +285,6 @@ def related_metrics(table_key: str, body: dict = Body(default=None),
286
  if (not isinstance(pin, dict) or not isinstance(pids, list)
287
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
288
  raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
289
- if virtual_sources.get(table_key) is not None:
290
- return virtual_sources.related_metrics(session, table_key, pin, pids)
291
  session.require(table_key)
292
  viewer = copy.copy(session)
293
  viewer.runtime = _MetricReadRuntime(session.runtime)
@@ -393,8 +390,6 @@ def _table_lend(runtime, table_key, defs_only=False):
393
  def _managed_or_refuse(session, table_key, *, mutation=False):
394
  """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
395
  from core import route_system
396
- if mutation:
397
- virtual_sources.refuse_write(table_key)
398
  definition = _ut().get_one(table_key, st=session.runtime)
399
  # Route/Zone are virtual system descriptors until their first dedicated write, so looking
400
  # only in `user_tables` would leave a generic mutation path open during that empty state.
@@ -589,7 +584,6 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=
589
  proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
590
  asserts the fast path was TAKEN.
591
  """
592
- virtual_sources.refuse_write(table_key)
593
  ut = _ut()
594
  # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
595
  # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
@@ -672,8 +666,8 @@ def _records_lock_message(table_key, definition, st):
672
  return f"This {name} is locked, so only user-created field values can be edited."
673
 
674
 
675
- def _editor_or_refuse(session, table_key, st=None):
676
- """Require editor authority on a database this session may see. `(defn, st)`."""
677
  # Share one snapshot across the read, editor and record-mode checks.
678
  st = st if st is not None else _ut().lend(session.runtime)
679
  # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
@@ -683,12 +677,6 @@ def _editor_or_refuse(session, table_key, st=None):
683
  defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
684
  if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
685
  raise err(403, "database_read_only", "Can edit access is required to change this database.")
686
- return defn, st
687
-
688
-
689
- def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
690
- """Require editor authority and reserve locked database edits for authored field values."""
691
- defn, st = _editor_or_refuse(session, table_key, st)
692
  if not _ut().records_mutable(table_key, st=st):
693
  if cell_updates and table_key != "ut_activity":
694
  from core import shared_overlay
@@ -1063,9 +1051,6 @@ def _activity_grid_rows(session):
1063
  @router.get('/tables/{table_key}/activity')
1064
  def table_activity(table_key: str, cursor: str = None, limit: int = 50,
1065
  session: Session = Depends(require_session)):
1066
- if virtual_sources.get(table_key) is not None:
1067
- from core import activity_log
1068
- return {'records': [], 'next_cursor': None, 'fields': activity_log.definition()['fields']}
1069
  if table_key.startswith('ut_'):
1070
  _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
1071
  elif table_key not in ('customer_data', 'product_data'):
@@ -2137,9 +2122,6 @@ def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None):
2137
  def ut_assembly(session: Session, table_key: str, storage_key: str = "",
2138
  consume_corrections: bool = False, with_rows: bool = True, st=None,
2139
  *, fields_only: bool = False, needed=None, cursor=None, limit=None):
2140
- if virtual_sources.get(table_key) is not None:
2141
- return virtual_sources.assembly(session, table_key, with_rows=with_rows and not fields_only,
2142
- storage_key=storage_key)
2143
  from core import grid_events
2144
  actor = grid_events.request_admission(session, table_key + '_table_workspace')
2145
  result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
@@ -3365,8 +3347,6 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
3365
  (found by this route's own gate check, not by luck)."""
3366
  import aios_grid
3367
 
3368
- if virtual_sources.get(table_key) is not None:
3369
- return virtual_sources.rows_payload(session, table_key, related=related)
3370
  if table_key == 'ut_activity':
3371
  if related:
3372
  raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
@@ -4541,8 +4521,7 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
4541
  from core import grid_events
4542
 
4543
  _managed_or_refuse(session, table_key, mutation=True)
4544
- # A locked database admits authored values only, so its record wall waits for the retained values.
4545
- _editor_or_refuse(session, table_key)
4546
  body = body if isinstance(body, dict) else {}
4547
  row_id = str(body.get("rowId") or "").strip()
4548
  if not row_id:
@@ -4572,7 +4551,6 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
4572
  raise err(404, "history_event_not_found", "that retained value version is unavailable")
4573
  if any(key in (g.get("hidden") or ()) for key in updates):
4574
  raise err(403, "field_hidden", "that retained version includes a hidden field")
4575
- _records_or_refuse(session, table_key, cell_updates=updates)
4576
  prior_history_ids = {
4577
  str(item.get("id")) for item in _ut().value_history(
4578
  table_key, row_id, st=session.runtime, retention_days=days,
 
56
  from deps import Session, err, require_session
57
  from harness.runtime import DatastoreMismatch
58
  from routes_grid import row_patch_receipt
 
59
 
60
  router = APIRouter(prefix="/api/v1")
61
 
 
285
  if (not isinstance(pin, dict) or not isinstance(pids, list)
286
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
287
  raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
 
 
288
  session.require(table_key)
289
  viewer = copy.copy(session)
290
  viewer.runtime = _MetricReadRuntime(session.runtime)
 
390
  def _managed_or_refuse(session, table_key, *, mutation=False):
391
  """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
392
  from core import route_system
 
 
393
  definition = _ut().get_one(table_key, st=session.runtime)
394
  # Route/Zone are virtual system descriptors until their first dedicated write, so looking
395
  # only in `user_tables` would leave a generic mutation path open during that empty state.
 
584
  proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
585
  asserts the fast path was TAKEN.
586
  """
 
587
  ut = _ut()
588
  # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
589
  # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
 
666
  return f"This {name} is locked, so only user-created field values can be edited."
667
 
668
 
669
+ def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
670
+ """Require editor authority and reserve locked database edits for authored field values."""
671
  # Share one snapshot across the read, editor and record-mode checks.
672
  st = st if st is not None else _ut().lend(session.runtime)
673
  # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
 
677
  defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
678
  if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
679
  raise err(403, "database_read_only", "Can edit access is required to change this database.")
 
 
 
 
 
 
680
  if not _ut().records_mutable(table_key, st=st):
681
  if cell_updates and table_key != "ut_activity":
682
  from core import shared_overlay
 
1051
  @router.get('/tables/{table_key}/activity')
1052
  def table_activity(table_key: str, cursor: str = None, limit: int = 50,
1053
  session: Session = Depends(require_session)):
 
 
 
1054
  if table_key.startswith('ut_'):
1055
  _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
1056
  elif table_key not in ('customer_data', 'product_data'):
 
2122
  def ut_assembly(session: Session, table_key: str, storage_key: str = "",
2123
  consume_corrections: bool = False, with_rows: bool = True, st=None,
2124
  *, fields_only: bool = False, needed=None, cursor=None, limit=None):
 
 
 
2125
  from core import grid_events
2126
  actor = grid_events.request_admission(session, table_key + '_table_workspace')
2127
  result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
 
3347
  (found by this route's own gate check, not by luck)."""
3348
  import aios_grid
3349
 
 
 
3350
  if table_key == 'ut_activity':
3351
  if related:
3352
  raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
 
4521
  from core import grid_events
4522
 
4523
  _managed_or_refuse(session, table_key, mutation=True)
4524
+ _records_or_refuse(session, table_key)
 
4525
  body = body if isinstance(body, dict) else {}
4526
  row_id = str(body.get("rowId") or "").strip()
4527
  if not row_id:
 
4551
  raise err(404, "history_event_not_found", "that retained value version is unavailable")
4552
  if any(key in (g.get("hidden") or ()) for key in updates):
4553
  raise err(403, "field_hidden", "that retained version includes a hidden field")
 
4554
  prior_history_ids = {
4555
  str(item.get("id")) for item in _ut().value_history(
4556
  table_key, row_id, st=session.runtime, retention_days=days,
api/sync_runtime.py CHANGED
@@ -6,7 +6,7 @@ publish is 878.8 s inside the 1,200 s limit, so a pass fits one job.
6
  Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
7
  every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
8
  inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
9
- live only in the worker. A resident job hands over to a new job only at a pass boundary of the
10
  resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
11
  timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
12
  The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
@@ -17,7 +17,7 @@ AIOS_SYNC_RUNTIME picks the runtime: `hf-job`, `external` (a separately run resi
17
  HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
18
 
19
  Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
20
- its work is due when per-pass. Worker side: `relay_at_boundary` hands queued work to the store.
21
  The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
22
  `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
23
  of RESYNC_STATE's.
@@ -29,7 +29,6 @@ import math
29
  import os
30
  import threading
31
  import time
32
- from copy import deepcopy
33
 
34
  RUNTIMES = ("hf-job", "external", "off")
35
  RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
@@ -234,51 +233,8 @@ def relay_due(now=None):
234
  return ""
235
 
236
 
237
- def _handoff_queued_requests():
238
- """Persist ordinary queued receipts without changing durable events or interrupting a claim."""
239
- from core import connector_sync
240
- import process_role
241
- with connector_sync._SYNC_REQUEST_LOCK:
242
- receipts = deepcopy(connector_sync._SYNC_REQUESTS)
243
- runtimes = dict(connector_sync._SYNC_REQUEST_RUNTIMES)
244
- if any(row.get("state") == "running" for row in receipts.values()):
245
- return None
246
- try:
247
- for row in receipts.values():
248
- if row.get("state") != "queued" or row.get("tables") is not None:
249
- continue
250
- tenant = row["tenant"]
251
- rt = runtimes.get(tenant) or process_role._status_runtime(tenant)
252
- if rt.key != tenant:
253
- raise ValueError("The relay desk does not match the receipt tenant.")
254
- connector_sync.queue_remote_request(rt, row, row.get("connectionId"))
255
- except Exception as exc:
256
- print(f"[sync-worker] job: relay handoff failed ({type(exc).__name__}); keeping this job running", flush=True)
257
- return None
258
- return receipts
259
-
260
-
261
- def relay_at_boundary(next_at=None, *, exit_fn=None):
262
- """Relay a resident job after its current write and projection finish, before another claim."""
263
- from core import connector_sync
264
- if not ONCE["on"] or ONCE["mode"] != "resident":
265
- return False
266
- reason = relay_due()
267
- if not reason:
268
- return False
269
- receipts = _handoff_queued_requests()
270
- if receipts is None:
271
- return False
272
- with connector_sync._SYNC_REQUEST_LOCK:
273
- # No receipt can be admitted or claimed between the final handoff check and exit.
274
- if connector_sync._SYNC_REQUESTS != receipts:
275
- return False
276
- finish(time.time() if next_at is None else next_at, reason, exit_fn=exit_fn)
277
- return True
278
-
279
-
280
  def idle_wait(event, seconds, *, exit_fn=None):
281
- """Wait between passes, relaying a resident job before a queued wake can start another pass."""
282
  if not ONCE["on"]:
283
  return event.wait(seconds)
284
  if seconds is None or not math.isfinite(float(seconds)):
@@ -286,21 +242,18 @@ def idle_wait(event, seconds, *, exit_fn=None):
286
  deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
287
  resident = ONCE["mode"] == "resident"
288
  while True:
289
- if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
290
- return False
291
  remaining = deadline - time.time()
292
  if remaining <= IDLE_GRACE_SECONDS:
293
  return event.wait(max(0.0, remaining))
294
  # Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
295
  if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
296
- if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
297
- return False
298
  return True
299
- if resident:
 
300
  continue
301
  busy, wake_at = _desk_state(time.time())
302
  if not busy:
303
- finish(deadline if wake_at is None else min(deadline, wake_at), "idle", exit_fn=exit_fn)
304
  return False
305
 
306
 
@@ -345,7 +298,7 @@ def no_lease_line(tenant, seconds=None):
345
 
346
 
347
  def _pass_in_flight():
348
- """True while the store resync loop runs and owns its pass boundaries."""
349
  return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
350
 
351
 
@@ -356,7 +309,9 @@ def run_once(*, exit_fn=None, alive=None):
356
  if alive():
357
  continue
358
  if ONCE["mode"] == "resident":
359
- if relay_at_boundary(exit_fn=exit_fn):
 
 
360
  return False
361
  continue
362
  finish(time.time() + MAX_IDLE_SECONDS,
 
6
  Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
7
  every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
8
  inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
9
+ live only in the worker. A resident job hands over to a new job only at an idle point of the
10
  resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
11
  timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
12
  The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
 
17
  HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
18
 
19
  Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
20
+ its work is due when per-pass. Worker side: `idle_wait` is the resync loop's only exit point.
21
  The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
22
  `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
23
  of RESYNC_STATE's.
 
29
  import os
30
  import threading
31
  import time
 
32
 
33
  RUNTIMES = ("hf-job", "external", "off")
34
  RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
 
233
  return ""
234
 
235
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
236
  def idle_wait(event, seconds, *, exit_fn=None):
237
+ """The resync loop's wait, and a job's only exit point. Outside a job it is `event.wait(seconds)`."""
238
  if not ONCE["on"]:
239
  return event.wait(seconds)
240
  if seconds is None or not math.isfinite(float(seconds)):
 
242
  deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
243
  resident = ONCE["mode"] == "resident"
244
  while True:
 
 
245
  remaining = deadline - time.time()
246
  if remaining <= IDLE_GRACE_SECONDS:
247
  return event.wait(max(0.0, remaining))
248
  # Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
249
  if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
 
 
250
  return True
251
+ reason = relay_due() if resident else "idle"
252
+ if not reason:
253
  continue
254
  busy, wake_at = _desk_state(time.time())
255
  if not busy:
256
+ finish(deadline if wake_at is None else min(deadline, wake_at), reason, exit_fn=exit_fn)
257
  return False
258
 
259
 
 
298
 
299
 
300
  def _pass_in_flight():
301
+ """True while the store resync loop runs: it relays at its own idle point."""
302
  return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
303
 
304
 
 
309
  if alive():
310
  continue
311
  if ONCE["mode"] == "resident":
312
+ reason = relay_due()
313
+ if reason and not _desk_state(time.time())[0]:
314
+ finish(time.time(), reason, exit_fn=exit_fn)
315
  return False
316
  continue
317
  finish(time.time() + MAX_IDLE_SECONDS,
api/test_odoo_rank_refresh.py CHANGED
@@ -1059,7 +1059,7 @@ class RankRefreshTests(unittest.TestCase):
1059
 
1060
  def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
1061
  reports=None, checkpoints=None, take=None, slot_flags=None, status=None,
1062
- stop_at=None):
1063
  """Drive the real loop and its real slot resolver on a clock that advances at each wait."""
1064
  import odoo_relational as rel
1065
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
@@ -1086,6 +1086,8 @@ class RankRefreshTests(unittest.TestCase):
1086
 
1087
  def sync(manual_flag, passes, entities):
1088
  events.append(('mirror', now[0], manual_flag))
 
 
1089
  if manual_flag:
1090
  return None, None, None
1091
  if reports:
@@ -1684,5 +1686,72 @@ class RankRefreshTests(unittest.TestCase):
1684
  self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
1685
 
1686
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1687
  if __name__ == '__main__':
1688
  unittest.main()
 
1059
 
1060
  def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
1061
  reports=None, checkpoints=None, take=None, slot_flags=None, status=None,
1062
+ stop_at=None, on_pass=None):
1063
  """Drive the real loop and its real slot resolver on a clock that advances at each wait."""
1064
  import odoo_relational as rel
1065
  source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
 
1086
 
1087
  def sync(manual_flag, passes, entities):
1088
  events.append(('mirror', now[0], manual_flag))
1089
+ if on_pass is not None:
1090
+ on_pass(store, now[0])
1091
  if manual_flag:
1092
  return None, None, None
1093
  if reports:
 
1686
  self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
1687
 
1688
 
1689
+ def test_h1_freshness_reads_the_keychain_default_slot(self):
1690
+ """H1: a pass written under the keychain default id reads fresh; no id keeps the default slot"""
1691
+ from datetime import datetime, timezone
1692
+ import odoo_relational as rel
1693
+ from core import connector_sync
1694
+ now = datetime(2026, 11, 18, 9, 0, tzinfo=timezone.utc)
1695
+ good = '2026-11-18T07:32:38+00:00'
1696
+ entry = {'lastGoodSyncAt': good, 'resync': {'connectionId': 'k_x'},
1697
+ 'pass': {'startedAt': '2026-11-18T07:00:00+00:00', 'endedAt': good,
1698
+ 'publishedAt': good, 'outcome': 'held'}}
1699
+ keyed = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'k_x': dict(entry)}}})
1700
+ with patch.object(rel, 'default_connection_id', return_value='k_x'):
1701
+ fresh = rel.sync_freshness(keyed, now=now)
1702
+ self.assertFalse(fresh['stale'])
1703
+ self.assertEqual(fresh['lastGoodSyncAt'], good)
1704
+ self.assertEqual((fresh['pass']['outcome'], fresh['resync']), ('held', {'connectionId': 'k_x'}))
1705
+ # Control: read through the "default" slot alone, the same store is stale (the live defect).
1706
+ with patch.object(rel, 'default_connection_id', return_value=None):
1707
+ self.assertTrue(rel.sync_freshness(keyed, now=now)['stale'])
1708
+ # Negative control: an empty keychain slot, or no keychain id, still reads the "default" slot.
1709
+ plain = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'default': dict(entry)}}})
1710
+ for cid in (None, 'k_x'):
1711
+ with patch.object(rel, 'default_connection_id', return_value=cid):
1712
+ fresh = rel.sync_freshness(plain, now=now)
1713
+ self.assertFalse(fresh['stale'], cid)
1714
+ self.assertEqual(fresh['lastGoodSyncAt'], good, cid)
1715
+
1716
+ def test_h1_a_held_pass_that_published_records_its_success(self):
1717
+ """H1: a held pass that published stamps lastSuccessAt and the source stage; a refused one writes nothing"""
1718
+ import odoo_relational as rel
1719
+ import routes_connectors
1720
+ from core import connector_sync
1721
+ last = self.utc('2026-10-22T06:05:00Z')
1722
+ boot = self.utc('2026-10-24T07:00:00Z')
1723
+ memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}
1724
+ held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}}
1725
+ marker = ('account_move_line.cols.amount_residual', 'running', 500, None, 100_000)
1726
+
1727
+ def publication(offset):
1728
+ def write(store, at):
1729
+ row = {'startedAt': connector_sync._utc_stamp(at), 'outcome': 'running'}
1730
+ if offset is not None:
1731
+ row.update(outcome='done', publishedAt=connector_sync._utc_stamp(at + offset))
1732
+ store.docs[connector_sync.STATUS_KEY] = {'odoo': {'k_x': {'pass': row}}}
1733
+ return write
1734
+ # Published during this pass; published by an earlier pass; publication refused.
1735
+ for offset, published in ((5, True), (-3600, False), (None, False)):
1736
+ stage = SimpleNamespace(calls=[])
1737
+ with patch.object(rel, 'default_connection_id', return_value='k_x'), \
1738
+ patch.object(routes_connectors, 'record_odoo_stage',
1739
+ lambda *a, **k: stage.calls.append((a, k))):
1740
+ self.run_slot_loop(memory, boot=boot, max_sleeps=1, reports=[held],
1741
+ checkpoints=[(marker,), (marker,)], on_pass=publication(offset))
1742
+ self.assertTrue(self.slot_state['publicationHeld'], offset)
1743
+ self.assertEqual(self.slot_state['connectionId'], 'k_x')
1744
+ self.assertIn('publication held; no vendor pulls or relational rebuild', self.loop_output)
1745
+ if published:
1746
+ self.assertEqual(self.slot_state.get('lastSuccessAt'), self.slot_state['lastPassAt'])
1747
+ self.assertEqual(len(stage.calls), 1)
1748
+ args, kwargs = stage.calls[0]
1749
+ self.assertEqual(args[1:], ('k_x', 'source', 'ready'))
1750
+ self.assertEqual(kwargs['at'], connector_sync._utc_stamp(boot + 5))
1751
+ else:
1752
+ self.assertNotIn('lastSuccessAt', self.slot_state, offset)
1753
+ self.assertEqual(stage.calls, [], offset)
1754
+
1755
+
1756
  if __name__ == '__main__':
1757
  unittest.main()
api/usage_ledger.py CHANGED
@@ -3,13 +3,16 @@
3
  R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
4
  does NOT cut anyone off this wave; over the allowance it still works, shown red."*
5
 
6
- ⛔ THIS FILE MEASURES AND NEVER ENFORCES. Token allowances remain report-only.
7
- Image-making limits live in `image_budget`, which reads measured image dollars and its own holds.
 
 
8
 
9
- **THE SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
10
  enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
11
- (a sentence becoming a draft flow) · `image_gen` (image making, edits and supporting reads or briefs).
12
- The gate in `verify_api.py` derives LLM entry points from the code rather than this list.
 
13
 
14
  ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
15
  obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
@@ -29,17 +32,17 @@ that hides the gap is worse than one that admits it.
29
  """
30
  from __future__ import annotations
31
 
32
- from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
33
  import os
34
  import time
35
 
36
- #: Weekly counters and image-settlement retry receipts contain no prompts or generated content.
 
37
  STORE_KEY = "ai_usage"
38
 
39
- #: The AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
40
  #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
41
  #: row nobody looks at.
42
- SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft", "image_gen")
43
 
44
  #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
45
  #: Env-tunable so a deployment can set a real number without a release.
@@ -129,39 +132,26 @@ def _int_or_none(value):
129
  return None
130
 
131
 
132
- def usd_micros(value):
133
- """Return nonnegative dollars as whole micro-dollars, rounded half up, or None if unknown."""
134
- if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)):
135
- return None
136
- try:
137
- amount = Decimal(str(value))
138
- if amount.is_finite() and amount >= 0:
139
- return int((amount * 1_000_000).to_integral_value(rounding=ROUND_HALF_UP))
140
- except (InvalidOperation, ValueError, OverflowError):
141
- return None
142
- return None
143
-
144
-
145
- def _book_dollars(row, micros, calls, entry_id):
146
- """Book measured dollars or a separate unknown count, resolving a single-call receipt once."""
147
- receipts = row.setdefault("_cost_entries", {}) if entry_id else {}
148
- repeated = bool(entry_id and entry_id in receipts)
149
- if repeated and (receipts[entry_id] is not None or micros is None):
150
- return False
151
- if repeated:
152
- row["usd_unmeasured"] = max(0, int(row.get("usd_unmeasured") or 0) - 1)
153
- if micros is None:
154
- row["usd_unmeasured"] = int(row.get("usd_unmeasured") or 0) + calls
155
- else:
156
- row["usd_micros"] = int(row.get("usd_micros") or 0) + micros
157
- if entry_id:
158
- receipts[entry_id] = micros
159
- return not repeated
160
-
161
-
162
  def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
163
- total=None, calls=1, cost_usd=None, st=None, user="", week=None, entry_id=""):
164
- """Report a call, with optional held week and single-call retry id, without enforcing a limit."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  key = str(surface or "").strip()
166
  if key not in SURFACES:
167
  # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
@@ -172,16 +162,15 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
172
  if tot is None and (ins is not None or outs is not None):
173
  tot = (ins or 0) + (outs or 0)
174
  n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
175
- if not n or (entry_id and n != 1):
176
  return False
177
  uname = str(user or "").strip().lower()
178
  if st is None:
179
  UNATTRIBUTED["calls"] += n
180
  UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
181
  return False
182
- week = week or week_of()
183
  prov, mdl = str(provider or "")[:40], str(model or "")[:80]
184
- micros = usd_micros(cost_usd)
185
 
186
  def _up(data):
187
  data = data if isinstance(data, dict) else {}
@@ -189,16 +178,18 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
189
  row = ((weeks.setdefault(week, {})
190
  .setdefault(uname or "-", {}))
191
  .setdefault(key, {}))
192
- if _book_dollars(row, micros, n, entry_id):
193
- row["calls"] = int(row.get("calls") or 0) + n
194
- if tot is None:
195
- row["unmeasured"] = int(row.get("unmeasured") or 0) + n
196
- else:
197
- row["tokens"] = int(row.get("tokens") or 0) + tot
198
- if ins is not None:
199
- row["tokens_in"] = int(row.get("tokens_in") or 0) + ins
200
- if outs is not None:
201
- row["tokens_out"] = int(row.get("tokens_out") or 0) + outs
 
 
202
  if prov:
203
  row["provider"] = prov
204
  if mdl:
@@ -210,8 +201,12 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
210
  return weeks
211
 
212
  try:
213
- # Image dollars must persist before their budget holds can disappear.
214
- st.update(STORE_KEY, _up, flush="sync" if key == "image_gen" else "async")
 
 
 
 
215
  return True
216
  except Exception: # noqa: BLE001
217
  UNATTRIBUTED["calls"] += n
@@ -220,8 +215,7 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
220
 
221
 
222
  def _blank_row():
223
- return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0,
224
- "usd_micros": 0, "usd_unmeasured": 0}
225
 
226
 
227
  def _fold(rows):
@@ -229,7 +223,7 @@ def _fold(rows):
229
  for row in rows:
230
  if not isinstance(row, dict):
231
  continue
232
- for field in out:
233
  got = row.get(field)
234
  if isinstance(got, int) and not isinstance(got, bool):
235
  out[field] += max(0, got)
 
3
  R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
4
  does NOT cut anyone off this wave; over the allowance it still works, shown red."*
5
 
6
+ ⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of
7
+ this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives
8
+ where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two
9
+ would make an over-allowance week silently stop somebody's enrichment run.
10
 
11
+ **THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
12
  enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
13
+ (a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate
14
+ in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point
15
+ is caught by the gate and not by somebody remembering to edit a tuple.
16
 
17
  ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
18
  obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
 
32
  """
33
  from __future__ import annotations
34
 
 
35
  import os
36
  import time
37
 
38
+ #: The store bucket. Small by construction — counters only, never a per-call log: a log would grow
39
+ #: without bound on a route that runs per row, and nothing in R9 asks a question a log answers.
40
  STORE_KEY = "ai_usage"
41
 
42
+ #: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
43
  #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
44
  #: row nobody looks at.
45
+ SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft")
46
 
47
  #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
48
  #: Env-tunable so a deployment can set a real number without a release.
 
132
  return None
133
 
134
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
  def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
136
+ total=None, calls=1, st=None, user=""):
137
+ """Book ONE AI call. Returns True when the line landed durably, False when it could not.
138
+
139
+ C7's signature is the five positional parameters; everything after `*` is this module's own
140
+ extension and each one has a reason:
141
+
142
+ · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field`
143
+ accumulates a per-run total through an `_ask` contract a gate injects against, and changing
144
+ that contract to carry a split would break the injected shape for no gain here).
145
+ · `calls` — so a RUN over many rows books one line with its real call count instead of one
146
+ store write per row.
147
+ · `st` / `user` — the target. Required in practice; see this module's header for the measured
148
+ reason a contextvar cannot supply them.
149
+
150
+ ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a
151
+ store blip during an enrichment run would abort the run and lose the cells.
152
+ ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which
153
+ `GET /usage` shows an admin.
154
+ """
155
  key = str(surface or "").strip()
156
  if key not in SURFACES:
157
  # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
 
162
  if tot is None and (ins is not None or outs is not None):
163
  tot = (ins or 0) + (outs or 0)
164
  n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
165
+ if not n:
166
  return False
167
  uname = str(user or "").strip().lower()
168
  if st is None:
169
  UNATTRIBUTED["calls"] += n
170
  UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
171
  return False
172
+ week = week_of()
173
  prov, mdl = str(provider or "")[:40], str(model or "")[:80]
 
174
 
175
  def _up(data):
176
  data = data if isinstance(data, dict) else {}
 
178
  row = ((weeks.setdefault(week, {})
179
  .setdefault(uname or "-", {}))
180
  .setdefault(key, {}))
181
+ row["calls"] = int(row.get("calls") or 0) + n
182
+ if tot is None:
183
+ # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be
184
+ # representable or the meter quietly under-reports. The dashboard shows it as a count of
185
+ # unmeasured calls beside the token total.
186
+ row["unmeasured"] = int(row.get("unmeasured") or 0) + n
187
+ else:
188
+ row["tokens"] = int(row.get("tokens") or 0) + tot
189
+ if ins is not None:
190
+ row["tokens_in"] = int(row.get("tokens_in") or 0) + ins
191
+ if outs is not None:
192
+ row["tokens_out"] = int(row.get("tokens_out") or 0) + outs
193
  if prov:
194
  row["provider"] = prov
195
  if mdl:
 
201
  return weeks
202
 
203
  try:
204
+ # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of
205
+ # every AI surface in the product (an enrichment run books one line per RUN, but the
206
+ # assistant books one per question), and a blocking hub upload per call would put a network
207
+ # round trip inside the answer a person is waiting for. The mutation applies to the
208
+ # in-process cache immediately, so `GET /usage` reads its own writes.
209
+ st.update(STORE_KEY, _up, flush="async")
210
  return True
211
  except Exception: # noqa: BLE001
212
  UNATTRIBUTED["calls"] += n
 
215
 
216
 
217
  def _blank_row():
218
+ return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0}
 
219
 
220
 
221
  def _fold(rows):
 
223
  for row in rows:
224
  if not isinstance(row, dict):
225
  continue
226
+ for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"):
227
  got = row.get(field)
228
  if isinstance(got, int) and not isinstance(got, bool):
229
  out[field] += max(0, got)
platform/core/amazon_projects.py CHANGED
@@ -26,7 +26,6 @@ STATUSES = ('Backlog', 'Planned', 'In progress', 'Completed', 'Canceled')
26
  COMPLETED = 'Completed'
27
  CANCELED = 'Canceled'
28
  PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
29
- TYPES = ('project', 'pdp')
30
  #: Health turns At risk this many days before the target date.
31
  AT_RISK_DAYS = 7
32
  CHUNK = 200
@@ -244,8 +243,6 @@ def _record(values, columns=PROJECT_COLUMNS):
244
  value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
245
  members = json.loads(value) if isinstance(value, str) else value
246
  result['contributors'] = sorted(members or [], key=lambda member: member['username'])
247
- result['type'] = values[len(columns) + 4] if len(values) > len(columns) + 4 else 'project'
248
- result['listed'] = bool(values[len(columns) + 5]) if len(values) > len(columns) + 5 else True
249
  return result
250
 
251
 
@@ -751,53 +748,11 @@ def _activity_target_preview(value, truncated):
751
  return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
752
 
753
 
754
- def _pdp_item_stages(value, pdp):
755
- """Accept complete typed workflow item identities before comparing historical stages."""
756
- if not isinstance(value, dict) or value.get('type') != 'pdp' or not isinstance(value.get('items'), list):
757
- return None
758
- stages = {}
759
- for item in value['items']:
760
- if (not isinstance(item, dict) or not isinstance(item.get('asin'), str)
761
- or not isinstance(item.get('stage'), str) or item['stage'] not in pdp.STAGE_PROGRESS
762
- or item['asin'] in stages):
763
- return None
764
- stages[item['asin']] = item['stage']
765
- return stages
766
-
767
-
768
- def _pdp_activity_facts(event, field, before, after):
769
- """Project only C4 action facts retained in the bounded stored event."""
770
- from core import amazon_pdp as pdp
771
- if event.get('via') != pdp.VIA:
772
- return None
773
- unavailable = {'action': 'unavailable'}
774
- if event.get('changes_truncated'):
775
- return unavailable
776
- old, new = (_pdp_item_stages(value, pdp) for value in (before, after))
777
- if field == 'setup' and before is None and new is not None:
778
- scope = after.get('scope')
779
- scope = scope if isinstance(scope, list) and scope and all(key in pdp.SCOPE_PARTS for key in scope) else None
780
- return {'action': 'created', 'scope': scope}
781
- if old is not None and new is not None:
782
- changed = {stage for asin, stage in new.items() if asin in old and old[asin] != stage}
783
- if len(changed) == 1:
784
- stage = next(iter(changed))
785
- if field == ('publish' if stage == 'published' else stage):
786
- return {'action': 'moved', 'stage': stage} if stage != 'published' else {'action': 'published'}
787
- return {'action': 'setup'} if field in ('setup', 'competitors') and not changed else unavailable
788
- if isinstance(after, dict) and isinstance(after.get('slot'), str) and after['slot'] in pdp.SLOTS:
789
- part = pdp.SLOTS[after['slot']]['key']
790
- if after.get('part') == part and field in ('draft', 'approve', 'request'):
791
- return {'action': {'draft': 'saved', 'approve': 'approved', 'request': 'requested'}[field]}
792
- return {'action': 'ai'} if field == 'ai' else unavailable
793
-
794
-
795
  def _activity_entries(event, actor_names, task_names):
796
  """Project one log event into named activity entries."""
797
  out = []
798
  for index, (field, change) in enumerate((event.get('changes') or {}).items()):
799
  before, after = change.get('before'), change.get('after')
800
- pdp_facts = _pdp_activity_facts(event, field, before, after)
801
  if field == 'project':
802
  action, field = ('created' if before is None else 'deleted'), None
803
  elif field == 'targets':
@@ -820,8 +775,6 @@ def _activity_entries(event, actor_names, task_names):
820
  'by': event['by'], 'byUsername': event['by'],
821
  'byName': actor_names.get(event['by'], event['by']),
822
  'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
823
- if pdp_facts is not None:
824
- out[-1]['pdp'] = pdp_facts
825
  return out
826
 
827
 
@@ -1142,8 +1095,6 @@ class PgProjectStore:
1142
  notes TEXT, completed_on DATE, source_id TEXT,
1143
  created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
1144
  rollup JSONB, impact JSONB,
1145
- type TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)})),
1146
- listed BOOLEAN NOT NULL DEFAULT TRUE,
1147
  PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
1148
  CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
1149
  OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
@@ -1161,11 +1112,8 @@ class PgProjectStore:
1161
  # One row's lookup is one indexed read on this index.
1162
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
1163
  f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
1164
- additions = {'rollup': 'JSONB', 'impact': 'JSONB',
1165
- 'type': f"TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)}))",
1166
- 'listed': 'BOOLEAN NOT NULL DEFAULT TRUE'}
1167
- for field, declaration in additions.items():
1168
- con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} {declaration}')
1169
 
1170
  @contextmanager
1171
  def _write(self):
@@ -1178,20 +1126,13 @@ class PgProjectStore:
1178
  cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
1179
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1180
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1181
- return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members}, '
1182
- f'{self._type_projection()}, {self._listed_projection()} '
1183
  f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
1184
 
1185
  def _rollup_projection(self):
1186
  # Old schemas read blank figures until a guarded write adds the columns.
1187
  return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
1188
 
1189
- def _type_projection(self):
1190
- return "COALESCE(to_jsonb(p)->>'type', 'project')"
1191
-
1192
- def _listed_projection(self):
1193
- return "COALESCE((to_jsonb(p)->>'listed')::boolean, TRUE)"
1194
-
1195
  def _members_projection(self, con):
1196
  if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
1197
  return 'NULL'
@@ -1237,19 +1178,15 @@ class PgProjectStore:
1237
  'WHERE tenant = %s AND id = %s',
1238
  (stamp, json.dumps(impact), self.tenant_slug, project_id))
1239
 
1240
- def _insert(self, con, record, pairs, stamp):
1241
- columns = (*PROJECT_COLUMNS, 'impact', 'type', 'listed')
1242
- values = (*[record[c] for c in PROJECT_COLUMNS],
1243
- json.dumps(_with_result_configs(None, record.get('result_configs', []))),
1244
- record.get('type', 'project'), record.get('listed', True))
1245
- con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
1246
- f'VALUES ({_marks(len(PROJECT_COLUMNS))}, %s::jsonb, %s, %s)', values)
1247
- self._insert_targets(con, record['id'], pairs, stamp)
1248
- self._replace_members(con, record['id'], record.get('contributors', []), [], stamp)
1249
-
1250
  def insert(self, record, pairs, stamp):
1251
  with self._write() as con:
1252
- self._insert(con, record, pairs, stamp)
 
 
 
 
 
 
1253
  return self._one(con, record['id'])
1254
 
1255
  def get(self, project_id):
@@ -1261,8 +1198,6 @@ class PgProjectStore:
1261
  def update(self, project_id, changes, stamp):
1262
  with self._write() as con:
1263
  current = self._one(con, project_id, lock=True)
1264
- if current['type'] == 'pdp' and set(changes) & {'progress', 'status', 'completed_on'}:
1265
- raise ValueError('PDP item stages control project progress and completion.')
1266
  candidate = _fields(changes, previous=current)
1267
  if 'contributors' in changes:
1268
  candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
@@ -1278,25 +1213,23 @@ class PgProjectStore:
1278
  self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
1279
  return self._one(con, project_id)
1280
 
1281
- def _delete(self, con, project_id):
1282
- for table in (MEMBERS, TARGETS):
1283
- con.execute(f'DELETE FROM {self.t(table)} WHERE tenant = %s AND project_id = %s',
1284
- (self.tenant_slug, project_id))
1285
- row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
1286
- (self.tenant_slug, project_id)).fetchone()
1287
- if row is None:
1288
- raise KeyError('Project not found.')
1289
-
1290
  def delete(self, project_id):
1291
  with self._write() as con:
1292
- self._delete(con, project_id)
 
 
 
 
 
 
 
1293
  return {'deleted': project_id}
1294
 
1295
  def list_all(self):
1296
  with self._connection() as con:
1297
  if not self._exists(con):
1298
  return []
1299
- rows = con.execute(self._select_sql(f' AND {self._listed_projection()} ORDER BY p.created_at, p.id',
1300
  members=self._members_projection(con)),
1301
  (self.tenant_slug,)).fetchall()
1302
  return [_record(row) for row in rows]
@@ -1320,12 +1253,10 @@ class PgProjectStore:
1320
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1321
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1322
  rows = con.execute(
1323
- f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members}, '
1324
- f'{self._type_projection()}, {self._listed_projection()} '
1325
  f'FROM {self.t(TARGETS)} AS t '
1326
  f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
1327
- f'WHERE t.tenant = %s AND {self._listed_projection()} AND ({match}) '
1328
- 'ORDER BY p.created_at, p.id',
1329
  (self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
1330
  for row in rows:
1331
  out[(row[0], row[1])].append(_record(row[2:]))
 
26
  COMPLETED = 'Completed'
27
  CANCELED = 'Canceled'
28
  PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
 
29
  #: Health turns At risk this many days before the target date.
30
  AT_RISK_DAYS = 7
31
  CHUNK = 200
 
243
  value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
244
  members = json.loads(value) if isinstance(value, str) else value
245
  result['contributors'] = sorted(members or [], key=lambda member: member['username'])
 
 
246
  return result
247
 
248
 
 
748
  return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
749
 
750
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
751
  def _activity_entries(event, actor_names, task_names):
752
  """Project one log event into named activity entries."""
753
  out = []
754
  for index, (field, change) in enumerate((event.get('changes') or {}).items()):
755
  before, after = change.get('before'), change.get('after')
 
756
  if field == 'project':
757
  action, field = ('created' if before is None else 'deleted'), None
758
  elif field == 'targets':
 
775
  'by': event['by'], 'byUsername': event['by'],
776
  'byName': actor_names.get(event['by'], event['by']),
777
  'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
 
 
778
  return out
779
 
780
 
 
1095
  notes TEXT, completed_on DATE, source_id TEXT,
1096
  created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
1097
  rollup JSONB, impact JSONB,
 
 
1098
  PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
1099
  CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
1100
  OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
 
1112
  # One row's lookup is one indexed read on this index.
1113
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
1114
  f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
1115
+ for field in ('rollup', 'impact'):
1116
+ con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} JSONB')
 
 
 
1117
 
1118
  @contextmanager
1119
  def _write(self):
 
1126
  cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
1127
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1128
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1129
+ return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} '
 
1130
  f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
1131
 
1132
  def _rollup_projection(self):
1133
  # Old schemas read blank figures until a guarded write adds the columns.
1134
  return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
1135
 
 
 
 
 
 
 
1136
  def _members_projection(self, con):
1137
  if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
1138
  return 'NULL'
 
1178
  'WHERE tenant = %s AND id = %s',
1179
  (stamp, json.dumps(impact), self.tenant_slug, project_id))
1180
 
 
 
 
 
 
 
 
 
 
 
1181
  def insert(self, record, pairs, stamp):
1182
  with self._write() as con:
1183
+ columns = (*PROJECT_COLUMNS, 'impact')
1184
+ values = (*[record[c] for c in PROJECT_COLUMNS],
1185
+ json.dumps(_with_result_configs(None, record.get('result_configs', []))))
1186
+ con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
1187
+ f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values)
1188
+ self._insert_targets(con, record['id'], pairs, stamp)
1189
+ self._replace_members(con, record['id'], record['contributors'], [], stamp)
1190
  return self._one(con, record['id'])
1191
 
1192
  def get(self, project_id):
 
1198
  def update(self, project_id, changes, stamp):
1199
  with self._write() as con:
1200
  current = self._one(con, project_id, lock=True)
 
 
1201
  candidate = _fields(changes, previous=current)
1202
  if 'contributors' in changes:
1203
  candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
 
1213
  self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
1214
  return self._one(con, project_id)
1215
 
 
 
 
 
 
 
 
 
 
1216
  def delete(self, project_id):
1217
  with self._write() as con:
1218
+ con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s',
1219
+ (self.tenant_slug, project_id))
1220
+ con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s',
1221
+ (self.tenant_slug, project_id))
1222
+ row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
1223
+ (self.tenant_slug, project_id)).fetchone()
1224
+ if row is None:
1225
+ raise KeyError('Project not found.')
1226
  return {'deleted': project_id}
1227
 
1228
  def list_all(self):
1229
  with self._connection() as con:
1230
  if not self._exists(con):
1231
  return []
1232
+ rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id',
1233
  members=self._members_projection(con)),
1234
  (self.tenant_slug,)).fetchall()
1235
  return [_record(row) for row in rows]
 
1253
  count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
1254
  'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
1255
  rows = con.execute(
1256
+ f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} '
 
1257
  f'FROM {self.t(TARGETS)} AS t '
1258
  f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
1259
+ f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id',
 
1260
  (self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
1261
  for row in rows:
1262
  out[(row[0], row[1])].append(_record(row[2:]))
platform/core/amazon_rivals.py CHANGED
@@ -1,11 +1,10 @@
1
- """Per-tenant PostgreSQL money-search rival pools and daily quotes.
2
 
3
  Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
4
  module (verify_layering) and no App module (rule 13). Tables are created on the first write.
5
  """
6
 
7
  from contextlib import contextmanager
8
- from dataclasses import dataclass
9
  from datetime import date, datetime, timedelta, timezone
10
  import json
11
  import re
@@ -20,6 +19,8 @@ RUNS = 'amazon_rival_runs'
20
  WANTED = 'amazon_rival_wanted'
21
  TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
22
 
 
 
23
  #: Brand Analytics keeps the top 3 clicked ASINs of each search term.
24
  TOP_CLICKED = 3
25
  SOURCES = frozenset(('auto', 'added'))
@@ -28,35 +29,16 @@ CHUNK = 200
28
  BA_TABLE = 'amzn_brand_analytics'
29
  LISTINGS_TABLE = 'amzn_listings'
30
  PRICING_TABLE = 'amzn_pricing'
31
- SQP_TABLE = 'amzn_search_query_performance'
32
- RANKS_TABLE = 'amzn_apify_rank_observations'
33
- SNAPSHOTS_TABLE = 'amzn_competitor_snapshots'
34
- IMAGES_TABLE = 'amzn_competitor_images'
35
- POOL_SNAPSHOT_FIELDS = ('snapshot_date', 'brand', 'title', 'bullets', 'price', 'currency', 'prime', 'offers')
36
- POOL_IMAGE_FIELDS = ('variant', 'link', 'width', 'height')
37
 
38
  NO_TOP3 = "Not in any search term's top 3 clicked"
39
  NO_WEEK = 'Brand Analytics has no dated weekly reports'
40
- NO_MONTH = 'No completed monthly Search Query Performance report for this product yet.'
41
- NO_MONEY = 'No searches with market money for this product yet.'
42
- NO_KEYWORD_RANKS = "No keyword ranks for this product's searches yet."
43
- NO_PRODUCT_TYPE = 'Product type not read yet. The next daily run reads it.'
44
- WAITING_PRODUCT_TYPE = '{count} found, waiting for the daily read of their product type.'
45
  NOT_OURS = 'Not an ASIN in your Amazon listings'
46
  PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
47
- NO_POOL = 'No search report or keyword ranks for this product yet.'
48
  ASIN_RE = re.compile(r'[A-Z0-9]{10}')
49
 
50
- POOL_FIELDS = {
51
- RIVALS: {'origin': ('TEXT', "'click'"), 'money_won': ('DOUBLE PRECISION', '0'),
52
- 'terms_won': ('INTEGER', '0'), 'term_rank': ('INTEGER', 'NULL'),
53
- 'eligible': ('BOOLEAN', 'TRUE')},
54
- DISCOVERY: {'month': ('DATE', 'NULL'), 'keyword_ranks_on': ('DATE', 'NULL'),
55
- 'product_type': ('TEXT', 'NULL'), 'held': ('INTEGER', '0')},
56
- }
57
  RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
58
- 'added_by', 'updated_at', *POOL_FIELDS[RIVALS])
59
- DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count', *POOL_FIELDS[DISCOVERY])
60
  QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
61
  'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
62
  RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
@@ -193,355 +175,96 @@ def children(cur, parent):
193
  return [r[0] for r in rows if r[0]]
194
 
195
 
196
- def _money_sql(cur):
197
- """Newest completed monthly searches per ASIN, including zero-money searches for freshness."""
198
- required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
199
- 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
200
- if not required <= _columns(cur, SQP_TABLE):
201
- return ('SELECT NULL::VARCHAR AS asin, NULL::VARCHAR AS term, NULL::DATE AS month, '
202
- 'NULL::DOUBLE AS market_money WHERE FALSE')
203
- return f'''
204
- WITH reports AS (
205
- SELECT upper(trim(CAST(asin AS VARCHAR))) AS asin,
206
- lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS term,
207
- TRY_CAST(start_date AS DATE) AS start_on, TRY_CAST(end_date AS DATE) AS end_on,
208
- TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
209
- TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS market_money
210
- FROM {SQP_TABLE}
211
- WHERE date_diff('day', TRY_CAST(start_date AS DATE), TRY_CAST(end_date AS DATE)) > 7
212
- AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
213
- latest AS (
214
- SELECT * FROM reports QUALIFY end_on = max(end_on) OVER (PARTITION BY asin))
215
- SELECT asin, term, CAST(date_trunc('month', start_on) AS DATE) AS month,
216
- max(market_money) AS market_money
217
- FROM latest GROUP BY asin, term, month'''
218
-
219
-
220
- def _click_sql(cur):
221
- """Normalized dated top-three clicks, with percentage shares kept in their source units."""
222
  cols = _columns(cur, BA_TABLE)
223
- if not {'search_term', 'clicked_asin', 'click_share_rank', 'click_share',
224
- '_period_start', '_period_end'} <= cols:
225
- return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DOUBLE AS share, '
226
- 'NULL::VARCHAR AS title, NULL::INTEGER AS term_rank, NULL::DATE AS week WHERE FALSE')
227
- title = 'CAST(clicked_item_name AS VARCHAR)' if 'clicked_item_name' in cols else 'NULL::VARCHAR'
228
- return f'''SELECT lower(trim(CAST(search_term AS VARCHAR))) AS term,
229
- upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
230
- TRY_CAST(click_share AS DOUBLE) AS share, {title} AS title,
231
- TRY_CAST(click_share_rank AS INTEGER) AS term_rank, {_week_sql()} AS week
232
- FROM {BA_TABLE} WHERE {weekly_period_sql()}
233
- AND TRY_CAST(click_share_rank AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}'''
234
-
235
-
236
- def _rank_rows_sql(cur):
237
- """Raw Keyword ranks observations include rivals absent from the own-placement view."""
238
- if not {'keyword', 'asin', 'date', 'organic_rank', 'is_sponsored'} <= _columns(cur, RANKS_TABLE):
239
- return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DATE AS observed_on, '
240
- 'NULL::INTEGER AS term_rank, NULL::BOOLEAN AS sponsored WHERE FALSE')
241
- return f'''SELECT lower(trim(CAST(keyword AS VARCHAR))) AS term,
242
- upper(trim(CAST(asin AS VARCHAR))) AS rival, TRY_CAST(date AS DATE) AS observed_on,
243
- TRY_CAST(organic_rank AS INTEGER) AS term_rank,
244
- TRY_CAST(is_sponsored AS BOOLEAN) AS sponsored FROM {RANKS_TABLE}'''
245
-
246
-
247
- def _search_dates(cur, targets):
248
- rows = _read(cur, f'''
249
- WITH money AS ({_money_sql(cur)}), observations AS ({_rank_rows_sql(cur)})
250
- SELECT m.asin, max(m.month), max(o.observed_on), count(DISTINCT m.term)
251
- FILTER (WHERE m.market_money > 0)
252
- FROM money m LEFT JOIN observations o ON o.term = m.term AND m.market_money > 0
253
- WHERE m.asin IN {_in(targets)} GROUP BY m.asin''', targets)
254
- return {a: dict(month=_day(m), keyword_ranks_on=_day(k), money_terms=n) for a, m, k, n in rows}
255
-
256
-
257
- def _newest_snapshots_sql(cur, fields=('product_type', 'brand')):
258
- if not {'asin', 'snapshot_date', 'product_type', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
259
- return 'SELECT NULL::VARCHAR AS asin, ' + ', '.join(f'NULL::VARCHAR AS {f}' for f in fields) + ' WHERE FALSE'
260
- return (f'SELECT upper(trim(asin)) AS asin, {", ".join(fields)} FROM {SNAPSHOTS_TABLE} '
261
- 'QUALIFY row_number() OVER (PARTITION BY upper(trim(asin)) ORDER BY snapshot_date DESC) = 1')
262
-
263
-
264
- def _brand(value):
265
- return ''.join((value or '').casefold().split())
266
-
267
-
268
- def _same_type(target_type, rival_type):
269
- return bool(target_type and rival_type and target_type == rival_type)
270
-
271
-
272
- def _other_brand(brand, own_brands):
273
- return not _brand(brand) or _brand(brand) not in own_brands
274
-
275
-
276
- @dataclass
277
- class PoolFacts:
278
- """Newest ASIN facts and all historical own brands determine pool eligibility."""
279
-
280
- latest: dict
281
- own_brands: set
282
- own_asins: set
283
-
284
- def product_type(self, target):
285
- return self.latest.get(target, {}).get('product_type')
286
-
287
- def eligible(self, target, rival):
288
- fact = self.latest.get(rival, {})
289
- return (rival not in self.own_asins and _same_type(self.product_type(target), fact.get('product_type'))
290
- and _other_brand(fact.get('brand'), self.own_brands))
291
-
292
-
293
- def _pool_facts(cur, values, own, prefetched=None, candidate_rows=()):
294
- latest = {r[1]: dict(product_type=r[-2], brand=r[-1]) for r in candidate_rows}
295
- for part in _chunks(sorted(values - latest.keys())):
296
- rows = _read(cur, f'SELECT asin, product_type, brand FROM ({_newest_snapshots_sql(cur)}) '
297
- f'WHERE asin IN {_in(part)}', part)
298
- latest.update({a: dict(product_type=t, brand=b) for a, t, b in rows})
299
- brands = set()
300
- if {'asin', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
301
- rows = _read(cur, f'SELECT DISTINCT s.brand FROM {SNAPSHOTS_TABLE} s '
302
- f'JOIN {LISTINGS_TABLE} l ON upper(trim(s.asin)) = upper(trim(l.asin1))')
303
- brands.update(_brand(row[0]) for row in rows if _brand(row[0]))
304
- latest.update(prefetched or {})
305
- brands.update(_brand(latest[a].get('brand')) for a in own if a in latest and _brand(latest[a].get('brand')))
306
- return PoolFacts(latest, brands, own)
307
-
308
-
309
- def _candidates(cur, targets, week):
310
- """Every non-own clicked or organic page-one ASIN on each target's monthly money searches."""
311
  sql = f'''
312
- WITH money AS ({_money_sql(cur)}), clicks AS ({_click_sql(cur)}),
313
- observations AS ({_rank_rows_sql(cur)}),
314
- newest_observations AS (
315
- SELECT * FROM observations
316
- QUALIFY observed_on = max(observed_on) OVER (PARTITION BY term)),
317
- click AS (
318
- SELECT term, rival, max(share) AS share, min(term_rank) AS term_rank, max(title) AS title
319
- FROM clicks WHERE week = CAST(? AS DATE) GROUP BY term, rival),
320
- page1 AS (
321
- SELECT term, rival, min(term_rank) AS term_rank FROM newest_observations
322
- WHERE sponsored = FALSE GROUP BY term, rival),
323
- placements AS (
324
- SELECT coalesce(c.term, p.term) AS term, coalesce(c.rival, p.rival) AS rival,
325
- c.share, c.title, coalesce(c.term_rank, p.term_rank) AS term_rank,
326
- c.term IS NOT NULL AS clicked, p.term IS NOT NULL AS page1
327
- FROM click c FULL OUTER JOIN page1 p ON p.term = c.term AND p.rival = c.rival),
328
  hits AS (
329
- SELECT m.asin, p.*, m.market_money * coalesce(p.share, 0) / 100 AS money_won
330
- FROM money m JOIN placements p ON p.term = m.term
331
- WHERE m.asin IN {_in(targets)} AND m.market_money > 0 AND p.rival <> ''
332
- AND p.rival NOT IN (SELECT upper(trim(CAST(asin1 AS VARCHAR)))
333
- FROM {LISTINGS_TABLE} WHERE asin1 IS NOT NULL)),
334
- totals AS (
335
- SELECT asin, rival, max(share) AS share, sum(money_won) AS money_won,
336
- count(*) FILTER (WHERE clicked AND term_rank = 1) AS terms_won,
337
- CASE WHEN bool_or(clicked) AND bool_or(page1) THEN 'both'
338
- WHEN bool_or(clicked) THEN 'click' ELSE 'page1' END AS origin
339
- FROM hits GROUP BY asin, rival),
340
- representative AS (
341
- SELECT asin, rival, term, title, term_rank FROM hits
342
- QUALIFY row_number() OVER (PARTITION BY asin, rival
343
- ORDER BY money_won DESC, term_rank, term) = 1)
344
- SELECT t.asin, t.rival, t.share, r.term, r.title, t.origin, t.money_won, t.terms_won, r.term_rank,
345
- s.product_type, s.brand
346
- FROM totals t JOIN representative r USING (asin, rival)
347
- LEFT JOIN ({_newest_snapshots_sql(cur)}) s ON s.asin = t.rival
348
- ORDER BY t.asin, t.money_won DESC, t.rival'''
349
- return _read(cur, sql, (week, *targets))
350
 
351
 
352
  # --------------------------------------------------------------------------- discovery
353
 
354
- def discover(rt, cur, asins=None, *, candidate_rows=None, prefetched=None):
355
- """Refresh uncapped money-search pools while preserving removed and hand-added rivals."""
 
 
356
  backend = _backend(rt)
357
  requested = None if asins is None else _asins(asins)
358
  targets = sorted(our_asins(cur, requested))
359
  stamp = _now()
360
  discovery = {}
361
  for missing in sorted(set(requested or ()) - set(targets)):
362
- discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS,
363
- auto_count=0, month=None, keyword_ranks_on=None, product_type=None, held=0)
364
  week = newest_week(cur) if targets else None
365
- dates = _search_dates(cur, targets) if targets else {}
366
- rows = (_candidates(cur, targets, week) if candidate_rows is None else candidate_rows) if targets else []
367
- stored = backend.pool_rows(targets)
368
- before = backend.rivals_of(targets)
369
- values = set(targets) | {r[1] for r in rows} | {r['rival_asin'] for r in stored}
370
- facts = _pool_facts(cur, values, our_asins(cur), prefetched, rows)
371
- found, held = _eligible_candidates(rows, stored, facts, week, stamp)
 
 
 
 
 
 
 
372
  for target in targets:
373
- count = (len(found.get(target, ())) if facts.product_type(target) else
374
- sum(r['source'] == 'auto' for r in stored if r['asin'] == target and not r['removed']))
375
- meta = dates.get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0)
376
- cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, count)
377
- discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count,
378
- month=meta['month'], keyword_ranks_on=meta['keyword_ranks_on'],
379
- product_type=facts.product_type(target), held=held.get(target, 0))
380
- ready = [t for t in targets if facts.product_type(t)]
381
- backend.replace_auto(ready, [row for t in ready for row in found.get(t, ())], list(discovery.values()))
382
- backend.set_eligibility(stored, facts)
383
- members = backend.rivals_of(targets)
384
- pools = {target: len(members.get(target, ())) for target in targets}
385
  return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
386
- 'pools': pools, 'rowsRead': len(rows), 'held': held,
387
- 'changed': tuple(t for t in targets if set(before.get(t, ())) != set(members.get(t, ()))),
388
- 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
389
-
390
-
391
- def _eligible_candidates(rows, stored, facts, week, stamp):
392
- kept = {(r['asin'], r['rival_asin']) for r in stored if r['removed'] or r['source'] == 'added'}
393
- candidates = {(r[0], r[1]) for r in rows} | {
394
- (r['asin'], r['rival_asin']) for r in stored if r['source'] == 'added' and not r['removed']}
395
- removed = {(r['asin'], r['rival_asin']) for r in stored if r['removed']}
396
- held = {}
397
- for target, rival in candidates - removed:
398
- if (rival not in facts.own_asins and not facts.product_type(rival)
399
- and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)):
400
- held[target] = held.get(target, 0) + 1
401
- found = {}
402
- for target, rival, share, term, title, origin, money, terms, term_rank, _type, _name in rows:
403
- if (target, rival) in kept or not facts.eligible(target, rival):
404
- continue
405
- found.setdefault(target, []).append(dict(
406
- asin=target, rival_asin=rival, source='auto', removed=False, eligible=True,
407
- click_share=None if share is None else float(share), term=term, title=title,
408
- week=week, added_by=None, updated_at=stamp, origin=origin,
409
- money_won=float(money), terms_won=int(terms), term_rank=term_rank))
410
- return found, held
411
-
412
-
413
- def _discovery_cause(week, meta, count):
414
- if meta['month'] is None:
415
- return NO_MONTH
416
- if not meta['money_terms']:
417
- return NO_MONEY
418
- if meta['keyword_ranks_on'] is None:
419
- return NO_KEYWORD_RANKS
420
- if week is None:
421
- return NO_WEEK
422
- return None if count else NO_TOP3
423
-
424
-
425
- def _pool_cause(product_type, held, week, meta, count):
426
- if not product_type:
427
- return NO_PRODUCT_TYPE
428
- if held:
429
- return WAITING_PRODUCT_TYPE.format(count=held)
430
- return _discovery_cause(week, meta, count)
431
 
432
 
433
  def stale(rt, cur, target):
434
- """True when the ASIN's discovery predates a weekly, monthly or Keyword ranks source."""
435
  row = _backend(rt).discovery([target]).get(target)
436
  if row is None:
437
  return True
438
- types = _read(cur, f'SELECT product_type FROM ({_newest_snapshots_sql(cur)}) WHERE asin = ?', [target])
439
- product_type = types[0][0] if types else None
440
- if row['product_type'] != product_type or (not product_type and row['cause'] != NO_PRODUCT_TYPE):
441
- return True
442
- newest = {'week': newest_week(cur), **(_search_dates(cur, [target]).get(target) or {})}
443
- return any(newest.get(key) is not None and
444
- (row.get(key) is None or row[key] < newest[key])
445
- for key in ('week', 'month', 'keyword_ranks_on'))
446
 
447
 
448
  # --------------------------------------------------------------------------- doors
449
 
450
  def rivals(rt, target):
451
- """One ASIN's rivals: hand-added first, then money won and rank average, with discovery dates."""
452
  target = asin(target)
453
  backend = _backend(rt)
454
  rows = backend.rivals(target)
455
  found = backend.discovery([target]).get(target)
456
  return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
457
  'week': found['week'] if found else None,
458
- 'month': found['month'] if found else None,
459
- 'keywordRanksOn': found['keyword_ranks_on'] if found else None,
460
  'discoveredAt': found['discovered_at'] if found else None,
461
- 'cause': found['cause'] if found else None,
462
- 'productType': found['product_type'] if found else None,
463
- 'held': found['held'] if found else 0}
464
-
465
-
466
- def pool(rt, cur, target):
467
- """Read eligible pool rows through the caller's mirror cursor without writes or provider reads."""
468
- target = asin(target)
469
- backend = _backend(rt)
470
- if cur is None:
471
- return {**rivals(rt, target), 'rivals': [], 'competitors': [], 'dataAsOf': None, 'cause': NO_POOL}
472
- own = our_asins(cur)
473
- week = newest_week(cur)
474
- sources = _candidates(cur, [target], week) if target in own else []
475
- stored = backend.pool_rows([target])
476
- facts = _pool_facts(cur, {target} | {r[1] for r in sources} | {r['rival_asin'] for r in stored},
477
- own, candidate_rows=sources)
478
- found, held = _eligible_candidates(sources, stored, facts, week, _now())
479
- if facts.product_type(target):
480
- rows = found.get(target, []) + [dict(r, eligible=True) for r in stored
481
- if r['source'] == 'added' and not r['removed']
482
- and facts.eligible(target, r['rival_asin'])]
483
- else:
484
- rows = [r for r in stored if r['eligible'] and not r['removed'] and r['rival_asin'] not in own
485
- and _other_brand(facts.latest.get(r['rival_asin'], {}).get('brand'), facts.own_brands)]
486
- if target not in own:
487
- rows = []
488
- averages = backend.rank_averages([r['rival_asin'] for r in rows])
489
- rows = [{**r, **averages.get(r['rival_asin'], {'rank_average': None, 'rank_days': 0})} for r in rows]
490
- meta = _search_dates(cur, [target]).get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0)
491
- cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, len(rows))
492
- rows = sorted(rows, key=_pool_order)
493
- snapshots = _pool_snapshots(cur, [target, *(r['rival_asin'] for r in rows)])
494
- images = _pool_images(cur, [r['rival_asin'] for r in rows])
495
- dates = [s['snapshot_date'] for s in snapshots.values() if s['snapshot_date'] is not None]
496
- return {**rivals(rt, target), 'rivals': rows, 'week': week,
497
- 'month': meta['month'], 'keywordRanksOn': meta['keyword_ranks_on'],
498
- 'productType': facts.product_type(target), 'held': held.get(target, 0),
499
- 'cause': cause if target in own else NOT_OURS, 'dataAsOf': max(dates, default=None),
500
- 'competitors': [_pool_competitor(r, snapshots.get(r['rival_asin'], {}),
501
- images.get(r['rival_asin'], [])) for r in rows]}
502
-
503
-
504
- def _pool_snapshots(cur, values):
505
- snapshots = {}
506
- for part in _chunks(sorted(set(values))):
507
- rows = _read(cur, f'SELECT * FROM ({_newest_snapshots_sql(cur, POOL_SNAPSHOT_FIELDS)}) '
508
- f'WHERE asin IN {_in(part)}', part)
509
- for target, *fields in rows:
510
- snapshot = dict(zip(POOL_SNAPSHOT_FIELDS, fields))
511
- snapshot['snapshot_date'] = _day(snapshot['snapshot_date'])
512
- snapshot['bullets'] = json.loads(snapshot['bullets'] or '[]')
513
- snapshots[target] = snapshot
514
- return snapshots
515
-
516
-
517
- def _pool_images(cur, values):
518
- """Resolve only the newest snapshot's variants against changed-link history."""
519
- images = {}
520
- if not {'asin', *POOL_IMAGE_FIELDS} <= _columns(cur, IMAGES_TABLE):
521
- return images
522
- for part in _chunks(values):
523
- rows = _read(cur, f'''
524
- SELECT s.asin, i.variant, i.link, i.width, i.height
525
- FROM ({_newest_snapshots_sql(cur, ('image_variants',))}) s,
526
- json_each(s.image_variants) v, {IMAGES_TABLE} i
527
- WHERE s.asin IN {_in(part)} AND i.asin = s.asin
528
- AND i.variant = json_extract_string(v.value, '$.variant')
529
- AND i.link = json_extract_string(v.value, '$.link')
530
- ORDER BY s.asin, CAST(v.key AS INTEGER)''', part)
531
- for target, *fields in rows:
532
- images.setdefault(target, []).append(dict(zip(POOL_IMAGE_FIELDS, fields)))
533
- return images
534
-
535
-
536
- def _pool_competitor(row, snapshot, images):
537
- days = row['rank_days']
538
- return {'asin': row['rival_asin'], 'brand': snapshot.get('brand'), 'title': snapshot.get('title'),
539
- 'origin': row['origin'], 'moneyWon': row['money_won'], 'termsWon': row['terms_won'],
540
- 'term': row['term'], 'termRank': row['term_rank'], 'bestSellerRank': row['rank_average'],
541
- 'rankDays': days, 'rankCause': f'Rank average needs 7 days of ranks. {days} so far.' if days < 7 else None,
542
- 'price': snapshot.get('price'), 'currency': snapshot.get('currency'), 'prime': snapshot.get('prime'),
543
- 'offers': snapshot.get('offers'), 'imageCount': len(images), 'images': images,
544
- 'bullets': snapshot.get('bullets', []), 'snapshotOn': snapshot.get('snapshot_date')}
545
 
546
 
547
  def add(rt, target, rival, *, user, title=None):
@@ -549,10 +272,9 @@ def add(rt, target, rival, *, user, title=None):
549
  target, rival = asin(target), asin(rival, 'rival')
550
  if rival == target:
551
  raise ValueError('An ASIN cannot be its own rival.')
552
- return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, eligible=False,
553
  click_share=None, term=None, title=title, week=None,
554
- added_by=str(user or '') or None, updated_at=_now(),
555
- origin='added', money_won=0, terms_won=0, term_rank=None))
556
 
557
 
558
  def remove(rt, target, rival, *, user):
@@ -616,19 +338,6 @@ def runs(rt, limit=10):
616
  return _backend(rt).runs(int(limit))
617
 
618
 
619
- def rank_average(rt, asins):
620
- """Mean daily rank in each ASIN's newest display group, requiring seven of its newest 30 days."""
621
- values = _asins(asins)
622
- found = _backend(rt).rank_averages(values) if values else {}
623
- return {a: (found.get(a) or {}).get('rank_average') for a in values}
624
-
625
-
626
- def _pool_order(row):
627
- average = row.get('rank_average')
628
- return (-float(row.get('money_won') or 0),
629
- average is None, average if average is not None else 0, row['rival_asin'])
630
-
631
-
632
  # --------------------------------------------------------------------------- the daily quotes run
633
 
634
  #: The run row gets a progress marker after each block of this many ASINs.
@@ -642,8 +351,7 @@ MAX_BACKOFF_SECONDS = 300.0
642
  #: An unfinished run, or a finished run with a cause, is retried after this many hours.
643
  RETRY_RUN_HOURS = 3
644
 
645
- READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'facts': 'catalog facts'}
646
- FACT_COUNT_COLUMNS = {'title': 'title', 'product_type': 'product_type', 'image': 'image', 'rank': 'sales_rank'}
647
  NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
648
  NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
649
  NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
@@ -741,7 +449,7 @@ def _summary_quote(response):
741
 
742
 
743
  def _keyed(kind, got):
744
- if kind == 'facts':
745
  return {str(k).upper(): v for k, v in (got or {}).items()}
746
  if kind == 'offers':
747
  return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
@@ -781,16 +489,7 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
781
  attempt = 0
782
  while True:
783
  try:
784
- got = _keyed(kind, fn(part))
785
- answers.update(got)
786
- if kind == 'facts':
787
- for target, fact in got.items():
788
- if fact.get('cause'):
789
- causes[target] = fact['cause']
790
- if '(429)' in fact['cause']:
791
- counts['throttled'] += 1
792
- elif 'Amazon refused' in fact['cause']:
793
- counts['refused'] += 1
794
  break
795
  except Exception as exc: # noqa: BLE001
796
  if _throttled(exc) and attempt < THROTTLE_RETRIES:
@@ -806,19 +505,14 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
806
  return answers, causes
807
 
808
 
809
- def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts,
810
- prefetched=None, fact_causes=None):
811
  """One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
812
  mine = [a for a in block if a in own]
813
  others = [a for a in block if a not in own]
814
  offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
815
  summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
816
  counts=counts)
817
- cached = prefetched or {}
818
- facts, rank_causes = _read_batches('facts', readers['facts'], [a for a in block if a not in cached],
819
- sleep=sleep, log=log, counts=counts)
820
- facts.update({a: cached[a] for a in block if a in cached})
821
- rank_causes.update(fact_causes or {})
822
  stamp = _now()
823
  rows = []
824
  for target in block:
@@ -837,31 +531,20 @@ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, c
837
  row.update(_summary_quote(summed[target]))
838
  else:
839
  row['buy_box_cause'] = NO_PRICE
840
- rank = facts.get(target) or {}
841
  value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
842
  row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
843
- rank_cause=None if value is not None else rank_causes.get(target) or NO_RANK)
844
  rows.append(row)
845
- return rows, [_snapshot(row, facts.get(row['asin']) or {}, row['asin'] in own) for row in rows]
846
-
847
-
848
- def _snapshot(quote, facts, own):
849
- """C3 facts with price and rank taken from the exact persisted quote row."""
850
- return {**{key: facts.get(key) for key in ('product_type', 'brand', 'title', 'bullets',
851
- 'image_variants', 'prime', 'offers', 'rank_cat', 'image')},
852
- 'asin': quote['asin'], 'snapshot_date': quote['quote_on'],
853
- 'price': quote['offer_price' if own else 'buy_box_price'], 'currency': quote['currency'],
854
- 'sales_rank': quote['sales_rank'], 'display_group': quote['display_group']}
855
 
856
 
857
- def _tally(counts, rows, own, snapshots):
858
  for row in rows:
859
  counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
860
  or row['buy_box_price'] is not None)
861
  counts['ranked'] += int(row['sales_rank'] is not None)
862
  counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
863
- for key, field in FACT_COUNT_COLUMNS.items():
864
- counts['facts'][key] += sum(row.get(field) is not None and row[field] != '' for row in snapshots)
865
 
866
 
867
  def _changed(before, row):
@@ -869,47 +552,35 @@ def _changed(before, row):
869
  return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
870
 
871
 
872
- def _prepare_pools(rt, open_mirror, facts_reader, sleep, log, counts):
873
- with open_mirror() as cur:
874
- if cur is None:
875
- return None
876
- own = our_asins(cur)
877
- sources = _candidates(cur, sorted(own), newest_week(cur)) if own else []
878
- stored = _backend(rt).pool_rows(sorted(own))
879
- values = own | {r[1] for r in sources} | {r['rival_asin'] for r in stored if not r['removed']}
880
- known = _pool_facts(cur, values, own, candidate_rows=sources)
881
- missing = sorted(a for a in values if not known.product_type(a))
882
- prefetched, causes = _read_batches('facts', facts_reader, missing, sleep=sleep, log=log, counts=counts)
883
- prefetched = {a: prefetched.get(a, {}) for a in missing}
884
- with open_mirror() as cur:
885
- if cur is None:
886
- return None
887
- found = discover(rt, cur, candidate_rows=sources, prefetched=prefetched)
888
- return own, found, prefetched, causes
889
-
890
-
891
- def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run_id=None, now=None,
892
  log=None, sleep=time.sleep):
893
- """Discover, read unknown types outside the mirror, filter pools, then quote with cached facts."""
 
 
 
 
 
 
894
  _backend(rt)
895
  log = log or (lambda _message: None)
896
  now = now or datetime.now(timezone.utc)
897
  quote_on = now.date().isoformat()
898
  run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
899
  counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
900
- backoffs=0, discovered=0, week=None, pools={}, facts=dict.fromkeys(FACT_COUNT_COLUMNS, 0))
901
  start_run(rt, run_id, asin_total=0)
902
  log(f'run {run_id} started')
903
  done = 0
904
  try:
905
  try:
906
- prepared = _prepare_pools(rt, open_mirror, facts, sleep, log, counts)
907
- if prepared is None:
908
- return _finish(rt, run_id, done, counts, NO_MIRROR, log)
909
- own, found, prefetched, fact_causes = prepared
 
910
  except MirrorUnreadable as exc:
911
  return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
912
- counts.update(discovered=found['rivals'], week=found['week'], pools=found['pools'])
913
  others = sorted(set(rival_asins(rt)) - own)
914
  extra = sorted(set(wanted(rt)) - own - set(others))
915
  targets = sorted(own) + others + extra
@@ -918,23 +589,18 @@ def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run
918
  log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
919
  f'{len(extra)} wanted')
920
  before = newest_quotes(rt, targets) if targets else {}
921
- readers = {'offers': offers, 'summaries': summaries, 'facts': facts}
922
- changed, snapshots = list(found['changed']), []
923
  for block in _chunks(targets, PROGRESS_EVERY):
924
- rows, captured = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts,
925
- prefetched, fact_causes)
926
  put_quotes(rt, rows)
927
- snapshots.extend(captured)
928
- _tally(counts, rows, own, captured)
929
  changed += [row['asin'] for row in rows if _changed(before, row)]
930
  done += len(block)
931
  mark_run(rt, run_id, progress=done, counts=counts)
932
  log(f'run {run_id} progress {done}/{len(targets)}')
933
- cause = (f'Amazon could not finish every read at its rate or refused a read. '
934
- f'Retry the run within {RETRY_RUN_HOURS} hours after the rate limit delay.'
935
- if counts['throttled'] or counts['refused'] else None)
936
- result = _finish(rt, run_id, done, counts, cause, log)
937
- result.update(changed=tuple(dict.fromkeys(changed)), snapshots=snapshots)
938
  return result
939
  except Exception as exc:
940
  mark_run(rt, run_id, progress=done, finished=True, counts=counts,
@@ -947,7 +613,7 @@ def _finish(rt, run_id, done, counts, cause, log):
947
  mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
948
  log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
949
  f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
950
- return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': (), 'snapshots': []}
951
 
952
 
953
  # --------------------------------------------------------------------------- competition status and rollups
@@ -1217,70 +883,6 @@ def validate(rt, tables, plan):
1217
  'rows': len(parents), 'mismatches': bad['parent']}]
1218
 
1219
 
1220
- # --------------------------------------------------------------------------- pool validation
1221
-
1222
- def _pool_expected(cur, targets):
1223
- """Recompute raw click money independently of discovery's candidate and money queries."""
1224
- required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
1225
- 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
1226
- clicks = {'search_term', 'clicked_asin', 'click_share', 'click_share_rank', '_period_start', '_period_end'}
1227
- if not required <= _columns(cur, SQP_TABLE) or not clicks <= _columns(cur, BA_TABLE):
1228
- return {}
1229
- rows = _read(cur, f'''
1230
- WITH monthly AS (
1231
- SELECT upper(trim(CAST(asin AS VARCHAR))) AS owner,
1232
- lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS query,
1233
- TRY_CAST(end_date AS DATE) AS ends,
1234
- TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
1235
- TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS money
1236
- FROM {SQP_TABLE}
1237
- WHERE TRY_CAST(end_date AS DATE) > TRY_CAST(start_date AS DATE) + INTERVAL '7 days'
1238
- AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
1239
- newest_month AS (SELECT owner, max(ends) AS ends FROM monthly GROUP BY owner),
1240
- searches AS (
1241
- SELECT m.owner, m.query, max(m.money) AS money FROM monthly m
1242
- JOIN newest_month n ON n.owner = m.owner AND n.ends = m.ends
1243
- GROUP BY m.owner, m.query HAVING max(m.money) > 0),
1244
- report AS (
1245
- SELECT lower(trim(CAST(search_term AS VARCHAR))) AS query,
1246
- upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
1247
- TRY_CAST(click_share AS DOUBLE) AS share,
1248
- TRY_CAST(click_share_rank AS INTEGER) AS position, {_week_sql()} AS starts
1249
- FROM {BA_TABLE} WHERE {weekly_period_sql()}),
1250
- term_clicks AS (
1251
- SELECT query, rival, max(share) AS share, min(position) AS position FROM report
1252
- WHERE starts = (SELECT max(starts) FROM report) AND position BETWEEN 1 AND 3
1253
- GROUP BY query, rival)
1254
- SELECT s.owner, c.rival, sum(s.money * coalesce(c.share, 0) * 0.01),
1255
- sum(CASE WHEN c.position = 1 THEN 1 ELSE 0 END)
1256
- FROM searches s JOIN term_clicks c ON c.query = s.query
1257
- WHERE s.owner IN {_in(targets)} GROUP BY s.owner, c.rival''', targets)
1258
- return {(a, r): (float(m), int(n)) for a, r, m, n in rows}
1259
-
1260
-
1261
- def validate_pool(cur, rt, targets):
1262
- """Tie stored auto-pool money and term wins to an independent raw-source aggregation."""
1263
- targets = _asins(targets)
1264
- expected = _pool_expected(cur, targets) if targets else {}
1265
- bad_money, bad_terms, count = [], [], 0
1266
- backend = _backend(rt)
1267
- for target in targets:
1268
- for row in backend.rivals(target):
1269
- if row['source'] != 'auto':
1270
- continue
1271
- count += 1
1272
- money, terms = expected.get((target, row['rival_asin']), (0, 0))
1273
- label = f'{target} {row["rival_asin"]}'
1274
- if _differs(row['money_won'], money, 0.000001):
1275
- bad_money.append(label)
1276
- if row['terms_won'] != terms:
1277
- bad_terms.append(label)
1278
- return [{'check': 'pool money won ties to Brand Analytics and search report rows',
1279
- 'ok': not bad_money, 'rows': count, 'mismatches': bad_money},
1280
- {'check': 'pool terms won ties to Brand Analytics and search report rows',
1281
- 'ok': not bad_terms, 'rows': count, 'mismatches': bad_terms}]
1282
-
1283
-
1284
  # --------------------------------------------------------------------------- storage
1285
 
1286
  def _ddl(table):
@@ -1351,10 +953,6 @@ class PgRivalStore:
1351
  """Create every rival table and index, only on a write."""
1352
  for name in TABLES:
1353
  con.execute(_ddl(self.t(name))[name])
1354
- for name, fields in POOL_FIELDS.items():
1355
- for column, (kind, default) in fields.items():
1356
- con.execute(f'ALTER TABLE {self.t(name)} ADD COLUMN IF NOT EXISTS {column} {kind} '
1357
- f'DEFAULT {default}')
1358
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
1359
 
1360
  @contextmanager
@@ -1364,49 +962,29 @@ class PgRivalStore:
1364
  self.ensure_schema(con)
1365
  yield con
1366
 
1367
- def _select(self, con, name, columns, where, args, order='', *, include_ineligible=False):
1368
  if not self._exists(con, name):
1369
  return []
1370
- projection = columns
1371
- fields = POOL_FIELDS.get(name, {})
1372
- if fields.keys() & set(columns) or name == RIVALS:
1373
- existing = {r[0] for r in con.execute(
1374
- 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
1375
- (self.schema, name)).fetchall()}
1376
- projection = [f'{fields[c][1]} AS {c}' if c in fields and c not in existing else c for c in columns]
1377
- if name == RIVALS and 'eligible' in existing and not include_ineligible:
1378
- where = f'({where}) AND eligible = TRUE' if where else 'eligible = TRUE'
1379
- rows = con.execute(f'SELECT {", ".join(projection)} FROM {self.t(name)} WHERE tenant = %s'
1380
  f'{" AND " + where if where else ""}{order}',
1381
  (self.tenant_slug, *args)).fetchall()
1382
  return [dict(zip(columns, row)) for row in rows]
1383
 
1384
  @staticmethod
1385
  def _rival(row):
1386
- return {**row, 'removed': bool(row['removed']), 'eligible': bool(row['eligible']), 'week': _day(row['week']),
1387
- 'origin': 'added' if row['source'] == 'added' else row['origin'],
1388
  'click_share': None if row['click_share'] is None else float(row['click_share'])}
1389
 
1390
- def pool_rows(self, targets):
1391
- """Stored choices include hidden additions and removal tombstones."""
 
1392
  with self._connection() as con:
1393
- return [self._rival(row) for part in _chunks(list(targets)) for row in self._select(
1394
- con, RIVALS, RIVAL_COLUMNS, f'asin IN ({_marks(len(part))})', part,
1395
- include_ineligible=True)]
1396
-
1397
- def set_eligibility(self, rows, facts):
1398
- with self._write() as con:
1399
- for row in rows:
1400
- if row['removed']:
1401
- continue
1402
- target, rival = row['asin'], row['rival_asin']
1403
- allowed = (facts.eligible(target, rival) if facts.product_type(target) else
1404
- row['eligible'] and rival not in facts.own_asins
1405
- and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands))
1406
- if allowed != row['eligible']:
1407
- con.execute(f'UPDATE {self.t(RIVALS)} SET eligible = %s '
1408
- 'WHERE tenant = %s AND asin = %s AND rival_asin = %s',
1409
- (allowed, self.tenant_slug, target, rival))
1410
 
1411
  def replace_auto(self, targets, rows, discovery):
1412
  with self._write() as con:
@@ -1425,9 +1003,7 @@ class PgRivalStore:
1425
  + ', '.join(f'({_marks(len(cols))})' for _ in part)
1426
  + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
1427
  'discovered_at = excluded.discovered_at, cause = excluded.cause, '
1428
- 'auto_count = excluded.auto_count, month = excluded.month, '
1429
- 'keyword_ranks_on = excluded.keyword_ranks_on, '
1430
- 'product_type = excluded.product_type, held = excluded.held',
1431
  tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
1432
 
1433
  def discovery(self, targets):
@@ -1435,41 +1011,14 @@ class PgRivalStore:
1435
  rows = []
1436
  for part in _chunks(list(targets)):
1437
  rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
1438
- return {row['asin']: {**row, **{key: _day(row[key]) for key in
1439
- ('week', 'month', 'keyword_ranks_on')}} for row in rows}
1440
 
1441
  def rivals(self, target):
1442
  with self._connection() as con:
1443
- rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False))
1444
- averages = self.rank_averages([row['rival_asin'] for row in rows])
1445
- enriched = [{**self._rival(row), **averages.get(row['rival_asin'],
1446
- {'rank_average': None, 'rank_days': 0})} for row in rows]
1447
- return sorted(enriched, key=lambda row: (row['source'] != 'added', *_pool_order(row)))
1448
-
1449
- def rank_averages(self, values):
1450
- out = {}
1451
- with self._connection() as con:
1452
- if not self._exists(con, QUOTES):
1453
- return out
1454
- for part in _chunks(values):
1455
- rows = con.execute(f'''
1456
- WITH newest AS (
1457
- SELECT asin, max(quote_on) AS newest FROM {self.t(QUOTES)}
1458
- WHERE tenant = %s AND asin IN ({_marks(len(part))}) GROUP BY asin),
1459
- groups AS (
1460
- SELECT n.asin, n.newest, q.display_group FROM newest n
1461
- JOIN {self.t(QUOTES)} q ON q.asin = n.asin AND q.quote_on = n.newest
1462
- WHERE q.tenant = %s)
1463
- SELECT q.asin, CASE WHEN count(q.sales_rank) >= 7 THEN avg(q.sales_rank) END,
1464
- count(q.sales_rank)
1465
- FROM groups m JOIN {self.t(QUOTES)} q ON q.asin = m.asin
1466
- AND q.display_group = m.display_group AND q.quote_on >= m.newest - 29
1467
- AND q.quote_on <= m.newest
1468
- WHERE q.tenant = %s GROUP BY q.asin''',
1469
- (self.tenant_slug, *part, self.tenant_slug, self.tenant_slug)).fetchall()
1470
- out.update({a: dict(rank_average=None if avg is None else float(avg), rank_days=n)
1471
- for a, avg, n in rows})
1472
- return out
1473
 
1474
  def removed_count(self, target):
1475
  with self._connection() as con:
@@ -1483,15 +1032,11 @@ class PgRivalStore:
1483
  with self._write() as con:
1484
  got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
1485
  "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
1486
- "origin = 'added', money_won = 0, terms_won = 0, term_rank = NULL, "
1487
  'removed = excluded.removed, added_by = excluded.added_by, '
1488
  'updated_at = excluded.updated_at, '
1489
  'title = coalesce(excluded.title, title) '
1490
  f'RETURNING {", ".join(RIVAL_COLUMNS)}',
1491
  (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
1492
- if not dict(zip(RIVAL_COLUMNS, got))['eligible']:
1493
- con.execute(f'UPDATE {self.t(DISCOVERY)} SET week = NULL, month = NULL, keyword_ranks_on = NULL '
1494
- 'WHERE tenant = %s AND asin = %s', (self.tenant_slug, row['asin']))
1495
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1496
 
1497
  def remove(self, target, rival, user, stamp):
@@ -1505,13 +1050,15 @@ class PgRivalStore:
1505
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1506
 
1507
  def rivals_of(self, targets):
1508
- """`{asin: [rival ASIN]}` of the rivals not removed."""
1509
  out = {}
1510
  with self._connection() as con:
1511
  for part in _chunks(list(targets)):
1512
  for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
1513
  f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
1514
- ' ORDER BY asin, rival_asin'):
 
 
1515
  out.setdefault(row['asin'], []).append(row['rival_asin'])
1516
  return out
1517
 
@@ -1532,10 +1079,6 @@ class PgRivalStore:
1532
  with self._connection() as con:
1533
  if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
1534
  return own, rows
1535
- columns = {r[0] for r in con.execute(
1536
- 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
1537
- (self.schema, RIVALS)).fetchall()}
1538
- eligibility = 'AND r.eligible = TRUE ' if 'eligible' in columns else ''
1539
  for part in _chunks(list(targets)):
1540
  marks = _marks(len(part))
1541
  for target, rank, group in con.execute(
@@ -1546,8 +1089,7 @@ class PgRivalStore:
1546
  rows += [tuple(r) for r in con.execute(
1547
  f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
1548
  f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
1549
- f'WHERE r.tenant = %s AND r.removed = %s {eligibility}'
1550
- f'AND r.asin IN ({marks}) AND {newest}',
1551
  (self.tenant_slug, False, *part)).fetchall()]
1552
  return own, rows
1553
 
 
1
+ """Per-tenant PostgreSQL rivals of our ASINs, found from Brand Analytics, plus their daily quotes.
2
 
3
  Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
4
  module (verify_layering) and no App module (rule 13). Tables are created on the first write.
5
  """
6
 
7
  from contextlib import contextmanager
 
8
  from datetime import date, datetime, timedelta, timezone
9
  import json
10
  import re
 
19
  WANTED = 'amazon_rival_wanted'
20
  TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
21
 
22
+ #: R23: at most this many auto-found rivals per ASIN, by click share.
23
+ AUTO_LIMIT = 10
24
  #: Brand Analytics keeps the top 3 clicked ASINs of each search term.
25
  TOP_CLICKED = 3
26
  SOURCES = frozenset(('auto', 'added'))
 
29
  BA_TABLE = 'amzn_brand_analytics'
30
  LISTINGS_TABLE = 'amzn_listings'
31
  PRICING_TABLE = 'amzn_pricing'
 
 
 
 
 
 
32
 
33
  NO_TOP3 = "Not in any search term's top 3 clicked"
34
  NO_WEEK = 'Brand Analytics has no dated weekly reports'
 
 
 
 
 
35
  NOT_OURS = 'Not an ASIN in your Amazon listings'
36
  PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
 
37
  ASIN_RE = re.compile(r'[A-Z0-9]{10}')
38
 
 
 
 
 
 
 
 
39
  RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
40
+ 'added_by', 'updated_at')
41
+ DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count')
42
  QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
43
  'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
44
  RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
 
175
  return [r[0] for r in rows if r[0]]
176
 
177
 
178
+ def _candidates(cur, targets, week, per_asin):
179
+ """Ranked co-clicked ASINs per target, bounded in SQL to `per_asin` rows each."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  cols = _columns(cur, BA_TABLE)
181
+ share = 'TRY_CAST("click_share" AS DOUBLE)' if 'click_share' in cols else 'CAST(NULL AS DOUBLE)'
182
+ title = 'CAST("clicked_item_name" AS VARCHAR)' if 'clicked_item_name' in cols else 'CAST(NULL AS VARCHAR)'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  sql = f'''
184
+ WITH ours AS (
185
+ SELECT DISTINCT upper(trim(CAST("asin1" AS VARCHAR))) AS asin FROM {LISTINGS_TABLE}
186
+ WHERE "asin1" IS NOT NULL),
187
+ week AS (
188
+ SELECT "search_term" AS term, upper(trim(CAST("clicked_asin" AS VARCHAR))) AS clicked,
189
+ {share} AS share, {title} AS title
190
+ FROM {BA_TABLE}
191
+ WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}
192
+ AND ({weekly_period_sql()}) AND {_week_sql()} = CAST(? AS DATE)),
 
 
 
 
 
 
 
193
  hits AS (
194
+ SELECT DISTINCT term, clicked AS asin FROM week WHERE clicked IN {_in(targets)}),
195
+ cands AS (
196
+ SELECT h.asin, w.clicked AS rival, w.share, w.term, w.title
197
+ FROM hits h JOIN week w ON w.term = h.term
198
+ WHERE w.clicked <> '' AND w.clicked NOT IN (SELECT asin FROM ours)),
199
+ best AS (
200
+ SELECT asin, rival, max(share) AS share,
201
+ arg_max(term, coalesce(share, -1)) AS term, arg_max(title, coalesce(share, -1)) AS title
202
+ FROM cands GROUP BY asin, rival),
203
+ ranked AS (
204
+ SELECT *, row_number() OVER (PARTITION BY asin ORDER BY share DESC NULLS LAST, rival) AS rn
205
+ FROM best)
206
+ SELECT asin, rival, share, term, title FROM ranked WHERE rn <= ? ORDER BY asin, rn'''
207
+ return _read(cur, sql, (week, *targets, per_asin))
 
 
 
 
 
 
 
208
 
209
 
210
  # --------------------------------------------------------------------------- discovery
211
 
212
+ def discover(rt, cur, asins=None):
213
+ """Rewrite the auto rivals of our ASINs (or of `asins`) from the newest dated week.
214
+
215
+ A removed rival stays removed and a hand-added one stays added. Returns counts and `rowsRead`."""
216
  backend = _backend(rt)
217
  requested = None if asins is None else _asins(asins)
218
  targets = sorted(our_asins(cur, requested))
219
  stamp = _now()
220
  discovery = {}
221
  for missing in sorted(set(requested or ()) - set(targets)):
222
+ discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, auto_count=0)
 
223
  week = newest_week(cur) if targets else None
224
+ found = {}
225
+ rows_read = 0
226
+ if targets and week is not None:
227
+ kept = backend.kept(targets)
228
+ per_asin = AUTO_LIMIT + max((len(v) for v in kept.values()), default=0)
229
+ rows = _candidates(cur, targets, week, per_asin)
230
+ rows_read = len(rows)
231
+ for target, rival, share, term, title in rows:
232
+ if rival in kept.get(target, ()) or len(found.get(target, ())) >= AUTO_LIMIT:
233
+ continue
234
+ found.setdefault(target, []).append(dict(
235
+ asin=target, rival_asin=rival, source='auto', removed=False,
236
+ click_share=None if share is None else float(share), term=term, title=title,
237
+ week=week, added_by=None, updated_at=stamp))
238
  for target in targets:
239
+ count = len(found.get(target, ()))
240
+ cause = NO_WEEK if week is None else (None if count else NO_TOP3)
241
+ discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count)
242
+ backend.replace_auto(targets, [row for t in targets for row in found.get(t, ())], list(discovery.values()))
 
 
 
 
 
 
 
 
243
  return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
244
+ 'rowsRead': rows_read, 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
 
246
 
247
  def stale(rt, cur, target):
248
+ """True when this ASIN's discovery is missing or predates the newest dated week."""
249
  row = _backend(rt).discovery([target]).get(target)
250
  if row is None:
251
  return True
252
+ newest = newest_week(cur)
253
+ return newest is not None and (row['week'] is None or row['week'] < newest)
 
 
 
 
 
 
254
 
255
 
256
  # --------------------------------------------------------------------------- doors
257
 
258
  def rivals(rt, target):
259
+ """One ASIN's rivals: hand-added first, then auto by click share, with its discovery row."""
260
  target = asin(target)
261
  backend = _backend(rt)
262
  rows = backend.rivals(target)
263
  found = backend.discovery([target]).get(target)
264
  return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
265
  'week': found['week'] if found else None,
 
 
266
  'discoveredAt': found['discovered_at'] if found else None,
267
+ 'cause': found['cause'] if found else None}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
 
270
  def add(rt, target, rival, *, user, title=None):
 
272
  target, rival = asin(target), asin(rival, 'rival')
273
  if rival == target:
274
  raise ValueError('An ASIN cannot be its own rival.')
275
+ return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False,
276
  click_share=None, term=None, title=title, week=None,
277
+ added_by=str(user or '') or None, updated_at=_now()))
 
278
 
279
 
280
  def remove(rt, target, rival, *, user):
 
338
  return _backend(rt).runs(int(limit))
339
 
340
 
 
 
 
 
 
 
 
 
 
 
 
 
 
341
  # --------------------------------------------------------------------------- the daily quotes run
342
 
343
  #: The run row gets a progress marker after each block of this many ASINs.
 
351
  #: An unfinished run, or a finished run with a cause, is retried after this many hours.
352
  RETRY_RUN_HOURS = 3
353
 
354
+ READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'ranks': 'sales rank'}
 
355
  NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
356
  NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
357
  NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
 
449
 
450
 
451
  def _keyed(kind, got):
452
+ if kind == 'ranks':
453
  return {str(k).upper(): v for k, v in (got or {}).items()}
454
  if kind == 'offers':
455
  return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
 
489
  attempt = 0
490
  while True:
491
  try:
492
+ answers.update(_keyed(kind, fn(part)))
 
 
 
 
 
 
 
 
 
493
  break
494
  except Exception as exc: # noqa: BLE001
495
  if _throttled(exc) and attempt < THROTTLE_RETRIES:
 
505
  return answers, causes
506
 
507
 
508
+ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts):
 
509
  """One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
510
  mine = [a for a in block if a in own]
511
  others = [a for a in block if a not in own]
512
  offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
513
  summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
514
  counts=counts)
515
+ ranked, rank_causes = _read_batches('ranks', readers['ranks'], block, sleep=sleep, log=log, counts=counts)
 
 
 
 
516
  stamp = _now()
517
  rows = []
518
  for target in block:
 
531
  row.update(_summary_quote(summed[target]))
532
  else:
533
  row['buy_box_cause'] = NO_PRICE
534
+ rank = ranked.get(target) or {}
535
  value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
536
  row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
537
+ rank_cause=rank_causes.get(target) or (None if value is not None else NO_RANK))
538
  rows.append(row)
539
+ return rows
 
 
 
 
 
 
 
 
 
540
 
541
 
542
+ def _tally(counts, rows, own):
543
  for row in rows:
544
  counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
545
  or row['buy_box_price'] is not None)
546
  counts['ranked'] += int(row['sales_rank'] is not None)
547
  counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
 
 
548
 
549
 
550
  def _changed(before, row):
 
552
  return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
553
 
554
 
555
+ def run_quotes(rt, open_mirror, *, offers, summaries, ranks, seller_id=None, run_id=None, now=None,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
556
  log=None, sleep=time.sleep):
557
+ """Rediscover rivals, then quote our ASINs, every rival and every wanted ASIN for today.
558
+
559
+ `open_mirror()` is a context manager yielding a mirror cursor or None. It closes before the first
560
+ SP-API read. `offers`, `summaries` and `ranks` each take a list of at most 20 ASINs and return
561
+ `item_offers` responses, `competitive_summaries` responses and `{asin: {rank, group}}`. A raised
562
+ read is a cause on each ASIN of its batch, and a 429 backs off first. Returns the run's counts
563
+ and `changed`, the ASINs whose quote differs from their previous newest quote."""
564
  _backend(rt)
565
  log = log or (lambda _message: None)
566
  now = now or datetime.now(timezone.utc)
567
  quote_on = now.date().isoformat()
568
  run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
569
  counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
570
+ backoffs=0, discovered=0, week=None)
571
  start_run(rt, run_id, asin_total=0)
572
  log(f'run {run_id} started')
573
  done = 0
574
  try:
575
  try:
576
+ with open_mirror() as cur:
577
+ if cur is None:
578
+ return _finish(rt, run_id, done, counts, NO_MIRROR, log)
579
+ found = discover(rt, cur)
580
+ own = our_asins(cur)
581
  except MirrorUnreadable as exc:
582
  return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
583
+ counts.update(discovered=found['rivals'], week=found['week'])
584
  others = sorted(set(rival_asins(rt)) - own)
585
  extra = sorted(set(wanted(rt)) - own - set(others))
586
  targets = sorted(own) + others + extra
 
589
  log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
590
  f'{len(extra)} wanted')
591
  before = newest_quotes(rt, targets) if targets else {}
592
+ readers = {'offers': offers, 'summaries': summaries, 'ranks': ranks}
593
+ changed = []
594
  for block in _chunks(targets, PROGRESS_EVERY):
595
+ rows = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts)
 
596
  put_quotes(rt, rows)
597
+ _tally(counts, rows, own)
 
598
  changed += [row['asin'] for row in rows if _changed(before, row)]
599
  done += len(block)
600
  mark_run(rt, run_id, progress=done, counts=counts)
601
  log(f'run {run_id} progress {done}/{len(targets)}')
602
+ result = _finish(rt, run_id, done, counts, None, log)
603
+ result['changed'] = tuple(changed)
 
 
 
604
  return result
605
  except Exception as exc:
606
  mark_run(rt, run_id, progress=done, finished=True, counts=counts,
 
613
  mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
614
  log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
615
  f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
616
+ return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()}
617
 
618
 
619
  # --------------------------------------------------------------------------- competition status and rollups
 
883
  'rows': len(parents), 'mismatches': bad['parent']}]
884
 
885
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
886
  # --------------------------------------------------------------------------- storage
887
 
888
  def _ddl(table):
 
953
  """Create every rival table and index, only on a write."""
954
  for name in TABLES:
955
  con.execute(_ddl(self.t(name))[name])
 
 
 
 
956
  con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
957
 
958
  @contextmanager
 
962
  self.ensure_schema(con)
963
  yield con
964
 
965
+ def _select(self, con, name, columns, where, args, order=''):
966
  if not self._exists(con, name):
967
  return []
968
+ rows = con.execute(f'SELECT {", ".join(columns)} FROM {self.t(name)} WHERE tenant = %s'
 
 
 
 
 
 
 
 
 
969
  f'{" AND " + where if where else ""}{order}',
970
  (self.tenant_slug, *args)).fetchall()
971
  return [dict(zip(columns, row)) for row in rows]
972
 
973
  @staticmethod
974
  def _rival(row):
975
+ return {**row, 'removed': bool(row['removed']), 'week': _day(row['week']),
 
976
  'click_share': None if row['click_share'] is None else float(row['click_share'])}
977
 
978
+ def kept(self, targets):
979
+ """`{asin: {rival}}` of removed and hand-added rivals, which discovery must not touch."""
980
+ out = {}
981
  with self._connection() as con:
982
+ for part in _chunks(targets):
983
+ for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
984
+ f"asin IN ({_marks(len(part))}) AND (removed = %s OR source = 'added')",
985
+ (*part, True)):
986
+ out.setdefault(row['asin'], set()).add(row['rival_asin'])
987
+ return out
 
 
 
 
 
 
 
 
 
 
 
988
 
989
  def replace_auto(self, targets, rows, discovery):
990
  with self._write() as con:
 
1003
  + ', '.join(f'({_marks(len(cols))})' for _ in part)
1004
  + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
1005
  'discovered_at = excluded.discovered_at, cause = excluded.cause, '
1006
+ 'auto_count = excluded.auto_count',
 
 
1007
  tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
1008
 
1009
  def discovery(self, targets):
 
1011
  rows = []
1012
  for part in _chunks(list(targets)):
1013
  rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
1014
+ return {row['asin']: {**row, 'week': _day(row['week'])} for row in rows}
 
1015
 
1016
  def rivals(self, target):
1017
  with self._connection() as con:
1018
+ rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False),
1019
+ " ORDER BY CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
1020
+ 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, rival_asin')
1021
+ return [self._rival(row) for row in rows]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1022
 
1023
  def removed_count(self, target):
1024
  with self._connection() as con:
 
1032
  with self._write() as con:
1033
  got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
1034
  "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
 
1035
  'removed = excluded.removed, added_by = excluded.added_by, '
1036
  'updated_at = excluded.updated_at, '
1037
  'title = coalesce(excluded.title, title) '
1038
  f'RETURNING {", ".join(RIVAL_COLUMNS)}',
1039
  (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
 
 
 
1040
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1041
 
1042
  def remove(self, target, rival, user, stamp):
 
1050
  return self._rival(dict(zip(RIVAL_COLUMNS, got)))
1051
 
1052
  def rivals_of(self, targets):
1053
+ """`{asin: [rival ASIN]}` of the rivals not removed, hand-added first then by click share."""
1054
  out = {}
1055
  with self._connection() as con:
1056
  for part in _chunks(list(targets)):
1057
  for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
1058
  f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
1059
+ " ORDER BY asin, CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
1060
+ 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, '
1061
+ 'rival_asin'):
1062
  out.setdefault(row['asin'], []).append(row['rival_asin'])
1063
  return out
1064
 
 
1079
  with self._connection() as con:
1080
  if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
1081
  return own, rows
 
 
 
 
1082
  for part in _chunks(list(targets)):
1083
  marks = _marks(len(part))
1084
  for target, rank, group in con.execute(
 
1089
  rows += [tuple(r) for r in con.execute(
1090
  f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
1091
  f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
1092
+ f'WHERE r.tenant = %s AND r.removed = %s AND r.asin IN ({marks}) AND {newest}',
 
1093
  (self.tenant_slug, False, *part)).fetchall()]
1094
  return own, rows
1095
 
platform/core/connector_spec.py CHANGED
@@ -843,16 +843,6 @@ AMAZON_TABLES = (
843
  identity=("rank_id",),
844
  links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
845
  ),
846
- ConnectorTableSpec(
847
- key=_AMZN + "competitor_snapshots", native="competitor_run:amazon_quotes_loop.run_tenant",
848
- source="amzn_competitor_snapshots", label="Amazon Marketing: Competitor snapshots", pinned="asin",
849
- identity=("asin", "snapshot_date"),
850
- ),
851
- ConnectorTableSpec(
852
- key=_AMZN + "competitor_images", native="competitor_run:amazon_quotes_loop.run_tenant",
853
- source="amzn_competitor_images", label="Amazon Marketing: Competitor images", pinned="asin",
854
- identity=("asin", "variant", "link"),
855
- ),
856
  )
857
 
858
  #: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
@@ -883,38 +873,6 @@ AMAZON_TABLES = AMAZON_TABLES + tuple(
883
 
884
  # Local projections supplement measured vendor columns without changing the table contract.
885
  AMAZON_FIELDS = {
886
- _AMZN + "competitor_snapshots": tuple(
887
- {"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
888
- for name, kind, label in (
889
- ("asin", "VARCHAR", "ASIN"),
890
- ("snapshot_date", "DATE", "Snapshot date"),
891
- ("product_type", "VARCHAR", "Product type"),
892
- ("brand", "VARCHAR", "Brand"),
893
- ("title", "VARCHAR", "Title"),
894
- ("bullets", "VARCHAR", "Bullet points"),
895
- ("image_variants", "VARCHAR", "Image variants"),
896
- ("price", "DOUBLE", "Price"),
897
- ("currency", "VARCHAR", "Currency"),
898
- ("prime", "BOOLEAN", "Prime"),
899
- ("offers", "BIGINT", "Offers"),
900
- ("sales_rank", "BIGINT", "Sales rank"),
901
- ("display_group", "VARCHAR", "Display group"),
902
- ("rank_cat", "VARCHAR", "Rank category"),
903
- ("image", "VARCHAR", "Image"),
904
- )
905
- ),
906
- _AMZN + "competitor_images": tuple(
907
- {"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
908
- for name, kind, label in (
909
- ("asin", "VARCHAR", "ASIN"),
910
- ("variant", "VARCHAR", "Variant"),
911
- ("link", "VARCHAR", "Link"),
912
- ("sha256", "VARCHAR", "SHA256"),
913
- ("width", "BIGINT", "Width"),
914
- ("height", "BIGINT", "Height"),
915
- ("snapshot_date", "DATE", "Snapshot date"),
916
- )
917
- ),
918
  _AMZN + "keyword_ranks": tuple(
919
  {"name": name, "type": kind, "label": label, "source": source}
920
  for name, kind, label, source in (
@@ -3637,9 +3595,6 @@ AMAZON_DEFAULT_COLUMNS = {
3637
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3638
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
3639
  "sponsored_rank_basis", "sku", "asin"),
3640
- _AMZN + "competitor_snapshots": ("asin", "snapshot_date", "brand", "title", "product_type",
3641
- "price", "currency", "prime", "offers", "sales_rank", "rank_cat"),
3642
- _AMZN + "competitor_images": ("asin", "variant", "link", "sha256", "width", "height", "snapshot_date"),
3643
  }
3644
  # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
3645
  AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
@@ -3767,7 +3722,7 @@ AMAZON_GROUPS = {
3767
  **{_AMZN + t: "Advertising" for t in (
3768
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3769
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
3770
- "sku_keywords", "keyword_ranks", "competitor_snapshots", "competitor_images")},
3771
  }
3772
  AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
3773
 
 
843
  identity=("rank_id",),
844
  links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
845
  ),
 
 
 
 
 
 
 
 
 
 
846
  )
847
 
848
  #: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
 
873
 
874
  # Local projections supplement measured vendor columns without changing the table contract.
875
  AMAZON_FIELDS = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
876
  _AMZN + "keyword_ranks": tuple(
877
  {"name": name, "type": kind, "label": label, "source": source}
878
  for name, kind, label, source in (
 
3595
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3596
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
3597
  "sponsored_rank_basis", "sku", "asin"),
 
 
 
3598
  }
3599
  # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
3600
  AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
 
3722
  **{_AMZN + t: "Advertising" for t in (
3723
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3724
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
3725
+ "sku_keywords", "keyword_ranks")},
3726
  }
3727
  AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
3728
 
platform/core/external_tools.py CHANGED
@@ -267,7 +267,7 @@ _APPLICATION = {
267
  "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
268
  "amazon_operator", "restock_planner", "reimbursements", "aged_stock",
269
  "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
270
- "amazon_dashboard", "amazon_profit", "alerts", "reports"]},
271
  "fieldMap": {"type": "object", "additionalProperties": _STRING},
272
  "navPlacement": _object({"section": {"const": "app"},
273
  "order": {"type": "integer", "minimum": 0, "maximum": 999}},
 
267
  "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
268
  "amazon_operator", "restock_planner", "reimbursements", "aged_stock",
269
  "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
270
+ "amazon_dashboard", "amazon_profit", "alerts"]},
271
  "fieldMap": {"type": "object", "additionalProperties": _STRING},
272
  "navPlacement": _object({"section": {"const": "app"},
273
  "order": {"type": "integer", "minimum": 0, "maximum": 999}},
platform/core/shares.py CHANGED
@@ -80,7 +80,7 @@ SHARES_KEY = 'object_shares'
80
  #: here is this tuple member. That is the point of one registry: the new object's WALL is written
81
  #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
82
  #: permission decision in a fourth file ([[one-evaluator-per-question]]).
83
- KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook', 'report')
84
 
85
  #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
86
  #: Spelled as a single character so it can never collide with a username (usernames are lower-case
@@ -692,32 +692,13 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
692
  return repaired[0]
693
 
694
 
695
- def _synthetic_qa_publication(target, table_keys):
696
- """Identify private scratch publications outside the legacy account plane."""
697
- from core import user_tables, users
698
-
699
- qa = users.qa_identity(users._QA_USERNAME)
700
- if (not qa or not table_keys or getattr(target, 'key', None) != qa['tenant']
701
- or getattr(getattr(target, 'tenant', None), 'key', None) != qa['tenant']):
702
- return False
703
- for key in table_keys:
704
- if not isinstance(key, str) or not key.startswith(user_tables.KEY_PREFIX):
705
- return False
706
- definition = user_tables.get_definition(key, st=target)
707
- if (not isinstance(definition, dict) or definition.get('key') != key
708
- or definition.get('createdBy') != qa['username']
709
- or definition.get('source') != user_tables.BLANK_SOURCE):
710
- return False
711
- return True
712
-
713
-
714
  def _strict_accounts():
715
  """Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
716
  from core import account_rows, users
717
  # On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
718
  if account_rows._pg_source(users.store):
719
  return dict(account_rows._all(users.store))
720
- registry = users.registry()
721
  if account_rows.is_marker(registry):
722
  raise RuntimeError('the account registry is unavailable')
723
  return registry
@@ -727,9 +708,6 @@ def repair_published_database_grants(st=None, *, table_keys):
727
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
728
  from core import freeze_notice
729
  try:
730
- table_keys = tuple(table_keys)
731
- if _synthetic_qa_publication(_st(st), table_keys):
732
- return 0
733
  return reconcile_database_grants(st, table_keys=table_keys)
734
  except freeze_notice.DataInputFrozen:
735
  raise
 
80
  #: here is this tuple member. That is the point of one registry: the new object's WALL is written
81
  #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
82
  #: permission decision in a fourth file ([[one-evaluator-per-question]]).
83
+ KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook')
84
 
85
  #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
86
  #: Spelled as a single character so it can never collide with a username (usernames are lower-case
 
692
  return repaired[0]
693
 
694
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
695
  def _strict_accounts():
696
  """Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
697
  from core import account_rows, users
698
  # On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
699
  if account_rows._pg_source(users.store):
700
  return dict(account_rows._all(users.store))
701
+ registry = users.store._read_strict('users')
702
  if account_rows.is_marker(registry):
703
  raise RuntimeError('the account registry is unavailable')
704
  return registry
 
708
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
709
  from core import freeze_notice
710
  try:
 
 
 
711
  return reconcile_database_grants(st, table_keys=table_keys)
712
  except freeze_notice.DataInputFrozen:
713
  raise
platform/harness/amazon_client.py CHANGED
@@ -450,11 +450,8 @@ REPORT_TYPES = {
450
  "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
451
  "history": {"from": "2025-04-01",
452
  "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
453
- "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed (2026-11-14)",
454
- "periods": {"WEEK": {"from": "2025-07-13", "provisional": True,
455
- "evidence": "2026-11-14 probe refused 2025-07-06 and served 2025-10-05. "
456
- "The intervening twelve weeks are unasked. 2025-07-13 is the "
457
- "first unasked Sunday, not a proved earliest served floor"}}},
458
  "role": "Brand Analytics",
459
  "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
460
  "json_root": "dataByAsin", "monthly": True,
 
450
  "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
451
  "history": {"from": "2025-04-01",
452
  "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
453
+ "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the "
454
+ "week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"},
 
 
 
455
  "role": "Brand Analytics",
456
  "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
457
  "json_root": "dataByAsin", "monthly": True,
platform/harness/amazon_schema.py CHANGED
@@ -95,8 +95,6 @@ PROVENANCE = {
95
  "search_query_performance": "measured_at_sync",
96
  "sku_keywords": "derived",
97
  "keyword_ranks": "apify_dataset",
98
- "competitor_snapshots": "competitor_run",
99
- "competitor_images": "competitor_run",
100
  "fee_estimates_api": "measured_at_sync",
101
  }
102
 
@@ -107,13 +105,10 @@ PROVENANCE.update(dict.fromkeys((t + "_history" for t in connector_spec.AMAZON_H
107
  REVIEW_UNAVAILABLE_NOTE = (
108
  "Amazon's API does not supply written reviews, average star rating or review count. "
109
  "Read them on the product page or in Seller Central.")
110
- SQP_LIMIT_NOTE = "Amazon lists at most 100 searches a month for each product."
111
 
112
  #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
113
  #: screen and standing rule 2 governs it.
114
  PROVENANCE_COPY = {
115
- "competitor_run": ("Daily competitor facts saved by the competitor run. Every snapshot day is kept. "
116
- "Image rows keep each distinct variant and link, dated when first saved."),
117
  "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
118
  "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
119
  "Global position is separate. Unmatched observations remain in the raw mirror."),
@@ -308,8 +303,8 @@ def catalogue():
308
  descriptor = descriptors[key]
309
  label, native = descriptor.label, descriptor.native
310
  table[key] = {"label": label, "native": native}
311
- if provenance in {"derived", "apify_dataset", "competitor_run"}:
312
- table["columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
313
  if field["name"] != STORE]
314
  table["note"] = PROVENANCE_COPY[provenance]
315
  table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
@@ -319,7 +314,7 @@ def catalogue():
319
 
320
  def columns_for(table):
321
  """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
322
- if PROVENANCE.get(str(table)) in {"derived", "apify_dataset", "competitor_run"}:
323
  return []
324
  spec = (catalogue().get("tables") or {}).get(str(table)) or {}
325
  return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
@@ -382,17 +377,17 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
382
  fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
383
  spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
384
  spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
385
- local_sources = {
386
- "derived": ("listing_items", "search_query_performance"),
387
- "apify_dataset": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"),
388
- "competitor_run": ("amazon_rivals.run_quotes", "amazon_quotes_loop.run_tenant"),
389
- }
390
- provenance = PROVENANCE[table]
391
- if provenance in local_sources:
392
  spec["columns_source"] = {
393
  "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
394
- "provenance": provenance, "assumption": PROVENANCE_COPY[provenance],
395
- "sources": local_sources[provenance],
396
  }
397
  return spec
398
 
@@ -575,13 +570,9 @@ TABLE_SPECS.update({
575
  "browse_node_review_topics": _table_spec("browse_node_review_topics",
576
  ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
577
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
578
- (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_period_asin_query"),
579
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
580
  (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
581
- "competitor_snapshots": _table_spec("competitor_snapshots", ("asin", "snapshot_date"),
582
- grain="asin_day"),
583
- "competitor_images": _table_spec("competitor_images", ("asin", "variant", "link"),
584
- grain="asin_variant_link"),
585
  })
586
 
587
  # A history table is one row per version, identified by its version key and first valid date.
 
95
  "search_query_performance": "measured_at_sync",
96
  "sku_keywords": "derived",
97
  "keyword_ranks": "apify_dataset",
 
 
98
  "fee_estimates_api": "measured_at_sync",
99
  }
100
 
 
105
  REVIEW_UNAVAILABLE_NOTE = (
106
  "Amazon's API does not supply written reviews, average star rating or review count. "
107
  "Read them on the product page or in Seller Central.")
 
108
 
109
  #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
110
  #: screen and standing rule 2 governs it.
111
  PROVENANCE_COPY = {
 
 
112
  "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
113
  "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
114
  "Global position is separate. Unmatched observations remain in the raw mirror."),
 
303
  descriptor = descriptors[key]
304
  label, native = descriptor.label, descriptor.native
305
  table[key] = {"label": label, "native": native}
306
+ if provenance in {"derived", "apify_dataset"}:
307
+ table["columns"] = [dict(field) for field in AMAZON_FIELDS[key]
308
  if field["name"] != STORE]
309
  table["note"] = PROVENANCE_COPY[provenance]
310
  table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
 
314
 
315
  def columns_for(table):
316
  """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
317
+ if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"}:
318
  return []
319
  spec = (catalogue().get("tables") or {}).get(str(table)) or {}
320
  return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
 
377
  fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
378
  spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
379
  spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
380
+ if PROVENANCE[table] == "derived":
381
+ spec["columns_source"] = {
382
+ "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
383
+ "provenance": "derived", "assumption": PROVENANCE_COPY["derived"],
384
+ "sources": ("listing_items", "search_query_performance"),
385
+ }
386
+ if PROVENANCE[table] == "apify_dataset":
387
  spec["columns_source"] = {
388
  "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
389
+ "provenance": "apify_dataset", "assumption": PROVENANCE_COPY["apify_dataset"],
390
+ "sources": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"),
391
  }
392
  return spec
393
 
 
570
  "browse_node_review_topics": _table_spec("browse_node_review_topics",
571
  ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
572
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
573
+ (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
574
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
575
  (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
 
 
 
 
576
  })
577
 
578
  # A history table is one row per version, identified by its version key and first valid date.
platform/harness/amazon_store.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/harness/datastore.py CHANGED
@@ -3732,7 +3732,6 @@ REPLACE_EACH_PASS = {
3732
  "product_tag": {"domain": []},
3733
  }
3734
  REPLACE_PAGE = 5000
3735
- REPLACE_READ_ATTEMPTS = 3
3736
 
3737
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
3738
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
@@ -4019,13 +4018,13 @@ def _base_domain(spec):
4019
  return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
4020
 
4021
 
4022
- def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None):
4023
  """Publish a single requested entity through the same source boundary as a full pass."""
4024
  if source_paused():
4025
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4026
  candidate = getattr(_SYNC_TLS, "candidate", None)
4027
  if candidate is not None:
4028
- phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report)
4029
  # A recurring refresh may report changed pages to a bespoke cursor, but a
4030
  # tenant pass cannot claim completion until those private pages finish too.
4031
  pending = _connection_pending(candidate["con"], key)
@@ -4034,7 +4033,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None):
4034
  return phase, pulled
4035
  rt = _resident_odoo_runtime()
4036
  if rt is None:
4037
- return _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report)
4038
  report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
4039
  return report[key]["phase"], report[key]["pulled"]
4040
 
@@ -4184,17 +4183,16 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print):
4184
  con.close()
4185
 
4186
 
4187
- def _sync_one(key, max_batches=200, batch=BATCH, log=print, report=None):
4188
  """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
4189
  _connection_model(ENTITIES[key])
4190
  plan = REPLACE_EACH_PASS.get(key)
4191
  if plan is None or source_paused():
4192
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4193
  if plan.get("scope") != "internal":
4194
- return _replace_entity(key, log=log, report=report)
4195
  phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4196
- replaced, changed = _replace_entity(key, log=log, report=report)
4197
- return (phase if replaced == "live" else replaced), pulled + changed
4198
 
4199
 
4200
  def _replace_scope(con, key, plan):
@@ -4208,44 +4206,37 @@ def _replace_scope(con, key, plan):
4208
  return "(" + " OR ".join(terms) + ") AND " + _connection_where()
4209
 
4210
 
4211
- def _read_replace_rows(key, spec, domain):
4212
- """Read uncapped pages again if neither surrounding count matches, with a fixed attempt bound."""
4213
- fields = spec["fields"] + list(spec.get("m2m") or [])
4214
- for _ in range(REPLACE_READ_ATTEMPTS):
4215
- expected = int(O.get_odoo().search_count(spec["model"], domain))
4216
- rows, last = [], 0
4217
- while True:
4218
- recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
4219
- limit=REPLACE_PAGE, order="id asc")
4220
- if any(field not in rec for rec in recs for field in fields):
4221
- raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
4222
- rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
4223
- named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
4224
- # Only an empty page ends a read, since Odoo may return short pages before the end.
4225
- if not recs:
4226
- break
4227
- last = recs[-1]["id"]
4228
- if len(rows) == expected:
4229
- return rows
4230
- after = int(O.get_odoo().search_count(spec["model"], domain))
4231
- if len(rows) == after:
4232
- return rows
4233
- return None
4234
 
4235
-
4236
- def _replace_entity(key, log=print, report=None):
4237
- """Replace a complete read atomically, or hold a moving entity before changing rows or state."""
4238
  if source_paused():
4239
  _paused_notice(f"full read of {key}", log)
4240
  return PAUSED_PHASE, 0
4241
  spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
4242
- rows = _read_replace_rows(key, spec, list(plan["domain"]))
4243
- if rows is None:
4244
- if report is not None:
4245
- report["cause"] = {"classifier": "odoo_entity_moving", "entity": key,
4246
- "attempts": REPLACE_READ_ATTEMPTS}
4247
- log(f" {key}: Odoo changed during all {REPLACE_READ_ATTEMPTS} full reads. Previous rows are kept.")
4248
- return "backfill", 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4249
  con = connect()
4250
  try:
4251
  _ensure_table(con, key, spec)
@@ -4337,7 +4328,6 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
4337
  with binding:
4338
  for key in selected:
4339
  t0 = time.time()
4340
- row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
4341
  marker = _connection_state_key(key + ".pass")
4342
  previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
4343
  if entry and candidate.get("resuming") and previous == ("done",):
@@ -4346,11 +4336,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
4346
  if entry:
4347
  candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
4348
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
4349
- **({"batch": batch} if batch != BATCH else {}),
4350
- **({"report": row} if candidate is not None else {}))
4351
  if entry and phase == "live":
4352
  candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
4353
  [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
 
4354
  if phase != "live":
4355
  row["phase"] = phase
4356
  row["pulled"] += pulled
 
3732
  "product_tag": {"domain": []},
3733
  }
3734
  REPLACE_PAGE = 5000
 
3735
 
3736
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
3737
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
 
4018
  return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
4019
 
4020
 
4021
+ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
4022
  """Publish a single requested entity through the same source boundary as a full pass."""
4023
  if source_paused():
4024
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4025
  candidate = getattr(_SYNC_TLS, "candidate", None)
4026
  if candidate is not None:
4027
+ phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log)
4028
  # A recurring refresh may report changed pages to a bespoke cursor, but a
4029
  # tenant pass cannot claim completion until those private pages finish too.
4030
  pending = _connection_pending(candidate["con"], key)
 
4033
  return phase, pulled
4034
  rt = _resident_odoo_runtime()
4035
  if rt is None:
4036
+ return _sync_one(key, max_batches=max_batches, batch=batch, log=log)
4037
  report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
4038
  return report[key]["phase"], report[key]["pulled"]
4039
 
 
4183
  con.close()
4184
 
4185
 
4186
+ def _sync_one(key, max_batches=200, batch=BATCH, log=print):
4187
  """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
4188
  _connection_model(ENTITIES[key])
4189
  plan = REPLACE_EACH_PASS.get(key)
4190
  if plan is None or source_paused():
4191
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4192
  if plan.get("scope") != "internal":
4193
+ return _replace_entity(key, log=log)
4194
  phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
4195
+ return phase, pulled + _replace_entity(key, log=log)[1]
 
4196
 
4197
 
4198
  def _replace_scope(con, key, plan):
 
4206
  return "(" + " OR ".join(terms) + ") AND " + _connection_where()
4207
 
4208
 
4209
+ def _replace_entity(key, log=print):
4210
+ """Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4211
 
4212
+ Returns (phase, changed_rows). A failed or short read raises before any delete.
4213
+ """
 
4214
  if source_paused():
4215
  _paused_notice(f"full read of {key}", log)
4216
  return PAUSED_PHASE, 0
4217
  spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
4218
+ domain = list(plan["domain"])
4219
+ fields = spec["fields"] + list(spec.get("m2m") or [])
4220
+ expected = int(O.get_odoo().search_count(spec["model"], domain))
4221
+ rows, last = [], 0
4222
+ while True:
4223
+ recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
4224
+ limit=REPLACE_PAGE, order="id asc")
4225
+ if any(field not in rec for rec in recs for field in fields):
4226
+ raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
4227
+ rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
4228
+ named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
4229
+ # Stop on an EMPTY page: a server-side page cap returns short pages before the end.
4230
+ if not recs:
4231
+ break
4232
+ last = recs[-1]["id"]
4233
+ # Rows moved during the read when the read matches a second count. Refuse only a read
4234
+ # that disagrees with both counts, before any delete.
4235
+ if len(rows) != expected:
4236
+ after = int(O.get_odoo().search_count(spec["model"], domain))
4237
+ if len(rows) != after:
4238
+ raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were "
4239
+ f"{expected} and {after}. The mirror keeps its previous rows.")
4240
  con = connect()
4241
  try:
4242
  _ensure_table(con, key, spec)
 
4328
  with binding:
4329
  for key in selected:
4330
  t0 = time.time()
 
4331
  marker = _connection_state_key(key + ".pass")
4332
  previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
4333
  if entry and candidate.get("resuming") and previous == ("done",):
 
4336
  if entry:
4337
  candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
4338
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
4339
+ **({"batch": batch} if batch != BATCH else {}))
 
4340
  if entry and phase == "live":
4341
  candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
4342
  [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
4343
+ row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
4344
  if phase != "live":
4345
  row["phase"] = phase
4346
  row["pulled"] += pulled
platform/harness/mirror_store.py CHANGED
@@ -355,8 +355,7 @@ def _sync_result(connector, report):
355
  completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
356
  for row in report.values())
357
  return completed, completed and all(row["phase"] == "live" for row in report.values())
358
- if (report.get("problems") or report.get("audit_problems") or (connector == "amazon"
359
- and any(row.get("state") == "refused" for row in (report.get("tables") or {}).values()))):
360
  # A problem owned by one held table publishes the rest; it never makes the pass current.
361
  holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
362
  if holds is None or holds["blocked"]:
@@ -381,34 +380,16 @@ def _sync_result(connector, report):
381
  return completed, current
382
 
383
 
384
- def odoo_moving_entities(report):
385
- """Return only typed, exhausted moving-entity holds whose cause matches their entity."""
386
- if not isinstance(report, dict):
387
- return {}
388
- return {name: row["cause"] for name, row in (report or {}).items()
389
- if re.fullmatch(r"[a-z][a-z0-9_]{0,79}", str(name)) and isinstance(row, dict)
390
- and row.get("phase") == "backfill" and isinstance(row.get("cause"), dict)
391
- and row["cause"].get("classifier") == "odoo_entity_moving"
392
- and row["cause"].get("entity") == name
393
- and type(row["cause"].get("attempts")) is int and row["cause"]["attempts"] == 3}
394
-
395
-
396
- def record_sync_attempt(connector, rt):
397
- """Record writer activity without claiming a completed pass or replacing a previous refusal."""
398
- with _CATCH_UP_LOCK:
399
- state = _catch_up_source(rt, connector)
400
- if not state["current"]:
401
- state.update(pass_pending=True, attempt_at=time.time())
402
-
403
-
404
  def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
405
  """Count completed passes and require a clean result before clearing startup catch-up."""
406
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
407
  rt = runtime
408
  if rt is None:
409
  return pull(*args, **kwargs)
410
- record_sync_attempt(connector, rt)
411
  with _CATCH_UP_LOCK:
 
 
 
412
  active = (connector, _tenant_key(rt.key))
413
  _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
414
  try:
@@ -540,99 +521,6 @@ def _sql_literal(value):
540
  return "'" + str(value).replace("'", "''") + "'"
541
 
542
 
543
- def _merge_publication_entries(base, local, stored):
544
- """Merge keyed observations against their common base, naming conflicting local changes."""
545
- missing = object()
546
- merged, conflicts = {}, []
547
- for name in sorted(set(base) | set(local) | set(stored)):
548
- before, ours, theirs = (entries.get(name, missing) for entries in (base, local, stored))
549
- if _same_publication_entry(ours, before):
550
- value = theirs
551
- elif _same_publication_entry(ours, theirs):
552
- value = theirs
553
- elif _same_publication_entry(theirs, before):
554
- value = ours
555
- else:
556
- conflicts.append(name)
557
- continue
558
- if value is not missing:
559
- merged[name] = value
560
- return merged, conflicts
561
-
562
-
563
- def _same_publication_entry(left, right):
564
- """A content-addressed table retains its identity when only its write timestamp differs."""
565
- if isinstance(left, dict) and isinstance(right, dict):
566
- return {k: v for k, v in left.items() if k != "written_at"} == {
567
- k: v for k, v in right.items() if k != "written_at"}
568
- return left == right
569
-
570
-
571
- def _marker_freshness(row):
572
- """An invalid observation time cannot outrank a valid source-state observation."""
573
- try:
574
- return _utc(row[5])
575
- except (ValueError, OverflowError):
576
- return datetime.min.replace(tzinfo=timezone.utc)
577
-
578
-
579
- def _merge_publication_markers(base, local, stored):
580
- """Merge source state independently of observation time, retaining its newest valid freshness."""
581
- versions = [{row[0]: tuple(row) for row in rows} for rows in (base, local, stored)]
582
- states = [{name: row[:5] for name, row in version.items()} for version in versions]
583
- merged, conflicts = _merge_publication_entries(*states)
584
- for name, state in merged.items():
585
- matching = [version[name] for version in versions
586
- if name in version and version[name][:5] == state]
587
- merged[name] = max(matching, key=_marker_freshness)
588
- return merged, conflicts
589
-
590
-
591
- def _candidate_checkpoint(con):
592
- """Read the private candidate's common-base manifest and authoritative local markers."""
593
- if not con.execute("SELECT 1 FROM information_schema.tables "
594
- "WHERE table_name='_candidate_checkpoint'").fetchone():
595
- return None
596
- row = con.execute("SELECT document FROM _candidate_checkpoint").fetchone()
597
- return json.loads(row[0]) if row else None
598
-
599
-
600
- def _candidate_base_tables(checkpoint):
601
- """Keep initial local content provenance distinct from an actual saved manifest."""
602
- return {**checkpoint.get("local_tables", {}), **checkpoint["base"].get("tables", {})}
603
-
604
-
605
- def _save_candidate_checkpoint(con, base, markers, *, local_tables=None):
606
- """Keep per-table base provenance separate from the installed serving publication."""
607
- con.execute("CREATE TABLE IF NOT EXISTS _candidate_checkpoint (document VARCHAR)")
608
- con.execute("DELETE FROM _candidate_checkpoint")
609
- con.execute("INSERT INTO _candidate_checkpoint VALUES (?)", [json.dumps({
610
- "base": base, "markers": markers, "local_tables": local_tables or {}})])
611
-
612
-
613
- def _save_initial_candidate_checkpoint(store, rt, candidate, published):
614
- """Capture pre-manifest local table content before private source work changes it."""
615
- base = datastore._publication(published, store.connector)
616
- local_tables = {}
617
- missing = store._tables(published) - set(base.get("tables", {}))
618
- if missing:
619
- sealed = store._sealed_mode(rt)
620
- part_limit = store.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
621
- with tempfile.TemporaryDirectory(prefix="mirror-local-base-") as tmp:
622
- for name in sorted(missing):
623
- entry, _fresh = store._checkpoint_table(published, name, {}, set(), tmp,
624
- sealed=sealed, part_limit=part_limit)
625
- local_tables[name] = entry
626
- _save_candidate_checkpoint(candidate, base, store._state_rows(published), local_tables=local_tables)
627
-
628
-
629
- def _table_marker_rows(store, rows, tables):
630
- """Pair held data markers and direct table status with their table's rows."""
631
- names = {row[0] for row in store._held_state_rows(rows, tables)}
632
- names.update(row[0] for row in rows if row[0].split("__", 1)[-1] in tables)
633
- return [row for row in rows if row[0] in names]
634
-
635
-
636
  class MirrorStore:
637
  """One connector policy, with state cached separately for each tenant mirror."""
638
 
@@ -1410,232 +1298,162 @@ class MirrorStore:
1410
  return entry, fresh
1411
 
1412
 
1413
- def _rebase_publication(self, tenant_key, rt, con, base, manifest, rows, *, allowed, report):
1414
- """Keep stored changes and their markers, holding overlapping private changes for retry."""
1415
- raw = self._download(rt, _MANIFEST_KEY)
1416
- if raw is None:
1417
- # A local snapshot identity alone does not prove a saved manifest existed.
1418
- if "tables" in base:
1419
- raise PublicationHeld(f"{self.label} checkpoint held: the saved base publication is missing.")
1420
- return rows, False
1421
- stored = json.loads(raw)
1422
- if stored.get("tenant") != _tenant_key(tenant_key):
1423
- raise ValueError("Mirror manifest belongs to another tenant")
1424
- has_base = con.execute("SELECT 1 FROM information_schema.tables "
1425
- "WHERE table_name='_candidate_base'").fetchone()
1426
- candidate_base = con.execute("SELECT identity FROM _candidate_base").fetchone() if has_base else None
1427
- base_id = candidate_base[0] if candidate_base else base.get("snapshot_id", "unknown")
1428
- if base_id != base.get("snapshot_id", "unknown"):
1429
- raise PublicationHeld(f"{self.label} checkpoint held: private rows have a different base publication.")
1430
- if (stored.get("snapshot_id", "unknown") == base_id
1431
- and stored.get("tables") == base.get("tables")
1432
- and stored.get("sync_state") == base.get("sync_state")):
1433
- return rows, False
1434
- checkpoint = _candidate_checkpoint(con)
1435
- tables, conflicts = _merge_publication_entries(
1436
- _candidate_base_tables(checkpoint) if checkpoint else base.get("tables", {}),
1437
- manifest["tables"], stored.get("tables", {}))
1438
- if not allowed:
1439
- names = ", ".join(conflicts) or "the source's held tables"
1440
- raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed {names}. "
1441
- "Private rows keep their previous base for the next pass.")
1442
- with tempfile.TemporaryDirectory(prefix="mirror-rebase-") as tmp:
1443
- self._before_restore(tenant_key, rt)
1444
- stored, stored_rows, _paths, _total, _refused = self._validated_snapshot(
1445
- tenant_key, rt, con, raw, tmp)
1446
- if checkpoint is None:
1447
- raise PublicationHeld(f"{self.label} checkpoint held: common-base markers are unavailable.")
1448
- base_rows = checkpoint["markers"]
1449
- markers, marker_conflicts = _merge_publication_markers(base_rows, rows, stored_rows)
1450
- versions = base_rows + rows + stored_rows
1451
- conflicts = sorted(set(conflicts) | {
1452
- name for name in report.get("held_tables", ())
1453
- if set(marker_conflicts) & {row[0] for row in _table_marker_rows(self, versions, {name})}})
1454
- held_markers = {row[0] for row in _table_marker_rows(self, versions, set(conflicts))}
1455
- if set(marker_conflicts) - held_markers:
1456
- raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed markers "
1457
- + ", ".join(marker_conflicts) + ". Private rows stay for the next pass.")
1458
- if conflicts:
1459
- report["conflicts"] = conflicts
1460
- report["problems"].append(f"{self.label} checkpoint held tables: a newer stored publication changed "
1461
- + ", ".join(conflicts) + ". Private rows keep their previous base for the next pass.")
1462
- tables.update({name: stored["tables"][name] for name in conflicts if name in stored["tables"]})
1463
- markers = {name: row for name, row in markers.items() if name not in held_markers}
1464
- markers.update({row[0]: tuple(row) for row in stored_rows if row[0] in held_markers})
1465
- manifest["tables"] = tables
1466
- return list(markers.values()), True
1467
-
1468
-
1469
- def _checkpoint_table(self, con, name, prior, named, tmp, *, sealed, part_limit):
1470
- """Prepare changed immutable content, reusing every part the base already names."""
1471
- path = Path(tmp) / (name + ".parquet")
1472
- size = None
1473
- if "parts" not in prior:
1474
- con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
1475
- "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1476
- size = path.stat().st_size
1477
- if size is None or size > part_limit:
1478
- return self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
1479
- metadata = _blob_metadata(path.read_bytes())
1480
- if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
1481
- and (".sealed-v1." in prior["key"]) == sealed):
1482
- return None, []
1483
- suffix = ".sealed-v1.parquet" if sealed else ".parquet"
1484
- key = _content_key(name, suffix, metadata)
1485
- metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
1486
- written_at=_iso(datetime.now(timezone.utc)), key=key)
1487
- if key in named and named[key] != metadata["sha256"]:
1488
- raise ValueError(f"{self.label} mirror content key collides with a published blob")
1489
- return metadata, [] if key in named else [(key, path)]
1490
-
1491
-
1492
- def _checkpoint_tables(self, rt, con, manifest, dirty, named, tmp, *, sealed, part_limit):
1493
- """Prepare local changes and migrate legacy remote entries without dropping refused tables."""
1494
- tables, changed, pending = self._tables(con), [], []
1495
- legacy = {name for name, meta in manifest["tables"].items()
1496
- if (not meta.get("key") and "parts" not in meta)
1497
- or any((".sealed-v1." in key) != sealed for key, _ in self._entry_blob_keys(name, meta))}
1498
- for name in sorted(legacy - tables):
1499
- prior, migrated = manifest["tables"][name], []
1500
- for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
1501
- if (".sealed-v1." in key) == sealed:
1502
- migrated.append(dict(metadata))
1503
- continue
1504
- path = Path(tmp) / f"{name}.migrate{index}.parquet"
1505
- path.write_bytes(self._checked_mirror_blob(rt, key, metadata))
1506
- new_key = key.replace(".parquet", ".sealed-v1.parquet")
1507
- migrated.append({**metadata, "key": new_key})
1508
- pending.append((new_key, path))
1509
- manifest["tables"][name] = ({**prior, "parts": migrated} if "parts" in prior
1510
- else {**prior, **migrated[0]})
1511
- changed.append(name)
1512
- for name in sorted((dirty | legacy | (tables - set(manifest["tables"]))) & tables):
1513
- entry, fresh = self._checkpoint_table(con, name, manifest["tables"].get(name, {}), named, tmp,
1514
- sealed=sealed, part_limit=part_limit)
1515
- if entry is not None:
1516
- manifest["tables"][name] = entry
1517
- changed.append(name)
1518
- pending.extend(fresh)
1519
- return changed, pending
1520
-
1521
-
1522
- def _upload_checkpoint_parts(self, tenant_key, rt, pending, part_limit, progress):
1523
- """Ledger immutable keys before uploading bounded parts, retaining byte progress on failure."""
1524
- if any(path.stat().st_size > part_limit for _key, path in pending):
1525
- raise ValueError("mirror part exceeds the byte limit")
1526
- if pending:
1527
- self._ledger_add(rt, tenant_key, [key for key, _path in pending])
1528
- for key, path in pending:
1529
- data = path.read_bytes()
1530
- self._upload(rt, key, data)
1531
- progress["uploaded"] += len(data)
1532
-
1533
-
1534
- def _publish_checkpoint(self, tenant_key, rt, con, cache, manifest, changed, pending, tmp, *,
1535
- publication, hold, rebase_allowed, report, log, progress):
1536
- """Publish one complete manifest after content upload and a fresh stored-base comparison."""
1537
- checkpoint = _candidate_checkpoint(con)
1538
- old = checkpoint["base"] if checkpoint else cache["manifest"]
1539
- sealed = bool(manifest.get("blob_envelope_version"))
1540
- part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1541
- rows = self._state_rows(con)
1542
- if not self._state_complete(rows):
1543
- return None
1544
- suffix = ".sealed-v1" if sealed else ""
1545
- state_meta, _blobs = self._state_entry(rows, part_limit, suffix)
1546
- if not (publication or changed or state_meta != old.get("sync_state")
1547
- or manifest["tables"] != old.get("tables", {}) or old.get("version") != MANIFEST_VERSION):
1548
- return None
1549
- self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
1550
- rows, report["rebased"] = self._rebase_publication(
1551
- tenant_key, rt, con, old, manifest, rows,
1552
- allowed=rebase_allowed and publication is not None and hold is None, report=report)
1553
- state_meta, blobs = self._state_entry(rows, part_limit, suffix)
1554
- pending, named = [], self._manifest_keys(old)
1555
- for index, (key, state) in enumerate(blobs):
1556
- if key not in named:
1557
- path = Path(tmp) / f"sync_state.{index}.json"
1558
- path.write_bytes(state)
1559
- pending.append((key, path))
1560
- self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
1561
- stamp = _iso(datetime.now(timezone.utc))
1562
- manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, **(publication or {}))
1563
- if publication is not None:
1564
- manifest["completed_at"] = stamp
1565
- if hold is not None:
1566
- manifest["hold"] = hold
1567
- self._upload(rt, _HELD_KEY, _mirror_json(manifest))
1568
- return manifest
1569
- superseded = _superseded_hold(self, tenant_key, rt)
1570
- if superseded:
1571
- manifest["superseded_hold"] = superseded
1572
- published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
1573
- progress["uploaded"] += len(published)
1574
- if superseded is not None:
1575
- try:
1576
- rt.delete_path(self.prefix + _HELD_KEY)
1577
- except Exception as exc:
1578
- log(f"{self.label} superseded hold cleanup deferred ({type(exc).__name__}); "
1579
- "the next publication retries")
1580
- if publication is None:
1581
- datastore._write_publication(con, self.connector, manifest)
1582
- cache["manifest"] = manifest
1583
- cache["problems"].clear()
1584
- self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
1585
- return manifest
1586
-
1587
-
1588
  def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
1589
- _publication=None, _hold=None, _rebase_allowed=True):
1590
- """Checkpoint immutable parts and markers before publishing one manifest, retaining private work on failure."""
 
 
 
 
 
 
 
 
 
 
 
1591
  candidate = getattr(datastore._SYNC_TLS, "candidate", None)
1592
  if candidate is not None and _publication is None:
1593
  if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
1594
  raise ValueError("Checkpoint belongs to another source candidate")
1595
- return
1596
- progress = {"uploaded": 0}
1597
  with self.lock:
1598
  cache = self.restore_mirror(tenant_key, rt, log=log)
1599
  cache["dirty"].update(changed_tables)
1600
  try:
1601
  if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
1602
- raise PublicationHeld(
1603
- f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot "
1604
- "at start. The saved copy is kept; restore tenant-store access and restart to "
1605
- "restore it before checkpointing.")
1606
- checkpoint = _candidate_checkpoint(con)
1607
- old = checkpoint["base"] if checkpoint else cache["manifest"]
 
1608
  sealed = self._sealed_mode(rt)
1609
- if old.get("blob_envelope_version") and not sealed:
1610
- raise tenant_keys.TenantKeyLocked("A sealed mirror cannot publish without its tenant key.")
 
 
1611
  if old.get("snapshot_id") and _publication is None:
1612
  raise ValueError("An identified source requires the complete sync publication boundary")
1613
- tables = self._tables(con) | set(cache.get("refused_tables", ()))
 
 
1614
  manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
1615
- "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}}
 
1616
  if sealed:
1617
- manifest["blob_envelope_version"] = be.VERSION
1618
- part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1619
  with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
1620
- changed, pending = self._checkpoint_tables(
1621
- rt, con, manifest, cache["dirty"], self._manifest_keys(old), tmp,
1622
- sealed=sealed, part_limit=part_limit)
1623
- saved = self._publish_checkpoint(tenant_key, rt, con, cache, manifest, changed, pending, tmp,
1624
- publication=_publication, hold=_hold, rebase_allowed=_rebase_allowed,
1625
- report=report, log=log, progress=progress)
1626
- if _hold is not None:
1627
- return saved
1628
- if saved or not changed:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1629
  cache["dirty"].clear()
1630
- log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, "
1631
- f"{progress['uploaded']:,} bytes uploaded")
1632
- return saved
1633
- except PublicationHeld as exc:
1634
- report["problems"].append(str(exc))
1635
- log(str(exc))
1636
  except Exception as exc:
1637
- message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); "
1638
- f"{progress['uploaded']:,} bytes uploaded. "
1639
  "Local rows were kept; restore tenant-store access and resync before restarting.")
1640
  report["problems"].append(message)
1641
  log(message)
@@ -1820,20 +1638,8 @@ def _export_tables(store, con, directory):
1820
  return paths
1821
 
1822
 
1823
- def _candidate_matches(store, candidate, published, previous):
1824
- """Resume a known same-tenant common base even after another publication was installed."""
1825
- if not previous:
1826
- return False
1827
- installed = datastore._publication(published, store.connector)
1828
- checkpoint = _candidate_checkpoint(candidate)
1829
- if checkpoint is None:
1830
- return previous[0] == installed.get("snapshot_id", "unknown")
1831
- base = checkpoint["base"]
1832
- return previous[0] == base.get("snapshot_id", "unknown") and base.get("tenant") == installed.get("tenant")
1833
-
1834
-
1835
  def _open_candidate(store, tenant_key, rt, published):
1836
- """Resume known common-base provenance or seed private rows from the serving copy."""
1837
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1838
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1839
  datastore._ensure_sync_state(candidate)
@@ -1843,9 +1649,7 @@ def _open_candidate(store, tenant_key, rt, published):
1843
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
1844
  previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
1845
  base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1846
- if _candidate_matches(store, candidate, published, previous):
1847
- if _candidate_checkpoint(candidate) is None:
1848
- _save_initial_candidate_checkpoint(store, rt, candidate, published)
1849
  return candidate, previous[1]
1850
  started = _iso(datetime.now(timezone.utc))
1851
  try:
@@ -1860,7 +1664,6 @@ def _open_candidate(store, tenant_key, rt, published):
1860
  candidate.execute("DELETE FROM _candidate_pending")
1861
  candidate.execute("DELETE FROM _candidate_held")
1862
  candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
1863
- _save_initial_candidate_checkpoint(store, rt, candidate, published)
1864
  return candidate, started
1865
  except BaseException:
1866
  candidate.close()
@@ -1880,7 +1683,7 @@ def pending_sync_entities(connector, tenant_key, rt):
1880
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1881
  try:
1882
  base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
1883
- if not _candidate_matches(store, candidate, published, base):
1884
  return None
1885
  pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
1886
  if store.connector == "odoo":
@@ -1898,13 +1701,6 @@ def _holds_rows(holds):
1898
  return holds is not None and bool(holds["tables"] or holds.get("accounts"))
1899
 
1900
 
1901
- def _held_tables(holds):
1902
- """Retain whole private tables and their original bases even when only an account is held."""
1903
- if holds is None:
1904
- return set()
1905
- return set(holds["tables"]).union(*(hold["tables"] for hold in (holds.get("accounts") or {}).values()))
1906
-
1907
-
1908
  def _hold_previous(store, candidate, published, tables):
1909
  """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1910
  prior = store._tables(published)
@@ -1925,36 +1721,6 @@ def _hold_previous(store, candidate, published, tables):
1925
  candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1926
 
1927
 
1928
- def _retain_conflicting_candidate(store, candidate, published, manifest, conflicts):
1929
- """Refresh independent tables while retaining held private rows, markers and their original common bases."""
1930
- checkpoint = _candidate_checkpoint(candidate)
1931
- tables = set(conflicts)
1932
- private = _table_marker_rows(store, store._state_rows(candidate), tables)
1933
- saved_rows = store._state_rows(published)
1934
- held_names = {row[0] for row in _table_marker_rows(store, checkpoint["markers"] + saved_rows, tables)}
1935
- base_rows = [row for row in saved_rows if row[0] not in held_names]
1936
- base_rows += [row for row in checkpoint["markers"] if row[0] in held_names]
1937
- base = {**manifest, "snapshot_id": checkpoint["base"].get("snapshot_id", "unknown"),
1938
- "tables": {name: entry for name, entry in manifest["tables"].items() if name not in tables}}
1939
- base["tables"].update({name: entry for name, entry in _candidate_base_tables(checkpoint).items()
1940
- if name in tables})
1941
- candidate.execute("BEGIN TRANSACTION")
1942
- try:
1943
- _hold_previous(store, candidate, published, (store._tables(candidate) | store._tables(published)) - tables)
1944
- owned = store._state_rows(candidate)
1945
- if owned:
1946
- candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in owned])
1947
- rows = [row for row in saved_rows if row[0] not in held_names] + private
1948
- if rows:
1949
- candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows)
1950
- _save_candidate_checkpoint(candidate, base, base_rows)
1951
- candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", [[name] for name in conflicts])
1952
- candidate.execute("COMMIT")
1953
- except BaseException:
1954
- candidate.execute("ROLLBACK")
1955
- raise
1956
-
1957
-
1958
  def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1959
  """One log line: what a pass published and what it held, each with its reason."""
1960
  rows = report.get("tables") or {}
@@ -2124,7 +1890,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2124
  prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
2125
  metadata.pop("rows")
2126
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
2127
- result = {"problems": [], "held_tables": _held_tables(holds)}
2128
  save_started = time.monotonic()
2129
  with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
2130
  exporting = time.monotonic()
@@ -2142,18 +1908,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2142
  report["problems"].extend(problems)
2143
  return
2144
  raise RuntimeError("; ".join(problems))
2145
- if report is not None and "problems" in report:
2146
- report["problems"].extend(result["problems"])
2147
- retained = set(result.get("conflicts", ())) | result["held_tables"]
2148
- kept = bool(retained)
2149
  try:
2150
  installing = time.monotonic()
2151
- rows = store._state_rows(candidate)
2152
- if result.get("rebased"):
2153
- store._before_restore(tenant_key, rt)
2154
- _saved, rows, paths, _total, _refused = store._validated_snapshot(
2155
- tenant_key, rt, published, _mirror_json(manifest), tmp)
2156
- store._install_snapshot(published, manifest, rows, paths)
2157
  installed = True
2158
  store._after_publish(tenant_key, manifest)
2159
  except BaseException:
@@ -2163,19 +1920,20 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
2163
  note("install", installing)
2164
  for listener in list(_PUBLISH_LISTENERS):
2165
  try:
2166
- listener(store.connector, tenant_key, manifest, rows, paths)
2167
  except Exception as exc:
2168
  # The snapshot is saved and installed here. A failed hand-off only delays readers.
2169
  log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
2170
  if _holds_rows(holds):
2171
  candidate.execute("ROLLBACK")
2172
- if retained:
2173
- _retain_conflicting_candidate(store, candidate, published, manifest, retained)
2174
- if odoo and holds is not None:
2175
  # D-11329: a held entity keeps its private progress but never narrows the next pass.
2176
  names = [[name] for name in sorted(holds["held"])]
2177
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
2178
  candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
 
2179
  if holds is not None:
2180
  log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
2181
  if odoo:
 
355
  completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
356
  for row in report.values())
357
  return completed, completed and all(row["phase"] == "live" for row in report.values())
358
+ if report.get("problems") or report.get("audit_problems"):
 
359
  # A problem owned by one held table publishes the rest; it never makes the pass current.
360
  holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
361
  if holds is None or holds["blocked"]:
 
380
  return completed, current
381
 
382
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
383
  def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
384
  """Count completed passes and require a clean result before clearing startup catch-up."""
385
  connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
386
  rt = runtime
387
  if rt is None:
388
  return pull(*args, **kwargs)
 
389
  with _CATCH_UP_LOCK:
390
+ state = _catch_up_source(rt, connector)
391
+ if not state["current"]:
392
+ state.update(pass_pending=True, attempt_at=time.time())
393
  active = (connector, _tenant_key(rt.key))
394
  _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
395
  try:
 
521
  return "'" + str(value).replace("'", "''") + "'"
522
 
523
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
524
  class MirrorStore:
525
  """One connector policy, with state cached separately for each tenant mirror."""
526
 
 
1298
  return entry, fresh
1299
 
1300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1301
  def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
1302
+ _publication=None, _hold=None):
1303
+ """Checkpoint changed tables and the connector's markers, publishing the manifest last.
1304
+
1305
+ Candidates include partially written tables. Hashing their parquet locally avoids reuploading
1306
+ identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain
1307
+ dirty for retry on the next pass, while landed local rows and cursors stay usable.
1308
+
1309
+ S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a
1310
+ CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists
1311
+ every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row
1312
+ upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4)
1313
+ leaves the previous manifest naming only complete, untouched blobs.
1314
+ """
1315
  candidate = getattr(datastore._SYNC_TLS, "candidate", None)
1316
  if candidate is not None and _publication is None:
1317
  if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
1318
  raise ValueError("Checkpoint belongs to another source candidate")
1319
+ return # The enclosing source pass owns publication after every table completes.
1320
+ uploaded = 0
1321
  with self.lock:
1322
  cache = self.restore_mirror(tenant_key, rt, log=log)
1323
  cache["dirty"].update(changed_tables)
1324
  try:
1325
  if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
1326
+ message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot "
1327
+ "at start. The saved copy is kept; restore tenant-store access and restart to "
1328
+ "restore it before checkpointing.")
1329
+ report["problems"].append(message)
1330
+ log(message)
1331
+ return
1332
+ old = cache["manifest"]
1333
  sealed = self._sealed_mode(rt)
1334
+ if old.get('blob_envelope_version') and not sealed:
1335
+ raise tenant_keys.TenantKeyLocked('A sealed mirror cannot publish without its tenant key.')
1336
+ suffix = '.sealed-v1' if sealed else ''
1337
+ part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
1338
  if old.get("snapshot_id") and _publication is None:
1339
  raise ValueError("An identified source requires the complete sync publication boundary")
1340
+ named = self._manifest_keys(old)
1341
+ tables = self._tables(con)
1342
+ refused_tables = set(cache.get("refused_tables", ()))
1343
  manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
1344
+ "tables": {name: meta for name, meta in old.get("tables", {}).items()
1345
+ if name in tables or name in refused_tables}}
1346
  if sealed:
1347
+ manifest['blob_envelope_version'] = be.VERSION
1348
+ changed, pending = [], []
1349
  with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
1350
+ # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
1351
+ legacy = {name for name, meta in manifest["tables"].items()
1352
+ if (not meta.get("key") and "parts" not in meta)
1353
+ or any((".sealed-v1." in key) != sealed
1354
+ for key, _ in self._entry_blob_keys(name, meta))}
1355
+ # Refused local tables still name good remote parts: migrate those bytes too.
1356
+ for name in sorted(legacy - tables):
1357
+ prior = manifest['tables'][name]
1358
+ migrated = []
1359
+ for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
1360
+ if ('.sealed-v1.' in key) == sealed:
1361
+ migrated.append(dict(metadata))
1362
+ continue
1363
+ data = self._checked_mirror_blob(rt, key, metadata)
1364
+ path = Path(tmp) / f'{name}.migrate{index}.parquet'
1365
+ path.write_bytes(data)
1366
+ new_key = key.replace('.parquet', '.sealed-v1.parquet')
1367
+ migrated.append({**metadata, 'key': new_key})
1368
+ pending.append((new_key, path))
1369
+ manifest['tables'][name] = ({**prior, 'parts': migrated} if 'parts' in prior
1370
+ else {**prior, **migrated[0]})
1371
+ changed.append(name)
1372
+ for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
1373
+ if not self._valid_table(name):
1374
+ raise ValueError(f"{self.label} mirror table name is invalid")
1375
+ prior = manifest["tables"].get(name, {})
1376
+ path = Path(tmp) / (name + ".parquet")
1377
+ size = None
1378
+ if "parts" not in prior:
1379
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
1380
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
1381
+ size = path.stat().st_size
1382
+ if size is None or size > part_limit:
1383
+ entry, fresh = self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
1384
+ if entry is None:
1385
+ continue
1386
+ manifest["tables"][name] = entry
1387
+ changed.append(name)
1388
+ pending.extend(fresh)
1389
+ continue
1390
+ data = path.read_bytes()
1391
+ metadata = _blob_metadata(data)
1392
+ if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
1393
+ and (".sealed-v1." in prior["key"]) == sealed):
1394
+ continue
1395
+ key = _content_key(name, suffix + ".parquet", metadata)
1396
+ metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
1397
+ written_at=_iso(datetime.now(timezone.utc)), key=key)
1398
+ manifest["tables"][name] = metadata
1399
+ changed.append(name)
1400
+ if key in named:
1401
+ # The same content under its own name is already stored and published.
1402
+ if named[key] != metadata["sha256"]:
1403
+ raise ValueError(f"{self.label} mirror content key collides with a published blob")
1404
+ continue
1405
+ pending.append((key, path))
1406
+ rows = self._state_rows(con)
1407
+ state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
1408
+ published = None
1409
+ if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
1410
+ or manifest["tables"] != old.get("tables", {})
1411
+ or old.get("version") != MANIFEST_VERSION):
1412
+ for index, (state_key, state) in enumerate(state_blobs):
1413
+ if state_key not in named:
1414
+ state_path = Path(tmp) / f"sync_state.{index}.json"
1415
+ state_path.write_bytes(state)
1416
+ pending.append((state_key, state_path))
1417
+ if any(path.stat().st_size > part_limit for _key, path in pending):
1418
+ raise ValueError("mirror part exceeds the byte limit")
1419
+ if pending:
1420
+ self._ledger_add(rt, tenant_key, [key for key, _path in pending])
1421
+ for key, path in pending:
1422
+ data = path.read_bytes()
1423
+ self._upload(rt, key, data)
1424
+ uploaded += len(data)
1425
+ stamp = _iso(datetime.now(timezone.utc))
1426
+ manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp,
1427
+ **(_publication or {}))
1428
+ if _publication is not None:
1429
+ manifest["completed_at"] = stamp
1430
+ if _hold is not None:
1431
+ manifest["hold"] = _hold
1432
+ self._upload(rt, _HELD_KEY, _mirror_json(manifest))
1433
+ return manifest
1434
+ superseded = _superseded_hold(self, tenant_key, rt)
1435
+ if superseded:
1436
+ manifest["superseded_hold"] = superseded
1437
+ published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
1438
+ uploaded += len(published)
1439
+ if superseded is not None:
1440
+ # A newer publication replaces a stored hold. Nobody has to release it.
1441
+ try:
1442
+ rt.delete_path(self.prefix + _HELD_KEY)
1443
+ except Exception: # noqa: BLE001
1444
+ log(f"{self.label} superseded hold pointer stays until the next publication")
1445
+ if _publication is None:
1446
+ datastore._write_publication(con, self.connector, manifest)
1447
+ cache["manifest"] = manifest
1448
+ cache["problems"].clear()
1449
+ if published:
1450
+ self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
1451
+ if published or not changed:
1452
  cache["dirty"].clear()
1453
+ log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded")
1454
+ return manifest if published else None
 
 
 
 
1455
  except Exception as exc:
1456
+ message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. "
 
1457
  "Local rows were kept; restore tenant-store access and resync before restarting.")
1458
  report["problems"].append(message)
1459
  log(message)
 
1638
  return paths
1639
 
1640
 
 
 
 
 
 
 
 
 
 
 
 
 
1641
  def _open_candidate(store, tenant_key, rt, published):
1642
+ """Resume private rows only while their base publication still matches the serving copy."""
1643
  path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
1644
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1645
  datastore._ensure_sync_state(candidate)
 
1649
  candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
1650
  previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
1651
  base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1652
+ if previous and previous[0] == base:
 
 
1653
  return candidate, previous[1]
1654
  started = _iso(datetime.now(timezone.utc))
1655
  try:
 
1664
  candidate.execute("DELETE FROM _candidate_pending")
1665
  candidate.execute("DELETE FROM _candidate_held")
1666
  candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
 
1667
  return candidate, started
1668
  except BaseException:
1669
  candidate.close()
 
1683
  candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
1684
  try:
1685
  base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
1686
+ if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"):
1687
  return None
1688
  pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
1689
  if store.connector == "odoo":
 
1701
  return holds is not None and bool(holds["tables"] or holds.get("accounts"))
1702
 
1703
 
 
 
 
 
 
 
 
1704
  def _hold_previous(store, candidate, published, tables):
1705
  """Give each held table its published rows and data markers, inside the candidate's open transaction."""
1706
  prior = store._tables(published)
 
1721
  candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
1722
 
1723
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1724
  def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
1725
  """One log line: what a pass published and what it held, each with its reason."""
1726
  rows = report.get("tables") or {}
 
1890
  prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
1891
  metadata.pop("rows")
1892
  metadata["completed_at"] = _iso(datetime.now(timezone.utc))
1893
+ result = {"problems": []}
1894
  save_started = time.monotonic()
1895
  with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
1896
  exporting = time.monotonic()
 
1908
  report["problems"].extend(problems)
1909
  return
1910
  raise RuntimeError("; ".join(problems))
 
 
 
 
1911
  try:
1912
  installing = time.monotonic()
1913
+ store._install_snapshot(published, manifest, store._state_rows(candidate), paths)
 
 
 
 
 
1914
  installed = True
1915
  store._after_publish(tenant_key, manifest)
1916
  except BaseException:
 
1920
  note("install", installing)
1921
  for listener in list(_PUBLISH_LISTENERS):
1922
  try:
1923
+ listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths)
1924
  except Exception as exc:
1925
  # The snapshot is saved and installed here. A failed hand-off only delays readers.
1926
  log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
1927
  if _holds_rows(holds):
1928
  candidate.execute("ROLLBACK")
1929
+ candidate.execute("DELETE FROM _candidate_base")
1930
+ candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
1931
+ if odoo:
1932
  # D-11329: a held entity keeps its private progress but never narrows the next pass.
1933
  names = [[name] for name in sorted(holds["held"])]
1934
  candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
1935
  candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
1936
+ kept = True
1937
  if holds is not None:
1938
  log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
1939
  if odoo:
platform/modules/amazon_profit.py CHANGED
@@ -31,11 +31,9 @@ import re
31
  import shutil
32
  import tempfile
33
  import threading
34
- from decimal import Decimal, ROUND_HALF_UP
35
  from pathlib import Path
36
 
37
  from modules import amazon_sku_metrics as M
38
- from modules import amazon_part_dates as _part_dates
39
  import core.user_tables as _registry
40
  from core import module_tables
41
  from core import amazon_rivals as _rivals
@@ -121,58 +119,14 @@ def _field(key, label, ftype, description, *, computed, **extra):
121
  return out
122
 
123
 
124
- def _progress_fields(description, *, key="projects", label="Projects", empty_label="No projects"):
125
  """Project progress and its completed-count label, written by the Projects doors."""
126
- return (_field(key, label, "pct", description, computed=True,
127
- format={"display": "progress", "labelKey": key + "_label", "emptyLabel": empty_label}),
128
- _field(key + "_label", label + (" completed" if key == "projects" else " label"), "text",
129
- "Completed projects of the counted ones." if key == "projects" else "Open PDP workflow stage and progress.",
130
  computed=True))
131
 
132
 
133
- HISTORY_FIELD = _field("history", "History", "text", "Open the row's recorded changes.", computed=True)
134
- ASIN_APP_KEYS = ("asin", "projects", "pdp_update", "trend_score", "search_trend", "market_share_trend",
135
- "click_share_trend", "exposure", "funnel_score", "pdp_last_updated", "title_last_updated",
136
- "bullets_last_updated", "images_last_updated", "aplus_last_updated", "history")
137
- SEARCH_TRENDS = (("search", "search_trend", "Search trend", "int"),
138
- ("market_share", "market_share_trend", "Market share trend", "pct"),
139
- ("click_share", "click_share_trend", "Click share trend", "pct"))
140
- EXPOSURES = {"under": "Under-exposed", "match": "Matched", "over": "Over-exposed",
141
- "few": "Few", "none": "Not seen"}
142
- PDP_PART_DATE_KEYS = tuple(part + "_last_updated" for part in ("title", "bullets", "images", "aplus"))
143
- PDP_CELL_KEYS = ("pdp_update", "pdp_update_label", "pdp_update_more") + PDP_PART_DATE_KEYS
144
- _PDP_PROGRESS_FIELDS = _progress_fields("Progress of the earliest open PDP workflow.", key="pdp_update",
145
- label="PDP update", empty_label="No open workflow")
146
- _SIGNAL_FIELDS = (
147
- _field("trend_score", "Trend score", "int", "Search trend score from 0 to 100.", computed=True),
148
- *(_field(key, label, "pct", "Change against the same reported months last year.", computed=True)
149
- for _, key, label, _ in SEARCH_TRENDS),
150
- _field("exposure", "Exposure", "select", "Sales or click share against impression share.",
151
- computed=True, options=list(EXPOSURES.values())),
152
- _field("funnel_score", "Funnel score", "int", "Search-weighted funnel score, with 5 at the market.",
153
- computed=True, format={"decimals": 1}),
154
- )
155
- _ASIN_DATE_FIELDS = tuple(_field(key, label, "date", "Latest recorded listing or part change.", computed=True)
156
- for key, label in zip(("pdp_last_updated",) + PDP_PART_DATE_KEYS,
157
- ("PDP last updated", "Title last updated", "Bullets last updated",
158
- "Images last updated", "A+ last updated")))
159
- _SIGNAL_COMPANIONS = tuple(field for kind, key, label, now_type in SEARCH_TRENDS for field in (
160
- _field(key + "_series", label + " series", "text", "Every reported historical month, with raw values, ASIN gaps and reasons.", computed=True),
161
- _field(key + "_now", label + " now", now_type, "Searches in the newest complete store month." if kind == "search"
162
- else "Share aggregated over the twelve-month calculation window.", computed=True),
163
- _field(key + "_reason", label + " reason", "text", "Why the year comparison is unavailable.", computed=True),
164
- _field(key + "_now_reason", label + " now reason", "text", "Why the current value is unavailable.", computed=True),
165
- )) + tuple(_field(key + "_reason", label + " reason", "text", "Why this signal is unavailable.", computed=True)
166
- for key, label in (("trend_score", "Trend score"), ("funnel_score", "Funnel score"), ("exposure", "Exposure")))
167
- _ASIN_COMPANIONS = (_PDP_PROGRESS_FIELDS[1:]
168
- + (_field("pdp_update_more", "More PDP workflows", "int", "Other open PDP workflows.", computed=True),)
169
- + _SIGNAL_COMPANIONS
170
- + (_field("exposure_label", "Exposure detail", "text", "Exposure multiple and comparison base.", computed=True),
171
- _field("pdp_last_updated_reason", "PDP date reason", "text", "Why Amazon's listing date is unavailable.", computed=True)))
172
- _ASIN_SOURCE_KEYS = tuple(f["key"] for f in _SIGNAL_FIELDS + _SIGNAL_COMPANIONS) + (
173
- "exposure_label", "pdp_last_updated", "pdp_last_updated_reason")
174
-
175
-
176
  def _metric_fields(skip=()):
177
  """The sibling metric module's columns, as computed fields of this module."""
178
  return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
@@ -226,7 +180,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x)
226
  for k, label, t, d, x in _rivals.PARENT_COLUMNS)
227
  COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
228
  _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
229
- COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (HISTORY_FIELD, PICTURE_FIELD) + _ECON_FIELDS[1:] + _metric_fields()
230
  + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
231
  + COMPETITOR_FIELDS)
232
  #: Owned columns: people edit them and a refresh never writes them.
@@ -378,7 +332,7 @@ def cost_note(row, *, code=None, found=True):
378
  if source and row.get("cogs_per_unit") is None:
379
  return source if source.endswith(".") else source + "."
380
  return NO_COST_NOTE
381
- _IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label", "history")
382
  _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
383
 
384
  #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
@@ -397,12 +351,9 @@ def _grain_fields(key_column, grain):
397
  else (metric["parent_asin"], PICTURE_FIELD))
398
  where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
399
  progress = _progress_fields(f"Average progress of the projects on {where}.")
400
- app = (_PDP_PROGRESS_FIELDS[:1] + _SIGNAL_FIELDS + _ASIN_DATE_FIELDS + (HISTORY_FIELD,)
401
- if key_column == "asin" else (HISTORY_FIELD,))
402
- companions = _ASIN_COMPANIONS if key_column == "asin" else ()
403
- return (ids[:1] + progress[:1] + app + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
404
  + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
405
- + progress[1:] + companions)
406
 
407
 
408
  ASIN_FIELDS = _grain_fields("asin", "asin")
@@ -448,7 +399,7 @@ PROJECT_COMPUTED_KEYS = tuple(f["key"] for f in PROJECT_COMPUTED_FIELDS)
448
 
449
  def table_specs():
450
  """Every module table in creation order."""
451
- specs = (
452
  {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
453
  "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
454
  {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
@@ -456,10 +407,6 @@ def table_specs():
456
  {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
457
  "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
458
  )
459
- for spec in specs:
460
- fields = {f["key"]: f for f in spec["fields"]}
461
- spec["fields"] = tuple(fields[key] for key in _project_field_order(fields))
462
- return specs
463
 
464
 
465
  # ---------------------------------------------------------------- period and threshold
@@ -1760,7 +1707,7 @@ def _sync_cursor(tenant_key, rt):
1760
  else:
1761
  import duckdb
1762
  cur = duckdb.connect(str(path), read_only=True)
1763
- if _mirror_has_economics(cur) or {'sku', 'attributes', 'summaries'} <= _columns(cur, 'amzn_listing_items'):
1764
  return cur, cur.close
1765
  cur.close()
1766
  for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
@@ -1773,7 +1720,7 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
1773
  """Refresh the current period, apply decisions, then suggest for every unlinked SKU.
1774
  Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
1775
  synced = {str(t or "").strip().lower() for t in synced_tables or ()}
1776
- if not synced & {"economics", "listings", "listing_items"}:
1777
  return {"state": "skipped", "synced": sorted(synced)}
1778
  if st is None:
1779
  from harness import amazon_store
@@ -1785,17 +1732,6 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
1785
  con, close = _sync_cursor(tenant_key, st)
1786
  try:
1787
  report = {"state": "done"}
1788
- if synced & {"listings", "listing_items"}:
1789
- try:
1790
- report["partDates"] = _part_dates.observe(st, con, day=today or _today())
1791
- report["partDateCells"] = paint_asin_part_dates(st)
1792
- except Exception as exc: # quality-ok: Date storage failure must not block profit matching.
1793
- report['partDatesError'] = f'{type(exc).__name__}: {exc}'
1794
- _log(f"{tenant_key}: listing part dates failed ({report['partDatesError']}).")
1795
- if not synced & {'economics', 'listings'}:
1796
- return dict(report, state='failed', reason=report['partDatesError'])
1797
- if not synced & {"economics", "listings"}:
1798
- return report
1799
  try:
1800
  refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
1801
  report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
@@ -1871,16 +1807,12 @@ def _definition_fields(fields=None, owned_keys=None):
1871
 
1872
 
1873
  def _project_field_order(keys):
1874
- """Pin App fields in schema defaults and keep companions last, without changing saved layouts."""
1875
  keys = list(keys)
1876
- if not keys or keys[0] not in ("sku", "asin", "parent_asin"):
1877
- return keys
1878
- pinned = ASIN_APP_KEYS if keys[0] == "asin" else (keys[0], "projects", "history")
1879
- companions = (("projects_label",) + tuple(f["key"] for f in _ASIN_COMPANIONS)
1880
- if keys[0] == "asin" else ())
1881
- return ([key for key in pinned if key in keys]
1882
- + [key for key in keys if key not in pinned + companions]
1883
- + [key for key in companions if key in keys])
1884
 
1885
 
1886
  def _format_repairs(fields, wanted):
@@ -1900,9 +1832,9 @@ def _grain_word_repairs(fields, wanted):
1900
  continue
1901
  if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
1902
  out.setdefault("grain", {})["options"] = list(target["options"])
1903
- if field["key"] in ("projects", "pdp_update") and (field.get("type") != target["type"] or "link" in field
1904
  or (field.get("format") or {}) != (target.get("format") or {})):
1905
- out[field["key"]] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None}
1906
  return out
1907
 
1908
 
@@ -2093,9 +2025,6 @@ def write_rows(st, computed, *, table_key=TABLE_KEY, key_column="sku", label=TAB
2093
  for rid, row in table_rows(st, table_key).items()
2094
  if str(row.get(key_column) or "").strip() not in computed}
2095
  wanted = {rid: values for rid, values in wanted.items() if values}
2096
- for rid, row in table_rows(st, table_key).items():
2097
- if row.get("history") != "View":
2098
- wanted.setdefault(rid, {})["history"] = "View"
2099
  if wanted:
2100
  import core.user_tables as ut
2101
 
@@ -2639,7 +2568,7 @@ def grain_rows(rows, key, *, thin=THIN_DEFAULT, period_label="", fields=()):
2639
  flat["cost_note"] = _grain_cost_note(flat, children)
2640
  flat["period"] = period_label
2641
  out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
2642
- if k not in ("projects", "projects_label") + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS}
2643
  return out
2644
 
2645
 
@@ -2871,117 +2800,8 @@ def _withhold_untied(by_grain, checks):
2871
 
2872
  def _grain_period_keys(fields, key_column):
2873
  return tuple(f["key"] for f in fields
2874
- if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label", "history")
2875
- + COMPETITOR_KEYS + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS)
2876
-
2877
-
2878
- def _listing_dates(con):
2879
- """Read the newest Amazon lastUpdatedDate per ASIN from raw listing summaries."""
2880
- if "summaries" not in _columns(con, "amzn_listing_items"):
2881
- return {}
2882
- rows = con.execute("""SELECT json_extract_string(s.value, '$.asin'),
2883
- CAST(MAX(TRY_CAST(json_extract_string(s.value, '$.lastUpdatedDate') AS TIMESTAMPTZ))
2884
- AT TIME ZONE 'UTC' AS DATE)
2885
- FROM amzn_listing_items l, json_each(TRY_CAST(l.summaries AS JSON)) s GROUP BY 1""").fetchall()
2886
- return {asin: date.isoformat() for asin, date in rows if asin and date is not None}
2887
-
2888
-
2889
- def _exposure_detail(exposure):
2890
- """Format the recorded exposure multiple with the prototype's one-decimal rounding."""
2891
- if not exposure or exposure["multiple"] is None:
2892
- return None
2893
- multiple = Decimal.from_float(float(exposure["multiple"])).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)
2894
- return f"{multiple}x {exposure['base']} share"
2895
-
2896
-
2897
- def _asin_signal_cells(signal, history):
2898
- """Store scalar cells and raw monthly bars for every reported historical month."""
2899
- out = {key: stored_cell(_FIELD_BY_KEY[key], signal[key]) for key in ("trend_score", "funnel_score")}
2900
- out.update({key + "_reason": value for key, value in signal["reasons"].items()
2901
- if key in ("trend_score", "funnel_score", "exposure")})
2902
- exposure = signal["exposure"]
2903
- out["exposure"] = EXPOSURES[exposure["kind"]] if exposure else None
2904
- out["exposure_label"] = _exposure_detail(exposure)
2905
- for kind, key, _label, _now_type in SEARCH_TRENDS:
2906
- trend = signal["trends"][kind]
2907
- points = {point["month"]: point for point in trend["series"]}
2908
- series = [points[month] for month in history]
2909
- latest = series[-1] if series else {"value": None, "reason": signal["note"] or "No search report"}
2910
- now = latest["value"] if kind == "search" else signal[kind]
2911
- out.update({key: None if trend["change"] is None else str(trend["change"] * 100),
2912
- key + "_series": json.dumps(series, separators=(",", ":")),
2913
- key + "_now": (stored_cell(_FIELD_BY_KEY[key + "_now"], now) if kind == "search"
2914
- else None if now is None else str(now * 100)),
2915
- key + "_reason": trend["reason"],
2916
- key + "_now_reason": (signal["note"] or latest["reason"] if kind == "search"
2917
- else signal["reasons"][kind])})
2918
- return out
2919
-
2920
-
2921
- def paint_asin_signals(st, con, *, today):
2922
- """Refresh source signals and Amazon's date for every stored ASIN, independent of the profit period."""
2923
- from modules import amazon_search_signals as signals
2924
- asins = [str(row["asin"]).strip() for row in table_rows(st, ASIN_KEY).values() if row.get("asin")]
2925
- result = signals.signals(con, asins, today=today)
2926
- dates = _listing_dates(con)
2927
- parts = _part_dates.cells(st, asins)
2928
- computed = {asin: dict(_asin_signal_cells(signal, result["history"]), **parts[asin], pdp_last_updated=dates.get(asin),
2929
- pdp_last_updated_reason=None if asin in dates else "No listing lastUpdatedDate reported")
2930
- for asin, signal in result["asins"].items()}
2931
- return write_rows(st, computed, table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=())
2932
-
2933
-
2934
- def paint_asin_part_dates(st):
2935
- """Refresh recorded part dates without profit validation, search reports or Odoo matching."""
2936
- asins = [str(row['asin']).strip() for row in table_rows(st, ASIN_KEY).values() if row.get('asin')]
2937
- if not asins:
2938
- return {'rows': 0}
2939
- return write_rows(st, _part_dates.cells(st, asins), table_key=ASIN_KEY,
2940
- key_column='asin', label=ASIN_LABEL, period_keys=())
2941
-
2942
-
2943
- def asin_signal_checks(st, con, *, today=None):
2944
- """Tie stored signal cells to the signal reader's independently validated raw monthly inputs."""
2945
- from modules import amazon_search_signals as signals
2946
- rows = {str(row["asin"]).strip(): row for row in table_rows(st, ASIN_KEY).values() if row.get("asin")}
2947
- result = signals.signals(con, rows, today=today)
2948
- verified = signals.validate(con, result, today=today)
2949
- differences = sum(any(rows[asin].get(key) != value for key, value in _asin_signal_cells(signal, result["history"]).items())
2950
- for asin, signal in result["asins"].items())
2951
- return [{"check": "amazon_profit: stored ASIN search signals tie to raw monthly reports",
2952
- "ok": verified["ok"] and not differences, "rows": len(rows), "differences": differences,
2953
- "reason": verified["reason"]}]
2954
-
2955
-
2956
- def listing_date_checks(st, con):
2957
- """Independently tie stored PDP dates to the newest raw summary timestamp per ASIN."""
2958
- newest = {}
2959
- available = "summaries" in _columns(con, "amzn_listing_items")
2960
- if available:
2961
- cursor = con.execute("SELECT summaries FROM amzn_listing_items")
2962
- for batch in iter(lambda: cursor.fetchmany(2048), []):
2963
- for (raw,) in batch:
2964
- try:
2965
- summaries = json.loads(raw) if isinstance(raw, str) else raw
2966
- except (TypeError, ValueError):
2967
- continue
2968
- for summary in summaries if isinstance(summaries, list) else ():
2969
- if not isinstance(summary, dict) or not summary.get("asin"):
2970
- continue
2971
- try:
2972
- stamp = dt.datetime.fromisoformat(str(summary.get("lastUpdatedDate")).replace("Z", "+00:00"))
2973
- except ValueError:
2974
- continue
2975
- stamp = stamp.replace(tzinfo=stamp.tzinfo or dt.timezone.utc).astimezone(dt.timezone.utc)
2976
- asin = summary["asin"]
2977
- if asin not in newest or stamp > newest[asin]:
2978
- newest[asin] = stamp
2979
- rows = list(table_rows(st, ASIN_KEY).values())
2980
- differences = sum(row.get("pdp_last_updated") != (newest[row["asin"]].date().isoformat()
2981
- if row.get("asin") in newest else None) for row in rows)
2982
- return [{"check": "amazon_profit: PDP last updated ties to newest listing lastUpdatedDate per ASIN",
2983
- "ok": available and not differences, "rows": len(rows), "differences": differences,
2984
- "reason": None if available else "Listing summaries are unavailable"}]
2985
 
2986
 
2987
  def _date(value):
@@ -3322,7 +3142,6 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N
3322
  report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
3323
  key_column="parent_asin", label=PARENT_LABEL,
3324
  period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
3325
- report["asinSignals"] = paint_asin_signals(st, con, today=day)
3326
  report["blankReasons"] = blank_check(stored_tables(st))
3327
  bands = {sku: values.get("band") for sku, values in computed.items()}
3328
  report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
@@ -3468,7 +3287,6 @@ def _mirror_has_economics(con):
3468
 
3469
 
3470
  def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
3471
- "amzn_listing_items", "amzn_search_query_performance",
3472
  "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
3473
  "amzn_ads_sd_advertised_products")):
3474
  """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
@@ -3630,10 +3448,10 @@ def stored_grain_checks(st, con):
3630
 
3631
 
3632
  def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
3633
- typed_values=None, typed_period=None, st=None, today=None):
3634
  """Tie net proceeds components to independent Amazon sources within 2% over settled days.
3635
  Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
3636
- With `st`, stored grain figures, search signals and listing dates tie to their sources."""
3637
  month = month or last_complete_month()
3638
  bounds = period_bounds(month)
3639
  start, end = bounds["start"], bounds["end"]
@@ -3693,8 +3511,6 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No
3693
  checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
3694
  if st is not None:
3695
  checks.extend(stored_grain_checks(st, con))
3696
- checks.extend(listing_date_checks(st, con))
3697
- checks.extend(asin_signal_checks(st, con, today=today))
3698
  return checks
3699
 
3700
 
 
31
  import shutil
32
  import tempfile
33
  import threading
 
34
  from pathlib import Path
35
 
36
  from modules import amazon_sku_metrics as M
 
37
  import core.user_tables as _registry
38
  from core import module_tables
39
  from core import amazon_rivals as _rivals
 
119
  return out
120
 
121
 
122
+ def _progress_fields(description):
123
  """Project progress and its completed-count label, written by the Projects doors."""
124
+ return (_field("projects", "Projects", "pct", description, computed=True,
125
+ format={"display": "progress", "labelKey": "projects_label", "emptyLabel": "No projects"}),
126
+ _field("projects_label", "Projects completed", "text", "Completed projects of the counted ones.",
 
127
  computed=True))
128
 
129
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  def _metric_fields(skip=()):
131
  """The sibling metric module's columns, as computed fields of this module."""
132
  return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
 
180
  for k, label, t, d, x in _rivals.PARENT_COLUMNS)
181
  COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
182
  _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
183
+ COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields()
184
  + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
185
  + COMPETITOR_FIELDS)
186
  #: Owned columns: people edit them and a refresh never writes them.
 
332
  if source and row.get("cogs_per_unit") is None:
333
  return source if source.endswith(".") else source + "."
334
  return NO_COST_NOTE
335
+ _IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label")
336
  _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
337
 
338
  #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
 
351
  else (metric["parent_asin"], PICTURE_FIELD))
352
  where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
353
  progress = _progress_fields(f"Average progress of the projects on {where}.")
354
+ return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
 
 
 
355
  + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
356
+ + progress[1:])
357
 
358
 
359
  ASIN_FIELDS = _grain_fields("asin", "asin")
 
399
 
400
  def table_specs():
401
  """Every module table in creation order."""
402
+ return (
403
  {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
404
  "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
405
  {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
 
407
  {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
408
  "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
409
  )
 
 
 
 
410
 
411
 
412
  # ---------------------------------------------------------------- period and threshold
 
1707
  else:
1708
  import duckdb
1709
  cur = duckdb.connect(str(path), read_only=True)
1710
+ if _mirror_has_economics(cur):
1711
  return cur, cur.close
1712
  cur.close()
1713
  for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
 
1720
  """Refresh the current period, apply decisions, then suggest for every unlinked SKU.
1721
  Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
1722
  synced = {str(t or "").strip().lower() for t in synced_tables or ()}
1723
+ if not synced & {"economics", "listings"}:
1724
  return {"state": "skipped", "synced": sorted(synced)}
1725
  if st is None:
1726
  from harness import amazon_store
 
1732
  con, close = _sync_cursor(tenant_key, st)
1733
  try:
1734
  report = {"state": "done"}
 
 
 
 
 
 
 
 
 
 
 
1735
  try:
1736
  refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
1737
  report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
 
1807
 
1808
 
1809
  def _project_field_order(keys):
1810
+ """Place Projects beside a product identity in schema defaults, never saved user layouts."""
1811
  keys = list(keys)
1812
+ if keys and keys[0] in ("sku", "asin", "parent_asin") and "projects" in keys:
1813
+ keys.remove("projects")
1814
+ keys.insert(1, "projects")
1815
+ return keys
 
 
 
 
1816
 
1817
 
1818
  def _format_repairs(fields, wanted):
 
1832
  continue
1833
  if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
1834
  out.setdefault("grain", {})["options"] = list(target["options"])
1835
+ if field["key"] == "projects" and (field.get("type") != target["type"] or "link" in field
1836
  or (field.get("format") or {}) != (target.get("format") or {})):
1837
+ out["projects"] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None}
1838
  return out
1839
 
1840
 
 
2025
  for rid, row in table_rows(st, table_key).items()
2026
  if str(row.get(key_column) or "").strip() not in computed}
2027
  wanted = {rid: values for rid, values in wanted.items() if values}
 
 
 
2028
  if wanted:
2029
  import core.user_tables as ut
2030
 
 
2568
  flat["cost_note"] = _grain_cost_note(flat, children)
2569
  flat["period"] = period_label
2570
  out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
2571
+ if k not in ("projects", "projects_label")}
2572
  return out
2573
 
2574
 
 
2800
 
2801
  def _grain_period_keys(fields, key_column):
2802
  return tuple(f["key"] for f in fields
2803
+ if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label")
2804
+ + COMPETITOR_KEYS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2805
 
2806
 
2807
  def _date(value):
 
3142
  report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
3143
  key_column="parent_asin", label=PARENT_LABEL,
3144
  period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
 
3145
  report["blankReasons"] = blank_check(stored_tables(st))
3146
  bands = {sku: values.get("band") for sku, values in computed.items()}
3147
  report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
 
3287
 
3288
 
3289
  def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
 
3290
  "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
3291
  "amzn_ads_sd_advertised_products")):
3292
  """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
 
3448
 
3449
 
3450
  def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
3451
+ typed_values=None, typed_period=None, st=None):
3452
  """Tie net proceeds components to independent Amazon sources within 2% over settled days.
3453
  Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
3454
+ With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why."""
3455
  month = month or last_complete_month()
3456
  bounds = period_bounds(month)
3457
  start, end = bounds["start"], bounds["end"]
 
3511
  checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
3512
  if st is not None:
3513
  checks.extend(stored_grain_checks(st, con))
 
 
3514
  return checks
3515
 
3516
 
platform/modules/amazon_profit_account.py CHANGED
@@ -555,124 +555,6 @@ def validate_pace(con, bounds, pace, elapsed, month_days, reason):
555
  "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
556
 
557
 
558
- class _SourceRows:
559
- def __init__(self, rows):
560
- self.rows = rows
561
-
562
- def fetchone(self):
563
- return self.rows[0] if self.rows else None
564
-
565
- def fetchall(self):
566
- return list(self.rows)
567
-
568
-
569
- def _settlement_source(sql):
570
- """Parse distinct settlement labels and referenced order dates without losing source lines or null keys."""
571
- expressions = {"posted": ("posted_date", f"CAST({profit._moment_sql('s.posted_date')} AS DATE)")}
572
- for name, field in (("description", "amount_description"), ("kind", "amount_type"),
573
- ("transaction", "transaction_type")):
574
- expressions[name] = field, f"lower(regexp_replace(COALESCE({field}, ''), '[^a-zA-Z0-9]', '', 'g'))"
575
- fields = [field for field, _expression in expressions.values()]
576
- projection = ", ".join(expression.replace("s.posted_date", "posted_date") + " AS " + name
577
- for name, (_field, expression) in expressions.items())
578
- keys = ", ".join(f"{field} AS source_{field}" for field in fields)
579
- labels = (f"labels AS MATERIALIZED (SELECT {keys}, {projection} "
580
- f"FROM (SELECT DISTINCT {', '.join(fields)} FROM {profit.SETTLEMENTS})), ")
581
- for name, (_field, expression) in expressions.items():
582
- sql = sql.replace(expression + " AS " + name, "label." + name + " AS " + name)
583
- join = " AND ".join(f"s.{field} IS NOT DISTINCT FROM label.source_{field}" for field in fields)
584
- sql = sql.replace(f"FROM {profit.SETTLEMENTS} s JOIN streams",
585
- f"FROM {profit.SETTLEMENTS} s JOIN labels label ON {join} JOIN streams")
586
- sql = sql.replace("FROM amzn_orders GROUP BY amazon_order_id",
587
- f"FROM amzn_orders WHERE amazon_order_id IN (SELECT trim(order_id) "
588
- f"FROM {profit.SETTLEMENTS}) GROUP BY amazon_order_id")
589
- return sql.replace("WITH orders AS (", "WITH " + labels + "orders AS (", 1)
590
-
591
-
592
- class _ValidationSources:
593
- """Share settlement source preparation across this request's independent window sums."""
594
-
595
- def __init__(self, con, bounds):
596
- self.con, self.results, self.spans = con, {}, {}
597
- self.bounds = [span for bound in bounds for span in (bound, *_comparisons(bound).values())]
598
-
599
- def _spans(self, currency=None):
600
- if currency in self.spans:
601
- return self.spans[currency]
602
- spans = {(bound["start"], bound["end"]) for bound in self.bounds}
603
- if currency is not None:
604
- for start, end in tuple(spans):
605
- covered = profit.settlement_coverage(self, start, end, currency)
606
- spans.add((covered["start"] or start, covered["end"] or end))
607
- self.spans[currency] = sorted(spans)
608
- return self.spans[currency]
609
-
610
- def _components(self, sql, params):
611
- marker = "SELECT component, COALESCE(SUM(currency_amount), 0)"
612
- prefix, select = sql.split(marker, 1)
613
- aggregates, select = (marker + select).split("FROM dated WHERE", 1)
614
- aggregates = aggregates.removeprefix("SELECT component, ")
615
- # The grouping keys retain every input to the original window predicate and its bad-row counts.
616
- prefix += (" , prepared(component, basis, posted, dates, bad_dates, amount, source_rows, "
617
- "bad_amounts, unmatched_orders) AS (SELECT component, basis, posted, dates, bad_dates, "
618
- + aggregates + " FROM dated GROUP BY component, basis, posted, dates, bad_dates) ")
619
- spans = self._spans(params[0])
620
- for bound in ("lo", "hi", "lo", "hi"):
621
- select = select.replace("CAST(? AS DATE)", "span." + bound, 1)
622
- ranges = ", ".join(f"({index}, CAST(? AS DATE), CAST(? AS DATE))" for index in range(len(spans)))
623
- select = f"FROM prepared JOIN (VALUES {ranges}) AS span(id, lo, hi) ON" + select
624
- select = select.replace("GROUP BY component ORDER BY component",
625
- "GROUP BY span.id, component ORDER BY span.id, component")
626
- bindings = list(params[:2]) + [value for span in spans for value in span]
627
- rows = self.con.execute(_settlement_source(prefix) + "SELECT span.id, component, "
628
- "COALESCE(SUM(amount), 0), SUM(source_rows), SUM(bad_amounts), SUM(unmatched_orders) "
629
- + select, bindings).fetchall()
630
- for index, span in enumerate(spans):
631
- self.results[(sql, tuple(params[:2]) + span * 2)] = [row[1:] for row in rows if row[0] == index]
632
-
633
- def _unscoped(self, sql):
634
- marker = "AND (CAST("
635
- source, _date_filter = sql.split(marker, 1)
636
- source = source.replace("SELECT count(*)", f"SELECT CAST({profit._moment_sql('posted_date')} AS DATE) AS posted", 1)
637
- spans = self._spans(profit.economics_currency(self.con))
638
- sums = ["count(*) FILTER (WHERE posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)"
639
- for _span in spans]
640
- counts = self.con.execute("SELECT " + ", ".join(sums) + " FROM (" + source + ")",
641
- [value for span in spans for value in span]).fetchone()
642
- for index, count in enumerate(counts):
643
- self.results[(sql, spans[index])] = [(count,)]
644
-
645
- def execute(self, sql, params=()):
646
- params = tuple(params or ())
647
- key = sql, params
648
- if key in self.results:
649
- return _SourceRows(self.results[key])
650
- if sql == "SELECT column_name FROM information_schema.columns WHERE table_name = ?":
651
- self.results[key] = self.con.execute(sql, params).fetchall()
652
- return _SourceRows(self.results[key])
653
- components = (sql.startswith("WITH orders AS (") and len(params) == 6
654
- and sql.endswith(" GROUP BY component ORDER BY component"))
655
- unscoped = (sql.startswith(f"SELECT count(*) FROM {profit.SETTLEMENTS} s\n")
656
- and "WHERE h.settlement_id IS NULL" in sql and len(params) == 2)
657
- if components:
658
- self._components(sql, params)
659
- elif unscoped:
660
- self._unscoped(sql)
661
- if key in self.results:
662
- return _SourceRows(self.results[key])
663
- return self.con.execute(sql, params)
664
-
665
-
666
- def _account_bounds(chosen, today):
667
- bounds = [chosen, _bounds(today.replace(day=1), today - dt.timedelta(days=1), "mtd")]
668
- current = today.replace(day=1)
669
- for _ in range(2):
670
- last = current - dt.timedelta(days=1)
671
- current = last.replace(day=1)
672
- bounds.append(_bounds(current, last, current.strftime("%Y-%m")))
673
- return bounds
674
-
675
-
676
  def account_view(con, period=None, *, stored=None, settings=None, today=None):
677
  """Return selected, comparison and monthly account windows without refreshing stored inputs."""
678
  today = today or profit._today()
@@ -692,8 +574,6 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
692
  "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
693
  raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
694
  stored, settings = stored or {}, settings or {}
695
- periods = _account_bounds(chosen, today)
696
- con = _ValidationSources(con, periods)
697
  cache = {}
698
  def window(bounds):
699
  key = bounds["start"], bounds["end"]
@@ -702,8 +582,13 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
702
  return {**cache[key], "period": bounds}
703
  def compared(bounds):
704
  return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
705
- mtd = compared(periods[1])
706
- months = [compared(bounds) for bounds in periods[2:]]
 
 
 
 
 
707
  pace = {}
708
  elapsed = mtd["period"]["days"]
709
  month_days = calendar.monthrange(today.year, today.month)[1]
 
555
  "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
556
 
557
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
558
  def account_view(con, period=None, *, stored=None, settings=None, today=None):
559
  """Return selected, comparison and monthly account windows without refreshing stored inputs."""
560
  today = today or profit._today()
 
574
  "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
575
  raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
576
  stored, settings = stored or {}, settings or {}
 
 
577
  cache = {}
578
  def window(bounds):
579
  key = bounds["start"], bounds["end"]
 
582
  return {**cache[key], "period": bounds}
583
  def compared(bounds):
584
  return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
585
+ current = today.replace(day=1)
586
+ mtd = compared(_bounds(current, today - dt.timedelta(days=1), "mtd"))
587
+ months = []
588
+ for _ in range(2):
589
+ last = current - dt.timedelta(days=1)
590
+ current = last.replace(day=1)
591
+ months.append(compared(_bounds(current, last, current.strftime("%Y-%m"))))
592
  pace = {}
593
  elapsed = mtd["period"]["days"]
594
  month_days = calendar.monthrange(today.year, today.month)[1]
platform/modules/competitor_watch.py CHANGED
@@ -300,7 +300,7 @@ class _TermIndex(_DocumentIndex):
300
  self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
301
  total_params = ([self.slug, *([display_term(search)] if search else [])]
302
  if search or tracked else [self.key(KEY_TERM_INDEX)])
303
- tail = ("), "
304
  "bounds AS (SELECT total, CASE WHEN %s >= total THEN "
305
  "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
306
  "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
@@ -308,12 +308,6 @@ class _TermIndex(_DocumentIndex):
308
  + ("AND strpos(lower(label), %s) > 0 " if search else "") +
309
  'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
310
  "FROM bounds")
311
- if isinstance(total, str):
312
- statement = self.term_sql("WITH totals(total) AS (" + total + tail)
313
- else:
314
- # PostgreSQL's `self.sql` returns a psycopg Composed: compose it, never concatenate it as text.
315
- from psycopg import sql as psql
316
- statement = psql.SQL("WITH totals(total) AS (") + total + psql.SQL(self.term_sql(tail))
317
  params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
318
  *([display_term(search)] if search else []), limit]
319
  with self.connection() as con:
@@ -469,25 +463,15 @@ def _ba_columns(cur):
469
 
470
 
471
  def _sqp(cur, term=None):
472
- """Monthly SQP rows as (term, asin, impressions, end_date) for our ASINs."""
473
  need = {"search_query_data_search_query", "asin"}
474
  cols = _columns(cur, SQP_TABLE) if cur is not None else set()
475
  if not need <= cols:
476
  return []
477
- if "_report_period" in cols:
478
- period = '"_report_period" = \'MONTH\''
479
- elif {"start_date", "end_date"} <= cols:
480
- start = 'TRY_CAST("start_date" AS DATE)'
481
- end_date = 'TRY_CAST("end_date" AS DATE)'
482
- period = f'day({start}) = 1 AND {end_date} = last_day({start})'
483
- else:
484
- return []
485
  imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
486
  if "impression_data_asin_impression_count" in cols else "0")
487
  end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
488
- where = f" WHERE {period}"
489
- if term is not None:
490
- where += f' AND {_term_sql("search_query_data_search_query")} = ?'
491
  rows = cur.execute(
492
  f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
493
  [term] if term is not None else []).fetchall()
 
300
  self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
301
  total_params = ([self.slug, *([display_term(search)] if search else [])]
302
  if search or tracked else [self.key(KEY_TERM_INDEX)])
303
+ statement = self.term_sql("WITH totals(total) AS (" + total + "), "
304
  "bounds AS (SELECT total, CASE WHEN %s >= total THEN "
305
  "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
306
  "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
 
308
  + ("AND strpos(lower(label), %s) > 0 " if search else "") +
309
  'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
310
  "FROM bounds")
 
 
 
 
 
 
311
  params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
312
  *([display_term(search)] if search else []), limit]
313
  with self.connection() as con:
 
463
 
464
 
465
  def _sqp(cur, term=None):
466
+ """SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only."""
467
  need = {"search_query_data_search_query", "asin"}
468
  cols = _columns(cur, SQP_TABLE) if cur is not None else set()
469
  if not need <= cols:
470
  return []
 
 
 
 
 
 
 
 
471
  imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
472
  if "impression_data_asin_impression_count" in cols else "0")
473
  end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
474
+ where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else ""
 
 
475
  rows = cur.execute(
476
  f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
477
  [term] if term is not None else []).fetchall()
requirements.txt CHANGED
@@ -25,7 +25,6 @@ duckdb==1.5.4
25
  # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
26
  pytz>=2024.1
27
  pyyaml>=6.0
28
- jsonschema==4.26.0
29
  pillow>=10.0
30
  beautifulsoup4>=4.12
31
  lxml>=5.0
 
25
  # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
26
  pytz>=2024.1
27
  pyyaml>=6.0
 
28
  pillow>=10.0
29
  beautifulsoup4>=4.12
30
  lxml>=5.0
web/package-lock.json CHANGED
@@ -25,7 +25,6 @@
25
  "@types/react": "^18.3.12",
26
  "@types/react-dom": "^18.3.1",
27
  "@vitejs/plugin-react": "^6.0.3",
28
- "esbuild": "^0.27.0",
29
  "openapi-typescript": "^7.13.0",
30
  "oxlint": "^1.71.0",
31
  "typescript": "~6.0.2",
@@ -390,448 +389,6 @@
390
  "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
391
  "license": "MIT"
392
  },
393
- "node_modules/@esbuild/aix-ppc64": {
394
- "version": "0.27.7",
395
- "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz",
396
- "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==",
397
- "cpu": [
398
- "ppc64"
399
- ],
400
- "dev": true,
401
- "license": "MIT",
402
- "optional": true,
403
- "os": [
404
- "aix"
405
- ],
406
- "engines": {
407
- "node": ">=18"
408
- }
409
- },
410
- "node_modules/@esbuild/android-arm": {
411
- "version": "0.27.7",
412
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz",
413
- "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==",
414
- "cpu": [
415
- "arm"
416
- ],
417
- "dev": true,
418
- "license": "MIT",
419
- "optional": true,
420
- "os": [
421
- "android"
422
- ],
423
- "engines": {
424
- "node": ">=18"
425
- }
426
- },
427
- "node_modules/@esbuild/android-arm64": {
428
- "version": "0.27.7",
429
- "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz",
430
- "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==",
431
- "cpu": [
432
- "arm64"
433
- ],
434
- "dev": true,
435
- "license": "MIT",
436
- "optional": true,
437
- "os": [
438
- "android"
439
- ],
440
- "engines": {
441
- "node": ">=18"
442
- }
443
- },
444
- "node_modules/@esbuild/android-x64": {
445
- "version": "0.27.7",
446
- "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz",
447
- "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==",
448
- "cpu": [
449
- "x64"
450
- ],
451
- "dev": true,
452
- "license": "MIT",
453
- "optional": true,
454
- "os": [
455
- "android"
456
- ],
457
- "engines": {
458
- "node": ">=18"
459
- }
460
- },
461
- "node_modules/@esbuild/darwin-arm64": {
462
- "version": "0.27.7",
463
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz",
464
- "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==",
465
- "cpu": [
466
- "arm64"
467
- ],
468
- "dev": true,
469
- "license": "MIT",
470
- "optional": true,
471
- "os": [
472
- "darwin"
473
- ],
474
- "engines": {
475
- "node": ">=18"
476
- }
477
- },
478
- "node_modules/@esbuild/darwin-x64": {
479
- "version": "0.27.7",
480
- "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz",
481
- "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==",
482
- "cpu": [
483
- "x64"
484
- ],
485
- "dev": true,
486
- "license": "MIT",
487
- "optional": true,
488
- "os": [
489
- "darwin"
490
- ],
491
- "engines": {
492
- "node": ">=18"
493
- }
494
- },
495
- "node_modules/@esbuild/freebsd-arm64": {
496
- "version": "0.27.7",
497
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz",
498
- "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==",
499
- "cpu": [
500
- "arm64"
501
- ],
502
- "dev": true,
503
- "license": "MIT",
504
- "optional": true,
505
- "os": [
506
- "freebsd"
507
- ],
508
- "engines": {
509
- "node": ">=18"
510
- }
511
- },
512
- "node_modules/@esbuild/freebsd-x64": {
513
- "version": "0.27.7",
514
- "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz",
515
- "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==",
516
- "cpu": [
517
- "x64"
518
- ],
519
- "dev": true,
520
- "license": "MIT",
521
- "optional": true,
522
- "os": [
523
- "freebsd"
524
- ],
525
- "engines": {
526
- "node": ">=18"
527
- }
528
- },
529
- "node_modules/@esbuild/linux-arm": {
530
- "version": "0.27.7",
531
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz",
532
- "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==",
533
- "cpu": [
534
- "arm"
535
- ],
536
- "dev": true,
537
- "license": "MIT",
538
- "optional": true,
539
- "os": [
540
- "linux"
541
- ],
542
- "engines": {
543
- "node": ">=18"
544
- }
545
- },
546
- "node_modules/@esbuild/linux-arm64": {
547
- "version": "0.27.7",
548
- "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz",
549
- "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==",
550
- "cpu": [
551
- "arm64"
552
- ],
553
- "dev": true,
554
- "license": "MIT",
555
- "optional": true,
556
- "os": [
557
- "linux"
558
- ],
559
- "engines": {
560
- "node": ">=18"
561
- }
562
- },
563
- "node_modules/@esbuild/linux-ia32": {
564
- "version": "0.27.7",
565
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz",
566
- "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==",
567
- "cpu": [
568
- "ia32"
569
- ],
570
- "dev": true,
571
- "license": "MIT",
572
- "optional": true,
573
- "os": [
574
- "linux"
575
- ],
576
- "engines": {
577
- "node": ">=18"
578
- }
579
- },
580
- "node_modules/@esbuild/linux-loong64": {
581
- "version": "0.27.7",
582
- "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz",
583
- "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==",
584
- "cpu": [
585
- "loong64"
586
- ],
587
- "dev": true,
588
- "license": "MIT",
589
- "optional": true,
590
- "os": [
591
- "linux"
592
- ],
593
- "engines": {
594
- "node": ">=18"
595
- }
596
- },
597
- "node_modules/@esbuild/linux-mips64el": {
598
- "version": "0.27.7",
599
- "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz",
600
- "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==",
601
- "cpu": [
602
- "mips64el"
603
- ],
604
- "dev": true,
605
- "license": "MIT",
606
- "optional": true,
607
- "os": [
608
- "linux"
609
- ],
610
- "engines": {
611
- "node": ">=18"
612
- }
613
- },
614
- "node_modules/@esbuild/linux-ppc64": {
615
- "version": "0.27.7",
616
- "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz",
617
- "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==",
618
- "cpu": [
619
- "ppc64"
620
- ],
621
- "dev": true,
622
- "license": "MIT",
623
- "optional": true,
624
- "os": [
625
- "linux"
626
- ],
627
- "engines": {
628
- "node": ">=18"
629
- }
630
- },
631
- "node_modules/@esbuild/linux-riscv64": {
632
- "version": "0.27.7",
633
- "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz",
634
- "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==",
635
- "cpu": [
636
- "riscv64"
637
- ],
638
- "dev": true,
639
- "license": "MIT",
640
- "optional": true,
641
- "os": [
642
- "linux"
643
- ],
644
- "engines": {
645
- "node": ">=18"
646
- }
647
- },
648
- "node_modules/@esbuild/linux-s390x": {
649
- "version": "0.27.7",
650
- "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz",
651
- "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==",
652
- "cpu": [
653
- "s390x"
654
- ],
655
- "dev": true,
656
- "license": "MIT",
657
- "optional": true,
658
- "os": [
659
- "linux"
660
- ],
661
- "engines": {
662
- "node": ">=18"
663
- }
664
- },
665
- "node_modules/@esbuild/linux-x64": {
666
- "version": "0.27.7",
667
- "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz",
668
- "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==",
669
- "cpu": [
670
- "x64"
671
- ],
672
- "dev": true,
673
- "license": "MIT",
674
- "optional": true,
675
- "os": [
676
- "linux"
677
- ],
678
- "engines": {
679
- "node": ">=18"
680
- }
681
- },
682
- "node_modules/@esbuild/netbsd-arm64": {
683
- "version": "0.27.7",
684
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz",
685
- "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==",
686
- "cpu": [
687
- "arm64"
688
- ],
689
- "dev": true,
690
- "license": "MIT",
691
- "optional": true,
692
- "os": [
693
- "netbsd"
694
- ],
695
- "engines": {
696
- "node": ">=18"
697
- }
698
- },
699
- "node_modules/@esbuild/netbsd-x64": {
700
- "version": "0.27.7",
701
- "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz",
702
- "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==",
703
- "cpu": [
704
- "x64"
705
- ],
706
- "dev": true,
707
- "license": "MIT",
708
- "optional": true,
709
- "os": [
710
- "netbsd"
711
- ],
712
- "engines": {
713
- "node": ">=18"
714
- }
715
- },
716
- "node_modules/@esbuild/openbsd-arm64": {
717
- "version": "0.27.7",
718
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz",
719
- "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==",
720
- "cpu": [
721
- "arm64"
722
- ],
723
- "dev": true,
724
- "license": "MIT",
725
- "optional": true,
726
- "os": [
727
- "openbsd"
728
- ],
729
- "engines": {
730
- "node": ">=18"
731
- }
732
- },
733
- "node_modules/@esbuild/openbsd-x64": {
734
- "version": "0.27.7",
735
- "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz",
736
- "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==",
737
- "cpu": [
738
- "x64"
739
- ],
740
- "dev": true,
741
- "license": "MIT",
742
- "optional": true,
743
- "os": [
744
- "openbsd"
745
- ],
746
- "engines": {
747
- "node": ">=18"
748
- }
749
- },
750
- "node_modules/@esbuild/openharmony-arm64": {
751
- "version": "0.27.7",
752
- "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz",
753
- "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==",
754
- "cpu": [
755
- "arm64"
756
- ],
757
- "dev": true,
758
- "license": "MIT",
759
- "optional": true,
760
- "os": [
761
- "openharmony"
762
- ],
763
- "engines": {
764
- "node": ">=18"
765
- }
766
- },
767
- "node_modules/@esbuild/sunos-x64": {
768
- "version": "0.27.7",
769
- "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz",
770
- "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==",
771
- "cpu": [
772
- "x64"
773
- ],
774
- "dev": true,
775
- "license": "MIT",
776
- "optional": true,
777
- "os": [
778
- "sunos"
779
- ],
780
- "engines": {
781
- "node": ">=18"
782
- }
783
- },
784
- "node_modules/@esbuild/win32-arm64": {
785
- "version": "0.27.7",
786
- "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz",
787
- "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==",
788
- "cpu": [
789
- "arm64"
790
- ],
791
- "dev": true,
792
- "license": "MIT",
793
- "optional": true,
794
- "os": [
795
- "win32"
796
- ],
797
- "engines": {
798
- "node": ">=18"
799
- }
800
- },
801
- "node_modules/@esbuild/win32-ia32": {
802
- "version": "0.27.7",
803
- "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz",
804
- "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==",
805
- "cpu": [
806
- "ia32"
807
- ],
808
- "dev": true,
809
- "license": "MIT",
810
- "optional": true,
811
- "os": [
812
- "win32"
813
- ],
814
- "engines": {
815
- "node": ">=18"
816
- }
817
- },
818
- "node_modules/@esbuild/win32-x64": {
819
- "version": "0.27.7",
820
- "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz",
821
- "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==",
822
- "cpu": [
823
- "x64"
824
- ],
825
- "dev": true,
826
- "license": "MIT",
827
- "optional": true,
828
- "os": [
829
- "win32"
830
- ],
831
- "engines": {
832
- "node": ">=18"
833
- }
834
- },
835
  "node_modules/@glideapps/glide-data-grid": {
836
  "version": "6.0.3",
837
  "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
@@ -2292,48 +1849,6 @@
2292
  "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
2293
  "license": "MIT"
2294
  },
2295
- "node_modules/esbuild": {
2296
- "version": "0.27.7",
2297
- "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz",
2298
- "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==",
2299
- "dev": true,
2300
- "hasInstallScript": true,
2301
- "license": "MIT",
2302
- "bin": {
2303
- "esbuild": "bin/esbuild"
2304
- },
2305
- "engines": {
2306
- "node": ">=18"
2307
- },
2308
- "optionalDependencies": {
2309
- "@esbuild/aix-ppc64": "0.27.7",
2310
- "@esbuild/android-arm": "0.27.7",
2311
- "@esbuild/android-arm64": "0.27.7",
2312
- "@esbuild/android-x64": "0.27.7",
2313
- "@esbuild/darwin-arm64": "0.27.7",
2314
- "@esbuild/darwin-x64": "0.27.7",
2315
- "@esbuild/freebsd-arm64": "0.27.7",
2316
- "@esbuild/freebsd-x64": "0.27.7",
2317
- "@esbuild/linux-arm": "0.27.7",
2318
- "@esbuild/linux-arm64": "0.27.7",
2319
- "@esbuild/linux-ia32": "0.27.7",
2320
- "@esbuild/linux-loong64": "0.27.7",
2321
- "@esbuild/linux-mips64el": "0.27.7",
2322
- "@esbuild/linux-ppc64": "0.27.7",
2323
- "@esbuild/linux-riscv64": "0.27.7",
2324
- "@esbuild/linux-s390x": "0.27.7",
2325
- "@esbuild/linux-x64": "0.27.7",
2326
- "@esbuild/netbsd-arm64": "0.27.7",
2327
- "@esbuild/netbsd-x64": "0.27.7",
2328
- "@esbuild/openbsd-arm64": "0.27.7",
2329
- "@esbuild/openbsd-x64": "0.27.7",
2330
- "@esbuild/openharmony-arm64": "0.27.7",
2331
- "@esbuild/sunos-x64": "0.27.7",
2332
- "@esbuild/win32-arm64": "0.27.7",
2333
- "@esbuild/win32-ia32": "0.27.7",
2334
- "@esbuild/win32-x64": "0.27.7"
2335
- }
2336
- },
2337
  "node_modules/escalade": {
2338
  "version": "3.2.0",
2339
  "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
 
25
  "@types/react": "^18.3.12",
26
  "@types/react-dom": "^18.3.1",
27
  "@vitejs/plugin-react": "^6.0.3",
 
28
  "openapi-typescript": "^7.13.0",
29
  "oxlint": "^1.71.0",
30
  "typescript": "~6.0.2",
 
389
  "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
390
  "license": "MIT"
391
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
  "node_modules/@glideapps/glide-data-grid": {
393
  "version": "6.0.3",
394
  "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
 
1849
  "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
1850
  "license": "MIT"
1851
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1852
  "node_modules/escalade": {
1853
  "version": "3.2.0",
1854
  "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
web/package-lock.linux.sha256 CHANGED
@@ -1,2 +1,2 @@
1
- 9cd826154f5fecd9784f6632ad62e9da94db63b1ed343ab137fbd29ce5690c07 package-lock.json
2
- # resolved on linux/x64 by HF job 6afd11b54643bac7ac631093 on 2026-11-18, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170).
 
1
+ 31cb494538043db42bdf698846c2262f5eb234fb686aa03d9cf3ab1cbfdbe2b8 package-lock.json
2
+ # resolved on linux/x64 in the SDLC 72 lane A HF container on 2026-10-09 (S72-T05, npm install --save-exact @phosphor-icons/react@2.1.10); npm ci --ignore-scripts passed there (228 packages). Regenerate ONLY with lockfile_linux.py (W-170).
web/package.json CHANGED
@@ -39,7 +39,6 @@
39
  "@types/react": "^18.3.12",
40
  "@types/react-dom": "^18.3.1",
41
  "@vitejs/plugin-react": "^6.0.3",
42
- "esbuild": "^0.27.0",
43
  "openapi-typescript": "^7.13.0",
44
  "oxlint": "^1.71.0",
45
  "typescript": "~6.0.2",
 
39
  "@types/react": "^18.3.12",
40
  "@types/react-dom": "^18.3.1",
41
  "@vitejs/plugin-react": "^6.0.3",
 
42
  "openapi-typescript": "^7.13.0",
43
  "oxlint": "^1.71.0",
44
  "typescript": "~6.0.2",
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 7c3abd53bb8053c1ff1dee68532366490bc632e10176ced049a6e6c163b5f300
 
1
+ f4813286cc57af9ab94a458dafeabfd9f77075e08fa73b573fd34b3ec376779d
web/src/apps/amazon-profit/AppView.tsx CHANGED
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
2
  import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
3
  import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
4
  OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
5
- import type { AppCellDraw, AppTableHost, AppViewProps, Row } from "../../customer-grid/seam";
6
  import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
7
  CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
8
  import { LogoLoader } from "../../ui/LogoLoader";
@@ -11,36 +11,12 @@ import PicturePopup from "./PicturePopup";
11
  import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
12
  import type { ProjectsTarget } from "./Projects";
13
  import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
14
- import GridFilter, { SIGNAL_SORTS } from "./pdp/GridFilter";
15
- import HistoryPopup from "./pdp/HistoryPopup";
16
- import PdpUpdatePopup, { type PdpListing, type PdpNavigation } from "./pdp/PdpUpdatePopup";
17
- import WorkflowWindow from "./pdp/WorkflowWindow";
18
- import type { PdpProjectNavigation } from "./pdp/ContinueWorkflow";
19
- import type { PdpProjectPane } from "./pdp/contract";
20
  import "../../styles/sdlc81/command-center-app.css";
21
- import "../../styles/sdlc86/project-manager-pdp.css";
22
 
23
  const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
24
  const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
25
  /** The App's 56 px rows (R5). Table keeps its own default. */
26
  const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
27
- const ASIN_COLUMNS = { rowHeight: 56, headerTone: { projects: "primary", pdp_update: "primary", history: "primary" },
28
- widths: { asin: 150, projects: 210, pdp_update: 170, trend_score: 150, search_trend: 210,
29
- market_share_trend: 240, click_share_trend: 240, exposure: 190, funnel_score: 150, history: 150 } } as const;
30
- const noReport = (reasonKey: string) => ({ reasonKey, equals: "No search report", text: "No search report" });
31
- const signalTrend = (key: string, share: boolean): AppCellDraw => ({ kind: "trend", seriesKey: key + "_series",
32
- valueKey: key + "_now", changeKey: key, format: share ? "percent" : "number", showValue: share,
33
- omitMissingValue: true, valuePrecision: "adaptive", changeDecimals: 0, changeNeutralBand: .5, fitLabels: true, barTone: "primary",
34
- unavailable: noReport(key + "_reason") });
35
- const SIGNAL_CELLS: AppTableHost["cellRenderers"] = {
36
- trend_score: { draw: { kind: "ring", labelPosition: "after", unavailable: noReport("trend_score_reason") } },
37
- search_trend: { draw: signalTrend("search_trend", false) },
38
- market_share_trend: { draw: signalTrend("market_share_trend", true) },
39
- click_share_trend: { draw: signalTrend("click_share_trend", true) },
40
- exposure: { draw: { kind: "badge", subKey: "exposure_label", emptyText: "None", unavailable: noReport("exposure_reason") } },
41
- funnel_score: { draw: { kind: "score", neutralRange: [4.5, 5.5], outerRange: [3, 7], emptyText: "Few",
42
- unavailable: noReport("funnel_score_reason") } },
43
- };
44
 
45
  /** Admission belongs to the selected tab's fresh probe. */
46
  type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
@@ -48,17 +24,12 @@ type TableProbe = { table: string; status: "ready"; tables: Record<string, boole
48
 
49
  // The Command Center App body: the Table module's own Grid, hosted under the App frame.
50
  // The tab strip switches `table` through the context, so the URL stays on the App.
51
- export default function ProjectManagementView(props: AppViewProps & PdpNavigation) {
52
  const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
53
  const [probe, setProbe] = useState<TableProbe>(null);
54
  const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
55
- const [projects, setProjects] = useState<(ProjectsTarget & {
56
- selectedProjectId?: string; originListing?: PdpListing; initialPane?: PdpProjectPane;
57
- }) | null>(null);
58
  const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
59
- const [history, setHistory] = useState<Row | null>(null);
60
- const [pdp, setPdp] = useState<PdpListing | null>(null);
61
- const [workflow, setWorkflow] = useState<{ id: string; listing: PdpListing; resumeRevision?: number } | null>(null);
62
  const [gridRevision, setGridRevision] = useState(0);
63
  const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
64
  const openTable = useCallback((next: string) => {
@@ -69,9 +40,6 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
69
  setPicture(null);
70
  setProjects(null);
71
  setCompetitors(null);
72
- setHistory(null);
73
- setPdp(null);
74
- setWorkflow(null);
75
  setTable(next);
76
  }, [table]);
77
  const reloadGrid = useCallback(() => {
@@ -120,37 +88,12 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
120
  }, [dataEntry, admitted, table]);
121
  const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
122
  const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
123
- const openProject = (id: string, asin: string, listing: PdpListing, initialPane?: PdpProjectPane) => {
124
- if (props.onOpenProject) { props.onOpenProject(id, asin); return; }
125
- setPdp(null);
126
- setProjects({ grain: "asin", rowKey: asin, title: listing.title, selectedProjectId: id, originListing: listing, initialPane });
127
- };
128
- const resumeWorkflow = () => {
129
- setProjects(null);
130
- setWorkflow(current => current ? { ...current, resumeRevision: (current.resumeRevision ?? 0) + 1 } : current);
131
- };
132
- const projectNavigation: PdpProjectNavigation = { onOpenWorkflow: (next, item, target) => {
133
- if (workflow?.id === next.id && workflow.listing.asin === item.asin) {
134
- resumeWorkflow();
135
- return;
136
- }
137
- setProjects(null);
138
- if (props.onOpenWorkflow) { props.onOpenWorkflow(next.id, item.asin); return; }
139
- setWorkflow({ id: next.id, listing: { asin: item.asin, title: target?.title ?? "", sku: target?.sku ?? null, brand: null } });
140
- } };
141
  // C2: progress colours, button doors and the Picture popup, declared once per table.
142
  // Only the Projects table keeps New record (`append`).
143
  const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
144
- ...(table === "ut_amazon_asin_profit" ? SIGNAL_CELLS : {}),
145
  "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
146
  projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
147
  competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
148
- history: { button: true, activate: row => setHistory(row) },
149
- pdp_update: { button: true, activate: row => {
150
- if (!row.asin) return false;
151
- setPdp({ asin: String(row.asin), title: String(row.title || ""),
152
- sku: row.sku ? String(row.sku) : null, brand: row.brand ? String(row.brand) : null });
153
- } },
154
  "type:image": { activate: (record, field) => {
155
  const ref = String(record[field.key] ?? "").trim();
156
  if (!ref) return false;
@@ -172,35 +115,17 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
172
  </div>
173
  : table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
174
  <CommandCenterTabStrip database={table} onOpenTable={openTable} />
175
- <ProjectsPage pdp={projectNavigation} />
176
  </div>
177
  : !entryChecked ? <LogoLoader data-app-reason="loading" /> :
178
  <AppTableGrid key={`${table}:${gridRevision}`}
179
  data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
180
- columns={table === "ut_amazon_asin_profit" ? ASIN_COLUMNS : COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers}
181
- history={false} filterPanel={table === "ut_amazon_asin_profit" ? props => <GridFilter {...props} /> : undefined}
182
- sortChoices={table === "ut_amazon_asin_profit" ? SIGNAL_SORTS : undefined}
183
  toolbar={<AmazonToolbar database={table} />} />}
184
  {picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
185
  onClose={() => setPicture(null)} />}
186
- {projects && <ProjectsPopup {...projects} pdp={projectNavigation}
187
- onClose={projects.initialPane ? resumeWorkflow : () => setProjects(null)} onChanged={reloadGrid}
188
- backLabel={projects.initialPane ? "Back to workflow" : projects.originListing ? "Back to projects" : undefined}
189
- onBack={projects.initialPane ? resumeWorkflow : projects.originListing
190
- ? () => { setPdp(projects.originListing ?? null); setProjects(null); setWorkflow(null); } : undefined} />}
191
  {competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
192
- {history && <HistoryPopup table={table} row={history} onClose={() => setHistory(null)} onChanged={reloadGrid} />}
193
- {pdp && <PdpUpdatePopup listing={pdp} onClose={() => setPdp(null)} onChanged={reloadGrid}
194
- onOpenWorkflow={props.onOpenWorkflow ?? ((id, asin) => {
195
- setWorkflow({ id, listing: { ...pdp, asin } }); setPdp(null);
196
- })} onAddWorkflow={props.onAddWorkflow}
197
- onOpenProject={(id, asin) => openProject(id, asin, pdp)} />}
198
- {workflow && <WorkflowWindow key={`${workflow.id}:${workflow.listing.asin}`} workflowId={workflow.id} listing={workflow.listing}
199
- resumeRevision={workflow.resumeRevision}
200
- onClose={() => setWorkflow(null)} onChanged={reloadGrid}
201
- onBack={() => { setPdp(workflow.listing); setWorkflow(null); }}
202
- onOpenProject={() => openProject(workflow.id, workflow.listing.asin, workflow.listing)}
203
- onOpenProjectPane={pane => openProject(workflow.id, workflow.listing.asin, workflow.listing, pane)} />}
204
  </OverlayProvider>
205
  </CommandCenterTableContext.Provider>;
206
  }
 
2
  import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
3
  import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
4
  OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
5
+ import type { AppTableHost, AppViewProps } from "../../customer-grid/seam";
6
  import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
7
  CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
8
  import { LogoLoader } from "../../ui/LogoLoader";
 
11
  import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
12
  import type { ProjectsTarget } from "./Projects";
13
  import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
 
 
 
 
 
 
14
  import "../../styles/sdlc81/command-center-app.css";
 
15
 
16
  const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
17
  const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
18
  /** The App's 56 px rows (R5). Table keeps its own default. */
19
  const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  /** Admission belongs to the selected tab's fresh probe. */
22
  type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
 
24
 
25
  // The Command Center App body: the Table module's own Grid, hosted under the App frame.
26
  // The tab strip switches `table` through the context, so the URL stays on the App.
27
+ export default function ProjectManagementView(_props: AppViewProps) {
28
  const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
29
  const [probe, setProbe] = useState<TableProbe>(null);
30
  const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
31
+ const [projects, setProjects] = useState<ProjectsTarget | null>(null);
 
 
32
  const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
 
 
 
33
  const [gridRevision, setGridRevision] = useState(0);
34
  const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
35
  const openTable = useCallback((next: string) => {
 
40
  setPicture(null);
41
  setProjects(null);
42
  setCompetitors(null);
 
 
 
43
  setTable(next);
44
  }, [table]);
45
  const reloadGrid = useCallback(() => {
 
88
  }, [dataEntry, admitted, table]);
89
  const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
90
  const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  // C2: progress colours, button doors and the Picture popup, declared once per table.
92
  // Only the Projects table keeps New record (`append`).
93
  const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
 
94
  "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
95
  projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
96
  competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
 
 
 
 
 
 
97
  "type:image": { activate: (record, field) => {
98
  const ref = String(record[field.key] ?? "").trim();
99
  if (!ref) return false;
 
115
  </div>
116
  : table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
117
  <CommandCenterTabStrip database={table} onOpenTable={openTable} />
118
+ <ProjectsPage />
119
  </div>
120
  : !entryChecked ? <LogoLoader data-app-reason="loading" /> :
121
  <AppTableGrid key={`${table}:${gridRevision}`}
122
  data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
123
+ columns={COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers}
 
 
124
  toolbar={<AmazonToolbar database={table} />} />}
125
  {picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
126
  onClose={() => setPicture(null)} />}
127
+ {projects && <ProjectsPopup {...projects} onClose={() => setProjects(null)} onChanged={reloadGrid} />}
 
 
 
 
128
  {competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
 
 
 
 
 
 
 
 
 
 
 
 
129
  </OverlayProvider>
130
  </CommandCenterTableContext.Provider>;
131
  }
web/src/apps/amazon-profit/ProjectOverview.tsx CHANGED
@@ -11,8 +11,6 @@ import { X } from "@phosphor-icons/react/dist/csr/X";
11
  import { useEscapeClose } from "../../ui/useEscapeClose";
12
  import ProjectTasks from "./ProjectTasks";
13
  import ProjectSkuPicker from "./ProjectSkuPicker";
14
- import { PdpApprovalRow, PdpOverviewRows, PdpStep, type ProjectPdp } from "./pdp/ContinueWorkflow";
15
- import { removeItems } from "./pdp/api";
16
  import {
17
  EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
18
  type Field, type FilterTree, type Row, type Sorts,
@@ -238,12 +236,11 @@ function ProjectDeleteDialog({ name, busy, error, onCancel, onConfirm }: {
238
  }
239
 
240
  export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
241
- error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope, pdp }: {
242
  id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
243
  error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
244
  setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
245
  rowScope?: RowScope | null; onClearRowScope?: () => void;
246
- pdp?: ProjectPdp;
247
  }) {
248
  const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
249
  const [people, setPeople] = useState<ProjectPerson[]>([]);
@@ -321,12 +318,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
321
  setBusy(true);
322
  const fallback = "Products were not removed.";
323
  try {
324
- if (pdp) {
325
- if (!pdp.detail?.canEdit) throw new Error("The workflow is unavailable. Reload the project.");
326
- pdp.accept((await removeItems(id, pairs.map(pair => pair.key))).workflow);
327
- setTick(value => value + 1);
328
- return { refused: [] };
329
- }
330
  const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
331
  { method: "POST", body: { targets: pairs } }, fallback);
332
  if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
@@ -352,10 +343,10 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
352
  writing.current = false;
353
  setBusy(false);
354
  }
355
- }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick, pdp]);
356
 
357
  const { project, targets } = detail;
358
- const canEdit = mayEdit && detail.canEdit && (!pdp || Boolean(pdp.detail?.canEdit));
359
  const deleteOpen = confirming && canEdit && active;
360
  useEffect(() => {
361
  if (!canEdit || !active) { setConfirming(false); setAdding(false); }
@@ -469,7 +460,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
469
  </section>
470
 
471
  <div className="lp-proj-setup-fields">
472
- {project.type === "pdp" && pdp ? <PdpOverviewRows pdp={pdp} /> : null}
473
  <SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
474
  {canEdit
475
  ? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
@@ -492,7 +482,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
492
  onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
493
  ? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
494
  </SetupRow>
495
- {project.type === "pdp" && pdp ? <PdpApprovalRow pdp={pdp} canEdit={canEdit} /> : null}
496
  <SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
497
  <div className="lp-proj-contributors">
498
  {contributors.map(person => <span key={person.username} className="lp-proj-contributor">
@@ -566,7 +555,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
566
  : null}
567
  </div>
568
  {canEdit && active && adding
569
- ? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)} pdp={pdp}
570
  onAdded={answer => {
571
  setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
572
  cause: answer.cause } : current);
@@ -577,7 +566,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
577
  : null}
578
  {targets.length
579
  ? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
580
- onRemove={removeTargets} pdp={pdp} />
581
  : null}
582
  </section>
583
  </SetupRow>
@@ -665,10 +654,9 @@ async function readLinked(table: string, grain: ProjectGrain, tree: FilterTree,
665
  }
666
  }
667
 
668
- function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove, pdp }: {
669
  targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
670
  onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
671
- pdp?: ProjectPdp;
672
  }) {
673
  const id = useId();
674
  const [chosen, setChosen] = useState<ProjectGrain>("sku");
@@ -698,14 +686,13 @@ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemov
698
  </div> : null}
699
  <SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
700
  targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
701
- onRemove={onRemove} pdp={pdp} />
702
  </div>;
703
  }
704
 
705
- function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove, pdp }: {
706
  grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
707
  busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
708
- pdp?: ProjectPdp;
709
  }) {
710
  const table = TABLES[grain];
711
  const label = GRAIN_LABEL[grain];
@@ -799,7 +786,7 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
799
  : new Set(Array.from(current).filter(key => shown.has(key))));
800
  }, [term, blocked, shownSignature]);
801
  const columns = LIST_COLUMNS[grain];
802
- const span = (canEdit ? 1 : 0) + 1 + columns.length + (pdp ? 2 : 0);
803
  const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
804
  const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
805
  const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
@@ -864,7 +851,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
864
  {canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
865
  <th scope="col">Product</th>
866
  {columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
867
- {pdp ? <><th scope="col">Step</th><th scope="col"><span className="lp-proj-sku-sr">Workflow</span></th></> : null}
868
  </tr></thead>
869
  <tbody>{groups.map(([value, members]) => <Fragment key={value}>
870
  {group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
@@ -887,7 +873,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
887
  return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
888
  {shown || <span className="lp-proj-muted">Not set</span>}</td>;
889
  })}
890
- {pdp ? <PdpSkuCells pdp={pdp} target={row.target} /> : null}
891
  </tr>;
892
  })}
893
  </Fragment>)}</tbody>
@@ -913,14 +898,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
913
  </div>;
914
  }
915
 
916
- function PdpSkuCells({ pdp, target }: { pdp: ProjectPdp; target: Target }) {
917
- const asin = target.grain === "asin" ? target.key : target.asin;
918
- const item = pdp.detail?.workflow.items.find(item => item.asin === asin);
919
- return <><td>{item ? <PdpStep item={item} /> : <span className="lp-proj-muted">Unavailable</span>}</td>
920
- <td><button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={!item || !pdp.open}
921
- onClick={() => { if (item) pdp.open?.(item); }}>Open</button></td></>;
922
- }
923
-
924
  export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
925
  targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
926
  onCancel: () => void; onForbidden: () => void;
 
11
  import { useEscapeClose } from "../../ui/useEscapeClose";
12
  import ProjectTasks from "./ProjectTasks";
13
  import ProjectSkuPicker from "./ProjectSkuPicker";
 
 
14
  import {
15
  EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
16
  type Field, type FilterTree, type Row, type Sorts,
 
236
  }
237
 
238
  export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
239
+ error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope }: {
240
  id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
241
  error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
242
  setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
243
  rowScope?: RowScope | null; onClearRowScope?: () => void;
 
244
  }) {
245
  const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
246
  const [people, setPeople] = useState<ProjectPerson[]>([]);
 
318
  setBusy(true);
319
  const fallback = "Products were not removed.";
320
  try {
 
 
 
 
 
 
321
  const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
322
  { method: "POST", body: { targets: pairs } }, fallback);
323
  if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
 
343
  writing.current = false;
344
  setBusy(false);
345
  }
346
+ }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick]);
347
 
348
  const { project, targets } = detail;
349
+ const canEdit = mayEdit && detail.canEdit;
350
  const deleteOpen = confirming && canEdit && active;
351
  useEffect(() => {
352
  if (!canEdit || !active) { setConfirming(false); setAdding(false); }
 
460
  </section>
461
 
462
  <div className="lp-proj-setup-fields">
 
463
  <SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
464
  {canEdit
465
  ? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
 
482
  onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
483
  ? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
484
  </SetupRow>
 
485
  <SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
486
  <div className="lp-proj-contributors">
487
  {contributors.map(person => <span key={person.username} className="lp-proj-contributor">
 
555
  : null}
556
  </div>
557
  {canEdit && active && adding
558
+ ? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)}
559
  onAdded={answer => {
560
  setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
561
  cause: answer.cause } : current);
 
566
  : null}
567
  {targets.length
568
  ? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
569
+ onRemove={removeTargets} />
570
  : null}
571
  </section>
572
  </SetupRow>
 
654
  }
655
  }
656
 
657
+ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove }: {
658
  targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
659
  onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
 
660
  }) {
661
  const id = useId();
662
  const [chosen, setChosen] = useState<ProjectGrain>("sku");
 
686
  </div> : null}
687
  <SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
688
  targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
689
+ onRemove={onRemove} />
690
  </div>;
691
  }
692
 
693
+ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove }: {
694
  grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
695
  busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
 
696
  }) {
697
  const table = TABLES[grain];
698
  const label = GRAIN_LABEL[grain];
 
786
  : new Set(Array.from(current).filter(key => shown.has(key))));
787
  }, [term, blocked, shownSignature]);
788
  const columns = LIST_COLUMNS[grain];
789
+ const span = (canEdit ? 1 : 0) + 1 + columns.length;
790
  const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
791
  const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
792
  const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
 
851
  {canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
852
  <th scope="col">Product</th>
853
  {columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
 
854
  </tr></thead>
855
  <tbody>{groups.map(([value, members]) => <Fragment key={value}>
856
  {group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
 
873
  return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
874
  {shown || <span className="lp-proj-muted">Not set</span>}</td>;
875
  })}
 
876
  </tr>;
877
  })}
878
  </Fragment>)}</tbody>
 
898
  </div>;
899
  }
900
 
 
 
 
 
 
 
 
 
901
  export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
902
  targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
903
  onCancel: () => void; onForbidden: () => void;
web/src/apps/amazon-profit/ProjectSkuPicker.tsx CHANGED
@@ -11,9 +11,6 @@ import { FilterBuilderPanel, Popover, SortBuilder, countConditions,
11
  import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
12
  type Project, type ProjectGrain, type Target } from "./ProjectsShared";
13
  import "../../styles/sdlc85/project-sku-picker.css";
14
- import { projectClaimNames, readProjectClaims, type ProjectPdp } from "./pdp/ContinueWorkflow";
15
- import { addItems, PdpRequestError } from "./pdp/api";
16
- import type { PdpClaim } from "./pdp/contract";
17
 
18
  export type FilterTree = FilterBuilderPanelProps["filters"];
19
  type FilterNode = FilterTree["nodes"][number];
@@ -175,7 +172,7 @@ export type TaskTargetSource = {
175
  name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
176
  targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
177
  };
178
- type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void; pdp?: ProjectPdp };
179
  export type SkuPickerProps = PickerBase & (
180
  | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
181
  | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
@@ -204,12 +201,11 @@ function taskStart(task: TaskTargetSource | null) {
204
  done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
205
  }
206
 
207
- export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose, pdp }: {
208
  projectId: string; existing: Target[]; canEdit: boolean;
209
  onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
210
- pdp?: ProjectPdp;
211
  }) {
212
- return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose} pdp={pdp} />;
213
  }
214
 
215
  /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
@@ -217,15 +213,11 @@ export function SkuPicker(props: SkuPickerProps) {
217
  const { projectId, canEdit, onClose } = props;
218
  const task = props.mode === "task" ? props : null;
219
  const taskMode = task !== null;
220
- const pdp = props.mode === "add" ? props.pdp : undefined;
221
- const pdpMode = Boolean(pdp);
222
- const workflow = pdp?.detail?.workflow;
223
- const [taken, setTaken] = useState<PdpClaim[]>([]);
224
  const existing = props.mode === "add" ? props.existing : NO_TARGETS;
225
  const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
226
  const id = useId();
227
  const [start] = useState(() => taskStart(task?.task ?? null));
228
- const [grain, setGrain] = useState<ProjectGrain>(pdp ? "asin" : start.grain);
229
  const [query, setQuery] = useState("");
230
  const [search, setSearch] = useState("");
231
  const [filters, setFilters] = useState<FilterTree>(start.filter);
@@ -269,17 +261,15 @@ export function SkuPicker(props: SkuPickerProps) {
269
  const order = useMemo<Sorts>(() => group
270
  ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
271
  const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
272
- const holder = (key: string) => workflow ? projectClaimNames(taken, workflow, key) : "";
273
- const selectedCount = allMatching ? Math.max(0, available - excluded.size)
274
- : workflow ? Array.from(selected).filter(key => !holder(key)).length : selected.size;
275
- const ready = canEdit && (!pdp || Boolean(pdp.detail?.canEdit)) && !forbidden && !loading && !busy && query === search && match !== null && !error;
276
  const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
277
- const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key) && !holder(key));
278
  const checkedOnPage = pageKeys.filter(isSelected).length;
279
  const preview = taskMode && kind === "filter";
280
  const picks = selected.size + Object.entries(held)
281
  .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
282
- const grains = pdp ? ["asin"] as const : taskMode ? TASK_GRAINS : GRAINS;
283
 
284
  const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
285
  const resetRead = () => {
@@ -335,11 +325,6 @@ export function SkuPicker(props: SkuPickerProps) {
335
  if (counted && counted.total > 0 && !products.length) {
336
  throw new Error("Matching products did not load. Retry to refresh the matches.");
337
  }
338
- if (pdpMode) {
339
- const claims = await readProjectClaims(products.map(row => text(row.asin)), controller.signal);
340
- if (controller.signal.aborted) return;
341
- setTaken(current => cursor === null ? claims : [...current, ...claims]);
342
- }
343
  setFields(page.fields);
344
  setRows(current => {
345
  if (counted?.total === 0) return [];
@@ -370,7 +355,7 @@ export function SkuPicker(props: SkuPickerProps) {
370
  })
371
  .finally(() => { if (!controller.signal.aborted) setLoading(false); });
372
  return () => controller.abort();
373
- }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode, pdpMode]);
374
 
375
  const toggleKeys = (keys: string[], checked: boolean) => {
376
  if (allMatching) setExcluded(current => {
@@ -389,11 +374,6 @@ export function SkuPicker(props: SkuPickerProps) {
389
  if (!ready || !selectedCount || submitting.current) return;
390
  submitting.current = true; setBusy(true); setRefused([]);
391
  try {
392
- if (pdp) {
393
- pdp.accept((await addItems(projectId, Array.from(selected).filter(key => !holder(key)))).workflow);
394
- onClose();
395
- return;
396
- }
397
  // Exclusions apply after server ranking, so they cannot promote a different top-N row.
398
  const complete = allMatching && excluded.size
399
  ? await matchingKeys(projectId, grain, tree, match!.total) : null;
@@ -413,11 +393,7 @@ export function SkuPicker(props: SkuPickerProps) {
413
  } else onClose();
414
  } catch (reason) {
415
  setError(messageOf(reason, "Products were not added."));
416
- if (reason instanceof PdpRequestError && reason.status === 409) {
417
- try { setTaken(await readProjectClaims(rows.map(row => text(row.asin)))); }
418
- catch (refreshReason) { setError(messageOf(refreshReason, "Workflow claims did not load. Retry.")); }
419
- }
420
- if ((reason instanceof ProjectRequestError || reason instanceof PdpRequestError) && [401, 403].includes(reason.status)) {
421
  setForbidden(true); setRows([]); setFields([]); setMatch(null);
422
  }
423
  } finally { submitting.current = false; setBusy(false); }
@@ -438,10 +414,7 @@ export function SkuPicker(props: SkuPickerProps) {
438
  try {
439
  const every = await matchingKeys(projectId, grain, tree, current.total);
440
  if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
441
- const claims = workflow ? await readProjectClaims(every.keys) : [];
442
- if (workflow) setTaken(claims);
443
- const eligible = workflow ? every.keys.filter(key => !linked.has(key) && !projectClaimNames(claims, workflow, key)) : every.keys;
444
- setSelected(previous => new Set([...previous, ...eligible]));
445
  } catch (reason) { refuse(reason, "Matches did not load."); }
446
  finally { submitting.current = false; setBusy(false); }
447
  };
@@ -535,7 +508,7 @@ export function SkuPicker(props: SkuPickerProps) {
535
  {match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
536
  {match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
537
  onClick={() => {
538
- if (taskMode || pdp) { void selectMatching(); return; }
539
  setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
540
  }}>
541
  Select all {number(match.total)} matching
@@ -570,15 +543,13 @@ export function SkuPicker(props: SkuPickerProps) {
570
  {members.map(row => {
571
  const key = text(row[grain]);
572
  const alreadyLinked = linked.has(key);
573
- const heldBy = holder(key);
574
  const picture = pictureSrc(text(row.picture));
575
  return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
576
- {preview ? null : <td><SelectBox checked={isSelected(key) && !heldBy} disabled={!ready || alreadyLinked || Boolean(heldBy)}
577
  label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
578
  <td><div className="lp-proj-sku-product">
579
  {picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
580
- <span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}
581
- {heldBy ? <small>{heldBy} already changes the same parts</small> : null}</span>
582
  </div></td>
583
  <td className="lp-proj-sku-code" title={key}>{key}</td>
584
  <td className="lp-proj-sku-code">{grain === "parent_asin"
 
11
  import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
12
  type Project, type ProjectGrain, type Target } from "./ProjectsShared";
13
  import "../../styles/sdlc85/project-sku-picker.css";
 
 
 
14
 
15
  export type FilterTree = FilterBuilderPanelProps["filters"];
16
  type FilterNode = FilterTree["nodes"][number];
 
172
  name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
173
  targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
174
  };
175
+ type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void };
176
  export type SkuPickerProps = PickerBase & (
177
  | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
178
  | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
 
201
  done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
202
  }
203
 
204
+ export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose }: {
205
  projectId: string; existing: Target[]; canEdit: boolean;
206
  onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
 
207
  }) {
208
+ return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose} />;
209
  }
210
 
211
  /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
 
213
  const { projectId, canEdit, onClose } = props;
214
  const task = props.mode === "task" ? props : null;
215
  const taskMode = task !== null;
 
 
 
 
216
  const existing = props.mode === "add" ? props.existing : NO_TARGETS;
217
  const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
218
  const id = useId();
219
  const [start] = useState(() => taskStart(task?.task ?? null));
220
+ const [grain, setGrain] = useState<ProjectGrain>(start.grain);
221
  const [query, setQuery] = useState("");
222
  const [search, setSearch] = useState("");
223
  const [filters, setFilters] = useState<FilterTree>(start.filter);
 
261
  const order = useMemo<Sorts>(() => group
262
  ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
263
  const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
264
+ const selectedCount = allMatching ? Math.max(0, available - excluded.size) : selected.size;
265
+ const ready = canEdit && !forbidden && !loading && !busy && query === search && match !== null && !error;
 
 
266
  const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
267
+ const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key));
268
  const checkedOnPage = pageKeys.filter(isSelected).length;
269
  const preview = taskMode && kind === "filter";
270
  const picks = selected.size + Object.entries(held)
271
  .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
272
+ const grains = taskMode ? TASK_GRAINS : GRAINS;
273
 
274
  const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
275
  const resetRead = () => {
 
325
  if (counted && counted.total > 0 && !products.length) {
326
  throw new Error("Matching products did not load. Retry to refresh the matches.");
327
  }
 
 
 
 
 
328
  setFields(page.fields);
329
  setRows(current => {
330
  if (counted?.total === 0) return [];
 
355
  })
356
  .finally(() => { if (!controller.signal.aborted) setLoading(false); });
357
  return () => controller.abort();
358
+ }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode]);
359
 
360
  const toggleKeys = (keys: string[], checked: boolean) => {
361
  if (allMatching) setExcluded(current => {
 
374
  if (!ready || !selectedCount || submitting.current) return;
375
  submitting.current = true; setBusy(true); setRefused([]);
376
  try {
 
 
 
 
 
377
  // Exclusions apply after server ranking, so they cannot promote a different top-N row.
378
  const complete = allMatching && excluded.size
379
  ? await matchingKeys(projectId, grain, tree, match!.total) : null;
 
393
  } else onClose();
394
  } catch (reason) {
395
  setError(messageOf(reason, "Products were not added."));
396
+ if (reason instanceof ProjectRequestError && [401, 403].includes(reason.status)) {
 
 
 
 
397
  setForbidden(true); setRows([]); setFields([]); setMatch(null);
398
  }
399
  } finally { submitting.current = false; setBusy(false); }
 
414
  try {
415
  const every = await matchingKeys(projectId, grain, tree, current.total);
416
  if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
417
+ setSelected(previous => new Set([...previous, ...every.keys]));
 
 
 
418
  } catch (reason) { refuse(reason, "Matches did not load."); }
419
  finally { submitting.current = false; setBusy(false); }
420
  };
 
508
  {match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
509
  {match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
510
  onClick={() => {
511
+ if (taskMode) { void selectMatching(); return; }
512
  setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
513
  }}>
514
  Select all {number(match.total)} matching
 
543
  {members.map(row => {
544
  const key = text(row[grain]);
545
  const alreadyLinked = linked.has(key);
 
546
  const picture = pictureSrc(text(row.picture));
547
  return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
548
+ {preview ? null : <td><SelectBox checked={isSelected(key)} disabled={!ready || alreadyLinked}
549
  label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
550
  <td><div className="lp-proj-sku-product">
551
  {picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
552
+ <span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}</span>
 
553
  </div></td>
554
  <td className="lp-proj-sku-code" title={key}>{key}</td>
555
  <td className="lp-proj-sku-code">{grain === "parent_asin"
web/src/apps/amazon-profit/Projects.tsx CHANGED
@@ -1,16 +1,12 @@
1
- import { useCallback, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react";
2
  import {
3
- CenteredPopup, GRAIN_LABEL, LoadingRows, ProjectHeader, call, enc, messageOf,
4
  type Detail, type ProjectGrain, type ProjectList,
5
  } from "./ProjectsShared";
6
  import ProjectResults from "./ProjectResults";
7
  import { AddExisting, ProjectBrowser } from "./ProjectsList";
8
- import ProjectOverview, { ProjectForm, RowScopeChip, inRowScope, type RowScope } from "./ProjectOverview";
9
  import ProjectActivities from "./ProjectActivities";
10
- import PdpActivities from "./pdp/PdpActivities";
11
- import BrandRules from "./pdp/BrandRules";
12
- import ContinueWorkflow, { useProjectPdp, type PdpProjectNavigation } from "./pdp/ContinueWorkflow";
13
- import type { PdpProjectPane } from "./pdp/contract";
14
 
15
  export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
16
  export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
@@ -23,29 +19,16 @@ const PROJECT_PANES = [
23
  { key: "results", label: "Dashboard" },
24
  { key: "log", label: "Activities" },
25
  ] as const;
26
- type ExtraPane = { key: string; label: string; content: ReactNode };
27
- type ProjectExtras = {
28
- pdp?: PdpProjectNavigation; backLabel?: string; onBack?: () => void;
29
- initialPane?: PdpProjectPane;
30
- headerExtra?: ReactNode; extraPanes?: ExtraPane[];
31
- };
32
  const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
33
 
34
- function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel,
35
- pdp: navigation, headerExtra, extraPanes, initialPane }: {
36
  id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
37
- } & ProjectExtras) {
38
  const [detail, setDetail] = useState<Detail | null>(null);
39
  const [error, setError] = useState("");
40
  const [tick, setTick] = useState(0);
41
- const [pane, setPane] = useState<string>(initialPane ?? "setup");
42
- const projectChanged = () => { setTick(value => value + 1); onChanged(); };
43
- const pdp = useProjectPdp(detail?.project, detail?.targets ?? [], tick, mayEdit && Boolean(detail?.canEdit), projectChanged, navigation);
44
- const extras = detail?.project.type === "pdp" ? [
45
- { key: "brand", label: "Brand and rules", content: <BrandRules key={id} workflowId={id} projectName={detail.project.name}
46
- canEdit={mayEdit && detail.canEdit} active={pane === "brand"} /> }, ...(extraPanes ?? []),
47
- ] : [];
48
- const panes = extras.length ? [...PROJECT_PANES, ...extras] : PROJECT_PANES;
49
  // The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
50
  const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
51
  ? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
@@ -81,20 +64,20 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
81
 
82
  const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
83
  <div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
84
- {panes.map(({ key, label }, index) => <button key={key} type="button" role="tab"
85
  className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
86
  id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
87
  aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
88
  onKeyDown={event => {
89
  let next: number;
90
- if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % panes.length;
91
- else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + panes.length) % panes.length;
92
  else if (event.key === "Home") next = 0;
93
- else if (event.key === "End") next = panes.length - 1;
94
  else return;
95
  event.preventDefault();
96
  event.stopPropagation();
97
- setPane(panes[next].key);
98
  railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
99
  }}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
100
  </div>
@@ -114,19 +97,15 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
114
  </div>;
115
  const { project } = detail;
116
  const canEdit = mayEdit && detail.canEdit;
117
- const scopedAsins = rowScope ? [...new Set(detail.targets.filter(target => inRowScope(target, rowScope))
118
- .flatMap(target => target.grain === "asin" ? [target.key] : target.asin ? [target.asin] : []))] : [];
119
 
120
  return <div className="lp-proj-detail-layout">
121
- <ProjectHeader>{pdp ? <ContinueWorkflow pdp={pdp} targets={detail.targets}
122
- asin={scopedAsins.length === 1 ? scopedAsins[0] : undefined} /> : null}{headerExtra}</ProjectHeader>
123
  {rail}
124
  <div className="lp-proj-panes" ref={panesRef}>
125
  <section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
126
  aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
127
  <ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
128
  onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
129
- formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope} pdp={pdp} />
130
  </section>
131
  <section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
132
  aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
@@ -135,24 +114,17 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
135
  <ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
136
  updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
137
  </section>
138
- {project.type === "pdp"
139
- ? <PdpActivities project={project} formId={formId} active={pane === "log"} />
140
- : <ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />}
141
- {extras.map(extra => <section key={extra.key} className="lp-proj-pane" role="tabpanel" tabIndex={0}
142
- id={`${formId}-pane-${extra.key}`} aria-labelledby={`${formId}-tab-${extra.key}`} hidden={pane !== extra.key}>
143
- {extra.content}
144
- </section>)}
145
  </div>
146
  </div>;
147
  }
148
 
149
  // ---- The popup a Projects cell opens ----
150
 
151
- export interface ProjectsPopupProps extends ProjectExtras {
152
  grain: ProjectGrain;
153
  rowKey: string;
154
  title?: string;
155
- selectedProjectId?: string;
156
  onClose: () => void;
157
  onChanged?: () => void;
158
  }
@@ -160,13 +132,11 @@ export interface ProjectsPopupProps extends ProjectExtras {
160
  type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
161
 
162
  export function ProjectsPopup(props: ProjectsPopupProps) {
163
- return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}:${props.selectedProjectId ?? ""}:${props.initialPane ?? ""}`} {...props} />;
164
  }
165
 
166
- function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose, onChanged,
167
- pdp, backLabel, onBack, headerExtra, extraPanes, initialPane }: ProjectsPopupProps) {
168
- const [view, setView] = useState<PopupView>(() => selectedProjectId
169
- ? { kind: "detail", id: selectedProjectId } : { kind: "list" });
170
  const [read, setRead] = useState<ProjectList | null>(null);
171
  const [error, setError] = useState("");
172
  const [tick, setTick] = useState(0);
@@ -186,7 +156,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
186
  const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
187
 
188
  const actions = view.kind === "detail"
189
- ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={onBack ?? (() => setView({ kind: "list" }))}>{backLabel ?? "All projects"}</button>
190
  : view.kind === "list" && canEdit
191
  ? <>
192
  <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
@@ -213,8 +183,6 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
213
  : null}
214
  {view.kind === "detail"
215
  ? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
216
- initialPane={view.id === selectedProjectId ? initialPane : undefined}
217
- pdp={pdp} backLabel={backLabel} onBack={onBack} headerExtra={headerExtra} extraPanes={extraPanes}
218
  onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
219
  : null}
220
  {view.kind === "new" && canEdit
@@ -236,7 +204,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
236
 
237
  // ---- The Projects tab: every project, the same rows ----
238
 
239
- export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {}) {
240
  const [read, setRead] = useState<ProjectList | null>(null);
241
  const [error, setError] = useState("");
242
  const [tick, setTick] = useState(0);
@@ -273,7 +241,6 @@ export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {
273
  {open
274
  ? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
275
  <ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
276
- pdp={pdp} headerExtra={headerExtra} extraPanes={extraPanes}
277
  onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
278
  </CenteredPopup>
279
  : null}
 
1
+ import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
2
  import {
3
+ CenteredPopup, GRAIN_LABEL, LoadingRows, call, enc, messageOf,
4
  type Detail, type ProjectGrain, type ProjectList,
5
  } from "./ProjectsShared";
6
  import ProjectResults from "./ProjectResults";
7
  import { AddExisting, ProjectBrowser } from "./ProjectsList";
8
+ import ProjectOverview, { ProjectForm, RowScopeChip, type RowScope } from "./ProjectOverview";
9
  import ProjectActivities from "./ProjectActivities";
 
 
 
 
10
 
11
  export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
12
  export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
 
19
  { key: "results", label: "Dashboard" },
20
  { key: "log", label: "Activities" },
21
  ] as const;
22
+ type ProjectPane = (typeof PROJECT_PANES)[number]["key"];
 
 
 
 
 
23
  const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
24
 
25
+ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel }: {
 
26
  id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
27
+ }) {
28
  const [detail, setDetail] = useState<Detail | null>(null);
29
  const [error, setError] = useState("");
30
  const [tick, setTick] = useState(0);
31
+ const [pane, setPane] = useState<ProjectPane>("setup");
 
 
 
 
 
 
 
32
  // The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
33
  const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
34
  ? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
 
64
 
65
  const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
66
  <div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
67
+ {PROJECT_PANES.map(({ key, label }, index) => <button key={key} type="button" role="tab"
68
  className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
69
  id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
70
  aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
71
  onKeyDown={event => {
72
  let next: number;
73
+ if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % PROJECT_PANES.length;
74
+ else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + PROJECT_PANES.length) % PROJECT_PANES.length;
75
  else if (event.key === "Home") next = 0;
76
+ else if (event.key === "End") next = PROJECT_PANES.length - 1;
77
  else return;
78
  event.preventDefault();
79
  event.stopPropagation();
80
+ setPane(PROJECT_PANES[next].key);
81
  railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
82
  }}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
83
  </div>
 
97
  </div>;
98
  const { project } = detail;
99
  const canEdit = mayEdit && detail.canEdit;
 
 
100
 
101
  return <div className="lp-proj-detail-layout">
 
 
102
  {rail}
103
  <div className="lp-proj-panes" ref={panesRef}>
104
  <section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
105
  aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
106
  <ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
107
  onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
108
+ formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope} />
109
  </section>
110
  <section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
111
  aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
 
114
  <ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
115
  updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
116
  </section>
117
+ <ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />
 
 
 
 
 
 
118
  </div>
119
  </div>;
120
  }
121
 
122
  // ---- The popup a Projects cell opens ----
123
 
124
+ export interface ProjectsPopupProps {
125
  grain: ProjectGrain;
126
  rowKey: string;
127
  title?: string;
 
128
  onClose: () => void;
129
  onChanged?: () => void;
130
  }
 
132
  type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
133
 
134
  export function ProjectsPopup(props: ProjectsPopupProps) {
135
+ return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}`} {...props} />;
136
  }
137
 
138
+ function ProjectsPopupContent({ grain, rowKey, title, onClose, onChanged }: ProjectsPopupProps) {
139
+ const [view, setView] = useState<PopupView>({ kind: "list" });
 
 
140
  const [read, setRead] = useState<ProjectList | null>(null);
141
  const [error, setError] = useState("");
142
  const [tick, setTick] = useState(0);
 
156
  const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
157
 
158
  const actions = view.kind === "detail"
159
+ ? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "list" })}>All projects</button>
160
  : view.kind === "list" && canEdit
161
  ? <>
162
  <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
 
183
  : null}
184
  {view.kind === "detail"
185
  ? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
 
 
186
  onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
187
  : null}
188
  {view.kind === "new" && canEdit
 
204
 
205
  // ---- The Projects tab: every project, the same rows ----
206
 
207
+ export function ProjectsPage() {
208
  const [read, setRead] = useState<ProjectList | null>(null);
209
  const [error, setError] = useState("");
210
  const [tick, setTick] = useState(0);
 
241
  {open
242
  ? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
243
  <ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
 
244
  onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
245
  </CenteredPopup>
246
  : null}
web/src/apps/amazon-profit/ProjectsShared.tsx CHANGED
@@ -1,4 +1,4 @@
1
- import { createContext, useContext, useEffect, useId, useRef, useState, type KeyboardEvent, type ReactNode } from "react";
2
  import { createPortal } from "react-dom";
3
  import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
4
  import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
@@ -35,7 +35,6 @@ const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [
35
 
36
  export interface Project {
37
  id: string;
38
- type?: "project" | "pdp";
39
  name: string;
40
  status: Status;
41
  priority: Priority;
@@ -338,20 +337,6 @@ export function Owner({ name, compact = false }: { name: string | null; compact?
338
  // ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
339
 
340
  const ProjectFooterHost = createContext<HTMLElement | null>(null);
341
- export function projectMenuKeys(event: KeyboardEvent<HTMLElement>) {
342
- const items = Array.from(event.currentTarget.querySelectorAll<HTMLElement>('[role="menuitem"]:not(:disabled)'));
343
- const index = items.indexOf(document.activeElement as HTMLElement);
344
- const next = event.key === "ArrowDown" ? items[(index + 1) % items.length]
345
- : event.key === "ArrowUp" ? items[(index - 1 + items.length) % items.length]
346
- : event.key === "Home" ? items[0] : event.key === "End" ? items.at(-1) : undefined;
347
- if (next) { event.preventDefault(); next.focus(); }
348
- }
349
-
350
- const ProjectHeaderHost = createContext<HTMLElement | null>(null);
351
- export function ProjectHeader({ children }: { children: ReactNode }) {
352
- const host = useContext(ProjectHeaderHost);
353
- return host ? createPortal(children, host) : null;
354
- }
355
  export function ProjectFooter({ children }: { children: ReactNode }) {
356
  const host = useContext(ProjectFooterHost);
357
  return host ? createPortal(children, host) : <>{children}</>;
@@ -372,7 +357,6 @@ export interface CenteredPopupProps {
372
  export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
373
  const panelRef = useRef<HTMLDivElement>(null);
374
  const [footer, setFooter] = useState<HTMLDivElement | null>(null);
375
- const [header, setHeader] = useState<HTMLSpanElement | null>(null);
376
  const titleId = useId();
377
  useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
378
  return <BodyPortal>
@@ -388,14 +372,13 @@ export function CenteredPopup({ title, subtitle, actions, onClose, children, cla
388
  </div>
389
  <div className="lp-cpop-actions">
390
  {actions}
391
- {className?.includes("lp-proj-popup") ? <span className="pdp-project-header" ref={setHeader} /> : null}
392
  <button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
393
  </div>
394
  </header>
395
- <ProjectHeaderHost.Provider value={header}><ProjectFooterHost.Provider value={className?.includes("lp-proj-popup") ? footer : null}>
396
  <div className="lp-cpop-body">{children}</div>
397
  {className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
398
- </ProjectFooterHost.Provider></ProjectHeaderHost.Provider>
399
  </div>
400
  </div>
401
  </BodyPortal>;
 
1
+ import { createContext, useContext, useEffect, useId, useRef, useState, type ReactNode } from "react";
2
  import { createPortal } from "react-dom";
3
  import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
4
  import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
 
35
 
36
  export interface Project {
37
  id: string;
 
38
  name: string;
39
  status: Status;
40
  priority: Priority;
 
337
  // ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
338
 
339
  const ProjectFooterHost = createContext<HTMLElement | null>(null);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
340
  export function ProjectFooter({ children }: { children: ReactNode }) {
341
  const host = useContext(ProjectFooterHost);
342
  return host ? createPortal(children, host) : <>{children}</>;
 
357
  export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
358
  const panelRef = useRef<HTMLDivElement>(null);
359
  const [footer, setFooter] = useState<HTMLDivElement | null>(null);
 
360
  const titleId = useId();
361
  useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
362
  return <BodyPortal>
 
372
  </div>
373
  <div className="lp-cpop-actions">
374
  {actions}
 
375
  <button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
376
  </div>
377
  </header>
378
+ <ProjectFooterHost.Provider value={className?.includes("lp-proj-popup") ? footer : null}>
379
  <div className="lp-cpop-body">{children}</div>
380
  {className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
381
+ </ProjectFooterHost.Provider>
382
  </div>
383
  </div>
384
  </BodyPortal>;
web/src/apps/templateMarks.tsx CHANGED
@@ -85,10 +85,6 @@ export const TEMPLATE_MARKS: Readonly<Record<string, ReactElement>> = {
85
  alerts: <>
86
  <path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
87
  </>,
88
- reports: <>
89
- <path className={ACCENT} d="M14 3v5h5z" />
90
- <path className={INK} d="M14 3H5v18h14V8zM14 3v5h5M8 12h8M8 16h8" />
91
- </>,
92
  };
93
 
94
  const AMAZON_MARKS = new Set([
 
85
  alerts: <>
86
  <path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
87
  </>,
 
 
 
 
88
  };
89
 
90
  const AMAZON_MARKS = new Set([
web/src/automation/AutomationDetail.tsx CHANGED
@@ -22,7 +22,7 @@ import PanelToggleIcon from "../shell/PanelToggleIcon";
22
  // what this codebase treats as a defect rather than a summary.
23
  // ---------------------------------------------------------------------------
24
  import { useCallback, useEffect, useMemo, useRef, useState } from "react";
25
- import { useRefusalToast } from "../shell/ToastStack";
26
 
27
  import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
28
  import HarnessPanel from "./HarnessPanel";
@@ -335,6 +335,28 @@ function RetiredAutomationResult({ automation }: { automation: Automation }) {
335
  );
336
  }
337
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  export default function AutomationDetailView(props: Props) {
339
  return props.automation.retired
340
  ? <RetiredAutomationResult automation={props.automation} />
 
22
  // what this codebase treats as a defect rather than a summary.
23
  // ---------------------------------------------------------------------------
24
  import { useCallback, useEffect, useMemo, useRef, useState } from "react";
25
+ import { TOAST_EVENT } from "../apiContract";
26
 
27
  import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
28
  import HarnessPanel from "./HarnessPanel";
 
335
  );
336
  }
337
 
338
+ function useRefusalToast() {
339
+ const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
340
+ const sequence = useRef(0);
341
+ const delivered = useRef(0);
342
+ useEffect(() => {
343
+ if (!messages.length) return;
344
+ const through = messages[messages.length - 1].id;
345
+ for (const message of messages) {
346
+ if (message.id <= delivered.current) continue;
347
+ delivered.current = message.id;
348
+ const text = message.text;
349
+ window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
350
+ }
351
+ setMessages((current) => current.filter((message) => message.id > through));
352
+ }, [messages]);
353
+ return useCallback((text: string) => {
354
+ if (!text) return;
355
+ const message = { id: ++sequence.current, text };
356
+ setMessages((current) => [...current, message]);
357
+ }, []);
358
+ }
359
+
360
  export default function AutomationDetailView(props: Props) {
361
  return props.automation.retired
362
  ? <RetiredAutomationResult automation={props.automation} />
web/src/automation/AutomationSurface.tsx CHANGED
@@ -18,8 +18,7 @@ import type { KeyboardEvent as ReactKeyboardEvent } from "react";
18
  import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
19
 
20
  import type { AutomationOpenDetail } from "../apiContract";
21
- import { AUTOMATION_OPEN_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
22
- import { useRefusalToast } from "../shell/ToastStack";
23
  // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
24
  // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
25
  // C7 says consume the existing icon component and do not inline a new SVG path, and
@@ -212,6 +211,27 @@ if (typeof window !== "undefined") {
212
  * that a database can be created BY asking for an automation.
213
  */
214
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
215
  export default function AutomationSurface() {
216
  /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
217
  through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
 
18
  import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
19
 
20
  import type { AutomationOpenDetail } from "../apiContract";
21
+ import { AUTOMATION_OPEN_EVENT, TOAST_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
 
22
  // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
23
  // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
24
  // C7 says consume the existing icon component and do not inline a new SVG path, and
 
211
  * that a database can be created BY asking for an automation.
212
  */
213
 
214
+ function useRefusalToast() {
215
+ const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
216
+ const sequence = useRef(0);
217
+ const delivered = useRef(0);
218
+ useEffect(() => {
219
+ if (!messages.length) return;
220
+ const through = messages[messages.length - 1].id;
221
+ for (const { id, text } of messages) {
222
+ if (id <= delivered.current) continue;
223
+ delivered.current = id;
224
+ window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
225
+ }
226
+ setMessages((current) => current.filter((message) => message.id > through));
227
+ }, [messages]);
228
+ return (text: string) => {
229
+ if (!text) return;
230
+ const message = { id: ++sequence.current, text };
231
+ setMessages((current) => [...current, message]);
232
+ };
233
+ }
234
+
235
  export default function AutomationSurface() {
236
  /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
237
  through `data === null` and therefore never paints the skeleton. The lazy initialiser runs