fsanyoto commited on
Commit
948168f
·
verified ·
1 Parent(s): a60104f

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "f76135962",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "23787a8ce",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- f76135962
 
1
+ 23787a8ce
api/external_activity.py CHANGED
@@ -3,12 +3,20 @@
3
  from contextlib import contextmanager
4
  from contextvars import ContextVar
5
  import json
 
 
6
  from core import audit, external_access
7
 
8
 
9
  RETAINED_MAX = audit.MAX_EVENTS
10
  _CALL = ContextVar("external_activity_call", default=None)
11
 
 
 
 
 
 
 
12
 
13
  def _preview(session):
14
  return bool(getattr(session.runtime, "preview_only", False)
@@ -179,13 +187,7 @@ def _text(value):
179
  def _detail(session, database, outcome, code):
180
  claims = session.claims or {}
181
  key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
182
- label = ""
183
- if key_id:
184
- try:
185
- rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
186
- label = next((row.get("label") for row in rows if row.get("id") == key_id), "")
187
- except external_access.ExternalAccessError:
188
- pass
189
  return {
190
  "key_id": key_id, "key_label": _text(label), "database": _text(database),
191
  "outcome": outcome, "code": _text(code),
@@ -193,6 +195,30 @@ def _detail(session, database, outcome, code):
193
  }
194
 
195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
196
  def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
197
  method=None):
198
  if _preview(session) or _CALL.get() is not None:
 
3
  from contextlib import contextmanager
4
  from contextvars import ContextVar
5
  import json
6
+ import threading
7
+ import time
8
  from core import audit, external_access
9
 
10
 
11
  RETAINED_MAX = audit.MAX_EVENTS
12
  _CALL = ContextVar("external_activity_call", default=None)
13
 
14
+ #: (tenant, user, key id) -> (label, monotonic stamp). A rename shows in the audit within the TTL.
15
+ _LABELS = {}
16
+ _LABELS_LOCK = threading.Lock()
17
+ LABEL_TTL_SECONDS = 60.0
18
+ LABEL_CACHE_MAX = 512
19
+
20
 
21
  def _preview(session):
22
  return bool(getattr(session.runtime, "preview_only", False)
 
187
  def _detail(session, database, outcome, code):
188
  claims = session.claims or {}
189
  key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
190
+ label = _key_label(session, key_id) if key_id else ""
 
 
 
 
 
 
191
  return {
192
  "key_id": key_id, "key_label": _text(label), "database": _text(database),
193
  "outcome": outcome, "code": _text(code),
 
195
  }
196
 
197
 
198
+ def _key_label(session, key_id):
199
+ """The key's label, read from the store at most once per key per TTL in this process."""
200
+ owner = (str(session.tenant or "").lower(), str(session.uname or "").lower())
201
+ now = time.monotonic()
202
+ with _LABELS_LOCK:
203
+ hit = _LABELS.get((*owner, key_id))
204
+ if hit is not None and now - hit[1] < LABEL_TTL_SECONDS:
205
+ return hit[0]
206
+ try:
207
+ rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
208
+ except external_access.ExternalAccessError:
209
+ return ""
210
+ found = {str(row.get("id")): str(row.get("label") or "") for row in rows if row.get("id")}
211
+ with _LABELS_LOCK:
212
+ if len(_LABELS) + len(found) + 1 > LABEL_CACHE_MAX:
213
+ _LABELS.clear()
214
+ for row_id, row_label in found.items():
215
+ _LABELS[(*owner, row_id)] = (row_label, now)
216
+ # A key with no row (an OAuth credential, a revoked key) caches its miss for the TTL too.
217
+ if key_id not in found:
218
+ _LABELS[(*owner, key_id)] = ("", now)
219
+ return found.get(key_id, "")
220
+
221
+
222
  def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
223
  method=None):
224
  if _preview(session) or _CALL.get() is not None:
api/main.py CHANGED
@@ -1328,8 +1328,13 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
1328
  odoo._tlocal.tenant_ctx = None
1329
  from connector_onboarding import selected_mirror_entities
1330
  selected_entities = selected_mirror_entities(rt)
 
 
 
 
1331
  report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
1332
- log=lambda *a, **k: None)
 
1333
  finally:
1334
  odoo._tlocal.client = previous_client
1335
  odoo._tlocal.tenant_ctx = previous_context
@@ -2526,8 +2531,10 @@ def _capture_resync_source_counts(tenant):
2526
  rest = _rel_counts.enabled_buckets(rt) - set(during)
2527
  if rest:
2528
  counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
 
2529
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2530
- f"database(s) at mirror-pull time, {len(during)} beside their own copy", flush=True)
 
2531
  return counts or None
2532
  except Exception as e: # noqa: BLE001
2533
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
@@ -2803,7 +2810,8 @@ def _store_resync_loop():
2803
  # D-9131: count the source now, before the vendor pulls, so reconciliation compares the
2804
  # mirror with the source as of the same moment. A pass without a good pull counts nothing.
2805
  _source_counts = (_capture_resync_source_counts(_cad["tenant"])
2806
- if _err is None and sync_report and not _manual else None)
 
2807
  # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
2808
  # refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
2809
  # is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
 
1328
  odoo._tlocal.tenant_ctx = None
1329
  from connector_onboarding import selected_mirror_entities
1330
  selected_entities = selected_mirror_entities(rt)
1331
+ # Sync now counts each entity's source beside its copy, as the full pass does.
1332
+ # Looked up by name: extracted test fixtures of this function omit the counter.
1333
+ _counter = globals().get("_entity_source_counter")
1334
+ _after_entity = _counter(datastore) if _counter else None
1335
  report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
1336
+ log=lambda *a, **k: None,
1337
+ **({"after_entity": _after_entity} if _after_entity else {}))
1338
  finally:
1339
  odoo._tlocal.client = previous_client
1340
  odoo._tlocal.tenant_ctx = previous_context
 
2531
  rest = _rel_counts.enabled_buckets(rt) - set(during)
2532
  if rest:
2533
  counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
2534
+ per_entity = " ".join(f"{bucket}={count:,}" for bucket, (count, _mode) in sorted(during.items()))
2535
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2536
+ f"database(s) at mirror-pull time, {len(during)} beside their own copy"
2537
+ + (f"; source counts per entity: {per_entity}" if per_entity else ""), flush=True)
2538
  return counts or None
2539
  except Exception as e: # noqa: BLE001
2540
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
 
2810
  # D-9131: count the source now, before the vendor pulls, so reconciliation compares the
2811
  # mirror with the source as of the same moment. A pass without a good pull counts nothing.
2812
  _source_counts = (_capture_resync_source_counts(_cad["tenant"])
2813
+ if _err is None and ((sync_report and not _manual)
2814
+ or (_manual and _odoo_requested)) else None)
2815
  # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
2816
  # refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
2817
  # is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
api/registry_source.py CHANGED
@@ -185,12 +185,36 @@ def generation(session, table_key):
185
  if not callable(revision):
186
  return None # Ordinary offline readers retain their original snapshot semantics.
187
  tokens = {name: revision(name) for name in names}
188
- return copy.deepcopy((session.user, tokens))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
189
 
190
 
191
  def require_generation(session, table_key, before):
192
- if before is not None and generation(session, table_key) != before:
193
- raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
 
 
 
 
 
 
194
 
195
 
196
  class BatchStore:
 
185
  if not callable(revision):
186
  return None # Ordinary offline readers retain their original snapshot semantics.
187
  tokens = {name: revision(name) for name in names}
188
+ user = session.user if isinstance(session.user, dict) else {}
189
+ # A sign in or activity stamp never changes what the reader may see, so it never forces a rebuild.
190
+ policy = {key: value for key, value in user.items() if key not in USER_NOISE}
191
+ return copy.deepcopy((policy, tokens))
192
+
193
+
194
+ #: User record keys that change on sign in or use and never change what the reader may see.
195
+ USER_NOISE = frozenset(('last_login', 'last_active', 'updated_at', 'epoch', 'avatar', 'whatsapp'))
196
+
197
+
198
+ def moved_keys(before, after):
199
+ """Sorted names of the generation parts that differ: store keys, plus `user` for the policy."""
200
+ if not (isinstance(before, tuple) and isinstance(after, tuple)
201
+ and len(before) == len(after) == 2):
202
+ return ['unknown']
203
+ moved = ['user'] if before[0] != after[0] else []
204
+ tokens_before, tokens_after = before[1] or {}, after[1] or {}
205
+ return moved + sorted(str(name) for name in set(tokens_before) | set(tokens_after)
206
+ if tokens_before.get(name) != tokens_after.get(name))
207
 
208
 
209
  def require_generation(session, table_key, before):
210
+ if before is None:
211
+ return
212
+ after = generation(session, table_key)
213
+ if after != before:
214
+ exc = err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
215
+ # The rebuild log names the moved keys. They never reach the client.
216
+ exc.moved = moved_keys(before, after)
217
+ raise exc
218
 
219
 
220
  class BatchStore:
api/request_timing.py CHANGED
@@ -70,25 +70,46 @@ class DeadlineCapacityExceeded(RuntimeError):
70
 
71
 
72
  class _StatementAlarm:
 
 
 
73
  def __init__(self, owner, connection, acc):
74
  self.owner, self.connection, self.acc = owner, connection, acc
75
  self.lock = threading.Lock()
76
  self.active, self.expired = True, False
 
77
 
78
  def fire(self):
79
  with self.lock:
80
- if self.active and (self.acc.cancelled or time.monotonic() >= self.acc.deadline):
81
- self.expired = True
82
- self.active = False
 
 
 
 
 
 
 
 
 
83
  self.connection.interrupt()
 
 
 
84
 
85
  def close(self):
86
  # Serialize disarm with interrupt before the cursor can execute another statement.
87
  with self.lock:
88
  self.active = False
89
- expired = self.expired
90
  with self.owner.lock:
91
  self.owner.active.discard(self)
 
 
 
 
 
92
  return expired
93
 
94
 
 
70
 
71
 
72
  class _StatementAlarm:
73
+ #: Repeat interrupt() this often until disarm: DuckDB can lose a single one on a parameterized execute.
74
+ REFIRE = .05
75
+
76
  def __init__(self, owner, connection, acc):
77
  self.owner, self.connection, self.acc = owner, connection, acc
78
  self.lock = threading.Lock()
79
  self.active, self.expired = True, False
80
+ self.fired_at, self.last, self.sent = None, 0.0, 0
81
 
82
  def fire(self):
83
  with self.lock:
84
+ if not self.active:
85
+ return
86
+ now = time.monotonic()
87
+ if not self.expired:
88
+ if not (self.acc.cancelled or now >= self.acc.deadline):
89
+ return
90
+ self.expired, self.fired_at = True, now
91
+ elif now - self.last < self.REFIRE:
92
+ return
93
+ self.last = now
94
+ self.sent += 1
95
+ try:
96
  self.connection.interrupt()
97
+ except Exception:
98
+ self.active = False
99
+ raise
100
 
101
  def close(self):
102
  # Serialize disarm with interrupt before the cursor can execute another statement.
103
  with self.lock:
104
  self.active = False
105
+ expired, fired_at, sent = self.expired, self.fired_at, self.sent
106
  with self.owner.lock:
107
  self.owner.active.discard(self)
108
+ if expired and fired_at is not None:
109
+ now = time.monotonic()
110
+ LOG.warning("[deadline-interrupt] statement stopped %.0f ms after the first interrupt "
111
+ "(%d sent), %.0f ms past the request deadline", (now - fired_at) * 1000, sent,
112
+ (now - self.acc.deadline) * 1000)
113
  return expired
114
 
115
 
api/routes_opportunity_finder.py CHANGED
@@ -85,6 +85,12 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
85
  return session
86
 
87
 
 
 
 
 
 
 
88
  def _with_spark(of, st, items):
89
  """Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
90
  weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
@@ -102,10 +108,12 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
102
  of = _module()
103
  record = _app(session, app)
104
  st = session.runtime
105
- listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st)
 
106
  listed["items"] = _call(_with_spark, of, st, listed["items"])
107
  return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
108
- "weeks": listed["weeks"], "departments": listed["departments"], "counts": listed["counts"],
 
109
  "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
110
  "problems": listed["problems"], "statuses": list(of.STATUSES),
111
  "settings": _call(of.settings, _tenant(session), st=st)}
@@ -114,7 +122,8 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
114
  @router.get("/items/{item_id}")
115
  def item_detail(item_id: str, session: Session = Depends(_app_reader)):
116
  of = _module()
117
- return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
 
118
 
119
 
120
  @router.patch("/items/{item_id}")
@@ -123,14 +132,14 @@ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(N
123
  session: Session = Depends(_app_writer)):
124
  of = _module()
125
  return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
126
- due=due, st=session.runtime)
127
 
128
 
129
  @router.get("/terms/{item_id}")
130
  def term_trend(item_id: str, session: Session = Depends(_app_reader)):
131
  """A term's weekly rank trend and its newest top 3 clicked ASINs."""
132
  of = _module()
133
- return _call(of.term_detail, _tenant(session), item_id, st=session.runtime)
134
 
135
 
136
  @router.get("/settings")
@@ -156,4 +165,5 @@ def write_settings(body: dict = Body(...), session: Session = Depends(_app_write
156
  def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
157
  session: Session = Depends(_app_reader)):
158
  of = _module()
159
- return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime)
 
 
85
  return session
86
 
87
 
88
+ def _mirror_opener(session):
89
+ """The typed `ut_amzn_*` door's cursor factory: tenant asserted, serving mirror version. The module closes it."""
90
+ import routes_odoo_tables
91
+ return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
92
+
93
+
94
  def _with_spark(of, st, items):
95
  """Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
96
  weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
 
108
  of = _module()
109
  record = _app(session, app)
110
  st = session.runtime
111
+ listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
112
+ opener=_mirror_opener(session))
113
  listed["items"] = _call(_with_spark, of, st, listed["items"])
114
  return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
115
+ "weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
116
+ "departments": listed["departments"], "counts": listed["counts"],
117
  "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
118
  "problems": listed["problems"], "statuses": list(of.STATUSES),
119
  "settings": _call(of.settings, _tenant(session), st=st)}
 
122
  @router.get("/items/{item_id}")
123
  def item_detail(item_id: str, session: Session = Depends(_app_reader)):
124
  of = _module()
125
+ return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
126
+ opener=_mirror_opener(session))
127
 
128
 
129
  @router.patch("/items/{item_id}")
 
132
  session: Session = Depends(_app_writer)):
133
  of = _module()
134
  return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
135
+ due=due, st=session.runtime, opener=_mirror_opener(session))
136
 
137
 
138
  @router.get("/terms/{item_id}")
139
  def term_trend(item_id: str, session: Session = Depends(_app_reader)):
140
  """A term's weekly rank trend and its newest top 3 clicked ASINs."""
141
  of = _module()
142
+ return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session))
143
 
144
 
145
  @router.get("/settings")
 
165
  def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
166
  session: Session = Depends(_app_reader)):
167
  of = _module()
168
+ return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
169
+ opener=_mirror_opener(session))
api/routes_sheet.py CHANGED
@@ -266,15 +266,38 @@ def _row_scope_applies(session, database):
266
 
267
  def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
268
  """Rebuild when the source changed during the build. Only the last change refuses."""
 
 
 
 
 
 
 
 
 
 
269
  for attempt in range(_MIRROR_BUILD_TRIES):
 
 
270
  try:
271
  return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
272
  except HTTPException as exc:
273
  detail = exc.detail if isinstance(exc.detail, dict) else {}
274
- if ((detail.get("error") or {}).get("code") != "drill_population_changed"
275
- or attempt + 1 >= _MIRROR_BUILD_TRIES):
 
 
 
 
 
 
 
 
 
 
276
  raise
277
- print(f"[sheet] mirror {table_key} changed during its build, rebuild {attempt + 2}",
 
278
  flush=True)
279
 
280
 
@@ -4168,9 +4191,19 @@ def _refresh_desk_run(session, row):
4168
  return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
4169
  with _sheet_request_scope(session):
4170
  if args.get("kind") == "retry":
4171
- return _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
4172
- only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
4173
- return _refresh_sheet_scoped(workbook, session, only=only)
 
 
 
 
 
 
 
 
 
 
4174
 
4175
 
4176
  # ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
@@ -6577,7 +6610,9 @@ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
6577
  def view_rows(workbook_id: str, body: object = Body(default=None),
6578
  session: Session = Depends(require_session)):
6579
  """W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
 
6580
  from fastapi.responses import JSONResponse
 
6581
  from core import perm_scope, sheets, user_tables
6582
  from modules import sheet as sheet_mod
6583
 
@@ -6603,74 +6638,74 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
6603
 
6604
  # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
6605
  _refuse_unapplied_audience(session, database)
6606
- wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6607
- if perm_scope.row_scope_applies(session.user, database) else None)
6608
- wall_nodes, wall_conj = _fe_parts(wall_tree)
6609
- scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
6610
- scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
6611
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
6612
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
6613
  if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
6614
  return JSONResponse(status_code=400, content={
6615
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6616
  today = _sheet_today(session)
 
 
6617
  try:
6618
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
6619
- window = _ending_window(session, binding, book=book, today=today)
6620
- needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
6621
- | sheet_mod.filter_columns_named(wall_nodes))
6622
- related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6623
- cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
6624
- cols = dict(cols)
6625
- if spec.get("scope_applied"):
6626
- wall_nodes, wall_conj = [], "and"
6627
- related = related_bags(cur, spec, cols, window=window)
6628
- for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
6629
- if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
6630
- continue
6631
- if key in hidden or not (cols.get(key) or {}).get("sql"):
6632
- return JSONResponse(status_code=400, content={
6633
- "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6634
- # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
6635
- wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
6636
- if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
6637
- raise ValueError("unapplied row wall")
6638
- count_sql, params, limits = sheet_mod.build_aggregate_query(
6639
- spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6640
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
6641
- related=related, summary_default=False)
6642
- if limits:
6643
- raise ValueError("unapplied view filter")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6644
  except sheet_mod.SheetRefusal as exc:
6645
  return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
6646
  except ValueError:
6647
  raise err(409, "scope_not_applicable",
6648
  "This view uses a restriction that cannot be applied to this database's live connection.")
6649
-
6650
- # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
6651
- total = (cur.execute(count_sql, params).fetchone() or [0])[0]
6652
- # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
6653
- # A placed Grid over one of our own tables writes a record back, and the write door is Data's
6654
- # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
6655
- # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
6656
- # refreshes. So the id rides beside the values under the key `pid` while `fields` still names
6657
- # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
6658
- # uses, so the identity here and the identity the drill lists can never be two different things.
6659
- id_expr = str(spec.get("id") or "").strip()
6660
- projection = ", ".join([*(cols[key]["sql"] for key in requested),
6661
- *([id_expr] if id_expr else [])])
6662
- rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
6663
- if binding["sort"]:
6664
- # W62 C2 / R30: expressions come only from resolved columns; directions are validated.
6665
- order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
6666
- for item in binding["sort"])
6667
- rows_sql += " ORDER BY " + order
6668
- rows_sql += f" LIMIT {binding['limit']}"
6669
- keys = [*requested, *(["pid"] if id_expr else [])]
6670
- rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
6671
  # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
6672
  # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
6673
  editable = user_tables.records_mutable(database, st=session.runtime)
 
 
 
 
6674
  return {"fields": [{"key": key, "label": visible[key].get("label") or key,
6675
  "kind": visible[key].get("type") or visible[key].get("kind") or "text"}
6676
  for key in requested],
@@ -6678,6 +6713,94 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
6678
  "viewType": binding["viewType"], "editable": bool(editable)}
6679
 
6680
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6681
  _DRILL_MIRRORS = OrderedDict()
6682
  # The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
6683
  _DRILL_MIRROR_LOCK = threading.RLock()
 
266
 
267
  def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
268
  """Rebuild when the source changed during the build. Only the last change refuses."""
269
+ from copy import deepcopy
270
+ from time import perf_counter
271
+ from core.request_timing import remaining_budget
272
+ slot = ("mirror_refused", table_key, json.dumps(metric_window, sort_keys=True, default=str))
273
+ memo = getattr(session, "_sheet_read_memo", None)
274
+ refused = memo if isinstance(memo, dict) else {}
275
+ if slot in refused:
276
+ # ⛔ One refused build per database and window per request: every further cell would rebuild.
277
+ status, detail = refused[slot]
278
+ raise HTTPException(status_code=status, detail=deepcopy(detail))
279
  for attempt in range(_MIRROR_BUILD_TRIES):
280
+ started = perf_counter()
281
+ session._sheet_mirror_builds = getattr(session, "_sheet_mirror_builds", 0) + 1
282
  try:
283
  return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
284
  except HTTPException as exc:
285
  detail = exc.detail if isinstance(exc.detail, dict) else {}
286
+ if (detail.get("error") or {}).get("code") != "drill_population_changed":
287
+ raise
288
+ spent = perf_counter() - started
289
+ moved = ",".join(getattr(exc, "moved", None) or ["unknown"])
290
+ left = remaining_budget()
291
+ # A rebuild that cannot finish inside the budget only turns a held cell into a timeout.
292
+ short = left is not None and left < spent * 1.2
293
+ if attempt + 1 >= _MIRROR_BUILD_TRIES or short:
294
+ print(f"[sheet] mirror {table_key} changed during its build, refused "
295
+ f"moved={moved} build_ms={int(spent * 1000)}"
296
+ + (f" budget_left_ms={int(left * 1000)}" if short else ""), flush=True)
297
+ refused[slot] = (exc.status_code, deepcopy(exc.detail))
298
  raise
299
+ print(f"[sheet] mirror {table_key} changed during its build, rebuild "
300
+ f"{attempt + 2}/{_MIRROR_BUILD_TRIES} moved={moved} build_ms={int(spent * 1000)}",
301
  flush=True)
302
 
303
 
 
4191
  return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
4192
  with _sheet_request_scope(session):
4193
  if args.get("kind") == "retry":
4194
+ result = _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
4195
+ else:
4196
+ only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
4197
+ result = _refresh_sheet_scoped(workbook, session, only=only)
4198
+ if isinstance(result, dict) and isinstance(result.get("workbook"), dict):
4199
+ # Placed views read their rows after the run ends. Their mirror builds here, in a fresh
4200
+ # scope, so the view-rows call reuses it. The run's own result never depends on it.
4201
+ try:
4202
+ with _sheet_request_scope(session):
4203
+ _prewarm_view_mirrors(session, result["workbook"])
4204
+ except Exception as exc: # noqa: BLE001
4205
+ print(f"[sheet] view prewarm wb={workbook} failed {type(exc).__name__}", flush=True)
4206
+ return result
4207
 
4208
 
4209
  # ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
 
6610
  def view_rows(workbook_id: str, body: object = Body(default=None),
6611
  session: Session = Depends(require_session)):
6612
  """W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
6613
+ from time import perf_counter
6614
  from fastapi.responses import JSONResponse
6615
+ import odoo_relational as rel
6616
  from core import perm_scope, sheets, user_tables
6617
  from modules import sheet as sheet_mod
6618
 
 
6638
 
6639
  # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
6640
  _refuse_unapplied_audience(session, database)
6641
+ wall_nodes, wall_conj, scope_nodes, scope_conj = _view_scope(session, database, binding)
 
 
 
 
6642
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
6643
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
6644
  if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
6645
  return JSONResponse(status_code=400, content={
6646
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6647
  today = _sheet_today(session)
6648
+ started = perf_counter()
6649
+ builds = getattr(session, "_sheet_mirror_builds", 0)
6650
  try:
6651
+ scope_nodes, window, related_bags, door = _view_mirror_door(
6652
+ session, book, binding, database, scope_nodes, wall_nodes, today)
6653
+ with door as (made, _token):
6654
+ cur, spec, cols = made
6655
+ cols = dict(cols)
6656
+ if spec.get("scope_applied"):
6657
+ wall_nodes, wall_conj = [], "and"
6658
+ related = related_bags(cur, spec, cols, window=window)
6659
+ for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
6660
+ if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
6661
+ continue
6662
+ if key in hidden or not (cols.get(key) or {}).get("sql"):
6663
+ return JSONResponse(status_code=400, content={
6664
+ "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
6665
+ # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
6666
+ wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
6667
+ if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
6668
+ raise ValueError("unapplied row wall")
6669
+ count_sql, params, limits = sheet_mod.build_aggregate_query(
6670
+ spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6671
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
6672
+ related=related, summary_default=False)
6673
+ if limits:
6674
+ raise ValueError("unapplied view filter")
6675
+
6676
+ # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
6677
+ total = (cur.execute(count_sql, params).fetchone() or [0])[0]
6678
+ # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
6679
+ # A placed Grid over one of our own tables writes a record back, and the write door is Data's
6680
+ # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
6681
+ # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
6682
+ # refreshes. So the id rides beside the values under the key `pid` while `fields` still names
6683
+ # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
6684
+ # uses, so the identity here and the identity the drill lists can never be two different things.
6685
+ id_expr = str(spec.get("id") or "").strip()
6686
+ projection = ", ".join([*(cols[key]["sql"] for key in requested),
6687
+ *([id_expr] if id_expr else [])])
6688
+ rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
6689
+ if binding["sort"]:
6690
+ # W62 C2 / R30: expressions come only from resolved columns; directions are validated.
6691
+ order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
6692
+ for item in binding["sort"])
6693
+ rows_sql += " ORDER BY " + order
6694
+ rows_sql += f" LIMIT {binding['limit']}"
6695
+ keys = [*requested, *(["pid"] if id_expr else [])]
6696
+ rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
6697
  except sheet_mod.SheetRefusal as exc:
6698
  return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
6699
  except ValueError:
6700
  raise err(409, "scope_not_applicable",
6701
  "This view uses a restriction that cannot be applied to this database's live connection.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6702
  # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
6703
  # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
6704
  editable = user_tables.records_mutable(database, st=session.runtime)
6705
+ reused = (int(getattr(session, "_sheet_mirror_builds", 0) == builds)
6706
+ if database in rel._REGISTRY_GRID_FIELDS else "na")
6707
+ print(f"[sheet] view-rows wb={workbook_id} db={database} ms={int((perf_counter() - started) * 1000)} "
6708
+ f"reused={reused} rows={len(rows)} total={int(total or 0)}", flush=True)
6709
  return {"fields": [{"key": key, "label": visible[key].get("label") or key,
6710
  "kind": visible[key].get("type") or visible[key].get("kind") or "text"}
6711
  for key in requested],
 
6713
  "viewType": binding["viewType"], "editable": bool(editable)}
6714
 
6715
 
6716
+ def _view_scope(session, database, binding):
6717
+ """`(wall_nodes, wall_conj, scope_nodes, scope_conj)` of one placed view for this reader."""
6718
+ import core.perm_scope as perm_scope
6719
+ _refuse_unapplied_audience(session, database)
6720
+ wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6721
+ if perm_scope.row_scope_applies(session.user, database) else None)
6722
+ wall_nodes, wall_conj = _fe_parts(wall_tree)
6723
+ scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
6724
+ return wall_nodes, wall_conj, _rewrite_related_filters(session, database, scope_nodes), scope_conj
6725
+
6726
+
6727
+ def _view_mirror_door(session, book, binding, database, scope_nodes, wall_nodes, today):
6728
+ """`(scope_nodes, window, related_bags, door)`; `door` yields `((cur, spec, cols), token)`.
6729
+
6730
+ A compiled registry grid reads through the reader-keyed drill mirror cache, so a repeat view
6731
+ and a view the desk prewarmed reuse one build. Every other database builds per request.
6732
+ """
6733
+ import odoo_relational as rel
6734
+ from modules import sheet as sheet_mod
6735
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
6736
+ window = _ending_window(session, binding, book=book, today=today)
6737
+ needed = (set(binding["fields"]) | {item["field"] for item in binding["sort"]}
6738
+ | sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes))
6739
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6740
+ if database in rel._REGISTRY_GRID_FIELDS:
6741
+ databases = {database}
6742
+ databases.update(_binding_related_entry(session, database, via)["target"]
6743
+ for via in sheet_mod.related_vias(scope_nodes))
6744
+ return scope_nodes, window, related_bags, _drill_mirror(session, database, window, needed, databases)
6745
+ made = _mirror(session)(database, metric_window=window, needed=needed)
6746
+ return scope_nodes, window, related_bags, contextlib.nullcontext((made, None))
6747
+
6748
+
6749
+ #: A desk prewarm of placed views starts only with this much of the run's budget left.
6750
+ VIEW_PREWARM_MIN_SECONDS = 90.0
6751
+ #: And it stops this long before the run's own deadline.
6752
+ VIEW_PREWARM_MARGIN_SECONDS = 20.0
6753
+
6754
+
6755
+ def _prewarm_view_mirrors(session, book):
6756
+ """Build the drill mirror each registry-grid placed view of `book` reads, as this reader.
6757
+
6758
+ The client reads view rows after a Refresh within a 45 s request budget. A customer grid
6759
+ build costs more than that, so the desk builds it first and the view-rows call reuses it.
6760
+ A failed prewarm leaves the view to build on its own request.
6761
+ """
6762
+ import time
6763
+ from copy import deepcopy
6764
+ from time import perf_counter
6765
+ import odoo_relational as rel
6766
+ from core import request_timing, sheets
6767
+ left = request_timing.remaining_budget()
6768
+ warmed = 0
6769
+ for tab in book.get("sheets") or []:
6770
+ for addr, cell in (tab.get("cells") or {}).items():
6771
+ if not (isinstance(cell, dict) and cell.get("kind") == "bound" and _is_view_binding(cell)
6772
+ and str((cell.get("binding") or {}).get("database") or "") in rel._REGISTRY_GRID_FIELDS):
6773
+ continue
6774
+ if left is not None and left < VIEW_PREWARM_MIN_SECONDS:
6775
+ print(f"[sheet] view prewarm wb={book.get('id')} skipped budget_left_ms={int(left * 1000)}",
6776
+ flush=True)
6777
+ return warmed
6778
+ started = perf_counter()
6779
+ builds = getattr(session, "_sheet_mirror_builds", 0)
6780
+ try:
6781
+ binding = sheets.clean_view_binding(deepcopy(cell["binding"]))
6782
+ database = _require_database(session, binding["database"])
6783
+ _refuse_unapplied_audience(session, database)
6784
+ wall_nodes, _wall_conj, scope_nodes, _conj = _view_scope(session, database, binding)
6785
+ deadline = (None if left is None else
6786
+ time.monotonic() + left - VIEW_PREWARM_MARGIN_SECONDS)
6787
+ with (request_timing.budget_scope(deadline) if deadline is not None
6788
+ else contextlib.nullcontext()):
6789
+ _nodes, window, _bags, door = _view_mirror_door(
6790
+ session, book, binding, database, scope_nodes, wall_nodes, _sheet_today(session))
6791
+ # A second view of the same reader and window reuses the first build.
6792
+ with door as (_made, token):
6793
+ outcome = "cached" if token else "uncached"
6794
+ warmed += 1 if token else 0
6795
+ except Exception as exc: # noqa: BLE001
6796
+ outcome = "failed " + type(exc).__name__
6797
+ print(f"[sheet] view prewarm wb={book.get('id')} cell={tab.get('id')}!{addr} {outcome} "
6798
+ f"built={int(getattr(session, '_sheet_mirror_builds', 0) != builds)} "
6799
+ f"ms={int((perf_counter() - started) * 1000)}", flush=True)
6800
+ left = request_timing.remaining_budget()
6801
+ return warmed
6802
+
6803
+
6804
  _DRILL_MIRRORS = OrderedDict()
6805
  # The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
6806
  _DRILL_MIRROR_LOCK = threading.RLock()
platform/evals/royal_questions.yml CHANGED
@@ -91,7 +91,7 @@ questions:
91
  - id: l03
92
  kind: list
93
  ask: Past due receivables by customer
94
- oracle: {topic: receivables, measures: [ar_overdue], group: [partner], filters: [{field: due_date, op: before, value: today}, {field: payment_state, op: ne, value: paid}]}
95
  - id: l04
96
  kind: list
97
  ask: Payables balance by vendor
 
91
  - id: l03
92
  kind: list
93
  ask: Past due receivables by customer
94
+ oracle: {topic: gl_lines, measures: [ar_overdue], group: [partner], filters: [], scope: receivable_lines}
95
  - id: l04
96
  kind: list
97
  ask: Payables balance by vendor
platform/harness/amazon_store.py CHANGED
@@ -255,6 +255,8 @@ FIRST_LOAD_ROWS_PER_PASS = 3_000_000
255
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
256
  #: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
257
  RESYNC_PREFIX = "resync_"
 
 
258
  FIRST_LOAD_END_KEY = "sync_first_load_end"
259
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
260
  FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
@@ -3288,14 +3290,19 @@ def _queue_resync(con, mirror, types, where):
3288
  return rows, windows
3289
 
3290
 
3291
- def queued_resync(con, table):
3292
- """The oldest queued resync window of `table` as `(start, end)` datetimes, or None."""
3293
  saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
3294
  try:
3295
- start, end = json.loads(saved["detail"])["windows"][0]
3296
- return _utc(start), _utc(end)
3297
  except (TypeError, ValueError, KeyError, IndexError, AttributeError):
3298
- return None
 
 
 
 
 
 
3299
 
3300
 
3301
  def resync_landed(con, table, window):
@@ -5318,6 +5325,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5318
  if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
5319
  and not _interval_reason(table, ticks.get(table), end)):
5320
  ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
 
 
 
 
 
 
5321
  plan = sync_plan(ticks, want, now=end)
5322
  want = plan["tables"]
5323
  for table, reason in plan["deferred"].items():
@@ -5538,6 +5551,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5538
  sibling_written, sibling_cleared = 0, False
5539
  written, cols, cleared = 0, [], False
5540
  landed_window = False
 
5541
  pricing_asins = []
5542
  operations = {}
5543
  if first_load:
@@ -5625,12 +5639,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5625
  margin_windows = _month_windows(plan["start"], plan["end"])
5626
  else:
5627
  margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
5628
- resync = None if first_load or spec.get("snapshot") else queued_resync(con, table)
5629
- if resync is not None and read_watermark(con, table):
5630
- # A queued window takes this tick; the watermark stays, so no new window is skipped.
5631
- margin_windows = [resync]
 
5632
  plan["end"] = read_watermark(con, table)
5633
- plan["cause"] = f"resync of the stored window starting {_iso(resync[0])}"
 
5634
  log(f"{table}: {plan['cause']}")
5635
  elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
5636
  # One due window per table tick respects the vendor floor. The watermark
@@ -5655,8 +5671,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5655
  {"state": "access", "operation": operation}, rt, report, log)
5656
  count = _sync_margin_window(con, table, fields, w0, w1,
5657
  max_wait=max_wait, log=log)
5658
- if resync is not None and (w0, w1) == resync:
5659
- resync_landed(con, table, resync)
 
 
5660
  probe.finish(count)
5661
  spend_rows(count)
5662
  written += count
@@ -5670,6 +5688,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5670
  if first_load and w0 is not None:
5671
  record_backfill_floor(con, table, w0)
5672
  floors[table] = w0
 
 
 
5673
  elif table != "settlements":
5674
  period_spec = cx.REPORT_TYPES[table]
5675
  windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
@@ -5867,6 +5888,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5867
  "rows": total_rows, "columns": column_count, "provenance": prov,
5868
  "report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
5869
  "operations": operations}
 
 
5870
  _record_state(con, key, "done", total_rows)
5871
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
5872
  if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
@@ -7984,6 +8007,33 @@ def group_a_selftest(log=print, check=None):
7984
  and queued_resync(con, "long_term_storage_fees") is None
7985
  and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
7986
  "GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7987
  finally:
7988
  con.close()
7989
 
 
255
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
256
  #: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
257
  RESYNC_PREFIX = "resync_"
258
+ #: Queued resync windows one pass downloads at most: Amazon's createReport burst, less the pass's spend.
259
+ RESYNC_REPORTS_PER_PASS = FIRST_LOAD_REPORTS_PER_PASS
260
  FIRST_LOAD_END_KEY = "sync_first_load_end"
261
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
262
  FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
 
3290
  return rows, windows
3291
 
3292
 
3293
+ def queued_resyncs(con, table):
3294
+ """Every queued resync window of `table`, oldest first, as `(start, end)` datetimes."""
3295
  saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
3296
  try:
3297
+ return [(_utc(start), _utc(end)) for start, end in json.loads(saved["detail"])["windows"]]
 
3298
  except (TypeError, ValueError, KeyError, IndexError, AttributeError):
3299
+ return []
3300
+
3301
+
3302
+ def queued_resync(con, table):
3303
+ """The oldest queued resync window of `table` as `(start, end)` datetimes, or None."""
3304
+ windows = queued_resyncs(con, table)
3305
+ return windows[0] if windows else None
3306
 
3307
 
3308
  def resync_landed(con, table, window):
 
5325
  if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
5326
  and not _interval_reason(table, ticks.get(table), end)):
5327
  ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
5328
+ for table in want:
5329
+ tick = ticks.get(table)
5330
+ # A queued resync is a repair: only Amazon's own floor and the stagger hold it, never the daily floor.
5331
+ if (tick and tick["state"] == "done" and queued_resync(con, table) is not None
5332
+ and read_watermark(con, table) and not _interval_reason(table, tick, end)):
5333
+ ticks[table] = {**tick, "state": "resync"}
5334
  plan = sync_plan(ticks, want, now=end)
5335
  want = plan["tables"]
5336
  for table, reason in plan["deferred"].items():
 
5551
  sibling_written, sibling_cleared = 0, False
5552
  written, cols, cleared = 0, [], False
5553
  landed_window = False
5554
+ resync, resync_count, resync_left = [], 0, None
5555
  pricing_asins = []
5556
  operations = {}
5557
  if first_load:
 
5639
  margin_windows = _month_windows(plan["start"], plan["end"])
5640
  else:
5641
  margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
5642
+ resync = [] if first_load or spec.get("snapshot") else queued_resyncs(con, table)
5643
+ if resync and read_watermark(con, table):
5644
+ # Queued windows take this tick, several within the quota left; the watermark stays.
5645
+ budget = report_budget if isinstance(report_budget, int) else RESYNC_REPORTS_PER_PASS
5646
+ margin_windows = resync[:max(1, budget - documents)]
5647
  plan["end"] = read_watermark(con, table)
5648
+ plan["cause"] = (f"resync of {len(margin_windows)} stored windows "
5649
+ f"from {_iso(margin_windows[0][0])}")
5650
  log(f"{table}: {plan['cause']}")
5651
  elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
5652
  # One due window per table tick respects the vendor floor. The watermark
 
5671
  {"state": "access", "operation": operation}, rt, report, log)
5672
  count = _sync_margin_window(con, table, fields, w0, w1,
5673
  max_wait=max_wait, log=log)
5674
+ if (w0, w1) in resync:
5675
+ # Per window, so a failure later in this tick never downloads a landed window again.
5676
+ resync_landed(con, table, (w0, w1))
5677
+ resync_count += 1
5678
  probe.finish(count)
5679
  spend_rows(count)
5680
  written += count
 
5688
  if first_load and w0 is not None:
5689
  record_backfill_floor(con, table, w0)
5690
  floors[table] = w0
5691
+ if resync_count:
5692
+ resync_left = len(queued_resyncs(con, table))
5693
+ log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left")
5694
  elif table != "settlements":
5695
  period_spec = cx.REPORT_TYPES[table]
5696
  windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
 
5888
  "rows": total_rows, "columns": column_count, "provenance": prov,
5889
  "report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
5890
  "operations": operations}
5891
+ if resync_count:
5892
+ report["tables"][table]["resync"] = {"landed": resync_count, "left": resync_left}
5893
  _record_state(con, key, "done", total_rows)
5894
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
5895
  if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
 
8007
  and queued_resync(con, "long_term_storage_fees") is None
8008
  and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
8009
  "GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
8010
+ # S79-T23: a scheduled pass inside the daily floor drains every queued window in one turn.
8011
+ queue = [list(window[:2]) for window in stored]
8012
+ _record_state(con, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1,
8013
+ json.dumps({"windows": queue}))
8014
+ _record_state(con, entity_key("sync_tick_long_term_storage_fees"), "done")
8015
+ clock[0] += timedelta(seconds=60)
8016
+ before = len(calls)
8017
+ held = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
8018
+ scheduled=True)["tables"]["long_term_storage_fees"]
8019
+ ck("NC: a queued resync still waits out Amazon's own report floor",
8020
+ held.get("state") == "deferred" and len(calls) == before
8021
+ and len(queued_resyncs(con, "long_term_storage_fees")) == len(queue))
8022
+ clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"])
8023
+ lines, before = [], len(calls)
8024
+ drained = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lines.append,
8025
+ scheduled=True)["tables"]["long_term_storage_fees"]
8026
+ ck("a scheduled resync turn drains several windows inside the daily floor, one log line",
8027
+ len(queue) > 1 and len(calls) - before == len(queue)
8028
+ and drained.get("resync") == {"landed": len(queue), "left": 0}
8029
+ and f"Amazon resync long_term_storage_fees: {len(queue)} windows landed, 0 left" in lines
8030
+ and queued_resync(con, "long_term_storage_fees") is None
8031
+ and read_watermark(con, "long_term_storage_fees") == mark)
8032
+ clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60)
8033
+ idle = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
8034
+ scheduled=True)["tables"]["long_term_storage_fees"]
8035
+ ck("NC: with the queue empty the daily floor defers the table again",
8036
+ idle.get("state") == "deferred" and "once a day" in idle.get("reason", ""))
8037
  finally:
8038
  con.close()
8039
 
platform/harness/datastore.py CHANGED
@@ -394,7 +394,9 @@ ENTITIES = {
394
  # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 —
395
  # a ('active','in',...) domain 500s); treat as non-archivable.
396
  "model": "account.account", "archivable": False,
397
- "fields": ["code", "name", "account_type", "write_date"],
 
 
398
  },
399
  # ⭐⭐ W37-T13 (owner: stock moved IN and OUT per SKU) — MEASURED FIRST in
400
  # `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess.
@@ -853,7 +855,7 @@ BATCH = 2000
853
  # (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
854
  # from "unknown".
855
  BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free",
856
- "is_internal_transfer")
857
 
858
  # ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence
859
  # shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec.
@@ -3255,7 +3257,7 @@ def _already(con, key, rec):
3255
  return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
3256
 
3257
 
3258
- def sync_all(entities=None, max_batches=200, log=print):
3259
  selected = list(ENTITIES) if entities is None else list(entities)
3260
  if entities is not None and not selected:
3261
  return {}
@@ -3279,6 +3281,12 @@ def sync_all(entities=None, max_batches=200, log=print):
3279
  t0 = time.time()
3280
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
3281
  out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
 
 
 
 
 
 
3282
  if rt is not None and out:
3283
  persist_odoo_mirror(rt.key, rt, out, log=log)
3284
  # A full pass that lands every entity in live mode latches process readiness in-process — so
 
394
  # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 —
395
  # a ('active','in',...) domain 500s); treat as non-archivable.
396
  "model": "account.account", "archivable": False,
397
+ # non_trade feeds the line-grain Overdue (ar.AGED_LINE_DOMAIN). Not a grid column, so unaudited.
398
+ "fields": ["code", "name", "account_type", "non_trade", "write_date"],
399
+ "backfill_fields": ["non_trade"],
400
  },
401
  # ⭐⭐ W37-T13 (owner: stock moved IN and OUT per SKU) — MEASURED FIRST in
402
  # `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess.
 
855
  # (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
856
  # from "unknown".
857
  BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free",
858
+ "is_internal_transfer", "non_trade")
859
 
860
  # ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence
861
  # shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec.
 
3257
  return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
3258
 
3259
 
3260
+ def sync_all(entities=None, max_batches=200, log=print, after_entity=None):
3261
  selected = list(ENTITIES) if entities is None else list(entities)
3262
  if entities is not None and not selected:
3263
  return {}
 
3281
  t0 = time.time()
3282
  phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
3283
  out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
3284
+ if after_entity is not None:
3285
+ # Beside this entity's copy, before the next pull. A hook failure never fails the pass.
3286
+ try:
3287
+ after_entity(key)
3288
+ except Exception as exc: # noqa: BLE001
3289
+ log(f" {key}: after-entity hook failed ({type(exc).__name__})")
3290
  if rt is not None and out:
3291
  persist_odoo_mirror(rt.key, rt, out, log=log)
3292
  # A full pass that lands every entity in live mode latches process readiness in-process — so
platform/harness/semantic.py CHANGED
@@ -134,6 +134,20 @@ class ModelError(Exception):
134
  """A model-file problem (unknown key, bad reference, unparseable expr)."""
135
 
136
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  class UnknownColumnError(ModelError):
138
  """A filter/group request named a column the topic does not declare."""
139
 
@@ -1019,11 +1033,10 @@ def _v_ar_overdue(date_from, date_to, team_id):
1019
  as_of = con.execute("SELECT CURRENT_DATE").fetchone()[0]
1020
  finally:
1021
  con.close()
1022
- tie = ar.overdue_tie(ar.aged_receivable(as_of))
1023
- # Only a line-grain binding can tie: a document sum leaves out unapplied payments, so the
1024
- # gap stays red until the metric reads account_move_line (Rule 8).
1025
  res = store_query(_model()["metrics"]["ar_overdue"]["topic"], ["ar_overdue"])
1026
  ours = res["rows"][0]["ar_overdue"] if res["rows"] else 0
 
1027
  return ours, tie["b"]
1028
 
1029
 
@@ -1191,7 +1204,11 @@ def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01, *
1191
  if key == 'amazon_order_count':
1192
  tol = 0 # Order counts reconcile exactly; a money tolerance cannot forgive missing orders.
1193
  else:
1194
- ours, independent = fn(date_from, date_to, team_id)
 
 
 
 
1195
  if m.get('date_age'):
1196
  gap = 0 if ours == independent else 1
1197
  elif m.get('format') == 'date':
@@ -1231,6 +1248,21 @@ def _store_con():
1231
  return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention
1232
 
1233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1234
  def _store_unready_cause(DS):
1235
  """D-513: the condition `ready()` actually hit. The old sentence always blamed a warm-up after
1236
  a restart, which is rarely the cause (another process holding the file is the usual one) and
@@ -1769,6 +1801,12 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
1769
  raise ModelError(f'Standard metric source {source!r} needs its declared field backfill')
1770
  if metric_key == 'shipped':
1771
  _check_shipped_links(date_from, date_to)
 
 
 
 
 
 
1772
  foreign_vals = {}
1773
  for k in foreign:
1774
  sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
 
134
  """A model-file problem (unknown key, bad reference, unparseable expr)."""
135
 
136
 
137
+ class NotSyncedError(ModelError):
138
+ """A metric reads mirror fields whose declared Odoo backfill has not finished."""
139
+
140
+ code = "not_synced"
141
+
142
+ def __init__(self, metric, sources):
143
+ self.metric = str(metric.get("key") or "")
144
+ self.sources = list(sources)
145
+ super().__init__(
146
+ f"{metric.get('label') or self.metric} is not available yet. The Odoo fields it reads "
147
+ f"have not finished syncing into this data cache ({', '.join(self.sources)}). "
148
+ "Run Sync now on the Odoo connector, or ask again after the next scheduled sync.")
149
+
150
+
151
  class UnknownColumnError(ModelError):
152
  """A filter/group request named a column the topic does not declare."""
153
 
 
1033
  as_of = con.execute("SELECT CURRENT_DATE").fetchone()[0]
1034
  finally:
1035
  con.close()
1036
+ # The store side first: a mirror without the line residual refuses before any Odoo read.
 
 
1037
  res = store_query(_model()["metrics"]["ar_overdue"]["topic"], ["ar_overdue"])
1038
  ours = res["rows"][0]["ar_overdue"] if res["rows"] else 0
1039
+ tie = ar.overdue_tie(ar.aged_receivable(as_of))
1040
  return ours, tie["b"]
1041
 
1042
 
 
1204
  if key == 'amazon_order_count':
1205
  tol = 0 # Order counts reconcile exactly; a money tolerance cannot forgive missing orders.
1206
  else:
1207
+ try:
1208
+ ours, independent = fn(date_from, date_to, team_id)
1209
+ except NotSyncedError as exc:
1210
+ return {"check": f"semantic.{key}: mirror fields not synced", "ok": False, "gap": None,
1211
+ "state": exc.code, "reason": str(exc), "sources": exc.sources}
1212
  if m.get('date_age'):
1213
  gap = 0 if ours == independent else 1
1214
  elif m.get('format') == 'date':
 
1248
  return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention
1249
 
1250
 
1251
+ def _pending_backfills(sources, cur=None):
1252
+ """The listed mirror entities whose declared column backfill has not finished."""
1253
+ if not sources:
1254
+ return []
1255
+ from harness import datastore
1256
+ con = cur if cur is not None else _store_con()
1257
+ try:
1258
+ # A column ALTERed in after the last pass stays absent or NULL until its marker reads done.
1259
+ return [s for s in sources
1260
+ if not datastore._columns_backfilled(con, s, datastore.ENTITIES[s])]
1261
+ finally:
1262
+ if cur is None:
1263
+ con.close()
1264
+
1265
+
1266
  def _store_unready_cause(DS):
1267
  """D-513: the condition `ready()` actually hit. The old sentence always blamed a warm-up after
1268
  a restart, which is rarely the cause (another process holding the file is the usual one) and
 
1801
  raise ModelError(f'Standard metric source {source!r} needs its declared field backfill')
1802
  if metric_key == 'shipped':
1803
  _check_shipped_links(date_from, date_to)
1804
+ for metric_key in base_keys:
1805
+ if metric_key in _STANDARD_METRICS:
1806
+ continue
1807
+ pending = _pending_backfills(_model()['metrics'][metric_key].get('required_backfills'), cur=cur)
1808
+ if pending:
1809
+ raise NotSyncedError(_model()['metrics'][metric_key], pending)
1810
  foreign_vals = {}
1811
  for k in foreign:
1812
  sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
platform/model/metrics/receivables.yml CHANGED
@@ -1,5 +1,5 @@
1
  # Metrics: receivables (topic receivables) — AR exposure, matching the Collections conventions
2
- # (amount_residual_signed nets credit notes; overdue = past invoice_date_due as of today).
3
  topic: receivables
4
  metrics:
5
  - key: customer_open_documents
@@ -41,17 +41,22 @@ metrics:
41
  method: ar_outstanding
42
  note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent."
43
 
44
- # R37 / C10: this metric names the Overdue definition. The validated number is the customer
45
- # field `ar_overdue` (ar.aged_receivable, tied by ar.overdue_tie). This document-grain store sum
46
- # has no validate contract because the line mirror lacks amount_residual.
47
  - key: ar_overdue
 
48
  label: Overdue
49
  agg: sum
50
- field: amount_residual_signed
51
- store_filter_sql: "CAST(m.invoice_date_due AS DATE) < CURRENT_DATE AND m.payment_state != 'paid'"
 
52
  format: usd
53
  description: "Overdue, defined as Odoo Aged Receivable overdue: open receivable past its due date with no grace, New Customer Hold due at once, credits and unapplied payments netted per customer."
54
- ai_context: "For the total that ties to Odoo, sum the customer database field ar_overdue (label Overdue). This document sum leaves out unapplied payments. The 5-day grace number is the field ar_overdue_past_grace (label Overdue past grace)."
 
 
 
55
 
56
  - key: open_invoices
57
  label: Open invoices
 
1
  # Metrics: receivables (topic receivables) — AR exposure, matching the Collections conventions
2
+ # (amount_residual_signed nets credit notes). ar_overdue reads receivable lines (topic gl_lines).
3
  topic: receivables
4
  metrics:
5
  - key: customer_open_documents
 
41
  method: ar_outstanding
42
  note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent."
43
 
44
+ # R37 / C10: Overdue at LINE grain, the Odoo Aged Receivable basis `_v_ar_overdue` ties to
45
+ # `ar.overdue_tie` (AGED_LINE_DOMAIN plus the past-due clause). Line grain keeps unapplied payments.
46
+ # A mirror whose line residual or account non_trade backfill is unfinished refuses (not_synced).
47
  - key: ar_overdue
48
+ topic: gl_lines
49
  label: Overdue
50
  agg: sum
51
+ field: amount_residual
52
+ store_filter_sql: "a.account_type = 'asset_receivable' AND NOT COALESCE(CAST(a.non_trade AS BOOLEAN), FALSE) AND l.parent_state = 'posted' AND NOT COALESCE(CAST(l.reconciled AS BOOLEAN), FALSE) AND CAST(l.date AS DATE) <= CURRENT_DATE AND (CAST(l.date_maturity AS DATE) < CURRENT_DATE OR (l.date_maturity IS NULL AND CAST(l.date AS DATE) < CURRENT_DATE))"
53
+ required_backfills: [account_move_line, account_account]
54
  format: usd
55
  description: "Overdue, defined as Odoo Aged Receivable overdue: open receivable past its due date with no grace, New Customer Hold due at once, credits and unapplied payments netted per customer."
56
+ ai_context: "Sums the residual of open trade receivable journal lines past their due date, so credits and unapplied payments net in. Matches the customer database field ar_overdue (label Overdue). The 5-day grace number is the field ar_overdue_past_grace (label Overdue past grace)."
57
+ validate:
58
+ method: ar_overdue
59
+ note: "Store-path line residual past due must equal Odoo account.move.line amount_residual over the Aged Receivable domain, to the cent."
60
 
61
  - key: open_invoices
62
  label: Open invoices
platform/model/skills/odoo/overdue-receivables.md CHANGED
@@ -24,14 +24,16 @@ Wait for a completed source snapshot. Record its timestamp and the query window
24
  Do not store the answer's figures in this playbook.
25
 
26
  ## Query
27
- Use the registered ar_overdue metric. Resolve $today to the date used by that metric. Filter due_date strictly before it and payment_state unequal to paid, retaining both customer document types.
 
28
  The front-matter query is a parameterized read request, never an instruction to change a database.
29
 
30
  ## Traps
31
- A due date equal to today is not overdue. Missing due dates need review, not an invented date. Signed overdue credits reduce the balance. Today's residuals cannot reconstruct a past aging report.
32
 
33
  ## Check the answer
34
- Read-group account.move amount_residual_signed:sum with the same cutoff and posted customer domain in source_check. Require a zero rounded currency delta. Report the cutoff timezone and timestamp.
 
35
  Use read-only source aggregates through the workspace's authorized source reader.
36
  Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
37
  An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
 
24
  Do not store the answer's figures in this playbook.
25
 
26
  ## Query
27
+ Use the registered ar_overdue metric. Resolve $today to the date used by that metric. It sums the residual of open trade receivable journal lines due strictly before $today, so credits and unapplied payments net in. A not_synced answer means its mirror fields are still syncing, so the metric is unavailable.
28
+ The front-matter query reads invoice documents only. Use it only when the metric is unavailable, and say it leaves out unapplied payments.
29
  The front-matter query is a parameterized read request, never an instruction to change a database.
30
 
31
  ## Traps
32
+ A due date equal to today is not overdue. A line with no due date is due on its entry date. Credits and unapplied payments reduce the balance. Today's residuals cannot reconstruct a past aging report.
33
 
34
  ## Check the answer
35
+ For the metric, read-group account.move.line amount_residual:sum over posted, unreconciled trade receivable lines dated on or before $today and due before it. Require a zero rounded currency delta. Report the cutoff timezone and timestamp.
36
+ The front-matter source_check ties the document fallback only.
37
  Use read-only source aggregates through the workspace's authorized source reader.
38
  Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
39
  An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
platform/model/topics/gl_lines.yml CHANGED
@@ -38,3 +38,5 @@ ai_context: >
38
  negates them to read positively. This topic is company-level: business-unit filters do not
39
  apply here (use sales_lines for BU questions). The newest month is often still being booked —
40
  a sharply lower final month usually means an OPEN period, not a cost win (close-lag).
 
 
 
38
  negates them to read positively. This topic is company-level: business-unit filters do not
39
  apply here (use sales_lines for BU questions). The newest month is often still being booked —
40
  a sharply lower final month usually means an OPEN period, not a cost win (close-lag).
41
+ It also carries ar_overdue (Overdue): open trade receivable lines past their due date, the
42
+ Odoo Aged Receivable basis, as an as-of-today balance that takes no date window.
platform/modules/competitor_watch.py CHANGED
@@ -31,6 +31,9 @@ HOLDING = ("Open", "Watching", "Ignore")
31
  DEFAULTS = {"undercut_pct": 5.0, "rank_surge_pct": 50.0}
32
  DUE_DAYS = 7
33
  NOT_READY = "Comparison starts after the next weekly pull."
 
 
 
34
  LIMIT = ("Each competitor watch document is limited to 128 MB by the store. About 200,000 alerts "
35
  "fit in one. Split the alerts document by year when it nears that size.")
36
 
@@ -80,9 +83,15 @@ def _mirror_path(tenant_key, st):
80
  return amazon_store._tenant_mirror_path(tenant_key, rt)
81
 
82
 
 
 
 
 
83
  @contextlib.contextmanager
84
  def _reader(tenant_key, st):
85
- """A read-only cursor on the tenant mirror, or None when the mirror file does not exist."""
 
 
86
  path = Path(_mirror_path(tenant_key, st))
87
  if not path.exists():
88
  yield None
@@ -93,27 +102,27 @@ def _reader(tenant_key, st):
93
  except Exception: # noqa: BLE001
94
  shared = False
95
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
96
- if shared:
97
- cur = datastore.ro_cursor()
98
- try:
99
- yield cur
100
- finally:
101
- cur.close()
102
- return
103
- import duckdb
104
- con = duckdb.connect(str(path), read_only=True)
105
  try:
106
- yield con
 
 
 
 
 
 
 
 
107
  finally:
108
- con.close()
109
 
110
 
111
  def _columns(cur, table):
 
112
  try:
113
  return {r[0] for r in cur.execute(
114
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
115
- except Exception: # noqa: BLE001
116
- return set()
117
 
118
 
119
  def _sqp(cur):
@@ -415,7 +424,8 @@ def on_amazon_sync(tenant_key, synced_tables):
415
  try:
416
  def _note(doc):
417
  doc = doc if isinstance(doc, dict) else {}
418
- doc["last_error"] = f"The weekly comparison failed ({type(exc).__name__}): {exc}"
 
419
  return doc
420
  _rt(tenant_key).update(KEY_WEEKS, _note)
421
  except Exception: # noqa: BLE001
 
31
  DEFAULTS = {"undercut_pct": 5.0, "rank_surge_pct": 50.0}
32
  DUE_DAYS = 7
33
  NOT_READY = "Comparison starts after the next weekly pull."
34
+ #: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
35
+ READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so no search term was compared. "
36
+ "The next Amazon sync reads it again.")
37
  LIMIT = ("Each competitor watch document is limited to 128 MB by the store. About 200,000 alerts "
38
  "fit in one. Split the alerts document by year when it nears that size.")
39
 
 
83
  return amazon_store._tenant_mirror_path(tenant_key, rt)
84
 
85
 
86
+ class _ReadFailed(Exception):
87
+ """A mirror read failed, was locked or busy. Never read as an absent table."""
88
+
89
+
90
  @contextlib.contextmanager
91
  def _reader(tenant_key, st):
92
+ """A read-only cursor on the tenant mirror, None when the mirror file does not exist.
93
+
94
+ An open failure raises `_ReadFailed` naming its error class."""
95
  path = Path(_mirror_path(tenant_key, st))
96
  if not path.exists():
97
  yield None
 
102
  except Exception: # noqa: BLE001
103
  shared = False
104
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
 
 
 
 
 
 
 
 
 
105
  try:
106
+ if shared:
107
+ cur = datastore.ro_cursor()
108
+ else:
109
+ import duckdb
110
+ cur = duckdb.connect(str(path), read_only=True)
111
+ except Exception as exc: # noqa: BLE001
112
+ raise _ReadFailed(type(exc).__name__) from exc
113
+ try:
114
+ yield cur
115
  finally:
116
+ cur.close()
117
 
118
 
119
  def _columns(cur, table):
120
+ """The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
121
  try:
122
  return {r[0] for r in cur.execute(
123
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
124
+ except Exception as exc: # noqa: BLE001
125
+ raise _ReadFailed(type(exc).__name__) from exc
126
 
127
 
128
  def _sqp(cur):
 
424
  try:
425
  def _note(doc):
426
  doc = doc if isinstance(doc, dict) else {}
427
+ doc["last_error"] = (READ_FAILED.format(cause=exc) if isinstance(exc, _ReadFailed) else
428
+ f"The weekly comparison failed ({type(exc).__name__}): {exc}")
429
  return doc
430
  _rt(tenant_key).update(KEY_WEEKS, _note)
431
  except Exception: # noqa: BLE001
platform/modules/opportunity_finder.py CHANGED
@@ -22,6 +22,7 @@ import contextlib
22
  import contextvars
23
  import datetime as dt
24
  import hashlib
 
25
  import os
26
  import threading
27
  from pathlib import Path
@@ -68,12 +69,16 @@ LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3
68
 
69
  #: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
70
  PROBE_SECONDS = 20.0
 
 
71
  #: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
72
  MEMO_KEEP = 16
73
 
74
  #: Accounts that sign in for a process, never a person who can own an item.
75
  SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
76
 
 
 
77
 
78
  class UnknownItem(KeyError, ValueError):
79
  """An item id the finder does not hold. A KeyError for the router, a ValueError for C4."""
@@ -145,14 +150,16 @@ _MEMO = {}
145
  _MEMO_LOCK = threading.Lock()
146
 
147
 
148
- def _remember(key, build):
149
- """`build()` once per key. A failed build is not kept, so the next request reads again."""
150
  if key is None:
151
  return build()
152
  with _MEMO_LOCK:
153
  if key in _MEMO:
154
  return _MEMO[key]
155
  value = build()
 
 
156
  with _MEMO_LOCK:
157
  _MEMO[key] = value
158
  while len(_MEMO) > MEMO_KEEP:
@@ -160,13 +167,18 @@ def _remember(key, build):
160
  return value
161
 
162
 
163
- def _mark(path, shared):
164
- """Moves whenever the mirror rows can have changed: an install, a file swap or a rewrite."""
 
 
 
165
  from harness import datastore
 
 
166
  generation = None
167
  if shared:
168
  with contextlib.suppress(Exception):
169
- generation = datastore.mirror_generation()
170
  stamps = []
171
  for part in (Path(path), Path(str(path) + ".wal")):
172
  try:
@@ -178,36 +190,45 @@ def _mark(path, shared):
178
 
179
 
180
  @contextlib.contextmanager
181
- def _reader(tenant_key, st):
182
  """A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
183
 
184
- The web process reads through the same door as the typed `ut_amzn_*` reads: assert the tenant, then
185
- `datastore.ro_cursor()` on the serving instance, which follows the installed mirror version."""
 
186
  from harness import datastore
187
  path = Path(_mirror_path(tenant_key, st))
188
  try:
189
  shared = Path(datastore.DB_PATH).resolve() == path.resolve()
190
  except Exception: # noqa: BLE001
191
  shared = False
192
- if not path.exists():
 
 
 
 
 
193
  yield None
194
  return
195
  cur = None
196
  try:
197
- if shared:
 
 
198
  if hasattr(st, "assert_datastore_matches"):
199
  st.assert_datastore_matches()
200
  cur = datastore.ro_cursor()
201
  else:
202
  import duckdb
203
- cur = duckdb.connect(str(path), read_only=True)
 
204
  except Exception as exc: # noqa: BLE001
205
  if cur is not None:
206
  with contextlib.suppress(Exception):
207
  cur.close()
208
- yield _Unreadable(type(exc).__name__)
209
  return
210
- token = _MARK.set(_mark(path, shared))
211
  try:
212
  yield cur
213
  finally:
@@ -221,12 +242,22 @@ def _interrupt(cur):
221
 
222
 
223
  def _bounded(cur, sql, params=()):
224
- """Every row of one probe query, interrupted past PROBE_SECONDS. Any failure raises `_ReadFailed`.
225
 
226
- Pass a query without parameters: DuckDB ignores interrupt() on a parameterized execute."""
227
  if cur is None or isinstance(cur, _Unreadable):
228
  raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
229
- timer = threading.Timer(PROBE_SECONDS, _interrupt, [cur])
 
 
 
 
 
 
 
 
 
 
230
  timer.daemon = True
231
  timer.start()
232
  try:
@@ -234,6 +265,8 @@ def _bounded(cur, sql, params=()):
234
  except Exception as exc: # noqa: BLE001
235
  raise _ReadFailed(type(exc).__name__) from exc
236
  finally:
 
 
237
  timer.cancel()
238
 
239
 
@@ -241,19 +274,13 @@ def _types(cur, table):
241
  """{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
242
  if not table.replace("_", "").isalnum():
243
  raise ValueError(f"{table!r} is not a table name.")
244
- # A literal, not a parameter: DuckDB ignores interrupt() on a parameterized execute.
245
  return {str(n): str(t) for n, t in _bounded(
246
  cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
247
 
248
 
249
  def _columns(cur, table):
250
- if cur is None or isinstance(cur, _Unreadable):
251
- return set()
252
- try:
253
- return {r[0] for r in cur.execute(
254
- "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
255
- except Exception: # noqa: BLE001
256
- return set()
257
 
258
 
259
  def iso_period(value):
@@ -269,11 +296,11 @@ def _our_asins(cur):
269
  """SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
270
  ours = set()
271
  if "asin" in _columns(cur, SQP_TABLE):
272
- ours |= {_text(r[0]) for r in cur.execute(
273
- f'SELECT DISTINCT CAST("asin" AS VARCHAR) FROM {SQP_TABLE} WHERE "asin" IS NOT NULL').fetchall()}
274
  if "asin1" in _columns(cur, LISTINGS_TABLE):
275
- ours |= {_text(r[0]) for r in cur.execute(
276
- f'SELECT DISTINCT CAST("asin1" AS VARCHAR) FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL').fetchall()}
277
  return {a for a in ours if a}
278
 
279
 
@@ -297,18 +324,25 @@ def _derived_departments(cur, ours):
297
  """R49 default: every department where a brand analytics row clicked one of our ASINs, any period."""
298
  if not ours or "clicked_asin" not in _columns(cur, BA_TABLE):
299
  return []
300
- rows = cur.execute(
301
- f'SELECT DISTINCT {_DEPT} FROM {BA_TABLE} '
302
  f'WHERE trim(CAST("clicked_asin" AS VARCHAR)) IN (SELECT unnest(?::VARCHAR[])) '
303
- f'AND "department_name" IS NOT NULL', [sorted(ours)]).fetchall()
304
  return sorted(_text(r[0]) for r in rows if _text(r[0]))
305
 
306
 
 
 
 
 
307
  def _facts(cur, fresh=False):
308
- """The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived."""
 
 
309
  def build():
310
  types = _types(cur, BA_TABLE)
311
- facts = {"columns": set(types), "period": None, "ours": set(), "derived": []}
 
312
  if not set(BA_NEED) <= facts["columns"]:
313
  return facts
314
  facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
@@ -317,16 +351,22 @@ def _facts(cur, fresh=False):
317
  try:
318
  facts["ours"] = _our_asins(cur)
319
  facts["derived"] = _derived_departments(cur, facts["ours"])
 
 
320
  except Exception as exc: # noqa: BLE001
321
  raise _ReadFailed(type(exc).__name__) from exc
322
  return facts
323
  mark = _MARK.get()
324
- return _remember(None if mark is None or fresh else ("facts", mark), build)
 
 
 
 
325
 
326
 
327
  def _source(cur, st, fresh=False):
328
  """What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
329
- out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
330
  if cur is None:
331
  out["problems"].append(NO_MIRROR)
332
  return out
@@ -335,7 +375,8 @@ def _source(cur, st, fresh=False):
335
  except _ReadFailed as exc:
336
  out["problems"].append(READ_FAILED.format(cause=exc))
337
  return out
338
- if not set(BA_NEED) <= facts["columns"] or facts["period"] is None:
 
339
  out["problems"].append(NOT_SYNCED)
340
  return out
341
  if not iso_period(facts["period"]):
@@ -391,7 +432,7 @@ def _rising_rows(cur, period, departments, prior, min_pct):
391
  mark = _MARK.get()
392
  key = (None if mark is None else
393
  ("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
394
- return _remember(key, lambda: cur.execute(sql, params).fetchall())
395
 
396
 
397
  def _top3_list(raw, ours):
@@ -563,16 +604,19 @@ def _prior_week(st, period):
563
  return before[-1], _doc(st, WEEK_PREFIX + before[-1]).get("terms") or {}
564
 
565
 
566
- def find(tenant_key, *, st=None):
567
  """Every rising in-department term against the prior held week, sorted by rise. No cap (rule 1)."""
568
  st = _rt(tenant_key, st)
569
  min_pct = settings(tenant_key, st=st)["min_rise_pct"]
570
- with _reader(tenant_key, st) as cur:
571
  src = _source(cur, st)
572
  prior_week, prior = _prior_week(st, src["period"])
573
  rows = []
574
  if src["period"] and src["departments"] and prior:
575
- rows = _rising_rows(cur, src["period"], src["departments"], prior, min_pct)
 
 
 
576
  problems = list(src["problems"])
577
  if src["period"] and src["departments"] and not prior_week:
578
  problems.append(NOT_READY if _index(st) else NO_HISTORY)
@@ -592,7 +636,7 @@ def find(tenant_key, *, st=None):
592
  "reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
593
  return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
594
  "priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
595
- "windows": windows, "rising": found, "problems": problems}
596
 
597
 
598
  def _windows(st, period, prior_week):
@@ -644,12 +688,12 @@ def _page(rows, cursor, limit):
644
  return rows[start:end], (str(end) if end < len(rows) else None)
645
 
646
 
647
- def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None):
648
  """C4: rising terms and held opportunities, one page. Counts cover every item."""
649
  if status is not None and status not in STATUSES:
650
  raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
651
  st = _rt(tenant_key, st)
652
- found = find(tenant_key, st=st)
653
  every = _all_items(st, found)
654
  counts = {s: 0 for s in STATUSES}
655
  for item in every:
@@ -658,7 +702,8 @@ def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMI
658
  page, nxt = _page(wanted, cursor, limit)
659
  return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
660
  "priorWeek": found["priorWeek"], "departments": found["departments"],
661
- "problems": found["problems"], "weeks": _index(st), "windows": found["windows"]}
 
662
 
663
 
664
  def _lookup(st, iid, found=None):
@@ -678,13 +723,13 @@ def _lookup(st, iid, found=None):
678
  return fields, state if isinstance(state, dict) else None, term
679
 
680
 
681
- def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None):
682
  """C4: move one item, optionally its owner and due date. Every change appends a history event."""
683
  st = _rt(tenant_key, st)
684
- return _move(tenant_key, st, _editor(st, user), iid, status, note=note, owner=owner, due=due)
685
 
686
 
687
- def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None):
688
  if status not in STATUSES:
689
  raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
690
  if due is not None:
@@ -692,7 +737,7 @@ def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None):
692
  due = dt.date.fromisoformat(_text(due)[:10]).isoformat()
693
  except ValueError:
694
  raise ValueError("Due must be a date such as 2026-10-30.") from None
695
- fields, state, term = _lookup(st, iid, find(tenant_key, st=st))
696
  if fields is None and state is None:
697
  raise UnknownItem(UNKNOWN_ITEM)
698
  owner = _text(owner) or None
@@ -753,25 +798,25 @@ def on_decision(tenant_key, iid, decision, reason, *, st=None):
753
  return _move(tenant_key, st, "Amazon operator", iid, DECISIONS[decision], note=_text(reason) or None)
754
 
755
 
756
- def item_detail(tenant_key, user, iid, *, st=None):
757
  """One item with its history, its weekly rank trend and its newest top 3."""
758
  st = _rt(tenant_key, st)
759
- found = find(tenant_key, st=st)
760
  fields, state, term = _lookup(st, iid, found)
761
  row = next((r for r in _all_items(st, found) if r["id"] == iid), None)
762
  if row is None:
763
  row = _row({"id": iid, "term": term, "title": term, "rising": False, "reason": NOT_RISING,
764
  "asin": None, "top3": []}, state or {})
765
  return {"item": row, "history": list((state or {}).get("history") or ()),
766
- **term_detail(tenant_key, iid, st=st, _term=term, _found=found)}
767
 
768
 
769
- def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
770
  """The weekly rank trend from history, oldest first, plus the newest week's top 3 clicked ASINs."""
771
  st = _rt(tenant_key, st)
772
  term = _term
773
  if term is None:
774
- _fields, _state, term = _lookup(st, iid, _found or find(tenant_key, st=st))
775
  if term is None:
776
  raise UnknownItem(UNKNOWN_ITEM)
777
  trend = []
@@ -780,7 +825,7 @@ def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
780
  if term in terms:
781
  trend.append({"week": week, "rank": terms[term]})
782
  top3, period = [], None
783
- with _reader(tenant_key, st) as cur:
784
  src = _source(cur, st)
785
  period = src["period"]
786
  if period and src["departments"] and {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE):
@@ -789,19 +834,19 @@ def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
789
  share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL"
790
  conv = ('TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)'
791
  if "conversion_share" in cols else "NULL")
792
- rows = cur.execute(
793
- f'SELECT trim(CAST("clicked_asin" AS VARCHAR)), {pos}, {share}, {conv} FROM {BA_TABLE} '
794
  f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {_TERM} = ? AND {pos} BETWEEN 1 AND 3 '
795
- f'ORDER BY 2', [period, src["departments"], term]).fetchall()
796
  top3 = _top3_list([{"asin": a, "pos": p, "share": s, "conv": c} for a, p, s, c in rows], src["ours"])
797
  return {"id": iid, "term": term, "trend": trend, "top3": top3,
798
  "week": period[:10] if period else None}
799
 
800
 
801
- def validate(tenant_key, *, week=None, start=None, end=None, st=None):
802
  """Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
803
  st = _rt(tenant_key, st)
804
- with _reader(tenant_key, st) as cur:
805
  src = _source(cur, st, fresh=True)
806
  period = src["period"]
807
  if period is None:
@@ -816,9 +861,9 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
816
  "detail": OLDER_WEEK}
817
  independent = 0
818
  if src["departments"]:
819
- independent = int(cur.execute(
820
- f'SELECT COUNT(DISTINCT {_TERM}) FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS}',
821
- [period, src["departments"]]).fetchone()[0])
822
  doc = _doc(st, WEEK_PREFIX + newest)
823
  if not doc:
824
  return {"ok": False, "app_total": None, "independent_total": independent, "week": newest,
 
22
  import contextvars
23
  import datetime as dt
24
  import hashlib
25
+ import logging
26
  import os
27
  import threading
28
  from pathlib import Path
 
69
 
70
  #: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
71
  PROBE_SECONDS = 20.0
72
+ #: A fired probe bound repeats interrupt() this often: DuckDB can lose a single one on a parameterized execute.
73
+ INTERRUPT_EVERY = 0.05
74
  #: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
75
  MEMO_KEEP = 16
76
 
77
  #: Accounts that sign in for a process, never a person who can own an item.
78
  SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
79
 
80
+ LOG = logging.getLogger("aios.opportunity_finder")
81
+
82
 
83
  class UnknownItem(KeyError, ValueError):
84
  """An item id the finder does not hold. A KeyError for the router, a ValueError for C4."""
 
150
  _MEMO_LOCK = threading.Lock()
151
 
152
 
153
+ def _remember(key, build, keep=None):
154
+ """`build()` once per key. A failed build, or one `keep` refuses, is not kept: the next request reads again."""
155
  if key is None:
156
  return build()
157
  with _MEMO_LOCK:
158
  if key in _MEMO:
159
  return _MEMO[key]
160
  value = build()
161
+ if keep is not None and not keep(value):
162
+ return value
163
  with _MEMO_LOCK:
164
  _MEMO[key] = value
165
  while len(_MEMO) > MEMO_KEEP:
 
167
  return value
168
 
169
 
170
+ def _mark(cur, shared):
171
+ """Moves whenever the mirror rows can have changed: an install, a version swap or a rewrite.
172
+
173
+ The file is the one the cursor reads, so a reader that followed the serving pointer to
174
+ `<stem>.v<N>.duckdb` gets a new mark, even where the logical path and its stats stay put."""
175
  from harness import datastore
176
+ rows = _bounded(cur, "SELECT path FROM duckdb_databases() WHERE database_name = current_database()")
177
+ path = rows[0][0] if rows and rows[0][0] else ""
178
  generation = None
179
  if shared:
180
  with contextlib.suppress(Exception):
181
+ generation = (datastore.mirror_generation(), str(datastore.serving_version().get("version")))
182
  stamps = []
183
  for part in (Path(path), Path(str(path) + ".wal")):
184
  try:
 
190
 
191
 
192
  @contextlib.contextmanager
193
+ def _reader(tenant_key, st, opener=None):
194
  """A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
195
 
196
+ `opener` is the typed `ut_amzn_*` door the route hands in (modules never import api): it asserts the
197
+ tenant and returns `datastore.ro_cursor()` on the serving mirror version. Without it, the shared
198
+ instance serves when it is this mirror, else the file the mirror's serving pointer names."""
199
  from harness import datastore
200
  path = Path(_mirror_path(tenant_key, st))
201
  try:
202
  shared = Path(datastore.DB_PATH).resolve() == path.resolve()
203
  except Exception: # noqa: BLE001
204
  shared = False
205
+ try:
206
+ physical = path if shared or opener is not None else Path(datastore.serving_file(path)[0])
207
+ present = physical.exists() or (opener is not None and Path(datastore.serving_file()[0]).exists())
208
+ except Exception: # noqa: BLE001
209
+ physical, present = path, path.exists()
210
+ if not present:
211
  yield None
212
  return
213
  cur = None
214
  try:
215
+ if opener is not None:
216
+ cur = opener()
217
+ elif shared:
218
  if hasattr(st, "assert_datastore_matches"):
219
  st.assert_datastore_matches()
220
  cur = datastore.ro_cursor()
221
  else:
222
  import duckdb
223
+ cur = duckdb.connect(str(physical), read_only=True)
224
+ mark = _mark(cur, shared or opener is not None)
225
  except Exception as exc: # noqa: BLE001
226
  if cur is not None:
227
  with contextlib.suppress(Exception):
228
  cur.close()
229
+ yield _Unreadable(str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__)
230
  return
231
+ token = _MARK.set(mark)
232
  try:
233
  yield cur
234
  finally:
 
242
 
243
 
244
  def _bounded(cur, sql, params=()):
245
+ """Every row of one query, interrupted past PROBE_SECONDS. Any failure raises `_ReadFailed`.
246
 
247
+ Past the bound, interrupt() repeats every INTERRUPT_EVERY until the query ends: one call can be lost."""
248
  if cur is None or isinstance(cur, _Unreadable):
249
  raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
250
+ lock, done = threading.Lock(), threading.Event()
251
+
252
+ def _fire():
253
+ while True:
254
+ with lock:
255
+ if done.is_set():
256
+ return
257
+ _interrupt(cur)
258
+ if done.wait(INTERRUPT_EVERY):
259
+ return
260
+ timer = threading.Timer(PROBE_SECONDS, _fire)
261
  timer.daemon = True
262
  timer.start()
263
  try:
 
265
  except Exception as exc: # noqa: BLE001
266
  raise _ReadFailed(type(exc).__name__) from exc
267
  finally:
268
+ with lock:
269
+ done.set()
270
  timer.cancel()
271
 
272
 
 
274
  """{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
275
  if not table.replace("_", "").isalnum():
276
  raise ValueError(f"{table!r} is not a table name.")
 
277
  return {str(n): str(t) for n, t in _bounded(
278
  cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
279
 
280
 
281
  def _columns(cur, table):
282
+ """The column names of a mirror table, empty when the table is absent. A failed read raises `_ReadFailed`."""
283
+ return set(_types(cur, table))
 
 
 
 
 
284
 
285
 
286
  def iso_period(value):
 
296
  """SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
297
  ours = set()
298
  if "asin" in _columns(cur, SQP_TABLE):
299
+ ours |= {_text(r[0]) for r in _bounded(
300
+ cur, f'SELECT DISTINCT CAST("asin" AS VARCHAR) FROM {SQP_TABLE} WHERE "asin" IS NOT NULL')}
301
  if "asin1" in _columns(cur, LISTINGS_TABLE):
302
+ ours |= {_text(r[0]) for r in _bounded(
303
+ cur, f'SELECT DISTINCT CAST("asin1" AS VARCHAR) FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL')}
304
  return {a for a in ours if a}
305
 
306
 
 
324
  """R49 default: every department where a brand analytics row clicked one of our ASINs, any period."""
325
  if not ours or "clicked_asin" not in _columns(cur, BA_TABLE):
326
  return []
327
+ rows = _bounded(
328
+ cur, f'SELECT DISTINCT {_DEPT} FROM {BA_TABLE} '
329
  f'WHERE trim(CAST("clicked_asin" AS VARCHAR)) IN (SELECT unnest(?::VARCHAR[])) '
330
+ f'AND "department_name" IS NOT NULL', [sorted(ours)])
331
  return sorted(_text(r[0]) for r in rows if _text(r[0]))
332
 
333
 
334
+ def _synced(facts):
335
+ return set(BA_NEED) <= facts["columns"] and facts["period"] is not None
336
+
337
+
338
  def _facts(cur, fresh=False):
339
+ """The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived.
340
+
341
+ A not-synced answer is never kept, so a mirror that gains the table is read at the next request."""
342
  def build():
343
  types = _types(cur, BA_TABLE)
344
+ facts = {"columns": set(types), "period": None, "ours": set(), "derived": [],
345
+ "period_type": types.get("_period_start")}
346
  if not set(BA_NEED) <= facts["columns"]:
347
  return facts
348
  facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
 
351
  try:
352
  facts["ours"] = _our_asins(cur)
353
  facts["derived"] = _derived_departments(cur, facts["ours"])
354
+ except _ReadFailed:
355
+ raise
356
  except Exception as exc: # noqa: BLE001
357
  raise _ReadFailed(type(exc).__name__) from exc
358
  return facts
359
  mark = _MARK.get()
360
+ facts = _remember(None if mark is None or fresh else ("facts", mark), build, keep=_synced)
361
+ if not _synced(facts):
362
+ LOG.warning("[opportunity-finder] %s not synced: file %s, missing columns %s, newest period %r",
363
+ BA_TABLE, mark[0] if mark else "?", sorted(set(BA_NEED) - facts["columns"]), facts["period"])
364
+ return facts
365
 
366
 
367
  def _source(cur, st, fresh=False):
368
  """What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
369
+ out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": [], "periodType": None}
370
  if cur is None:
371
  out["problems"].append(NO_MIRROR)
372
  return out
 
375
  except _ReadFailed as exc:
376
  out["problems"].append(READ_FAILED.format(cause=exc))
377
  return out
378
+ out["periodType"] = facts.get("period_type")
379
+ if not _synced(facts):
380
  out["problems"].append(NOT_SYNCED)
381
  return out
382
  if not iso_period(facts["period"]):
 
432
  mark = _MARK.get()
433
  key = (None if mark is None else
434
  ("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
435
+ return _remember(key, lambda: _bounded(cur, sql, params))
436
 
437
 
438
  def _top3_list(raw, ours):
 
604
  return before[-1], _doc(st, WEEK_PREFIX + before[-1]).get("terms") or {}
605
 
606
 
607
+ def find(tenant_key, *, st=None, opener=None):
608
  """Every rising in-department term against the prior held week, sorted by rise. No cap (rule 1)."""
609
  st = _rt(tenant_key, st)
610
  min_pct = settings(tenant_key, st=st)["min_rise_pct"]
611
+ with _reader(tenant_key, st, opener) as cur:
612
  src = _source(cur, st)
613
  prior_week, prior = _prior_week(st, src["period"])
614
  rows = []
615
  if src["period"] and src["departments"] and prior:
616
+ try:
617
+ rows = _rising_rows(cur, src["period"], src["departments"], prior, min_pct)
618
+ except _ReadFailed as exc:
619
+ src["problems"].append(READ_FAILED.format(cause=exc))
620
  problems = list(src["problems"])
621
  if src["period"] and src["departments"] and not prior_week:
622
  problems.append(NOT_READY if _index(st) else NO_HISTORY)
 
636
  "reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
637
  return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
638
  "priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
639
+ "windows": windows, "periodType": src["periodType"], "rising": found, "problems": problems}
640
 
641
 
642
  def _windows(st, period, prior_week):
 
688
  return rows[start:end], (str(end) if end < len(rows) else None)
689
 
690
 
691
+ def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None, opener=None):
692
  """C4: rising terms and held opportunities, one page. Counts cover every item."""
693
  if status is not None and status not in STATUSES:
694
  raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
695
  st = _rt(tenant_key, st)
696
+ found = find(tenant_key, st=st, opener=opener)
697
  every = _all_items(st, found)
698
  counts = {s: 0 for s in STATUSES}
699
  for item in every:
 
702
  page, nxt = _page(wanted, cursor, limit)
703
  return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
704
  "priorWeek": found["priorWeek"], "departments": found["departments"],
705
+ "problems": found["problems"], "weeks": _index(st), "windows": found["windows"],
706
+ "periodType": found["periodType"]}
707
 
708
 
709
  def _lookup(st, iid, found=None):
 
723
  return fields, state if isinstance(state, dict) else None, term
724
 
725
 
726
+ def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None, opener=None):
727
  """C4: move one item, optionally its owner and due date. Every change appends a history event."""
728
  st = _rt(tenant_key, st)
729
+ return _move(tenant_key, st, _editor(st, user), iid, status, note=note, owner=owner, due=due, opener=opener)
730
 
731
 
732
+ def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None, opener=None):
733
  if status not in STATUSES:
734
  raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
735
  if due is not None:
 
737
  due = dt.date.fromisoformat(_text(due)[:10]).isoformat()
738
  except ValueError:
739
  raise ValueError("Due must be a date such as 2026-10-30.") from None
740
+ fields, state, term = _lookup(st, iid, find(tenant_key, st=st, opener=opener))
741
  if fields is None and state is None:
742
  raise UnknownItem(UNKNOWN_ITEM)
743
  owner = _text(owner) or None
 
798
  return _move(tenant_key, st, "Amazon operator", iid, DECISIONS[decision], note=_text(reason) or None)
799
 
800
 
801
+ def item_detail(tenant_key, user, iid, *, st=None, opener=None):
802
  """One item with its history, its weekly rank trend and its newest top 3."""
803
  st = _rt(tenant_key, st)
804
+ found = find(tenant_key, st=st, opener=opener)
805
  fields, state, term = _lookup(st, iid, found)
806
  row = next((r for r in _all_items(st, found) if r["id"] == iid), None)
807
  if row is None:
808
  row = _row({"id": iid, "term": term, "title": term, "rising": False, "reason": NOT_RISING,
809
  "asin": None, "top3": []}, state or {})
810
  return {"item": row, "history": list((state or {}).get("history") or ()),
811
+ **term_detail(tenant_key, iid, st=st, _term=term, _found=found, opener=opener)}
812
 
813
 
814
+ def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None, opener=None):
815
  """The weekly rank trend from history, oldest first, plus the newest week's top 3 clicked ASINs."""
816
  st = _rt(tenant_key, st)
817
  term = _term
818
  if term is None:
819
+ _fields, _state, term = _lookup(st, iid, _found or find(tenant_key, st=st, opener=opener))
820
  if term is None:
821
  raise UnknownItem(UNKNOWN_ITEM)
822
  trend = []
 
825
  if term in terms:
826
  trend.append({"week": week, "rank": terms[term]})
827
  top3, period = [], None
828
+ with _reader(tenant_key, st, opener) as cur:
829
  src = _source(cur, st)
830
  period = src["period"]
831
  if period and src["departments"] and {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE):
 
834
  share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL"
835
  conv = ('TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)'
836
  if "conversion_share" in cols else "NULL")
837
+ rows = _bounded(
838
+ cur, f'SELECT trim(CAST("clicked_asin" AS VARCHAR)), {pos}, {share}, {conv} FROM {BA_TABLE} '
839
  f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {_TERM} = ? AND {pos} BETWEEN 1 AND 3 '
840
+ f'ORDER BY 2', [period, src["departments"], term])
841
  top3 = _top3_list([{"asin": a, "pos": p, "share": s, "conv": c} for a, p, s, c in rows], src["ours"])
842
  return {"id": iid, "term": term, "trend": trend, "top3": top3,
843
  "week": period[:10] if period else None}
844
 
845
 
846
+ def validate(tenant_key, *, week=None, start=None, end=None, st=None, opener=None):
847
  """Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
848
  st = _rt(tenant_key, st)
849
+ with _reader(tenant_key, st, opener) as cur:
850
  src = _source(cur, st, fresh=True)
851
  period = src["period"]
852
  if period is None:
 
861
  "detail": OLDER_WEEK}
862
  independent = 0
863
  if src["departments"]:
864
+ independent = int(_bounded(
865
+ cur, f'SELECT COUNT(DISTINCT {_TERM}) FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS}',
866
+ [period, src["departments"]])[0][0])
867
  doc = _doc(st, WEEK_PREFIX + newest)
868
  if not doc:
869
  return {"ok": False, "app_total": None, "independent_total": independent, "week": newest,
platform/modules/ranking_diagnostic.py CHANGED
@@ -85,6 +85,9 @@ CAUSE_ORDER = {key: n for n, (key, _l, _f) in enumerate((*CAUSES, UNEXPLAINED))}
85
  ABSENT = "Not in this week's Sales and Traffic report."
86
  READ_ONLY = "Your account can view the ranking diagnostic but not change it."
87
  NOT_READY = "The diagnostic starts after two weekly syncs. One week is held so far."
 
 
 
88
  LIMIT = ("Each week of history is one store document, limited to 128 MB. About 300,000 ASINs fit in "
89
  "one week. Split a week by ASIN range when a catalogue nears that size.")
90
 
@@ -155,9 +158,15 @@ def _mirror_path(tenant_key, st):
155
  return amazon_store._tenant_mirror_path(tenant_key, rt)
156
 
157
 
 
 
 
 
158
  @contextlib.contextmanager
159
  def _reader(tenant_key, st):
160
- """A read-only cursor on the tenant mirror, or None when the mirror file does not exist."""
 
 
161
  path = Path(_mirror_path(tenant_key, st))
162
  if not path.exists():
163
  yield None
@@ -168,29 +177,29 @@ def _reader(tenant_key, st):
168
  except Exception: # noqa: BLE001
169
  shared = False
170
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
171
- if shared:
172
- cur = datastore.ro_cursor()
173
- try:
174
- yield cur
175
- finally:
176
- cur.close()
177
- return
178
- import duckdb
179
- con = duckdb.connect(str(path), read_only=True)
180
  try:
181
- yield con
 
 
 
 
 
 
 
 
182
  finally:
183
- con.close()
184
 
185
 
186
  def _columns(cur, table):
 
187
  if cur is None:
188
  return set()
189
  try:
190
  return {r[0] for r in cur.execute(
191
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
192
- except Exception: # noqa: BLE001
193
- return set()
194
 
195
 
196
  def _newest(cols, column, table):
@@ -254,6 +263,7 @@ def _pricing(cur):
254
  for a, doc, _at, sku, sessions, units, sales, box in rows:
255
  if not a or newest[a] != (landed[doc], doc):
256
  continue
 
257
  p = out.setdefault(a, {"sessions": 0.0, "units": 0.0, "sales": 0.0, "skus": [], "_box": []})
258
  p["sessions"] += sessions or 0.0
259
  p["units"] += units or 0.0
@@ -275,12 +285,14 @@ def source_totals(cur):
275
  """P10: sessions and units in SQL over each ASIN's newest document, apart from `_pricing`'s Python pick.
276
 
277
  Both apply one rule (newest document per ASIN), so the tie proves the two picks and sums agree.
278
- `supersededRows` and the raw sums show what the rule left out."""
 
279
  cols = _columns(cur, PRICING)
280
  asin = _asin_expr(cols)
281
  if asin is None or not {SESSIONS, UNITS} <= cols:
282
  return None
283
- sessions, units, count, superseded, raw_sessions, raw_units = cur.execute(
 
284
  f"WITH r AS (SELECT {asin} AS a, {_document(cols)} AS d, {_landing(cur)} AS i, "
285
  f"TRY_CAST(\"{SESSIONS}\" AS DOUBLE) AS s, TRY_CAST(\"{UNITS}\" AS DOUBLE) AS u FROM {PRICING}), "
286
  f"docs AS (SELECT d, MAX(i) AS landed FROM r GROUP BY d), "
@@ -289,10 +301,11 @@ def source_totals(cur):
289
  f"WHERE n = 1), kept AS (SELECT r.* FROM r JOIN best USING (a, d)) "
290
  f"SELECT COALESCE(SUM(s), 0), COALESCE(SUM(u), 0), COUNT(DISTINCT CASE WHEN u > 0 THEN a END), "
291
  f"(SELECT COUNT(*) FROM r WHERE a IS NOT NULL) - COUNT(*), (SELECT COALESCE(SUM(s), 0) FROM r), "
292
- f"(SELECT COALESCE(SUM(u), 0) FROM r) FROM kept").fetchone()
 
293
  return {"sessions": round(float(sessions), 4), "units": round(float(units), 4), "asinsWithUnits": int(count),
294
  "supersededRows": int(superseded), "rawSessions": round(float(raw_sessions), 4),
295
- "rawUnits": round(float(raw_units), 4)}
296
 
297
 
298
  def _rank(raw):
@@ -724,7 +737,8 @@ def on_amazon_sync(tenant_key, synced_tables, *, st=None, now=None):
724
  try:
725
  def _note(doc):
726
  doc = doc if isinstance(doc, dict) else {}
727
- doc["last_error"] = f"The weekly history could not be written ({type(exc).__name__}): {exc}"
 
728
  return doc
729
  _rt(tenant_key, st).update(KEY_INDEX, _note)
730
  except Exception: # noqa: BLE001
@@ -890,10 +904,14 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
890
  if not chosen:
891
  return {"ok": False, "app_total": None, "independent_total": None, "weeks": [],
892
  "detail": "No weekly history is held for that period, so nothing can be tied."}
893
- live = None
894
  if held and held[-1] in chosen:
895
- with _reader(tenant_key, st) as cur:
896
- live = source_totals(cur) if cur is not None else None
 
 
 
 
897
  rows, ok = [], True
898
  app = {"sessions": 0.0, "units": 0.0}
899
  ind = {"sessions": 0.0, "units": 0.0}
@@ -915,6 +933,8 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
915
  bad = [r["week"] for r in rows if not r["ok"]]
916
  detail = (f"{len(rows)} weeks tie sessions and units to the Sales and Traffic totals." if ok else
917
  "These weeks do not tie to the Sales and Traffic totals: " + ", ".join(bad) + ".")
 
 
918
  superseded = int((live or {}).get("supersededRows") or 0)
919
  if superseded:
920
  detail += (f" {superseded} older Sales and Traffic rows are left out because a newer report covers "
 
85
  ABSENT = "Not in this week's Sales and Traffic report."
86
  READ_ONLY = "Your account can view the ranking diagnostic but not change it."
87
  NOT_READY = "The diagnostic starts after two weekly syncs. One week is held so far."
88
+ #: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
89
+ READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so the weekly history was not updated. "
90
+ "The next Amazon sync reads it again.")
91
  LIMIT = ("Each week of history is one store document, limited to 128 MB. About 300,000 ASINs fit in "
92
  "one week. Split a week by ASIN range when a catalogue nears that size.")
93
 
 
158
  return amazon_store._tenant_mirror_path(tenant_key, rt)
159
 
160
 
161
+ class _ReadFailed(Exception):
162
+ """A mirror read failed, was locked or busy. Never read as an absent table."""
163
+
164
+
165
  @contextlib.contextmanager
166
  def _reader(tenant_key, st):
167
+ """A read-only cursor on the tenant mirror, None when the mirror file does not exist.
168
+
169
+ An open failure raises `_ReadFailed` naming its error class."""
170
  path = Path(_mirror_path(tenant_key, st))
171
  if not path.exists():
172
  yield None
 
177
  except Exception: # noqa: BLE001
178
  shared = False
179
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
 
 
 
 
 
 
 
 
 
180
  try:
181
+ if shared:
182
+ cur = datastore.ro_cursor()
183
+ else:
184
+ import duckdb
185
+ cur = duckdb.connect(str(path), read_only=True)
186
+ except Exception as exc: # noqa: BLE001
187
+ raise _ReadFailed(type(exc).__name__) from exc
188
+ try:
189
+ yield cur
190
  finally:
191
+ cur.close()
192
 
193
 
194
  def _columns(cur, table):
195
+ """The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
196
  if cur is None:
197
  return set()
198
  try:
199
  return {r[0] for r in cur.execute(
200
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
201
+ except Exception as exc: # noqa: BLE001
202
+ raise _ReadFailed(type(exc).__name__) from exc
203
 
204
 
205
  def _newest(cols, column, table):
 
263
  for a, doc, _at, sku, sessions, units, sales, box in rows:
264
  if not a or newest[a] != (landed[doc], doc):
265
  continue
266
+ # One child under two parents in one document is two traffic slices of that child: SUM, never pick.
267
  p = out.setdefault(a, {"sessions": 0.0, "units": 0.0, "sales": 0.0, "skus": [], "_box": []})
268
  p["sessions"] += sessions or 0.0
269
  p["units"] += units or 0.0
 
285
  """P10: sessions and units in SQL over each ASIN's newest document, apart from `_pricing`'s Python pick.
286
 
287
  Both apply one rule (newest document per ASIN), so the tie proves the two picks and sums agree.
288
+ `supersededRows` and the raw sums show what the rule left out. `summedAsins` counts the ASINs
289
+ whose kept document lists them on more than one row (one child under two parents)."""
290
  cols = _columns(cur, PRICING)
291
  asin = _asin_expr(cols)
292
  if asin is None or not {SESSIONS, UNITS} <= cols:
293
  return None
294
+ # Every kept row of an ASIN sums, the same rule as `_pricing`: a child under two parents counts once.
295
+ sessions, units, count, superseded, raw_sessions, raw_units, summed = cur.execute(
296
  f"WITH r AS (SELECT {asin} AS a, {_document(cols)} AS d, {_landing(cur)} AS i, "
297
  f"TRY_CAST(\"{SESSIONS}\" AS DOUBLE) AS s, TRY_CAST(\"{UNITS}\" AS DOUBLE) AS u FROM {PRICING}), "
298
  f"docs AS (SELECT d, MAX(i) AS landed FROM r GROUP BY d), "
 
301
  f"WHERE n = 1), kept AS (SELECT r.* FROM r JOIN best USING (a, d)) "
302
  f"SELECT COALESCE(SUM(s), 0), COALESCE(SUM(u), 0), COUNT(DISTINCT CASE WHEN u > 0 THEN a END), "
303
  f"(SELECT COUNT(*) FROM r WHERE a IS NOT NULL) - COUNT(*), (SELECT COALESCE(SUM(s), 0) FROM r), "
304
+ f"(SELECT COALESCE(SUM(u), 0) FROM r), "
305
+ f"(SELECT COUNT(*) FROM (SELECT a FROM kept GROUP BY a HAVING COUNT(*) > 1)) FROM kept").fetchone()
306
  return {"sessions": round(float(sessions), 4), "units": round(float(units), 4), "asinsWithUnits": int(count),
307
  "supersededRows": int(superseded), "rawSessions": round(float(raw_sessions), 4),
308
+ "rawUnits": round(float(raw_units), 4), "summedAsins": int(summed)}
309
 
310
 
311
  def _rank(raw):
 
737
  try:
738
  def _note(doc):
739
  doc = doc if isinstance(doc, dict) else {}
740
+ doc["last_error"] = (READ_FAILED.format(cause=exc) if isinstance(exc, _ReadFailed) else
741
+ f"The weekly history could not be written ({type(exc).__name__}): {exc}")
742
  return doc
743
  _rt(tenant_key, st).update(KEY_INDEX, _note)
744
  except Exception: # noqa: BLE001
 
904
  if not chosen:
905
  return {"ok": False, "app_total": None, "independent_total": None, "weeks": [],
906
  "detail": "No weekly history is held for that period, so nothing can be tied."}
907
+ live, failed = None, None
908
  if held and held[-1] in chosen:
909
+ try:
910
+ with _reader(tenant_key, st) as cur:
911
+ live = source_totals(cur) if cur is not None else None
912
+ except _ReadFailed as exc:
913
+ failed = (f"The Amazon mirror could not be read just now ({exc}), so the newest week cannot be "
914
+ "tied. Try again in a minute.")
915
  rows, ok = [], True
916
  app = {"sessions": 0.0, "units": 0.0}
917
  ind = {"sessions": 0.0, "units": 0.0}
 
933
  bad = [r["week"] for r in rows if not r["ok"]]
934
  detail = (f"{len(rows)} weeks tie sessions and units to the Sales and Traffic totals." if ok else
935
  "These weeks do not tie to the Sales and Traffic totals: " + ", ".join(bad) + ".")
936
+ if failed:
937
+ detail += " " + failed
938
  superseded = int((live or {}).get("supersededRows") or 0)
939
  if superseded:
940
  detail += (f" {superseded} older Sales and Traffic rows are left out because a newer report covers "
platform/modules/restock_planner.py CHANGED
@@ -37,6 +37,9 @@ REOPEN_FROM = ("Watching", "Done", "Dismissed")
37
  DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"}
38
  DEFAULTS = {"send_to_fba_days": 30.0, "overstock_days": 90.0, "due_days": 7}
39
  NOT_READY = "The restock plan starts after the next Amazon inventory sync."
 
 
 
40
  LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in "
41
  "the items document, and history splits by month. Split the items document by band when "
42
  "it nears that size.")
@@ -106,9 +109,15 @@ def _mirror_path(tenant_key, st):
106
  return amazon_store._tenant_mirror_path(tenant_key, rt)
107
 
108
 
 
 
 
 
109
  @contextlib.contextmanager
110
  def _reader(tenant_key, st):
111
- """A read-only cursor on the tenant mirror, or None when the mirror file does not exist."""
 
 
112
  path = Path(_mirror_path(tenant_key, st))
113
  if not path.exists():
114
  yield None
@@ -119,29 +128,29 @@ def _reader(tenant_key, st):
119
  except Exception: # noqa: BLE001
120
  shared = False
121
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
122
- if shared:
123
- cur = datastore.ro_cursor()
124
- try:
125
- yield cur
126
- finally:
127
- cur.close()
128
- return
129
- import duckdb
130
- con = duckdb.connect(str(path), read_only=True)
131
  try:
132
- yield con
 
 
 
 
 
 
 
 
133
  finally:
134
- con.close()
135
 
136
 
137
  def _columns(cur, table):
 
138
  if cur is None:
139
  return set()
140
  try:
141
  return {r[0] for r in cur.execute(
142
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
143
- except Exception: # noqa: BLE001
144
- return set()
145
 
146
 
147
  def _newest(cols, table):
@@ -466,11 +475,22 @@ def refresh(tenant_key, *, st=None, now=None):
466
  now = now or _now()
467
  conf = settings(tenant_key, st=st)
468
  links = _links(st)
469
- with _reader(tenant_key, st) as cur:
470
- if cur is None:
471
- rows, notices = None, ["The tenant mirror does not exist yet."]
472
- else:
473
- rows, notices = build_rows(cur, links, conf)
 
 
 
 
 
 
 
 
 
 
 
474
  if rows is None:
475
  message = notices[0] if notices else NOT_READY
476
 
@@ -734,15 +754,20 @@ def validate(tenant_key, *, st=None, **period):
734
  """Rule 8: FBA available over the planner's rows ties to the planning table's own total."""
735
  st = _rt(tenant_key, st)
736
  conf = settings(tenant_key, st=st)
737
- with _reader(tenant_key, st) as cur:
738
- if cur is None or "available" not in _columns(cur, PLANNING):
739
- return {"ok": False, "app_total": None, "independent_total": None,
740
- "detail": f"{SOURCE_NAMES[PLANNING]} has not synced yet."}
741
- rows, _notices = build_rows(cur, _links(st), conf)
742
- cols = _columns(cur, PLANNING)
743
- independent = cur.execute(
744
- f'SELECT SUM(TRY_CAST("available" AS DOUBLE)), COUNT(DISTINCT "sku") FROM {PLANNING} '
745
- f'WHERE {_newest(cols, PLANNING)}').fetchone()
 
 
 
 
 
746
  planned = [r for r in rows or () if r["availableSource"] == "planning"]
747
  app_total = float(sum(r["fbaAvailable"] or 0 for r in planned))
748
  independent_total = float(independent[0] or 0)
 
37
  DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"}
38
  DEFAULTS = {"send_to_fba_days": 30.0, "overstock_days": 90.0, "due_days": 7}
39
  NOT_READY = "The restock plan starts after the next Amazon inventory sync."
40
+ #: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
41
+ READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so the restock plan was not refreshed. "
42
+ "The next Amazon sync reads it again.")
43
  LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in "
44
  "the items document, and history splits by month. Split the items document by band when "
45
  "it nears that size.")
 
109
  return amazon_store._tenant_mirror_path(tenant_key, rt)
110
 
111
 
112
+ class _ReadFailed(Exception):
113
+ """A mirror read failed, was locked or busy. Never read as an absent table."""
114
+
115
+
116
  @contextlib.contextmanager
117
  def _reader(tenant_key, st):
118
+ """A read-only cursor on the tenant mirror, None when the mirror file does not exist.
119
+
120
+ An open failure raises `_ReadFailed` naming its error class."""
121
  path = Path(_mirror_path(tenant_key, st))
122
  if not path.exists():
123
  yield None
 
128
  except Exception: # noqa: BLE001
129
  shared = False
130
  # The process holds one writer on its own mirror: a separate read-only connect would refuse.
 
 
 
 
 
 
 
 
 
131
  try:
132
+ if shared:
133
+ cur = datastore.ro_cursor()
134
+ else:
135
+ import duckdb
136
+ cur = duckdb.connect(str(path), read_only=True)
137
+ except Exception as exc: # noqa: BLE001
138
+ raise _ReadFailed(type(exc).__name__) from exc
139
+ try:
140
+ yield cur
141
  finally:
142
+ cur.close()
143
 
144
 
145
  def _columns(cur, table):
146
+ """The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
147
  if cur is None:
148
  return set()
149
  try:
150
  return {r[0] for r in cur.execute(
151
  "SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
152
+ except Exception as exc: # noqa: BLE001
153
+ raise _ReadFailed(type(exc).__name__) from exc
154
 
155
 
156
  def _newest(cols, table):
 
475
  now = now or _now()
476
  conf = settings(tenant_key, st=st)
477
  links = _links(st)
478
+ try:
479
+ with _reader(tenant_key, st) as cur:
480
+ if cur is None:
481
+ rows, notices = None, ["The tenant mirror does not exist yet."]
482
+ else:
483
+ rows, notices = build_rows(cur, links, conf)
484
+ except _ReadFailed as exc:
485
+ # The held plan stays as it was: a failed read is never "not synced" and never an empty plan.
486
+ failed = READ_FAILED.format(cause=exc)
487
+
488
+ def _failed(doc):
489
+ doc = doc if isinstance(doc, dict) else {}
490
+ doc.update(last_error=failed, checkedAt=_stamp(now))
491
+ return doc
492
+ st.update(KEY_ITEMS, _failed)
493
+ return {"state": "read_failed", "message": failed}
494
  if rows is None:
495
  message = notices[0] if notices else NOT_READY
496
 
 
754
  """Rule 8: FBA available over the planner's rows ties to the planning table's own total."""
755
  st = _rt(tenant_key, st)
756
  conf = settings(tenant_key, st=st)
757
+ try:
758
+ with _reader(tenant_key, st) as cur:
759
+ if cur is None or "available" not in _columns(cur, PLANNING):
760
+ return {"ok": False, "app_total": None, "independent_total": None,
761
+ "detail": f"{SOURCE_NAMES[PLANNING]} has not synced yet."}
762
+ rows, _notices = build_rows(cur, _links(st), conf)
763
+ cols = _columns(cur, PLANNING)
764
+ independent = cur.execute(
765
+ f'SELECT SUM(TRY_CAST("available" AS DOUBLE)), COUNT(DISTINCT "sku") FROM {PLANNING} '
766
+ f'WHERE {_newest(cols, PLANNING)}').fetchone()
767
+ except _ReadFailed as exc:
768
+ return {"ok": False, "app_total": None, "independent_total": None,
769
+ "detail": f"The Amazon mirror could not be read just now ({exc}), so nothing can be tied. "
770
+ "Try again in a minute."}
771
  planned = [r for r in rows or () if r["availableSource"] == "planning"]
772
  app_total = float(sum(r["fbaAvailable"] or 0 for r in planned))
773
  independent_total = float(independent[0] or 0)
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 7d1ba4682927d550a33ca880be112e51699b4a79459a302274e1e09cd13ef434
 
1
+ 0ae8b932683b55f2b15d76ac37e314ae821910fcbbc157e63bc0fe6cacf75cfe