fsanyoto commited on
Commit
4f4f4bb
Β·
verified Β·
1 Parent(s): 0bdfaeb

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "27a848090b",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "fc37ff799b",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 27a848090b
 
1
+ fc37ff799b
api/main.py CHANGED
@@ -1249,6 +1249,11 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1249
  return detail
1250
  classifier, sentence = _sync_now_cause(cause)
1251
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
 
 
 
 
 
1252
  suffix = f" (cause code: {classifier})"
1253
  # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
1254
  budget = 300 - len(suffix)
@@ -1357,9 +1362,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
1357
  "refresh_loaded": True} if first_load else {}
1358
  report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1359
  scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1360
- **target_options, **options, log=lambda *a, **k: None)
 
1361
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1362
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
 
 
1363
  for table, row in (report.get("tables") or {}).items()))
1364
  from connectors_amazon import assert_safe_status
1365
  first_load_progress = None if targeted else assert_safe_status(report.get("firstLoad"))
 
1249
  return detail
1250
  classifier, sentence = _sync_now_cause(cause)
1251
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
1252
+ if isinstance(cause, BaseException):
1253
+ import traceback
1254
+ locations = " > ".join(f"{os.path.basename(frame.filename)}:{frame.lineno}:{frame.name}"
1255
+ for frame in traceback.extract_tb(cause.__traceback__))
1256
+ print(f"[aios-api] sync failure location {type(cause).__name__}: {locations}")
1257
  suffix = f" (cause code: {classifier})"
1258
  # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
1259
  budget = 300 - len(suffix)
 
1362
  "refresh_loaded": True} if first_load else {}
1363
  report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1364
  scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1365
+ **target_options, **options,
1366
+ log=lambda message: print(f"[aios-api] amazon sync desk: {message}", flush=True))
1367
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1368
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1369
+ + (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
1370
+ if "resync" in row else "")
1371
  for table, row in (report.get("tables") or {}).items()))
1372
  from connectors_amazon import assert_safe_status
1373
  first_load_progress = None if targeted else assert_safe_status(report.get("firstLoad"))
api/registry_source.py CHANGED
@@ -249,6 +249,46 @@ class BatchStore:
249
  return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
250
  [[*path, pid] for pid in ids])) if value is not None}
251
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  def update(self, *args, **kwargs):
253
  raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
254
 
@@ -329,8 +369,12 @@ class BatchStore:
329
  [*operand.params, *selected]).fetchall()]
330
  else:
331
  # Metadata consumes no row memberships; retain only the curated witness.
332
- with closing(values()) as source:
333
- next(source, None)
 
 
 
 
334
  kept = []
335
  if saw_any[0] and first[0] is None:
336
  # Config sanitization drops invalid entries; [] would mean all rows.
@@ -343,9 +387,9 @@ class BatchStore:
343
  for owner, ws in meta.items():
344
  if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
345
  continue
346
- values = paths(self._runtime, name, [[owner, 'overlays', pid] for pid in self.row_ids])
347
  # Only this owner's selected cells have been read, so admit a new bounded document.
348
- meta[owner] = dict(ws, overlays={pid: value for pid, value in zip(self.row_ids, values)
349
  if isinstance(value, dict)})
350
  return meta
351
 
@@ -432,17 +476,23 @@ class CanonicalStore(BatchStore):
432
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
433
  for key, meta in definitions.items() if isinstance(meta, dict)}}
434
  elif name == 'loopable_route_system':
435
- value = object_values(self._runtime, name, omit=('routes', 'zones'))
 
 
 
 
 
 
 
 
 
436
  value['routes'], value['zones'] = [], []
437
- for batch in array_values(self._runtime, name, ['routes'], drop=('stops',)):
438
- for index, route in batch:
439
- if isinstance(route, dict):
440
- route = dict(route, stops=members(self._runtime, name, ['routes', index, 'stops'],
441
- self.row_ids, id_key='customerPid'))
442
- value['routes'].append(route)
443
- for batch in array_values(self._runtime, name, ['zones'], drop=(
444
- 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
445
- value['zones'].extend(meta for _, meta in batch)
446
  elif name == 'table_zones':
447
  value = {self.table_key: []}
448
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
@@ -466,12 +516,13 @@ class CanonicalStore(BatchStore):
466
  return super().get_entry(name, entry)
467
 
468
 
469
- def canonical_metadata(session, table_key):
470
  """Read the canonical permitted definitions without constructing source rows."""
471
  import routes_customers as customers
472
  import routes_products as products
473
  session.require(table_key)
474
- lent = CanonicalStore(session, table_key, [], cur=None, metadata={}, needed=frozenset())
 
475
  actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
476
  build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
477
  return build(actor, scope='product' if table_key == 'product_data' else 'customer',
@@ -506,8 +557,8 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
506
  session.require(table_key)
507
  team, agent = perm_scope.derive_pool_scope(session.user, table_key)
508
  before = generation(session, table_key)
509
- metadata = {}
510
- definition = canonical_metadata(session, table_key)
511
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
512
  tree = perm_scope.entry(session.user, table_key) or {}
513
  wall = tree.get('filter') or {}
 
249
  return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
250
  [[*path, pid] for pid in ids])) if value is not None}
251
 
252
+ def _selected_json(self, name, path, *, array=False, id_key=None):
253
+ """Reuse one request-owned SQL copy of an overlay or route membership across batches."""
254
+ import json
255
+ cur = getattr(self, '_canonical_source_cursor', None)
256
+ if cur is None or not self.metadata.get('_scan'):
257
+ if array:
258
+ return members(self._runtime, name, path, self.row_ids, id_key=id_key)
259
+ return self.selected_values(name, path, self.row_ids)
260
+ key = ('selected_json', name, tuple(path), array, id_key)
261
+ if key not in self.metadata:
262
+ from routes_odoo_tables import _RelationLease
263
+ lease = _RelationLease(cur)
264
+ table = lease.table('_json')
265
+ cur.execute(f'CREATE TEMP TABLE "{table}" (ordinal BIGINT, pid VARCHAR, body JSON)')
266
+ source = array_values(self._runtime, name, path) if array else items(self._runtime, name, path)
267
+ offset = 0
268
+ with closing(source):
269
+ for batch in source:
270
+ records = []
271
+ for pid, value in batch:
272
+ raw = value.get(id_key) if id_key and isinstance(value, dict) else value
273
+ try:
274
+ key_pid = str(int(raw)) if array else str(pid)
275
+ except (TypeError, ValueError):
276
+ continue
277
+ records.append((key_pid, json.dumps(value)))
278
+ if records:
279
+ cur.execute(f'INSERT INTO "{table}" SELECT unnest(?::BIGINT[]), '
280
+ 'unnest(?::VARCHAR[]), unnest(?::JSON[])',
281
+ [list(range(offset, offset + len(records))),
282
+ [pid for pid, _ in records], [body for _, body in records]])
283
+ offset += len(records)
284
+ self.metadata[key] = table, lease
285
+ table, _lease = self.metadata[key]
286
+ rows = cur.execute(f'SELECT pid, body FROM "{table}" WHERE pid IN '
287
+ '(SELECT unnest(?::VARCHAR[])) '
288
+ 'QUALIFY row_number() OVER (PARTITION BY pid ORDER BY ordinal) = 1 '
289
+ 'ORDER BY ordinal', [self.row_ids]).fetchall()
290
+ return [json.loads(body) for _, body in rows] if array else {pid: json.loads(body) for pid, body in rows}
291
+
292
  def update(self, *args, **kwargs):
293
  raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
294
 
 
369
  [*operand.params, *selected]).fetchall()]
370
  else:
371
  # Metadata consumes no row memberships; retain only the curated witness.
372
+ witness_key = ('view_witness', name, owner, vid)
373
+ if witness_key not in self.metadata:
374
+ with closing(values()) as source:
375
+ next(source, None)
376
+ self.metadata[witness_key] = first[0], saw_any[0]
377
+ first[0], saw_any[0] = self.metadata[witness_key]
378
  kept = []
379
  if saw_any[0] and first[0] is None:
380
  # Config sanitization drops invalid entries; [] would mean all rows.
 
387
  for owner, ws in meta.items():
388
  if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
389
  continue
390
+ values = self._selected_json(name, [owner, 'overlays'])
391
  # Only this owner's selected cells have been read, so admit a new bounded document.
392
+ meta[owner] = dict(ws, overlays={pid: value for pid, value in values.items()
393
  if isinstance(value, dict)})
394
  return meta
395
 
 
476
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
477
  for key, meta in definitions.items() if isinstance(meta, dict)}}
478
  elif name == 'loopable_route_system':
479
+ meta_key = ('route_definitions', name)
480
+ if meta_key not in self.metadata or fresh:
481
+ base = object_values(self._runtime, name, omit=('routes', 'zones'))
482
+ base['routes'] = [(index, route) for batch in array_values(self._runtime, name,
483
+ ['routes'], drop=('stops',)) for index, route in batch]
484
+ base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
485
+ 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
486
+ self.metadata[meta_key] = base
487
+ saved = self.metadata[meta_key]
488
+ value = {key: copy.deepcopy(val) for key, val in saved.items() if key not in {'routes', 'zones'}}
489
  value['routes'], value['zones'] = [], []
490
+ for index, route in saved['routes']:
491
+ if isinstance(route, dict):
492
+ route = dict(route, stops=self._selected_json(name, ['routes', index, 'stops'],
493
+ array=True, id_key='customerPid'))
494
+ value['routes'].append(route)
495
+ value['zones'] = copy.deepcopy(saved['zones'])
 
 
 
496
  elif name == 'table_zones':
497
  value = {self.table_key: []}
498
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
 
516
  return super().get_entry(name, entry)
517
 
518
 
519
+ def canonical_metadata(session, table_key, *, metadata=None):
520
  """Read the canonical permitted definitions without constructing source rows."""
521
  import routes_customers as customers
522
  import routes_products as products
523
  session.require(table_key)
524
+ lent = CanonicalStore(session, table_key, [], cur=None,
525
+ metadata={} if metadata is None else metadata, needed=frozenset())
526
  actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
527
  build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
528
  return build(actor, scope='product' if table_key == 'product_data' else 'customer',
 
557
  session.require(table_key)
558
  team, agent = perm_scope.derive_pool_scope(session.user, table_key)
559
  before = generation(session, table_key)
560
+ metadata = {'_scan': ids is None}
561
+ definition = canonical_metadata(session, table_key, metadata=metadata)
562
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
563
  tree = perm_scope.entry(session.user, table_key) or {}
564
  wall = tree.get('filter') or {}
api/routes_competitor_watch.py CHANGED
@@ -153,7 +153,7 @@ def _not_ready_message(cw, periods):
153
  return cw.NOT_READY
154
 
155
 
156
- def screen_payload(cw, detail, terms, term, can_edit):
157
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
158
  periods = list(detail.get("periods") or ())
159
  top3 = list(detail.get("top3") or ())
@@ -200,18 +200,21 @@ def screen_payload(cw, detail, terms, term, can_edit):
200
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
201
  "comparisonReady": ready,
202
  "comparisonMessage": "" if ready else _not_ready_message(cw, periods),
203
- "top3": rows, "trend": trend, "problems": []}
 
 
204
 
205
 
206
  @router.get("/screen")
207
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
 
208
  session: Session = Depends(_app_reader)):
209
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
210
  cw = _module()
211
  tenant = _tenant(session)
212
  record = _app(session, app)
213
- terms = [row["term"] for row in _call(cw.terms_summary, tenant, st=session.runtime)["terms"]]
214
- picked = " ".join(str(term or "").split()).lower()
215
- picked = picked if picked in terms else (terms[0] if terms else None)
216
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
217
- return screen_payload(cw, detail, terms, picked, _may_edit(session, record))
 
153
  return cw.NOT_READY
154
 
155
 
156
+ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
157
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
158
  periods = list(detail.get("periods") or ())
159
  top3 = list(detail.get("top3") or ())
 
200
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
201
  "comparisonReady": ready,
202
  "comparisonMessage": "" if ready else _not_ready_message(cw, periods),
203
+ "top3": rows, "trend": trend, "problems": list((page or {}).get("problems") or []),
204
+ **{key: (page or {}).get(key, default) for key, default in
205
+ (("termTotal", len(terms)), ("termOffset", 0), ("termLimit", len(terms)))}}
206
 
207
 
208
  @router.get("/screen")
209
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
210
+ q: str = "", offset: int = 0, limit: int = 50,
211
  session: Session = Depends(_app_reader)):
212
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
213
  cw = _module()
214
  tenant = _tenant(session)
215
  record = _app(session, app)
216
+ page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
217
+ limit=limit, selected=term)
218
+ terms, picked = page["terms"], page["term"]
219
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
220
+ return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
platform/harness/amazon_store.py CHANGED
@@ -2915,11 +2915,6 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
2915
  def part_max_bytes(self):
2916
  return MIRROR_PART_MAX_BYTES
2917
 
2918
- def _install_snapshot(self, con, manifest, rows, paths, only=None):
2919
- result = super()._install_snapshot(con, manifest, rows, paths, only=only)
2920
- _note_published() # C2: AFTER_SYNC runs only for a pass that published.
2921
- return result
2922
-
2923
  def _valid_table(self, name):
2924
  # Preserve older Amazon tables even when they are no longer advertised.
2925
  return isinstance(name, str) and bool(re.fullmatch(r"amzn_[a-z0-9_]+", name))
@@ -2944,8 +2939,12 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
2944
  return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
2945
 
2946
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
2947
- super()._install_snapshot(con, manifest, rows, paths, only=only)
2948
  refresh_sku_keywords(con)
 
 
 
 
2949
 
2950
  def _publication_holds(self, report, waiting):
2951
  """Publish what a refresh pass completed; hold each unfinished table at its previous rows.
@@ -5119,9 +5118,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
5119
  """Publish a whole Amazon source after its requested work completes."""
5120
  rt = rt if rt is not None else _runtime_for(tenant_key)
5121
  _sync_credential(tenant_key, rt)
5122
- if tables is None and os.environ.get("AIOS_AMAZON_BRAND_ANALYTICS") != "1":
5123
- # β›” One brand analytics week is a 3.1 GB report; parsing it in the web process froze live
5124
- # (2026-10-21). It syncs only when named, until D-10430 moves sync to a worker.
5125
  tables = [name for name in TABLES if name != "brand_analytics"]
5126
  tables = None if tables is None else list(tables)
5127
  options = dict(tables=tables, since=since, log=log, rt=rt, max_wait=max_wait,
@@ -5166,6 +5165,7 @@ def _after_sync(tenant_key, report, log):
5166
  import importlib
5167
  module_name, _, name = target.partition(":")
5168
  getattr(importlib.import_module(module_name), name)(tenant_key, synced)
 
5169
  except Exception as exc: # noqa: BLE001
5170
  log(f"Amazon after-sync callback {target} failed ({type(exc).__name__}); the sync result is unchanged.")
5171
 
@@ -5514,6 +5514,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5514
  continue
5515
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
5516
  root = cx.REPORT_TYPES[table].get("json_root")
 
 
 
 
 
 
 
 
5517
  log(f"{table}: {plan['cause']}")
5518
  sibling = SHARED_REPORT_SIBLINGS.get(table)
5519
  sibling_plan = None
@@ -5539,6 +5547,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5539
  changed_tables.add(table_name(sibling))
5540
  proof_key = entity_key("reconcile_" + table)
5541
  previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None
 
5542
  try:
5543
  if previous_proof:
5544
  # The recorder accepts an empty object; only a parsed document can replace it with proof.
@@ -5551,7 +5560,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5551
  sibling_written, sibling_cleared = 0, False
5552
  written, cols, cleared = 0, [], False
5553
  landed_window = False
5554
- resync, resync_count, resync_left = [], 0, None
5555
  pricing_asins = []
5556
  operations = {}
5557
  if first_load:
@@ -5688,9 +5696,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5688
  if first_load and w0 is not None:
5689
  record_backfill_floor(con, table, w0)
5690
  floors[table] = w0
5691
- if resync_count:
5692
- resync_left = len(queued_resyncs(con, table))
5693
- log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left")
5694
  elif table != "settlements":
5695
  period_spec = cx.REPORT_TYPES[table]
5696
  windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
@@ -5888,8 +5893,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5888
  "rows": total_rows, "columns": column_count, "provenance": prov,
5889
  "report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
5890
  "operations": operations}
5891
- if resync_count:
5892
- report["tables"][table]["resync"] = {"landed": resync_count, "left": resync_left}
5893
  _record_state(con, key, "done", total_rows)
5894
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
5895
  if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
@@ -5971,6 +5974,11 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5971
  _sync_event("amazon.sync", tenant_key, sibling,
5972
  {"state": problem["state"], "rows": 0}, rt, report, log)
5973
  finally:
 
 
 
 
 
5974
  if previous_proof:
5975
  proof = _read_state(con, proof_key) or {}
5976
  if proof.get("state") == "unproven" and proof.get("detail") == "{}":
 
2915
  def part_max_bytes(self):
2916
  return MIRROR_PART_MAX_BYTES
2917
 
 
 
 
 
 
2918
  def _valid_table(self, name):
2919
  # Preserve older Amazon tables even when they are no longer advertised.
2920
  return isinstance(name, str) and bool(re.fullmatch(r"amzn_[a-z0-9_]+", name))
 
2939
  return super().persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log, **kwargs)
2940
 
2941
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
2942
+ result = super()._install_snapshot(con, manifest, rows, paths, only=only)
2943
  refresh_sku_keywords(con)
2944
+ return result
2945
+
2946
+ def _after_publish(self, tenant_key, manifest):
2947
+ _note_published()
2948
 
2949
  def _publication_holds(self, report, waiting):
2950
  """Publish what a refresh pass completed; hold each unfinished table at its previous rows.
 
5118
  """Publish a whole Amazon source after its requested work completes."""
5119
  rt = rt if rt is not None else _runtime_for(tenant_key)
5120
  _sync_credential(tenant_key, rt)
5121
+ if (tables is None and os.environ.get("AIOS_AMAZON_BRAND_ANALYTICS") != "1"
5122
+ and os.environ.get("AIOS_PROCESS_ROLE") != "worker"):
5123
+ # Large weekly reports belong to the dedicated sync worker.
5124
  tables = [name for name in TABLES if name != "brand_analytics"]
5125
  tables = None if tables is None else list(tables)
5126
  options = dict(tables=tables, since=since, log=log, rt=rt, max_wait=max_wait,
 
5165
  import importlib
5166
  module_name, _, name = target.partition(":")
5167
  getattr(importlib.import_module(module_name), name)(tenant_key, synced)
5168
+ log(f"Amazon after-sync callback {target} completed")
5169
  except Exception as exc: # noqa: BLE001
5170
  log(f"Amazon after-sync callback {target} failed ({type(exc).__name__}); the sync result is unchanged.")
5171
 
 
5514
  continue
5515
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
5516
  root = cx.REPORT_TYPES[table].get("json_root")
5517
+ if not first_load and cx.REPORT_TYPES[table].get("weekly") and not {
5518
+ "_period_start", "_period_end"} <= set(_columns_of(con, mirror)):
5519
+ # Legacy rows have no trustworthy week. Fetch a real completed period before replacing them.
5520
+ repair_weeks = _week_windows(end - timedelta(days=21), end)
5521
+ if repair_weeks:
5522
+ repair_start, repair_end = repair_weeks[-1]
5523
+ plan = {**plan, "start": repair_start, "end": repair_end,
5524
+ "cause": "repair missing period fields from the latest completed week"}
5525
  log(f"{table}: {plan['cause']}")
5526
  sibling = SHARED_REPORT_SIBLINGS.get(table)
5527
  sibling_plan = None
 
5547
  changed_tables.add(table_name(sibling))
5548
  proof_key = entity_key("reconcile_" + table)
5549
  previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None
5550
+ resync, resync_count = [], 0
5551
  try:
5552
  if previous_proof:
5553
  # The recorder accepts an empty object; only a parsed document can replace it with proof.
 
5560
  sibling_written, sibling_cleared = 0, False
5561
  written, cols, cleared = 0, [], False
5562
  landed_window = False
 
5563
  pricing_asins = []
5564
  operations = {}
5565
  if first_load:
 
5696
  if first_load and w0 is not None:
5697
  record_backfill_floor(con, table, w0)
5698
  floors[table] = w0
 
 
 
5699
  elif table != "settlements":
5700
  period_spec = cx.REPORT_TYPES[table]
5701
  windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
 
5893
  "rows": total_rows, "columns": column_count, "provenance": prov,
5894
  "report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
5895
  "operations": operations}
 
 
5896
  _record_state(con, key, "done", total_rows)
5897
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
5898
  if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
 
5974
  _sync_event("amazon.sync", tenant_key, sibling,
5975
  {"state": problem["state"], "rows": 0}, rt, report, log)
5976
  finally:
5977
+ if resync:
5978
+ resync_left = len(queued_resyncs(con, table))
5979
+ report["tables"].setdefault(table, {})["resync"] = {
5980
+ "landed": resync_count, "left": resync_left}
5981
+ log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left")
5982
  if previous_proof:
5983
  proof = _read_state(con, proof_key) or {}
5984
  if proof.get("state") == "unproven" and proof.get("detail") == "{}":
platform/harness/datastore.py CHANGED
@@ -2879,6 +2879,7 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal
2879
  if not fields:
2880
  return 0
2881
  marker = f"{key}.{'refresh' if repeat else 'cols'}.{','.join(sorted(fields))}"
 
2882
  operation = "column refresh" if repeat else "column backfill"
2883
  st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
2884
  [marker]).fetchone()
@@ -2886,6 +2887,29 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal
2886
  if not repeat:
2887
  return 0
2888
  st = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2889
  # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
2890
  # an hour of work to one dropped connection gets skipped by the next person under time
2891
  # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
@@ -2895,7 +2919,6 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal
2895
  if last:
2896
  log(f" {key}: {operation} resuming at id {last:,} ({n:,} rows already done)")
2897
  cols = [c for f in fields for c in _field_cols(f, named_m2m=spec.get("named_m2m"))]
2898
- setter = ", ".join(f"{c}=?" for c in cols)
2899
  pages, changed = 0, 0
2900
  while True:
2901
  recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
@@ -2905,27 +2928,29 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal
2905
  if any(field not in rec for rec in recs for field in fields):
2906
  raise RuntimeError(f"{key}: column backfill response omitted a requested field")
2907
  rows = _flatten_records(recs, fields, named_m2m=spec.get("named_m2m"))
 
 
 
 
 
 
 
 
 
 
 
 
2908
  if repeat:
2909
- # UPDATE ... RETURNING counts actual changed mirror rows. Fetched,
2910
- # identical and absent rows must not trigger a product-store rebuild.
2911
- values = ','.join('(' + ','.join('?' for _ in range(len(cols) + 1)) + ')'
2912
- for _ in rows)
2913
- assignments = ', '.join(f'{c}=src.{c}' for c in cols)
2914
- differs = ' OR '.join(f'dest.{c} IS DISTINCT FROM src.{c}' for c in cols)
2915
- updated = con.execute(
2916
- f'UPDATE {key} AS dest SET {assignments} '
2917
- f'FROM (VALUES {values}) AS src(id, {", ".join(cols)}) '
2918
- f'WHERE dest.id=src.id AND ({differs}) RETURNING dest.id',
2919
- [value for row in rows for value in [row['id']] + [row.get(c) for c in cols]]
2920
- ).fetchall()
2921
  changed += len(updated)
2922
- else:
2923
- con.executemany(f"UPDATE {key} SET {setter} WHERE id=?",
2924
- [[r.get(c) for c in cols] + [r["id"]] for r in rows])
2925
  last, n = recs[-1]["id"], n + len(recs)
2926
  pages += 1
2927
  con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2928
  [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
 
 
 
 
 
2929
  log(f" {key}: {operation} {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
2930
  # β›” RETURN, NOT `break`: falling out of the loop writes 'done', and declaring completion
2931
  # a bounded pass never reached is the one failure this whole helper's checkpointing
@@ -2935,6 +2960,12 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=Fal
2935
  return changed
2936
  con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2937
  [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
 
 
 
 
 
 
2938
  log(f" {key}: {operation} complete -> {n:,} rows")
2939
  return changed
2940
 
 
2879
  if not fields:
2880
  return 0
2881
  marker = f"{key}.{'refresh' if repeat else 'cols'}.{','.join(sorted(fields))}"
2882
+ full_marker, remaining_progress = marker, []
2883
  operation = "column refresh" if repeat else "column backfill"
2884
  st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
2885
  [marker]).fetchone()
 
2887
  if not repeat:
2888
  return 0
2889
  st = None
2890
+ if not repeat:
2891
+ # A field addition does not invalidate completed migrations. Legacy combined markers
2892
+ # prove each field they name; unfinished supersets retain their existing ID checkpoint.
2893
+ prefix = f"{key}.cols."
2894
+ prior = con.execute("SELECT entity, phase, last_id, rows FROM _sync_state "
2895
+ "WHERE starts_with(entity, ?)", [prefix]).fetchall()
2896
+ completed = {field for name, phase, _last, _rows in prior if phase == "done"
2897
+ for field in name[len(prefix):].split(",")}
2898
+ fields = [field for field in fields if field not in completed]
2899
+ if not fields:
2900
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2901
+ [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
2902
+ return 0
2903
+ progress = {field: max(((last_id or 0, count or 0) for name, phase, last_id, count in prior
2904
+ if phase != "done" and field in name[len(prefix):].split(",")),
2905
+ default=(0, 0)) for field in fields}
2906
+ # Finish the furthest-ahead cohort first. A new field at zero must not rewind
2907
+ # another field's running migration; each cohort keeps its own durable marker.
2908
+ last_id, count = max(progress.values())
2909
+ remaining_progress = [point for point in progress.values() if point[0] != last_id]
2910
+ fields = [field for field in fields if progress[field][0] == last_id]
2911
+ marker = prefix + ','.join(sorted(fields))
2912
+ st = ("running", last_id, count)
2913
  # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
2914
  # an hour of work to one dropped connection gets skipped by the next person under time
2915
  # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
 
2919
  if last:
2920
  log(f" {key}: {operation} resuming at id {last:,} ({n:,} rows already done)")
2921
  cols = [c for f in fields for c in _field_cols(f, named_m2m=spec.get("named_m2m"))]
 
2922
  pages, changed = 0, 0
2923
  while True:
2924
  recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
 
2928
  if any(field not in rec for rec in recs for field in fields):
2929
  raise RuntimeError(f"{key}: column backfill response omitted a requested field")
2930
  rows = _flatten_records(recs, fields, named_m2m=spec.get("named_m2m"))
2931
+ # One set-based write per source page also bounds million-row migration elapsed time;
2932
+ # executemany paid a DuckDB transaction for each individual row.
2933
+ values = ','.join('(' + ','.join('?' for _ in range(len(cols) + 1)) + ')'
2934
+ for _ in rows)
2935
+ assignments = ', '.join(f'{c}=src.{c}' for c in cols)
2936
+ differs = ' OR '.join(f'dest.{c} IS DISTINCT FROM src.{c}' for c in cols)
2937
+ updated = con.execute(
2938
+ f'UPDATE {key} AS dest SET {assignments} '
2939
+ f'FROM (VALUES {values}) AS src(id, {", ".join(cols)}) '
2940
+ f'WHERE dest.id=src.id AND ({differs}) RETURNING dest.id',
2941
+ [value for row in rows for value in [row['id']] + [row.get(c) for c in cols]]
2942
+ ).fetchall()
2943
  if repeat:
 
 
 
 
 
 
 
 
 
 
 
 
2944
  changed += len(updated)
 
 
 
2945
  last, n = recs[-1]["id"], n + len(recs)
2946
  pages += 1
2947
  con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2948
  [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
2949
+ if full_marker != marker:
2950
+ full_last, full_count = min([*remaining_progress, (last, n)])
2951
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2952
+ [full_marker, "running", full_last, None, full_count,
2953
+ time.strftime("%Y-%m-%d %H:%M:%S")])
2954
  log(f" {key}: {operation} {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
2955
  # β›” RETURN, NOT `break`: falling out of the loop writes 'done', and declaring completion
2956
  # a bounded pass never reached is the one failure this whole helper's checkpointing
 
2960
  return changed
2961
  con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2962
  [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
2963
+ if full_marker != marker:
2964
+ phase = "running" if remaining_progress else "done"
2965
+ full_last, full_count = min(remaining_progress) if remaining_progress else (last, n)
2966
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
2967
+ [full_marker, phase, full_last, None, full_count,
2968
+ time.strftime("%Y-%m-%d %H:%M:%S")])
2969
  log(f" {key}: {operation} complete -> {n:,} rows")
2970
  return changed
2971
 
platform/harness/mirror_store.py CHANGED
@@ -420,6 +420,10 @@ class MirrorStore:
420
  def part_max_bytes(self):
421
  return MIRROR_PART_MAX_BYTES
422
 
 
 
 
 
423
  def _valid_table(self, name):
424
  return isinstance(name, str) and name in self.spec.tables
425
 
@@ -1463,6 +1467,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
1463
  installing = time.monotonic()
1464
  store._install_snapshot(published, manifest, store._state_rows(candidate), paths)
1465
  installed = True
 
1466
  except BaseException:
1467
  store.cache.pop(store._mirror_cache_key(tenant_key, rt), None)
1468
  raise
 
420
  def part_max_bytes(self):
421
  return MIRROR_PART_MAX_BYTES
422
 
423
+ def _after_publish(self, tenant_key, manifest):
424
+ """Connector notification for a new durable publication, never restore or candidate copy."""
425
+ pass
426
+
427
  def _valid_table(self, name):
428
  return isinstance(name, str) and name in self.spec.tables
429
 
 
1467
  installing = time.monotonic()
1468
  store._install_snapshot(published, manifest, store._state_rows(candidate), paths)
1469
  installed = True
1470
+ store._after_publish(tenant_key, manifest)
1471
  except BaseException:
1472
  store.cache.pop(store._mirror_cache_key(tenant_key, rt), None)
1473
  raise
platform/model/skills/odoo/overdue-receivables.md CHANGED
@@ -4,13 +4,12 @@ section: "playbooks"
4
  owner: "role:admin"
5
  approvers: ["role:admin"]
6
  approvals_needed: 1
7
- reads: ["ut_odoo_invoices"]
8
  tools: []
9
  description: "Odoo: How much customer debt is overdue today?"
10
- required_fields: {"ut_odoo_invoices":["odoo_id","move_type","residual","due_date","payment_state"]}
11
  metric: "ar_overdue"
12
- query: {"database":"ut_odoo_invoices","fields":["odoo_id","move_type","residual","due_date","payment_state"],"filters":{"conj":"and","nodes":[{"colId":"move_type","op":"isAnyOf","values":["out_invoice","out_refund"]},{"colId":"due_date","op":"lt","value":"$today"},{"colId":"payment_state","op":"neq","value":"paid"}]},"aggregates":[{"fn":"sum","field":"residual"}],"limit":1}
13
- source_check: {"model":"account.move","domain":[["company_id","in","$company_ids"],["state","=","posted"],["move_type","in",["out_invoice","out_refund"]],["invoice_date_due","<","$today"],["payment_state","!=","paid"]],"aggregate":"sum","field":"amount_residual_signed"}
14
  ---
15
  # How much customer debt is overdue today?
16
 
@@ -25,15 +24,14 @@ Do not store the answer's figures in this playbook.
25
 
26
  ## Query
27
  Use the registered ar_overdue metric. Resolve $today to the date used by that metric. It sums the residual of open trade receivable journal lines due strictly before $today, so credits and unapplied payments net in. A not_synced answer means its mirror fields are still syncing, so the metric is unavailable.
28
- The front-matter query reads invoice documents only. Use it only when the metric is unavailable, and say it leaves out unapplied payments.
29
- The front-matter query is a parameterized read request, never an instruction to change a database.
30
 
31
  ## Traps
32
  A due date equal to today is not overdue. A line with no due date is due on its entry date. Credits and unapplied payments reduce the balance. Today's residuals cannot reconstruct a past aging report.
33
 
34
  ## Check the answer
35
  For the metric, read-group account.move.line amount_residual:sum over posted, unreconciled trade receivable lines dated on or before $today and due before it. Require a zero rounded currency delta. Report the cutoff timezone and timestamp.
36
- The front-matter source_check ties the document fallback only.
37
  Use read-only source aggregates through the workspace's authorized source reader.
38
  Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
39
  An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
 
4
  owner: "role:admin"
5
  approvers: ["role:admin"]
6
  approvals_needed: 1
7
+ reads: []
8
  tools: []
9
  description: "Odoo: How much customer debt is overdue today?"
 
10
  metric: "ar_overdue"
11
+ query: {"topic":"gl_lines","measures":["ar_overdue"],"group":[],"filters":[]}
12
+ source_check: {"model":"account.move.line","domain":[["company_id","in","$company_ids"],["parent_state","=","posted"],["account_id.account_type","=","asset_receivable"],["account_id.non_trade","=",false],["reconciled","=",false],["date","<=","$today"],"|",["date_maturity","<","$today"],"&",["date_maturity","=",false],["date","<","$today"]],"aggregate":"sum","field":"amount_residual"}
13
  ---
14
  # How much customer debt is overdue today?
15
 
 
24
 
25
  ## Query
26
  Use the registered ar_overdue metric. Resolve $today to the date used by that metric. It sums the residual of open trade receivable journal lines due strictly before $today, so credits and unapplied payments net in. A not_synced answer means its mirror fields are still syncing, so the metric is unavailable.
27
+ The front-matter query uses the semantic metric door. If it is unavailable, report that state and wait for the backfill; invoice documents omit unapplied payments and cannot substitute for this metric.
 
28
 
29
  ## Traps
30
  A due date equal to today is not overdue. A line with no due date is due on its entry date. Credits and unapplied payments reduce the balance. Today's residuals cannot reconstruct a past aging report.
31
 
32
  ## Check the answer
33
  For the metric, read-group account.move.line amount_residual:sum over posted, unreconciled trade receivable lines dated on or before $today and due before it. Require a zero rounded currency delta. Report the cutoff timezone and timestamp.
34
+ The front-matter source_check uses the same line population and includes unapplied payments.
35
  Use read-only source aggregates through the workspace's authorized source reader.
36
  Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
37
  An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
platform/model/topics/odoo_invoices.yml CHANGED
@@ -43,8 +43,8 @@ store:
43
  #
44
  # β›” ONE KEY, AND IT IS THE VALIDATED ONE. `ar_outstanding` carries
45
  # `validate: {method: ar_outstanding}`, which reconciles the store's signed-residual sum against
46
- # the live Odoo aggregate over the same domain, to the cent. `ar_overdue` and `open_invoices`
47
- # resolve here too and are left out: neither declares an independent contract.
48
  #
49
  # ⚠ KEYING PROVEN: 31,418 groups on tenant #0, every one an `account.move.id` in the posted
50
  # customer-document population β€” the same integer `read_invoices` uses as a row's pid.
@@ -53,8 +53,11 @@ measures:
53
  dim: move
54
  keys: [ar_outstanding]
55
  not_yet:
56
- - key: ar_overdue / open_invoices
57
- cause: "no `validate:` contract, so neither ties to an independent Odoo aggregate (standing rule 8)"
 
 
 
58
  fix: "add the validate method + a `_v_*` reconciliation, then add the key here"
59
 
60
  # key / label / type / kind, derived from the grid contract. `kind` says where the
 
43
  #
44
  # β›” ONE KEY, AND IT IS THE VALIDATED ONE. `ar_outstanding` carries
45
  # `validate: {method: ar_outstanding}`, which reconciles the store's signed-residual sum against
46
+ # the live Odoo aggregate over the same domain, to the cent. `ar_overdue` uses journal-line
47
+ # grain and its own validation, so it cannot use this document topic's move dimension.
48
  #
49
  # ⚠ KEYING PROVEN: 31,418 groups on tenant #0, every one an `account.move.id` in the posted
50
  # customer-document population β€” the same integer `read_invoices` uses as a row's pid.
 
53
  dim: move
54
  keys: [ar_outstanding]
55
  not_yet:
56
+ - key: ar_overdue
57
+ cause: "line-grain metric on gl_lines; includes unapplied payments with no invoice document"
58
+ fix: "use the validated ar_overdue metric or customer Overdue field, not an invoice rollup"
59
+ - key: open_invoices
60
+ cause: "no `validate:` contract, so it does not tie to an independent Odoo aggregate (standing rule 8)"
61
  fix: "add the validate method + a `_v_*` reconciliation, then add the key here"
62
 
63
  # key / label / type / kind, derived from the grid contract. `kind` says where the
platform/modules/competitor_watch.py CHANGED
@@ -274,6 +274,27 @@ def terms_summary(tenant_key, *, st=None):
274
  "openAlerts": open_counts.get(t, 0)} for t in terms]}
275
 
276
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  # --------------------------------------------------------------------------- settings and owner
278
 
279
  #: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit).
 
274
  "openAlerts": open_counts.get(t, 0)} for t in terms]}
275
 
276
 
277
+ def term_page(tenant_key, *, st=None, search="", offset=0, limit=50, selected=None):
278
+ """Page the complete watched vocabulary without comparing every term on a screen read."""
279
+ st = _rt(tenant_key, st)
280
+ if not _doc(st, KEY_TERMS).get("seeded_at"):
281
+ seed_terms(tenant_key, st=st)
282
+ terms = watched_terms(tenant_key, st=st)
283
+ picked = _norm(selected or "")
284
+ picked = picked if picked in terms else None
285
+ needle = _norm(search or "")
286
+ matches = [term for term in terms if needle in term] if needle else terms
287
+ size = max(1, min(int(limit), 100))
288
+ start = max(0, int(offset))
289
+ if start >= len(matches):
290
+ start = max(0, (len(matches) - 1) // size * size)
291
+ page = matches[start:start + size]
292
+ error = _doc(st, KEY_WEEKS).get("last_error")
293
+ return {"terms": page, "term": picked or next(iter(page), None),
294
+ "termTotal": len(matches), "termOffset": start, "termLimit": size,
295
+ "problems": [str(error)] if error else []}
296
+
297
+
298
  # --------------------------------------------------------------------------- settings and owner
299
 
300
  #: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit).
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- ee51eb80c947243429e7e05e7d985de86dedd26caaed908fff1344d36f8217ed
 
1
+ b21cdba82c094292d4af5f6af53511bd5108795a7634589b0c65f0295116cbfe
web/src/apps/competitor-watch/CompetitorWatchView.tsx CHANGED
@@ -47,6 +47,9 @@ interface TrendSeries {
47
  interface TermScreen {
48
  canEdit: boolean;
49
  terms: string[];
 
 
 
50
  term: string | null;
51
  week: Week | null;
52
  previousWeek: Week | null;
@@ -96,6 +99,9 @@ function parseTermScreen(raw: unknown): TermScreen {
96
  return {
97
  canEdit: body.canEdit === true,
98
  terms: Array.isArray(body.terms) ? body.terms.map(text).filter(Boolean) : [],
 
 
 
99
  term: text(body.term) || null,
100
  week: parseWeek(body.week),
101
  previousWeek: parseWeek(body.previousWeek),
@@ -282,24 +288,40 @@ function useAppQuery(appId: string) {
282
  function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean }) {
283
  const url = useAppQuery(appId);
284
  const [term, setTerm] = useState("");
 
 
 
 
285
  const [state, setState] = useState<Load<TermScreen>>({ phase: "loading" });
286
  const [attempt, setAttempt] = useState(0);
287
  const [draft, setDraft] = useState("");
288
  const [termError, setTermError] = useState("");
289
  const [busy, setBusy] = useState(false);
290
 
 
 
 
 
 
291
  useEffect(() => {
292
  const controller = new AbortController();
293
  setState({ phase: "loading" });
294
- void fetch(url("/screen", term ? { term } : {}), { credentials: CREDENTIALS, signal: controller.signal })
 
 
295
  .then((response) => readJson(response, "The competitor watch could not be loaded."))
296
- .then((body) => { if (!controller.signal.aborted) setState({ phase: "ready", value: parseTermScreen(body) }); })
 
 
 
 
 
297
  .catch((reason: unknown) => {
298
  if (!controller.signal.aborted) setState({ phase: "error",
299
  message: reason instanceof Error ? reason.message : "The competitor watch could not be loaded." });
300
  });
301
  return () => controller.abort();
302
- }, [url, term, attempt]);
303
 
304
  const screen = state.phase === "ready" ? state.value : null;
305
  const canEdit = canEditApp && screen?.canEdit === true;
@@ -331,14 +353,28 @@ function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean }
331
 
332
  return <div className="cw-view">
333
  <div className="cw-filters" role="group" aria-label="Term and week">
 
 
 
 
 
334
  <label className="cw-field">
335
  <span className="cw-field-label">Search term</span>
336
  <select className="cw-select" value={selected} disabled={!screen || !screen.terms.length}
337
  onChange={(event) => setTerm(event.target.value)}>
338
- {!screen?.terms.length && <option value="">No watched terms</option>}
 
339
  {screen?.terms.map((option) => <option key={option} value={option}>{option}</option>)}
340
  </select>
341
  </label>
 
 
 
 
 
 
 
 
342
  {screen?.week && <span className="cw-period">
343
  Week of {weekLabel(screen.week)}{screen.previousWeek ? ` against ${weekLabel(screen.previousWeek)}` : ""}
344
  </span>}
 
47
  interface TermScreen {
48
  canEdit: boolean;
49
  terms: string[];
50
+ termTotal: number;
51
+ termOffset: number;
52
+ termLimit: number;
53
  term: string | null;
54
  week: Week | null;
55
  previousWeek: Week | null;
 
99
  return {
100
  canEdit: body.canEdit === true,
101
  terms: Array.isArray(body.terms) ? body.terms.map(text).filter(Boolean) : [],
102
+ termTotal: num(body.termTotal) ?? 0,
103
+ termOffset: num(body.termOffset) ?? 0,
104
+ termLimit: num(body.termLimit) ?? 50,
105
  term: text(body.term) || null,
106
  week: parseWeek(body.week),
107
  previousWeek: parseWeek(body.previousWeek),
 
288
  function TermsView({ appId, canEditApp }: { appId: string; canEditApp: boolean }) {
289
  const url = useAppQuery(appId);
290
  const [term, setTerm] = useState("");
291
+ const displayedTerm = useRef("");
292
+ const [search, setSearch] = useState("");
293
+ const [query, setQuery] = useState("");
294
+ const [offset, setOffset] = useState(0);
295
  const [state, setState] = useState<Load<TermScreen>>({ phase: "loading" });
296
  const [attempt, setAttempt] = useState(0);
297
  const [draft, setDraft] = useState("");
298
  const [termError, setTermError] = useState("");
299
  const [busy, setBusy] = useState(false);
300
 
301
+ useEffect(() => {
302
+ const timer = setTimeout(() => { setQuery(search); setOffset(0); }, 200);
303
+ return () => clearTimeout(timer);
304
+ }, [search]);
305
+
306
  useEffect(() => {
307
  const controller = new AbortController();
308
  setState({ phase: "loading" });
309
+ const requestedTerm = term || displayedTerm.current;
310
+ void fetch(url("/screen", { ...(requestedTerm ? { term: requestedTerm } : {}), q: query, offset: String(offset), limit: "50" }),
311
+ { credentials: CREDENTIALS, signal: controller.signal })
312
  .then((response) => readJson(response, "The competitor watch could not be loaded."))
313
+ .then((body) => {
314
+ if (controller.signal.aborted) return;
315
+ const value = parseTermScreen(body);
316
+ displayedTerm.current = value.term ?? "";
317
+ setState({ phase: "ready", value });
318
+ })
319
  .catch((reason: unknown) => {
320
  if (!controller.signal.aborted) setState({ phase: "error",
321
  message: reason instanceof Error ? reason.message : "The competitor watch could not be loaded." });
322
  });
323
  return () => controller.abort();
324
+ }, [url, term, query, offset, attempt]);
325
 
326
  const screen = state.phase === "ready" ? state.value : null;
327
  const canEdit = canEditApp && screen?.canEdit === true;
 
353
 
354
  return <div className="cw-view">
355
  <div className="cw-filters" role="group" aria-label="Term and week">
356
+ <label className="cw-field">
357
+ <span className="cw-field-label">Find terms</span>
358
+ <input className="cw-input" type="search" value={search}
359
+ onChange={(event) => setSearch(event.target.value)} placeholder="Search watched terms" />
360
+ </label>
361
  <label className="cw-field">
362
  <span className="cw-field-label">Search term</span>
363
  <select className="cw-select" value={selected} disabled={!screen || !screen.terms.length}
364
  onChange={(event) => setTerm(event.target.value)}>
365
+ {!screen?.terms.length && <option value="">{query ? "No matching terms" : "No watched terms"}</option>}
366
+ {selected && !screen?.terms.includes(selected) && <option value={selected}>{selected}</option>}
367
  {screen?.terms.map((option) => <option key={option} value={option}>{option}</option>)}
368
  </select>
369
  </label>
370
+ <div className="cw-pager" aria-label="Term pages">
371
+ <button className="cw-button is-quiet" type="button" disabled={!screen || screen.termOffset === 0}
372
+ onClick={() => setOffset(Math.max(0, screen!.termOffset - screen!.termLimit))}>Previous</button>
373
+ <span className="cw-period" role="status">{screen ? `${screen.termTotal ? screen.termOffset + 1 : 0} to ${screen.termOffset + screen.terms.length} of ${screen.termTotal.toLocaleString()}` : "Loading"}</span>
374
+ <button className="cw-button is-quiet" type="button"
375
+ disabled={!screen || screen.termOffset + screen.terms.length >= screen.termTotal}
376
+ onClick={() => setOffset(screen!.termOffset + screen!.termLimit)}>Next</button>
377
+ </div>
378
  {screen?.week && <span className="cw-period">
379
  Week of {weekLabel(screen.week)}{screen.previousWeek ? ` against ${weekLabel(screen.previousWeek)}` : ""}
380
  </span>}
web/src/sheet/SheetGrid.tsx CHANGED
@@ -163,7 +163,7 @@ import { holdsTwoOrMore, selectionRects, selectionStatsSteps } from "./sheetSele
163
  import type { SelectionStats } from "./sheetSelectionStats";
164
  import { colToIndex, columnRangeA1, indexToCol, rowRangeA1, shiftForStructure } from "./a1";
165
  import {
166
- fetchViewRows, viewBlockGeometry, viewChromeRefusal, viewHeaderAt, viewRefreshOp, writeViewCell,
167
  } from "./viewPlacement";
168
  import type { FormulaBarProps } from "./FormulaBar";
169
  // β›”β›” W51-T18 β€” THE RULE ENGINE AND THE VALIDATION CHECKER ARE PURE AND SHEET LOCAL, and that is
@@ -1229,6 +1229,7 @@ export default function SheetGrid({
1229
  const viewContext = useRef({ storedWorkbook, storedTab, onStructure, onNotice });
1230
  viewContext.current = { storedWorkbook, storedTab, onStructure, onNotice };
1231
  const viewWritePending = useRef(false);
 
1232
  const [viewWriteRefresh, setViewWriteRefresh] = useState(0);
1233
  useEffect(() => {
1234
  const state = viewRefresh.current;
@@ -1251,7 +1252,7 @@ export default function SheetGrid({
1251
  && (cell.binding.viewType === "grid" || !cell.binding.block))
1252
  .map(([addr]) => addr);
1253
  if (!anchors.length) { state.generation++; state.key = ""; return; }
1254
- const key = JSON.stringify([storedWorkbook.sheets, storedWorkbook.namedRanges]);
1255
  // History restores the original sheets array. Do not immediately redo an undone push.
1256
  if (!refreshed && (key === state.key || state.seen.has(storedWorkbook.sheets))) {
1257
  if (key !== state.key) state.generation++; // Undo also cancels a pending resolver answer.
@@ -1274,7 +1275,7 @@ export default function SheetGrid({
1274
  const cell = (target.cells ?? {})[origin];
1275
  const at = splitAddress(origin);
1276
  if (!at || cell?.kind !== "bound" || cell.binding?.mode !== "view") continue;
1277
- const got = await fetchViewRows(projected.id, cell.binding as ViewBinding);
1278
  if (!current()) return;
1279
  if (!got.ok) { report(got.message); continue; }
1280
  const op = viewRefreshOp(target, origin, got.answer);
@@ -1292,7 +1293,7 @@ export default function SheetGrid({
1292
  if (!op.delta && !op.place!.also?.delta && Object.entries(cells).every(([addr, value]) =>
1293
  JSON.stringify((target.cells ?? {})[addr] ?? null) === JSON.stringify(value))) continue;
1294
  projected = applyStructureWorkbook(projected, target.id, op);
1295
- state.key = JSON.stringify([projected.sheets, projected.namedRanges]);
1296
  // The prop returns the page's async structure operation at runtime. Awaiting it keeps
1297
  // hydration/commit ordering intact without adding another history or save door.
1298
  await viewContext.current.onStructure(op);
@@ -2533,7 +2534,7 @@ export default function SheetGrid({
2533
  const projected = applyStructureWorkbook(base, target.id, op);
2534
  const state = viewRefresh.current;
2535
  state.seen.add(base.sheets);
2536
- state.key = JSON.stringify([projected.sheets, projected.namedRanges]);
2537
  await latest.onStructure(op);
2538
  return true;
2539
  }, []);
 
163
  import type { SelectionStats } from "./sheetSelectionStats";
164
  import { colToIndex, columnRangeA1, indexToCol, rowRangeA1, shiftForStructure } from "./a1";
165
  import {
166
+ createViewRowsReader, fetchViewRows, viewBlockGeometry, viewChromeRefusal, viewHeaderAt, viewReadKey, viewRefreshOp, writeViewCell,
167
  } from "./viewPlacement";
168
  import type { FormulaBarProps } from "./FormulaBar";
169
  // β›”β›” W51-T18 β€” THE RULE ENGINE AND THE VALIDATION CHECKER ARE PURE AND SHEET LOCAL, and that is
 
1229
  const viewContext = useRef({ storedWorkbook, storedTab, onStructure, onNotice });
1230
  viewContext.current = { storedWorkbook, storedTab, onStructure, onNotice };
1231
  const viewWritePending = useRef(false);
1232
+ const readViewRows = useRef(createViewRowsReader());
1233
  const [viewWriteRefresh, setViewWriteRefresh] = useState(0);
1234
  useEffect(() => {
1235
  const state = viewRefresh.current;
 
1252
  && (cell.binding.viewType === "grid" || !cell.binding.block))
1253
  .map(([addr]) => addr);
1254
  if (!anchors.length) { state.generation++; state.key = ""; return; }
1255
+ const key = viewReadKey(storedTab);
1256
  // History restores the original sheets array. Do not immediately redo an undone push.
1257
  if (!refreshed && (key === state.key || state.seen.has(storedWorkbook.sheets))) {
1258
  if (key !== state.key) state.generation++; // Undo also cancels a pending resolver answer.
 
1275
  const cell = (target.cells ?? {})[origin];
1276
  const at = splitAddress(origin);
1277
  if (!at || cell?.kind !== "bound" || cell.binding?.mode !== "view") continue;
1278
+ const got = await readViewRows.current(projected.id, cell.binding as ViewBinding);
1279
  if (!current()) return;
1280
  if (!got.ok) { report(got.message); continue; }
1281
  const op = viewRefreshOp(target, origin, got.answer);
 
1293
  if (!op.delta && !op.place!.also?.delta && Object.entries(cells).every(([addr, value]) =>
1294
  JSON.stringify((target.cells ?? {})[addr] ?? null) === JSON.stringify(value))) continue;
1295
  projected = applyStructureWorkbook(projected, target.id, op);
1296
+ state.key = viewReadKey(projected.sheets.find(sheet => sheet.id === storedTab.id)!);
1297
  // The prop returns the page's async structure operation at runtime. Awaiting it keeps
1298
  // hydration/commit ordering intact without adding another history or save door.
1299
  await viewContext.current.onStructure(op);
 
2534
  const projected = applyStructureWorkbook(base, target.id, op);
2535
  const state = viewRefresh.current;
2536
  state.seen.add(base.sheets);
2537
+ state.key = viewReadKey(projected.sheets.find(sheet => sheet.id === target.id)!);
2538
  await latest.onStructure(op);
2539
  return true;
2540
  }, []);
web/src/sheet/viewPlacement.ts CHANGED
@@ -13,6 +13,26 @@ import { fetchNav } from "../shell/nav";
13
  /** Record identity travels with the displayed row, never inferred from its sheet position. */
14
  type ViewRecordChild = NonNullable<Cell["viewChild"]> & { pid?: number };
15
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  export const viewChromeRefusal = (cell: Cell | null | undefined): string | null => {
17
  if (cell?.viewChild?.line === -2) return "The Grid count row cannot be edited.";
18
  if (cell?.viewChild?.line === -1 || (cell?.kind === "bound"
 
13
  /** Record identity travels with the displayed row, never inferred from its sheet position. */
14
  type ViewRecordChild = NonNullable<Cell["viewChild"]> & { pid?: number };
15
 
16
+ /** Only view definitions invalidate automatic reads; hydrated answers and formulas do not. */
17
+ export const viewReadKey = (tab: SheetTab): string => JSON.stringify(
18
+ Object.entries(tab.cells ?? {}).filter(([, cell]) => cell.kind === "bound"
19
+ && cell.binding?.mode === "view" && (cell.binding.viewType === "grid" || !cell.binding.block))
20
+ .map(([addr, cell]) => [addr, cell.binding]));
21
+
22
+ /** An instance belongs to one mounted grid; settled reads and write refreshes are never cached. */
23
+ export const createViewRowsReader = () => {
24
+ const pending = new Map<string, ReturnType<typeof fetchViewRows>>();
25
+ return (workbookId: string, binding: ViewBinding) => {
26
+ const key = JSON.stringify([workbookId, binding]);
27
+ let request = pending.get(key);
28
+ if (!request) {
29
+ request = fetchViewRows(workbookId, binding).finally(() => { pending.delete(key); });
30
+ pending.set(key, request);
31
+ }
32
+ return request;
33
+ };
34
+ };
35
+
36
  export const viewChromeRefusal = (cell: Cell | null | undefined): string | null => {
37
  if (cell?.viewChild?.line === -2) return "The Grid count row cannot be edited.";
38
  if (cell?.viewChild?.line === -1 || (cell?.kind === "bound"
web/src/styles/sdlc78/competitor-watch.css CHANGED
@@ -62,6 +62,7 @@
62
  font-variant-numeric: tabular-nums;
63
  }
64
  .cw-term-form { display: inline-flex; flex-wrap: wrap; gap: 8px; margin-left: auto; }
 
65
 
66
  .cw-select,
67
  .cw-input {
@@ -94,7 +95,7 @@
94
  }
95
  .cw-button.is-quiet { background: transparent; color: var(--lp-muted); font-weight: 400; }
96
  .cw-button.is-quiet:hover { background: var(--lp-surface-2); color: var(--lp-ink); }
97
- .cw-button:disabled { opacity: 0.5; cursor: default; }
98
  .cw-link {
99
  border: 0;
100
  background: transparent;
 
62
  font-variant-numeric: tabular-nums;
63
  }
64
  .cw-term-form { display: inline-flex; flex-wrap: wrap; gap: 8px; margin-left: auto; }
65
+ .cw-pager { display: inline-flex; align-items: center; gap: 4px; flex-wrap: wrap; }
66
 
67
  .cw-select,
68
  .cw-input {
 
95
  }
96
  .cw-button.is-quiet { background: transparent; color: var(--lp-muted); font-weight: 400; }
97
  .cw-button.is-quiet:hover { background: var(--lp-surface-2); color: var(--lp-ink); }
98
+ .cw-button:disabled { opacity: 0.5; cursor: not-allowed; }
99
  .cw-link {
100
  border: 0;
101
  background: transparent;