fsanyoto commited on
Commit
a60104f
·
verified ·
1 Parent(s): 4e84128

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "ed6e607f3",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "f76135962",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- ed6e607f3
 
1
+ f76135962
api/main.py CHANGED
@@ -2267,8 +2267,10 @@ def _resync_mirror_pass(manual, passes, entities=None):
2267
  print(f"[aios-api] store resync: the absent tables first failed "
2268
  f"({type(exc).__name__}); the full pass below still syncs them")
2269
  _resync_log("full pass start")
 
2270
  sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
2271
- entities=entities, log=_resync_log)
 
2272
  _log_entities("full", sync_report)
2273
  for key, row in first.items():
2274
  if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
@@ -2463,20 +2465,69 @@ def _resync_confirm_fresh(tenant, cadence_seconds):
2463
  print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
2464
 
2465
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2466
  def _capture_resync_source_counts(tenant):
2467
  """Count the default connection's Odoo source right after its mirror pull, or None.
2468
 
2469
  The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
2470
- time describes a later source than the mirror holds (D-9131). Never raises.
 
2471
  """
2472
  try:
2473
  if not tenant:
2474
  return None
2475
  import odoo_relational as _rel_counts
2476
  from harness import runtime as _runtime_counts
2477
- counts = _rel_counts.capture_source_counts(_runtime_counts.get_runtime(tenant))
 
 
 
 
 
 
 
 
2478
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2479
- f"database(s) at mirror-pull time", flush=True)
2480
  return counts or None
2481
  except Exception as e: # noqa: BLE001
2482
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
 
2267
  print(f"[aios-api] store resync: the absent tables first failed "
2268
  f"({type(exc).__name__}); the full pass below still syncs them")
2269
  _resync_log("full pass start")
2270
+ _after_entity = _entity_source_counter(_ds)
2271
  sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
2272
+ entities=entities, log=_resync_log,
2273
+ **({"after_entity": _after_entity} if _after_entity else {}))
2274
  _log_entities("full", sync_report)
2275
  for key, row in first.items():
2276
  if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
 
2465
  print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
2466
 
2467
 
2468
+ #: tenant -> {bucket: (count, mode)} counted beside each entity's copy during the last mirror pass.
2469
+ _PASS_SOURCE_COUNTS = {}
2470
+
2471
+
2472
+ def _entity_source_counter(ds):
2473
+ """A `sync_all` hook that counts an entity's source buckets right after its copy, or None.
2474
+
2475
+ None when `sync_all` takes no `after_entity` hook: the loop then counts once after the pass.
2476
+ """
2477
+ import inspect as _inspect_counts
2478
+ try:
2479
+ if "after_entity" not in _inspect_counts.signature(ds.sync_all).parameters:
2480
+ return None
2481
+ rt = ds._resident_odoo_runtime()
2482
+ except Exception: # noqa: BLE001
2483
+ return None
2484
+ tenant = getattr(rt, "key", None)
2485
+ if not tenant:
2486
+ return None
2487
+ counted = _PASS_SOURCE_COUNTS[tenant] = {}
2488
+
2489
+ def after_entity(key):
2490
+ # A volatile table (stock_quant) drifts between its copy and a count at the end of the pass.
2491
+ try:
2492
+ import odoo_relational as _rel_counts
2493
+ model = (ds.ENTITIES.get(key) or {}).get("model")
2494
+ buckets = _rel_counts.source_count_buckets(model) - set(counted)
2495
+ if not buckets:
2496
+ return
2497
+ got = _rel_counts.capture_source_counts(rt, buckets=buckets)
2498
+ counted.update(got)
2499
+ for bucket, (count, _mode) in sorted(got.items()):
2500
+ print(f"[aios-api] store resync: counted the Odoo source for {bucket} "
2501
+ f"beside the {key} copy: {count:,} rows", flush=True)
2502
+ except Exception as e: # noqa: BLE001
2503
+ print(f"[aios-api] store resync: the {key} source was not counted beside its copy "
2504
+ f"({type(e).__name__}); it is counted after the pass", flush=True)
2505
+ return after_entity
2506
+
2507
+
2508
  def _capture_resync_source_counts(tenant):
2509
  """Count the default connection's Odoo source right after its mirror pull, or None.
2510
 
2511
  The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
2512
+ time describes a later source than the mirror holds (D-9131). A bucket counted beside its
2513
+ entity's copy during the pass keeps that count. Never raises.
2514
  """
2515
  try:
2516
  if not tenant:
2517
  return None
2518
  import odoo_relational as _rel_counts
2519
  from harness import runtime as _runtime_counts
2520
+ during = dict((globals().get("_PASS_SOURCE_COUNTS") or {}).pop(tenant, None) or {})
2521
+ rt = _runtime_counts.get_runtime(tenant)
2522
+ counts = dict(during)
2523
+ if not during:
2524
+ counts.update(_rel_counts.capture_source_counts(rt))
2525
+ else:
2526
+ rest = _rel_counts.enabled_buckets(rt) - set(during)
2527
+ if rest:
2528
+ counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
2529
  print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
2530
+ f"database(s) at mirror-pull time, {len(during)} beside their own copy", flush=True)
2531
  return counts or None
2532
  except Exception as e: # noqa: BLE001
2533
  print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
api/odoo_relational.py CHANGED
@@ -3914,12 +3914,18 @@ def _source_domain_count(client, bucket):
3914
  raise Refused("the database source-count mode is not supported")
3915
 
3916
 
3917
- def capture_source_counts(rt, connection_id=None):
 
 
 
 
 
3918
  """`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
3919
 
3920
  The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
3921
- with the source as of the same moment. A bucket whose count raises is left out, and that
3922
- bucket counts fresh at reconciliation. No source or no client returns `{}`.
 
3923
  """
3924
  try:
3925
  source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
@@ -3936,7 +3942,10 @@ def capture_source_counts(rt, connection_id=None):
3936
  return {}
3937
  if client is None:
3938
  return {}
3939
- for bucket in sorted(enabled_buckets(rt, connection_id)):
 
 
 
3940
  try:
3941
  counts[bucket] = _source_domain_count(client, bucket)
3942
  except Exception: # noqa: BLE001
 
3914
  raise Refused("the database source-count mode is not supported")
3915
 
3916
 
3917
+ def source_count_buckets(model):
3918
+ """The curated buckets whose source population is counted on one Odoo model."""
3919
+ return {bucket for bucket, spec in _SOURCE_COUNT_SPECS.items() if spec["model"] == model}
3920
+
3921
+
3922
+ def capture_source_counts(rt, connection_id=None, buckets=None):
3923
  """`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
3924
 
3925
  The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
3926
+ with the source as of the same moment. `buckets` limits the count to those buckets. A bucket
3927
+ whose count raises is left out, and that bucket counts fresh at reconciliation. No source or
3928
+ no client returns `{}`.
3929
  """
3930
  try:
3931
  source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
 
3942
  return {}
3943
  if client is None:
3944
  return {}
3945
+ chosen = enabled_buckets(rt, connection_id)
3946
+ if buckets is not None:
3947
+ chosen = chosen & set(buckets)
3948
+ for bucket in sorted(chosen):
3949
  try:
3950
  counts[bucket] = _source_domain_count(client, bucket)
3951
  except Exception: # noqa: BLE001
api/routes_nav.py CHANGED
@@ -459,11 +459,39 @@ def _visible_database_keys(session):
459
  return keys, enumerated
460
 
461
 
462
- def _granted_view_sources(session):
463
- """`{view_id: owner}` for grants this caller may read, from one registry read.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
464
 
465
  `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
466
  source lookup for that case; modern grants point directly at the canonical personal stratum.
 
 
467
  """
468
  import core.shares as shares
469
  try:
@@ -476,9 +504,46 @@ def _granted_view_sources(session):
476
  # add a View to "shared with me"; a valid named or everyone grant did.
477
  if shares.role_from(record, session.uname) is not None:
478
  out[str(raw_id)] = owner
479
- return out
480
  except Exception:
481
  return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
482
 
483
 
484
  def _granted_view_ids(session):
 
459
  return keys, enumerated
460
 
461
 
462
+ def _folder_views_in(doc, folder_id, owner=None):
463
+ """`(owner, [view_id])` for the VIEWS folder `folder_id` in one held workspace doc, else None.
464
+
465
+ The same lookup as `TableStore.find_folder`, over a projection the caller already read.
466
+ """
467
+ import core.table_store as table_store
468
+ fid = str(folder_id or "").strip()
469
+ if not fid or not isinstance(doc, dict):
470
+ return None
471
+ for who, ws in doc.items():
472
+ if who in table_store.RESERVED_MEMBERS or not isinstance(ws, dict):
473
+ continue
474
+ if owner and str(who).strip().lower() != owner:
475
+ continue
476
+ rows = (ws.get("folders") or {}).get("views") or []
477
+ if not any(isinstance(f, dict) and str(f.get("id") or "") == fid for f in rows):
478
+ continue
479
+ # `itemFolders` is keyed by surface first: the views map sits under 'views'.
480
+ placed = (ws.get("itemFolders") or {}).get("views") or {}
481
+ placed = placed if isinstance(placed, dict) else {}
482
+ views = ws.get("views") or {}
483
+ return str(who), [str(vid) for vid, view in views.items()
484
+ if isinstance(view, dict) and str(placed.get(str(vid)) or "") == fid]
485
+ return None
486
+
487
+
488
+ def _granted_view_sources(session, docs=None):
489
+ """`{view_id: owner}` for grants this caller may read, from one registry read per kind.
490
 
491
  `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
492
  source lookup for that case; modern grants point directly at the canonical personal stratum.
493
+ `docs` is `{database: projected workspace doc}` the caller already holds; the folder leg
494
+ reads views from it, else from `TableStore.find_folder` per visible database.
495
  """
496
  import core.shares as shares
497
  try:
 
504
  # add a View to "shared with me"; a valid named or everyone grant did.
505
  if shares.role_from(record, session.uname) is not None:
506
  out[str(raw_id)] = owner
 
507
  except Exception:
508
  return {}
509
+ # Folder leg (the `grid_events._granted_views` D-37 rule): a View filed in a folder shared
510
+ # with the caller is shared with the caller. Direct grants win, so this leg only fills gaps.
511
+ # A failure here drops the folder leg alone and keeps the direct answer: narrower, never wider.
512
+ try:
513
+ folders = {}
514
+ for raw_fid, record in shares.grants_bulk("folder", st=session.runtime).items():
515
+ owner = str(record.get("owner") or "").strip().lower() or None
516
+ if owner == session.uname:
517
+ continue
518
+ if shares.role_from(record, session.uname) in ("view", "edit"):
519
+ folders[str(raw_fid)] = owner
520
+ if not folders:
521
+ return out
522
+ hits = []
523
+ if docs is not None:
524
+ for doc in docs.values():
525
+ for fid, owner in folders.items():
526
+ hits.append(_folder_views_in(doc, fid, owner))
527
+ else:
528
+ import core.table_store as table_store
529
+ import core.view_templates as view_templates
530
+ keys, _ = _visible_database_keys(session)
531
+ for key in sorted(keys):
532
+ ws_key = view_templates.workspace_key(key)
533
+ if not ws_key:
534
+ continue
535
+ tops = table_store.make(ws_key, st=session.runtime)
536
+ for fid in folders:
537
+ found = tops.find_folder(fid)
538
+ hits.append((found[0], list(found[2])) if found else None)
539
+ for hit in hits:
540
+ if not hit or str(hit[0]).strip().lower() == session.uname:
541
+ continue
542
+ for vid in hit[1]:
543
+ out.setdefault(str(vid), hit[0])
544
+ except Exception:
545
+ pass
546
+ return out
547
 
548
 
549
  def _granted_view_ids(session):
api/routes_products.py CHANGED
@@ -39,6 +39,8 @@ router = APIRouter(prefix="/api/v1")
39
  MODULE = "product_data"
40
 
41
  _CACHE_TTL = 900
 
 
42
 
43
 
44
  def _pool_for(rt, team_id):
@@ -105,14 +107,25 @@ def _pool_for(rt, team_id):
105
  # keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
106
  # branch on the overwhelmingly common/default-off path.
107
  from harness import canonical_source, runtime as _runtime
 
108
  # Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
109
  with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
110
  rows = (pd.pool(team_id=team_id,
111
  raw_fields=[field.get("rawName") for field in enabled_raw])
112
  if enabled_raw else pd.pool(team_id=team_id))
113
- return _project_enabled_raw_rows(rows, enabled_raw, drop_source=True)
114
-
115
- return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
 
 
 
 
 
 
 
 
 
 
116
 
117
 
118
  def scoped_pool(session: Session, *, with_rows: bool = True):
 
39
  MODULE = "product_data"
40
 
41
  _CACHE_TTL = 900
42
+ #: A product pool request slower than this logs its wait, so a cold scope is measurable on staging.
43
+ _SLOW_POOL_S = 5.0
44
 
45
 
46
  def _pool_for(rt, team_id):
 
107
  # keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
108
  # branch on the overwhelmingly common/default-off path.
109
  from harness import canonical_source, runtime as _runtime
110
+ started = time.perf_counter()
111
  # Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
112
  with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
113
  rows = (pd.pool(team_id=team_id,
114
  raw_fields=[field.get("rawName") for field in enabled_raw])
115
  if enabled_raw else pd.pool(team_id=team_id))
116
+ projected = _project_enabled_raw_rows(rows, enabled_raw, drop_source=True)
117
+ # The cold path's own receipt in the Space log: the build alone, apart from any wait.
118
+ print(f"[aios-api] product pool built team={team_id} rows={len(projected)} "
119
+ f"in {time.perf_counter() - started:.1f}s", flush=True)
120
+ return projected
121
+
122
+ asked = time.perf_counter()
123
+ served = scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
124
+ waited = time.perf_counter() - asked
125
+ if waited >= _SLOW_POOL_S:
126
+ # A wait far past the build line above is another build of this scope holding the lock.
127
+ print(f"[aios-api] product pool served team={team_id} after {waited:.1f}s", flush=True)
128
+ return served
129
 
130
 
131
  def scoped_pool(session: Session, *, with_rows: bool = True):
api/routes_sheet.py CHANGED
@@ -207,6 +207,119 @@ def _binding_catching_up(session, binding):
207
  return any(_database_catching_up(session, target) for target in targets)
208
 
209
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
210
  def _mirror(session: Session):
211
  """`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
212
 
@@ -278,7 +391,7 @@ def _mirror(session: Session):
278
  session._sheet_amzn_cursor = (cur, None)
279
  else:
280
  cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
281
- or rot._mirror_cur(session))
282
  except RuntimeError as exc:
283
  raise err(503, "store_not_ready", str(exc))
284
  if mismatch:
@@ -350,6 +463,10 @@ def _mirror(session: Session):
350
  session._sheet_mirrors = memo
351
  slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
352
  want = None if needed is None else {str(k) for k in needed}
 
 
 
 
353
  entry = memo.get(slot)
354
  if entry is not None:
355
  built, made = entry
@@ -357,10 +474,10 @@ def _mirror(session: Session):
357
  return made
358
  if want is not None:
359
  want = set(want) | built
360
- made = _registry_mirror(session, cur, table_key, metric_window, needed=want)
361
  memo[slot] = (None if want is None else set(want), made)
362
  return made
363
- spec = rot._source_for(cur, table_key)
364
  if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
365
  spec = _amazon_stored_spec(cur, table_key)
366
  if spec is None:
@@ -4449,6 +4566,7 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
4449
  return hit["cell"]
4450
  return None
4451
 
 
4452
  started = perf_counter()
4453
  resolved_count = 0
4454
  held_refusal = None
@@ -5564,6 +5682,7 @@ _BINDING_REFUSAL_DISPLAY = {
5564
  "store_not_ready": "#UPDATING",
5565
  "mirror_not_ready": "#UPDATING",
5566
  "amazon_mirror_not_loaded": "#UPDATING",
 
5567
  }
5568
 
5569
 
@@ -5572,6 +5691,7 @@ _TRANSIENT_REFUSAL_CODES = frozenset((
5572
  "store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
5573
  "mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
5574
  "sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
 
5575
  ))
5576
 
5577
  #: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
@@ -6016,7 +6136,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
6016
  # and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
6017
  _refuse_unapplied_audience(session, database)
6018
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6019
- if perm_scope.row_scope_applies(session.user, database) else None)
6020
  wall_nodes, wall_conj = _fe_parts(wall_tree)
6021
  scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
6022
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
 
207
  return any(_database_catching_up(session, target) for target in targets)
208
 
209
 
210
+ #: Builds of one registry mirror per cell. A source write during a build costs a rebuild.
211
+ _MIRROR_BUILD_TRIES = 3
212
+
213
+
214
+ def _request_memo(session):
215
+ """The Sheet request memo, or None outside a request scope or inside an isolated drill."""
216
+ memo = getattr(session, "_sheet_read_memo", None)
217
+ if not isinstance(memo, dict) or getattr(session, "_sheet_drill_isolated", False):
218
+ return None
219
+ return memo
220
+
221
+
222
+ def _request_mirror_cur(session):
223
+ """One gated mirror cursor per Sheet request and mirror install generation."""
224
+ import routes_odoo_tables as rot
225
+ memo = _request_memo(session)
226
+ generation = _install_generation() if memo is not None else None
227
+ if generation is None:
228
+ return rot._mirror_cur(session)
229
+ hit = memo.get("mirror_cur")
230
+ if hit is not None and hit[0] == generation:
231
+ return hit[1]
232
+ got = rot._mirror_cur(session)
233
+ if got[1] is None:
234
+ memo["mirror_cur"] = (generation, got)
235
+ return got
236
+
237
+
238
+ def _request_source_for(session, cur, table_key):
239
+ """`rot._source_for` once per Sheet request, cursor and database. Its catalog reads cost per call."""
240
+ import routes_odoo_tables as rot
241
+ memo = _request_memo(session)
242
+ if memo is None:
243
+ return rot._source_for(cur, table_key)
244
+ slot = ("source_for", table_key)
245
+ hit = memo.get(slot)
246
+ if hit is not None and hit[0] is cur:
247
+ return hit[1]
248
+ spec = rot._source_for(cur, table_key)
249
+ if spec is not None:
250
+ memo[slot] = (cur, spec)
251
+ return spec
252
+
253
+
254
+ def _row_scope_applies(session, database):
255
+ """`perm_scope.row_scope_applies` once per Sheet request: its storeless form reads a revision."""
256
+ import core.perm_scope as perm_scope
257
+ memo = _request_memo(session)
258
+ slot = ("row_scope_applies", database)
259
+ if memo is not None and slot in memo:
260
+ return memo[slot]
261
+ applies = perm_scope.row_scope_applies(session.user, database)
262
+ if memo is not None:
263
+ memo[slot] = applies
264
+ return applies
265
+
266
+
267
+ def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
268
+ """Rebuild when the source changed during the build. Only the last change refuses."""
269
+ for attempt in range(_MIRROR_BUILD_TRIES):
270
+ try:
271
+ return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
272
+ except HTTPException as exc:
273
+ detail = exc.detail if isinstance(exc.detail, dict) else {}
274
+ if ((detail.get("error") or {}).get("code") != "drill_population_changed"
275
+ or attempt + 1 >= _MIRROR_BUILD_TRIES):
276
+ raise
277
+ print(f"[sheet] mirror {table_key} changed during its build, rebuild {attempt + 2}",
278
+ flush=True)
279
+
280
+
281
+ def _hint_mirror_columns(session, book, only=None):
282
+ """Name every column one refresh reads per registry database, so its mirror builds once."""
283
+ import odoo_relational as rel
284
+ from modules import sheet as sheet_mod
285
+ from core.request_timing import raise_if_resource_error
286
+ context = getattr(session, "_sheet_scope_context", None)
287
+ if not isinstance(context, dict):
288
+ return
289
+ hint = {}
290
+ for tab in book.get("sheets") or []:
291
+ for addr, cell in (tab.get("cells") or {}).items():
292
+ if (not isinstance(cell, dict) or cell.get("kind") != "bound" or _is_view_binding(cell)
293
+ or (only is not None and f"{tab.get('id')}!{addr}" not in only)):
294
+ continue
295
+ try:
296
+ binding = cell.get("binding") if isinstance(cell.get("binding"), dict) else {}
297
+ database = str(binding.get("database") or "")
298
+ scope = binding.get("scope") if isinstance(binding.get("scope"), dict) else {}
299
+ nodes, _conj = _fe_parts(scope.get("filter"))
300
+ related = sheet_mod.related_filter_ids(nodes)
301
+ if database not in rel._REGISTRY_GRID_FIELDS and not related:
302
+ continue
303
+ columns = hint.setdefault(database, set())
304
+ columns.update(name for name in sheet_mod.filter_columns_named(nodes) - related
305
+ if ":" not in str(name))
306
+ field = (binding.get("aggregate") or {}).get("field")
307
+ if field:
308
+ columns.add(str(field))
309
+ for key in sorted(related):
310
+ via, field = sheet_mod.related_field_parts(key)
311
+ entry = _binding_related_entry(session, database, via)
312
+ if entry["via"].get("from"):
313
+ columns.add(str(entry["via"]["from"]))
314
+ hint.setdefault(entry["target"], set()).update(
315
+ {str(field), str(entry["via"].get("on") or field)})
316
+ except Exception as exc: # noqa: BLE001
317
+ # The cell's own resolution reports its refusal. A hint only widens a build.
318
+ raise_if_resource_error(exc)
319
+ continue
320
+ context[("mirror_columns",)] = hint
321
+
322
+
323
  def _mirror(session: Session):
324
  """`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
325
 
 
391
  session._sheet_amzn_cursor = (cur, None)
392
  else:
393
  cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
394
+ or _request_mirror_cur(session))
395
  except RuntimeError as exc:
396
  raise err(503, "store_not_ready", str(exc))
397
  if mismatch:
 
463
  session._sheet_mirrors = memo
464
  slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
465
  want = None if needed is None else {str(k) for k in needed}
466
+ context = getattr(session, "_sheet_scope_context", None)
467
+ hint = (context.get(("mirror_columns",)) or {}).get(table_key) if isinstance(context, dict) else None
468
+ if want is not None and hint:
469
+ want |= hint
470
  entry = memo.get(slot)
471
  if entry is not None:
472
  built, made = entry
 
474
  return made
475
  if want is not None:
476
  want = set(want) | built
477
+ made = _registry_mirror_settled(session, cur, table_key, metric_window, want)
478
  memo[slot] = (None if want is None else set(want), made)
479
  return made
480
+ spec = _request_source_for(session, cur, table_key)
481
  if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
482
  spec = _amazon_stored_spec(cur, table_key)
483
  if spec is None:
 
4566
  return hit["cell"]
4567
  return None
4568
 
4569
+ _hint_mirror_columns(session, book, only)
4570
  started = perf_counter()
4571
  resolved_count = 0
4572
  held_refusal = None
 
5682
  "store_not_ready": "#UPDATING",
5683
  "mirror_not_ready": "#UPDATING",
5684
  "amazon_mirror_not_loaded": "#UPDATING",
5685
+ "drill_population_changed": "#UPDATING",
5686
  }
5687
 
5688
 
 
5691
  "store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
5692
  "mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
5693
  "sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
5694
+ "drill_population_changed",
5695
  ))
5696
 
5697
  #: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
 
6136
  # and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
6137
  _refuse_unapplied_audience(session, database)
6138
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
6139
+ if _row_scope_applies(session, database) else None)
6140
  wall_nodes, wall_conj = _fe_parts(wall_tree)
6141
  scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
6142
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
api/routes_starred.py CHANGED
@@ -329,7 +329,8 @@ def _scan_starred_views(session, keys=None, cache=None, include=None, with_right
329
  import core.table_store as table_store
330
  import core.view_templates as view_templates
331
 
332
- granted = routes_nav._granted_view_sources(session)
 
333
  # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
334
  # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
335
  # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
 
329
  import core.table_store as table_store
330
  import core.view_templates as view_templates
331
 
332
+ # The docs this scan holds resolve folder-shared Views without a second read per database.
333
+ granted = routes_nav._granted_view_sources(session, docs=docs)
334
  # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
335
  # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
336
  # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
platform/core/audit.py CHANGED
@@ -1,7 +1,9 @@
1
  """Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
2
  from collections.abc import Mapping
 
3
  import math
4
  import re
 
5
  import time
6
 
7
  from harness import runtime
@@ -21,6 +23,11 @@ BUCKET = "security_events"
21
  MAX_EVENTS = 2000
22
  EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
23
  RETENTION_SECONDS = 400 * 24 * 60 * 60
 
 
 
 
 
24
  _ATTRIBUTION_MAX = 256
25
  _KEY_MAX = 128
26
 
@@ -161,37 +168,131 @@ def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]:
161
  return sorted(retained, key=lambda row: row["ts"])[-limit:]
162
 
163
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  def record(kind, tenant, principal, detail=None) -> dict | None:
165
  """Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
166
  item = event(kind, tenant, principal, detail)
167
  if freeze_notice.enabled():
168
  return None
169
  tenant = item["tenant"]
170
-
171
- def _append(doc):
172
- # Prune and cap INSIDE the atomic update, as users.note_login_failure does.
173
- rows = ordered(doc, tenant, MAX_EVENTS * 2)
174
- rows.append(item)
175
- rows = [row for row in rows if row["ts"] >= item["ts"] - RETENTION_SECONDS]
176
- rows.sort(key=lambda row: row["ts"])
177
- # Sign-ins and external calls each own their 2000-event retention partition.
178
- external = [row for row in rows if row["kind"] in EXTERNAL_KINDS][-MAX_EVENTS:]
179
- security = [row for row in rows if row["kind"] not in EXTERNAL_KINDS][-MAX_EVENTS:]
180
- return {"events": sorted(external + security, key=lambda row: row["ts"])}
181
-
182
  try:
183
- # Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
184
- # ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
185
- # flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
186
- # on the sign-in path: every login would then wait on a network round trip, and on the hf
187
- # backend every login would spend one of that repo's 256 commits per hour. The async
188
- # branch applies the mutation to the in-process cache and a coalescing worker commits it,
189
- # which is the same posture the table-workspace hot path has used since wave 7.
190
- # ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
191
- # loses the events still in its cache. That is the same window `record` already accepts by
192
- # swallowing a store failure, and it is the right side of the trade: an audit log must
193
- # never be the reason a person cannot sign in.
194
- runtime.get_runtime(str(tenant)).update(BUCKET, _append, flush="async")
195
  except Exception as exc:
196
  # Exception text can contain credentials from a provider or connection string.
197
  print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
@@ -204,6 +305,8 @@ def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
204
  """Read one tenant's retained events. Read failures propagate instead of claiming no history."""
205
  tenant = _attribution("tenant", tenant)
206
  now = time.time() if now is None else float(now)
 
 
207
  doc = runtime.get_runtime(str(tenant)).get(BUCKET)
208
  rows = ordered(doc, tenant, limit, kinds=kinds)
209
  rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
 
1
  """Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
2
  from collections.abc import Mapping
3
+ import atexit
4
  import math
5
  import re
6
+ import threading
7
  import time
8
 
9
  from harness import runtime
 
23
  MAX_EVENTS = 2000
24
  EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
25
  RETENTION_SECONDS = 400 * 24 * 60 * 60
26
+ #: Kinds a write-through store queues and writes in batches, so one query never pays a document write.
27
+ DEFERRED_KINDS = frozenset({"external.query"})
28
+ #: A tenant queue this long is written by the caller that fills it: the queue never grows past it.
29
+ QUEUE_MAX = 200
30
+ FLUSH_INTERVAL_S = 2.0
31
  _ATTRIBUTION_MAX = 256
32
  _KEY_MAX = 128
33
 
 
168
  return sorted(retained, key=lambda row: row["ts"])[-limit:]
169
 
170
 
171
+ def _well_formed(row, tenant):
172
+ """A stored row this writer may keep without revalidation. Readers still revalidate every row."""
173
+ return (isinstance(row, dict) and row.get("tenant") == tenant
174
+ and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS
175
+ and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict))
176
+
177
+
178
+ def _append_rows(doc, tenant, items):
179
+ """`{"events": rows}` with `items` added, pruned by age and capped per partition.
180
+
181
+ The common case is linear with no per-row validation. A document holding any row this tenant's
182
+ writer did not produce takes the full `ordered()` rebuild, which drops it.
183
+ """
184
+ stored = doc.get("events") if isinstance(doc, Mapping) else None
185
+ stored = stored if isinstance(stored, list) else []
186
+ if all(_well_formed(row, tenant) for row in stored):
187
+ rows = list(stored)
188
+ else:
189
+ rows = ordered(doc, tenant, MAX_EVENTS * 2)
190
+ rows.extend(items)
191
+ # A queued batch can land after a later sign-in, so restore ascending order when it did.
192
+ if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)):
193
+ rows.sort(key=lambda row: row["ts"])
194
+ cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS
195
+ if rows and rows[0]["ts"] < cutoff:
196
+ rows = [row for row in rows if row["ts"] >= cutoff]
197
+ # Sign-ins and external calls each own their 2000-event retention partition, oldest out first.
198
+ external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS)
199
+ drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)}
200
+ if drop[True] or drop[False]:
201
+ kept = []
202
+ for row in rows:
203
+ side = row["kind"] in EXTERNAL_KINDS
204
+ if drop[side]:
205
+ drop[side] -= 1
206
+ continue
207
+ kept.append(row)
208
+ rows = kept
209
+ return {"events": rows}
210
+
211
+
212
+ def _write(tenant, items):
213
+ # Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
214
+ # ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
215
+ # flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
216
+ # on the sign-in path: every login would then wait on a network round trip, and on the hf
217
+ # backend every login would spend one of that repo's 256 commits per hour. The async
218
+ # branch applies the mutation to the in-process cache and a coalescing worker commits it,
219
+ # which is the same posture the table-workspace hot path has used since wave 7.
220
+ # ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
221
+ # loses the events still in its cache. That is the same window `record` already accepts by
222
+ # swallowing a store failure, and it is the right side of the trade: an audit log must
223
+ # never be the reason a person cannot sign in.
224
+ runtime.get_runtime(str(tenant)).update(
225
+ BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async")
226
+
227
+
228
+ def _writes_through(tenant):
229
+ """True when the tenant store ignores flush="async" and commits the whole document per call (Postgres)."""
230
+ try:
231
+ backend = runtime.get_runtime(str(tenant))._document_store()
232
+ except Exception:
233
+ return False
234
+ return type(backend).__module__ == "core.store_pg"
235
+
236
+
237
+ _QUEUE = {}
238
+ _QUEUE_LOCK = threading.Lock()
239
+ _WAKE = threading.Event()
240
+ _FLUSHER = []
241
+
242
+
243
+ def _flush_loop():
244
+ while True:
245
+ _WAKE.wait(FLUSH_INTERVAL_S)
246
+ _WAKE.clear()
247
+ try:
248
+ flush_pending()
249
+ except Exception:
250
+ pass
251
+
252
+
253
+ def _enqueue(tenant, item):
254
+ with _QUEUE_LOCK:
255
+ pending = _QUEUE.setdefault(tenant, [])
256
+ pending.append(item)
257
+ full = len(pending) >= QUEUE_MAX
258
+ if not _FLUSHER:
259
+ _FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True))
260
+ _FLUSHER[0].start()
261
+ atexit.register(flush_pending)
262
+ if full:
263
+ flush_pending(tenant)
264
+
265
+
266
+ def flush_pending(tenant=None) -> int:
267
+ """Write queued events in one update per tenant. Returns how many rows were handed to the store."""
268
+ with _QUEUE_LOCK:
269
+ names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)]
270
+ batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)}
271
+ written = 0
272
+ for name, items in batches.items():
273
+ try:
274
+ _write(name, items)
275
+ written += len(items)
276
+ except Exception as exc:
277
+ # Exception text can contain credentials from a provider or connection string.
278
+ print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: "
279
+ f"store unavailable ({type(exc).__name__}).")
280
+ return written
281
+
282
+
283
  def record(kind, tenant, principal, detail=None) -> dict | None:
284
  """Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
285
  item = event(kind, tenant, principal, detail)
286
  if freeze_notice.enabled():
287
  return None
288
  tenant = item["tenant"]
289
+ # Sign-in and security kinds always write now. A query event on a write-through store is
290
+ # queued, because that store would otherwise rewrite the whole log on the request path.
291
+ if kind in DEFERRED_KINDS and _writes_through(tenant):
292
+ _enqueue(tenant, item)
293
+ return item
 
 
 
 
 
 
 
294
  try:
295
+ _write(tenant, [item])
 
 
 
 
 
 
 
 
 
 
 
296
  except Exception as exc:
297
  # Exception text can contain credentials from a provider or connection string.
298
  print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
 
305
  """Read one tenant's retained events. Read failures propagate instead of claiming no history."""
306
  tenant = _attribution("tenant", tenant)
307
  now = time.time() if now is None else float(now)
308
+ # Queued events are written first, so a reader sees every call this process accepted.
309
+ flush_pending(tenant)
310
  doc = runtime.get_runtime(str(tenant)).get(BUCKET)
311
  rows = ordered(doc, tenant, limit, kinds=kinds)
312
  rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
platform/harness/amazon_schema.py CHANGED
@@ -411,7 +411,8 @@ TABLE_SPECS = {
411
  ("msku", "listings", "seller_sku", "many_to_one"),
412
  ("asin", "listings", "asin1", "many_to_one"),
413
  )),
414
- "pricing": _table_spec("pricing", ("parent_asin", "child_asin", "sku"), (
 
415
  ("child_asin", "listings", "asin1", "many_to_one"),
416
  ("sku", "listings", "seller_sku", "many_to_one"),
417
  )),
 
411
  ("msku", "listings", "seller_sku", "many_to_one"),
412
  ("asin", "listings", "asin1", "many_to_one"),
413
  )),
414
+ # The period, not the parent, is identity: a re-parented child replaces its own window's row.
415
+ "pricing": _table_spec("pricing", ("child_asin", "sku", "_period_start", "_period_end"), (
416
  ("child_asin", "listings", "asin1", "many_to_one"),
417
  ("sku", "listings", "seller_sku", "many_to_one"),
418
  )),
platform/harness/amazon_store.py CHANGED
@@ -253,6 +253,8 @@ FIRST_LOAD_REPORTS_PER_PASS = 15
253
  #: minutes each) never hold the sync desk for hours (D-7081).
254
  FIRST_LOAD_ROWS_PER_PASS = 3_000_000
255
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
 
 
256
  FIRST_LOAD_END_KEY = "sync_first_load_end"
257
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
258
  FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
@@ -2984,7 +2986,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
2984
  def owner(entity):
2985
  if entity.startswith(entity_key("settlement_doc_")):
2986
  return table_name("settlements")
2987
- for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_"):
2988
  for name in names:
2989
  marker = entity_key(family + name[len(PREFIX):])
2990
  if entity == marker or entity.startswith(marker + "_"):
@@ -3003,8 +3005,15 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
3003
  def _after_restore(self, con, result, log):
3004
  _heal_snapshots(con, result, log)
3005
  _heal_quoted_values(con, result, log)
 
3006
  refresh_sku_keywords(con)
3007
 
 
 
 
 
 
 
3008
  def _restored_state_rows(self, rows, tables, refused):
3009
  return _restored_state_rows(rows, tables, refused)
3010
 
@@ -3062,7 +3071,8 @@ def _restored_state_rows(rows, tables, refused):
3062
  return rows
3063
  refused = set(refused)
3064
  names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
3065
- families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_")
 
3066
  shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
3067
 
3068
  def keep(row):
@@ -3208,6 +3218,7 @@ def _unquote_mirror_values(con, log=None):
3208
  stale = " OR ".join(f"{_quoted_identifier(c)} LIKE '\"%\"'" for c in identity)
3209
  if not con.execute(f"SELECT 1 FROM {table} WHERE {stale} LIMIT 1").fetchone():
3210
  continue
 
3211
  sets = []
3212
  dated = mirror[len(PREFIX):] in APP_DATE_TABLES
3213
  for col in text:
@@ -3220,6 +3231,7 @@ def _unquote_mirror_values(con, log=None):
3220
  con.execute("BEGIN TRANSACTION")
3221
  try:
3222
  healed[mirror] = con.execute(f"SELECT count(*) FROM {table} WHERE {stale}").fetchone()[0]
 
3223
  con.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE {stale}")
3224
  con.execute("COMMIT")
3225
  except Exception:
@@ -3228,9 +3240,139 @@ def _unquote_mirror_values(con, log=None):
3228
  raise
3229
  if log:
3230
  log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted")
 
 
 
3231
  return healed
3232
 
3233
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3234
  def _heal_quoted_values(con, result, log):
3235
  """Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean."""
3236
  try:
@@ -3595,7 +3737,8 @@ def _record_state(con, key, phase, rows=0, detail=None):
3595
  con.execute(
3596
  "INSERT OR REPLACE INTO _sync_state "
3597
  "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
3598
- [str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_"))
 
3599
  else str(detail)[:300]) if detail else None, int(rows or 0),
3600
  _iso(datetime.now(timezone.utc))])
3601
 
@@ -5202,12 +5345,17 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5202
  except Exception as exc: # noqa: BLE001
5203
  report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
5204
  "resync of each snapshot table replaces its earlier copies.")
5205
- # D-2682: a restored PUBLISHED snapshot skips `_after_restore`, so quoted rows heal here.
5206
  try:
5207
  changed_tables.update(_unquote_mirror_values(con, log))
5208
  except Exception as exc: # noqa: BLE001
5209
  report["problems"].append(f"Amazon quoted value heal failed ({type(exc).__name__}); the next "
5210
  "resync of each quoted table writes clean values.")
 
 
 
 
 
5211
 
5212
  def write_window(table, cols, types, rows):
5213
  if cx.REPORT_TYPES[table].get("weekly"):
@@ -5477,7 +5625,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5477
  margin_windows = _month_windows(plan["start"], plan["end"])
5478
  else:
5479
  margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
5480
- if table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
 
 
 
 
 
 
 
5481
  # One due window per table tick respects the vendor floor. The watermark
5482
  # advances only to that window's end, so the following tick loses no history.
5483
  remaining_windows = len(margin_windows) - 1
@@ -5500,6 +5655,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
5500
  {"state": "access", "operation": operation}, rt, report, log)
5501
  count = _sync_margin_window(con, table, fields, w0, w1,
5502
  max_wait=max_wait, log=log)
 
 
5503
  probe.finish(count)
5504
  spend_rows(count)
5505
  written += count
@@ -6927,6 +7084,91 @@ def mirror_selftest(log=print, check=None):
6927
  whole["state"] == "refused" and whole["refused_tables"] == []
6928
  and dump() == {"_sync_state": []}
6929
  and not any(path.endswith(".parquet") for path in downloaded))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6930
  finally:
6931
  forget()
6932
  con.close()
@@ -7725,6 +7967,23 @@ def group_a_selftest(log=print, check=None):
7725
  con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(refused_table)])
7726
  ck("NC: the same table never refused (no watermark) still owes a first load",
7727
  needs_first_load(rt.key, rt))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7728
  finally:
7729
  con.close()
7730
 
 
253
  #: minutes each) never hold the sync desk for hours (D-7081).
254
  FIRST_LOAD_ROWS_PER_PASS = 3_000_000
255
  BACKFILL_FLOOR_PREFIX = "sync_floor_"
256
+ #: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
257
+ RESYNC_PREFIX = "resync_"
258
  FIRST_LOAD_END_KEY = "sync_first_load_end"
259
  FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
260
  FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
 
2986
  def owner(entity):
2987
  if entity.startswith(entity_key("settlement_doc_")):
2988
  return table_name("settlements")
2989
+ for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_", RESYNC_PREFIX):
2990
  for name in names:
2991
  marker = entity_key(family + name[len(PREFIX):])
2992
  if entity == marker or entity.startswith(marker + "_"):
 
3005
  def _after_restore(self, con, result, log):
3006
  _heal_snapshots(con, result, log)
3007
  _heal_quoted_values(con, result, log)
3008
+ _heal_identity_duplicates(con, result, log)
3009
  refresh_sku_keywords(con)
3010
 
3011
+ def _heal_restored(self, con, result, log):
3012
+ # Heal the served rows without marking them dirty: a dirty set makes the retention sweep publish.
3013
+ healed = {"dirty": set(), "problems": result["problems"]}
3014
+ for heal in (_heal_snapshots, _heal_quoted_values, _heal_identity_duplicates):
3015
+ heal(con, healed, log)
3016
+
3017
  def _restored_state_rows(self, rows, tables, refused):
3018
  return _restored_state_rows(rows, tables, refused)
3019
 
 
3071
  return rows
3072
  refused = set(refused)
3073
  names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
3074
+ families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_",
3075
+ RESYNC_PREFIX)
3076
  shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
3077
 
3078
  def keep(row):
 
3218
  stale = " OR ".join(f"{_quoted_identifier(c)} LIKE '\"%\"'" for c in identity)
3219
  if not con.execute(f"SELECT 1 FROM {table} WHERE {stale} LIMIT 1").fetchone():
3220
  continue
3221
+ lost = _unhealable_sql(types, text)
3222
  sets = []
3223
  dated = mirror[len(PREFIX):] in APP_DATE_TABLES
3224
  for col in text:
 
3231
  con.execute("BEGIN TRANSACTION")
3232
  try:
3233
  healed[mirror] = con.execute(f"SELECT count(*) FROM {table} WHERE {stale}").fetchone()[0]
3234
+ resync = _queue_resync(con, mirror, types, f"({stale}) AND ({lost})") if lost else None
3235
  con.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE {stale}")
3236
  con.execute("COMMIT")
3237
  except Exception:
 
3240
  raise
3241
  if log:
3242
  log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted")
3243
+ if resync:
3244
+ log(f"Amazon resync {mirror}: {resync[0]:,} rows lost cells to the old parse; "
3245
+ f"{len(resync[1])} report windows from {resync[1][0][0]} download again on the next passes")
3246
  return healed
3247
 
3248
 
3249
+ def _unhealable_sql(types, text):
3250
+ """SQL true for a quoted row the unquote cannot repair, or "" when no such test applies.
3251
+
3252
+ A cell opening an unclosed quote is a value the old parse split at a tab, so every later cell
3253
+ shifted. A row whose every numeric measure is NULL lost them to the failed numeric cast.
3254
+ """
3255
+ split = [f"({q} LIKE '\"%' AND length(regexp_extract(substr({q}, 2), '\"*$')) % 2 = 0)"
3256
+ for q in map(_quoted_identifier, text)]
3257
+ measures = [_quoted_identifier(c) for c, t in types.items()
3258
+ if str(t).upper() != "VARCHAR" and not str(c).startswith("_")]
3259
+ if measures:
3260
+ split.append("(" + " AND ".join(f"{q} IS NULL" for q in measures) + ")")
3261
+ return " OR ".join(split)
3262
+
3263
+
3264
+ def _queue_resync(con, mirror, types, where):
3265
+ """Record the report windows of rows matching `where` for download again -> (rows, windows) or None.
3266
+
3267
+ Only a dated window table qualifies: its next pass replaces each queued `_period_start` whole.
3268
+ """
3269
+ table = mirror[len(PREFIX):]
3270
+ if (table not in PERIOD_TABLES or not {"_period_start", "_period_end"} <= set(types)
3271
+ or (_client().REPORT_TYPES.get(table) or {}).get("snapshot")):
3272
+ return None
3273
+ found = con.execute(
3274
+ f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) "
3275
+ f"FROM {_quoted_identifier(mirror)} WHERE ({where}) AND _period_start IS NOT NULL "
3276
+ f"AND _period_end IS NOT NULL GROUP BY 1, 2").fetchall()
3277
+ if not found:
3278
+ return None
3279
+ key = entity_key(RESYNC_PREFIX + table)
3280
+ saved = _read_state(con, key)
3281
+ try:
3282
+ queued = {tuple(w) for w in json.loads(saved["detail"])["windows"]} if saved else set()
3283
+ except (ValueError, KeyError, TypeError):
3284
+ queued = set()
3285
+ windows = sorted(queued | {(start, end) for start, end, _n in found})
3286
+ rows = sum(n for _s, _e, n in found)
3287
+ _record_state(con, key, "pending", rows, json.dumps({"windows": [list(w) for w in windows]}))
3288
+ return rows, windows
3289
+
3290
+
3291
+ def queued_resync(con, table):
3292
+ """The oldest queued resync window of `table` as `(start, end)` datetimes, or None."""
3293
+ saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
3294
+ try:
3295
+ start, end = json.loads(saved["detail"])["windows"][0]
3296
+ return _utc(start), _utc(end)
3297
+ except (TypeError, ValueError, KeyError, IndexError, AttributeError):
3298
+ return None
3299
+
3300
+
3301
+ def resync_landed(con, table, window):
3302
+ """Drop one landed window from the queue; the marker goes when the queue is empty."""
3303
+ key = entity_key(RESYNC_PREFIX + table)
3304
+ saved = _read_state(con, key)
3305
+ try:
3306
+ windows = [w for w in json.loads(saved["detail"])["windows"]
3307
+ if (_utc(w[0]), _utc(w[1])) != tuple(window)]
3308
+ except (TypeError, ValueError, KeyError, AttributeError):
3309
+ windows = []
3310
+ if windows:
3311
+ _record_state(con, key, "pending", saved["rows"], json.dumps({"windows": windows}))
3312
+ else:
3313
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
3314
+
3315
+
3316
+ #: Tables whose declared identity can hold rows from two documents after an identity change.
3317
+ _IDENTITY_COLLAPSE_TABLES = ("pricing",)
3318
+
3319
+
3320
+ def _collapse_identity(con, log=None):
3321
+ """Rows sharing a declared identity keep the newest landed document only -> {mirror: (before, after)}.
3322
+
3323
+ Newest is the document holding the highest `rowid` (landing order, the ranking diagnostic's
3324
+ rule), then the larger document ID. A row with no `_period_start` goes once the same child and
3325
+ SKU hold a dated row. Idempotent: one probe per table when nothing is doubled.
3326
+ """
3327
+ existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"}
3328
+ collapsed = {}
3329
+ for table in _IDENTITY_COLLAPSE_TABLES:
3330
+ mirror = table_name(table)
3331
+ if mirror not in existing:
3332
+ continue
3333
+ cols = _columns_of(con, mirror)
3334
+ keys = [c for c in replace_keys_for(table) if c in cols]
3335
+ if not keys or "_report_document_id" not in cols:
3336
+ continue
3337
+ q, target = _quoted_identifier, _quoted_identifier(mirror)
3338
+ part = ", ".join(q(c) for c in keys)
3339
+ doomed = (f"SELECT rid FROM (SELECT rid, d, first_value(d) OVER (PARTITION BY {part} "
3340
+ f"ORDER BY landed DESC, d DESC) AS keep FROM (SELECT rowid AS rid, {part}, "
3341
+ f"_report_document_id AS d, max(rowid) OVER (PARTITION BY _report_document_id) AS landed "
3342
+ f"FROM {target})) WHERE d IS DISTINCT FROM keep")
3343
+ if "_period_start" in keys:
3344
+ same = " AND ".join(f"o.{q(c)} IS NOT DISTINCT FROM t.{q(c)}" for c in keys
3345
+ if c not in ("_period_start", "_period_end")) or "TRUE"
3346
+ doomed += (f" UNION SELECT t.rowid FROM {target} AS t WHERE t._period_start IS NULL AND EXISTS "
3347
+ f"(SELECT 1 FROM {target} AS o WHERE o._period_start IS NOT NULL AND {same})")
3348
+ if not con.execute(f"SELECT 1 FROM ({doomed}) LIMIT 1").fetchone():
3349
+ continue
3350
+ before = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0]
3351
+ con.execute("BEGIN TRANSACTION")
3352
+ try:
3353
+ con.execute(f"DELETE FROM {target} WHERE rowid IN ({doomed})")
3354
+ con.execute("COMMIT")
3355
+ except Exception:
3356
+ con.execute("ROLLBACK")
3357
+ raise
3358
+ after = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0]
3359
+ collapsed[mirror] = (before, after)
3360
+ if log:
3361
+ log(f"Amazon identity {mirror}: {before:,} rows to {after:,}, one document per identity")
3362
+ return collapsed
3363
+
3364
+
3365
+ def _heal_identity_duplicates(con, result, log):
3366
+ """Collapse doubled identities after a restore; healed tables are dirty for the next checkpoint."""
3367
+ try:
3368
+ result["dirty"].update(_collapse_identity(con, log))
3369
+ except Exception as exc: # noqa: BLE001
3370
+ message = (f"Amazon identity collapse failed ({type(exc).__name__}); the next resync of each "
3371
+ "doubled table replaces its earlier document's rows.")
3372
+ result["problems"].append(message)
3373
+ log(message)
3374
+
3375
+
3376
  def _heal_quoted_values(con, result, log):
3377
  """Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean."""
3378
  try:
 
3737
  con.execute(
3738
  "INSERT OR REPLACE INTO _sync_state "
3739
  "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
3740
+ [str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_",
3741
+ "amzn_resync_"))
3742
  else str(detail)[:300]) if detail else None, int(rows or 0),
3743
  _iso(datetime.now(timezone.utc))])
3744
 
 
5345
  except Exception as exc: # noqa: BLE001
5346
  report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
5347
  "resync of each snapshot table replaces its earlier copies.")
5348
+ # Backstop: a candidate reused across restarts never passed through the restore heal.
5349
  try:
5350
  changed_tables.update(_unquote_mirror_values(con, log))
5351
  except Exception as exc: # noqa: BLE001
5352
  report["problems"].append(f"Amazon quoted value heal failed ({type(exc).__name__}); the next "
5353
  "resync of each quoted table writes clean values.")
5354
+ try:
5355
+ changed_tables.update(_collapse_identity(con, log))
5356
+ except Exception as exc: # noqa: BLE001
5357
+ report["problems"].append(f"Amazon identity collapse failed ({type(exc).__name__}); the next "
5358
+ "resync of each doubled table replaces its earlier document's rows.")
5359
 
5360
  def write_window(table, cols, types, rows):
5361
  if cx.REPORT_TYPES[table].get("weekly"):
 
5625
  margin_windows = _month_windows(plan["start"], plan["end"])
5626
  else:
5627
  margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
5628
+ resync = None if first_load or spec.get("snapshot") else queued_resync(con, table)
5629
+ if resync is not None and read_watermark(con, table):
5630
+ # A queued window takes this tick; the watermark stays, so no new window is skipped.
5631
+ margin_windows = [resync]
5632
+ plan["end"] = read_watermark(con, table)
5633
+ plan["cause"] = f"resync of the stored window starting {_iso(resync[0])}"
5634
+ log(f"{table}: {plan['cause']}")
5635
+ elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
5636
  # One due window per table tick respects the vendor floor. The watermark
5637
  # advances only to that window's end, so the following tick loses no history.
5638
  remaining_windows = len(margin_windows) - 1
 
5655
  {"state": "access", "operation": operation}, rt, report, log)
5656
  count = _sync_margin_window(con, table, fields, w0, w1,
5657
  max_wait=max_wait, log=log)
5658
+ if resync is not None and (w0, w1) == resync:
5659
+ resync_landed(con, table, resync)
5660
  probe.finish(count)
5661
  spend_rows(count)
5662
  written += count
 
7084
  whole["state"] == "refused" and whole["refused_tables"] == []
7085
  and dump() == {"_sync_state": []}
7086
  and not any(path.endswith(".parquet") for path in downloaded))
7087
+
7088
+ # S79-T19, S79-T23: a PUBLISHED snapshot and a hand-off heal the rows they serve.
7089
+ split_window = ("2026-03-20T18:01:25Z", "2026-04-19T18:01:25Z")
7090
+
7091
+ def seed_quoted():
7092
+ origin = "'GET_FIXTURE', 'Amazon Services API'"
7093
+ con.execute("CREATE TABLE amzn_pricing AS SELECT * FROM (VALUES "
7094
+ f"('P1', 'C1', 'S1', 5, 'doc-old', 1, {origin}), "
7095
+ f"('P1', 'C2', 'S2', 7, 'doc-old', 2, {origin}), "
7096
+ f"('P2', 'C1', 'S1', 9, 'doc-new', 1, {origin})) AS t(parent_asin, child_asin, sku, "
7097
+ "units, _report_document_id, _report_row_number, _report_type, _data_origin)")
7098
+ con.execute("CREATE TABLE amzn_inventory_ledger_detail AS SELECT * FROM (VALUES "
7099
+ "('2026-02-18T18:01:25Z', '2026-03-20T18:01:25Z', '\"SKU-1\"', '\"Plain\"', "
7100
+ "'\"Receipts\"', '\"12\"', 'doc-a', 1), "
7101
+ f"('{split_window[0]}', '{split_window[1]}', '\"SKU-2\"', '\"A vase', "
7102
+ "'with a tab\"', '\"Shipments\"', 'doc-b', 1)) AS t(_period_start, _period_end, msku, "
7103
+ "title, event_type, quantity, _report_document_id, _report_row_number)")
7104
+
7105
+ def served():
7106
+ return (con.execute("SELECT child_asin, parent_asin FROM amzn_pricing ORDER BY 1").fetchall(),
7107
+ con.execute("SELECT msku, quantity FROM amzn_inventory_ledger_detail "
7108
+ "ORDER BY _period_start").fetchall())
7109
+
7110
+ healed_rows = ([("C1", "P2"), ("C2", "P1")], [("SKU-1", "12"), ("SKU-2", "Shipments")])
7111
+ rt.blobs = {}
7112
+ wipe()
7113
+ restore()
7114
+ seed_quoted()
7115
+ record_watermark(con, "inventory_ledger_detail", datetime(2026, 10, 23, tzinfo=timezone.utc), 2)
7116
+ report = {"problems": []}
7117
+ _MIRROR_STORE.persist_mirror(rt.key, rt, con, {"amzn_pricing", "amzn_inventory_ledger_detail"},
7118
+ report=report, log=lambda *_a: None, _publication={
7119
+ "snapshot_id": uuid.uuid4().hex, "started_at": "2026-10-24T00:00:00Z",
7120
+ "state": "complete"})
7121
+ wipe()
7122
+ published = restore()
7123
+ ck("[restore] a published snapshot restore heals quoted and doubled rows, not marked dirty", lambda:
7124
+ not report["problems"] and published["state"] == "restored"
7125
+ and bool(published["manifest"].get("snapshot_id")) and not published["problems"]
7126
+ and not published["dirty"] and served() == healed_rows)
7127
+ ck("[restore] a split ledger row queues its own report window for download again", lambda:
7128
+ queued_resync(con, "inventory_ledger_detail") == (_utc(split_window[0]), _utc(split_window[1])))
7129
+ settled = dump()
7130
+ forget()
7131
+ again = restore()
7132
+ ck("[restore] the heal on a local restart of the same snapshot changes nothing", lambda:
7133
+ again["state"] == "local" and dump() == settled)
7134
+ resync_landed(con, "inventory_ledger_detail", queued_resync(con, "inventory_ledger_detail"))
7135
+ ck("[restore] a landed resync window leaves the queue", lambda:
7136
+ queued_resync(con, "inventory_ledger_detail") is None
7137
+ and _read_state(con, entity_key(RESYNC_PREFIX + "inventory_ledger_detail")) is None)
7138
+
7139
+ import tempfile as _tmp
7140
+ with _tmp.TemporaryDirectory(prefix="amazon-handoff-") as tmp:
7141
+ wipe()
7142
+ seed_quoted()
7143
+ paths = {}
7144
+ for name in ("amzn_pricing", "amzn_inventory_ledger_detail"):
7145
+ paths[name] = [Path(tmp) / (name + ".parquet")]
7146
+ con.execute(f"COPY {name} TO ? (FORMAT PARQUET)", [str(paths[name][0])])
7147
+ wipe()
7148
+ _mirrors.install_handoff("amazon", rt.key, rt, {"snapshot_id": uuid.uuid4().hex,
7149
+ "tables": {}}, [], paths)
7150
+ ck("[restore] a hand-off install heals the rows the web process serves", lambda:
7151
+ served() == healed_rows)
7152
+
7153
+ # A re-parented child replaces its own window's row; another window keeps its own.
7154
+ con.execute("DROP TABLE amzn_pricing")
7155
+
7156
+ def land_pricing(doc, parent, start):
7157
+ w0 = datetime(2026, 10, start, tzinfo=timezone.utc)
7158
+ rows = [_origin({"parent_asin": parent, "child_asin": "C1", "sku": "S1", "units": 3},
7159
+ doc, 1, "GET_SALES_AND_TRAFFIC_REPORT")]
7160
+ cols, rows = stamp_week(["parent_asin", "child_asin", "sku", "units", *_ORIGIN_TYPES],
7161
+ rows, w0, w0 + timedelta(days=7))
7162
+ types = _ensure(con, "amzn_pricing", cols, rows)
7163
+ return replace_window(con, "amzn_pricing", cols, types, rows, replace_keys_for("pricing"))
7164
+
7165
+ landed = [land_pricing("doc-1", "P1", 1), land_pricing("doc-2", "P2", 1)]
7166
+ ck("[restore] a re-parented pricing child replaces its older document's row", lambda:
7167
+ landed == [(1, None), (1, None)] and con.execute(
7168
+ "SELECT parent_asin, _report_document_id FROM amzn_pricing").fetchall() == [("P2", "doc-2")])
7169
+ land_pricing("doc-3", "P2", 8)
7170
+ ck("[restore] a later pricing window keeps its own row beside the earlier window", lambda:
7171
+ con.execute("SELECT count(*), count(DISTINCT _period_start) FROM amzn_pricing").fetchone() == (2, 2))
7172
  finally:
7173
  forget()
7174
  con.close()
 
7967
  con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(refused_table)])
7968
  ck("NC: the same table never refused (no watermark) still owes a first load",
7969
  needs_first_load(rt.key, rt))
7970
+ # S79-T23: a queued resync window downloads again alone; the watermark stays.
7971
+ fees = "amzn_long_term_storage_fees"
7972
+ stored = con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
7973
+ "GROUP BY 1, 2 ORDER BY 1").fetchall()
7974
+ mark = read_watermark(con, "long_term_storage_fees")
7975
+ _record_state(con, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1,
7976
+ json.dumps({"windows": [list(stored[0][:2])]}))
7977
+ clock[0] += timedelta(days=2)
7978
+ before = len(calls)
7979
+ _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None)
7980
+ asked = [kwargs for _report, kwargs in calls[before:]]
7981
+ ck("a queued resync window downloads again alone and keeps the watermark",
7982
+ len(asked) == 1 and str(asked[0].get("start", ""))[:10] == stored[0][0][:10]
7983
+ and read_watermark(con, "long_term_storage_fees") == mark
7984
+ and queued_resync(con, "long_term_storage_fees") is None
7985
+ and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
7986
+ "GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
7987
  finally:
7988
  con.close()
7989
 
platform/harness/mirror_store.py CHANGED
@@ -93,6 +93,9 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
93
  return False
94
  only = changed_tables(previous, manifest, store._tables(con), paths)
95
  store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
 
 
 
96
  newest = _record_newest(con, store.connector)
97
  finally:
98
  con.close()
@@ -453,6 +456,10 @@ class MirrorStore:
453
  def _after_restore(self, con, result, log):
454
  pass
455
 
 
 
 
 
456
  def _restored_state_rows(self, rows, tables, refused):
457
  # Without a connector-specific marker policy, invalidate all its sync
458
  # decisions after any table refusal. Good tables remain available.
@@ -728,6 +735,8 @@ class MirrorStore:
728
  result["manifest"] = saved
729
  if not result["manifest"].get("snapshot_id"):
730
  self._after_restore(con, result, log)
 
 
731
  with _CATCH_UP_LOCK:
732
  _catch_up_source(rt, self.connector)["pass_pending"] = True
733
  self.cache[cache_key] = result
@@ -759,6 +768,10 @@ class MirrorStore:
759
  self._log_restored(paths, total, rows, log)
760
  if not manifest.get("snapshot_id"):
761
  self._after_restore(con, result, log)
 
 
 
 
762
  table_states = {row[0]: row[1] for row in rows if row[0] in paths}
763
  current = (not refused and bool(paths) and set(paths) <= set(table_states)
764
  and all(row[1] in {"live", "done"} for row in rows))
 
93
  return False
94
  only = changed_tables(previous, manifest, store._tables(con), paths)
95
  store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
96
+ # The web process serves the hand-off as installed, so it heals its own copy too.
97
+ healed = {"dirty": set(), "problems": []}
98
+ store._heal_restored(con, healed, print)
99
  newest = _record_newest(con, store.connector)
100
  finally:
101
  con.close()
 
456
  def _after_restore(self, con, result, log):
457
  pass
458
 
459
+ def _heal_restored(self, con, result, log):
460
+ """Row heals for a published snapshot or hand-off: idempotent, one probe per table when clean."""
461
+ pass
462
+
463
  def _restored_state_rows(self, rows, tables, refused):
464
  # Without a connector-specific marker policy, invalidate all its sync
465
  # decisions after any table refusal. Good tables remain available.
 
735
  result["manifest"] = saved
736
  if not result["manifest"].get("snapshot_id"):
737
  self._after_restore(con, result, log)
738
+ else:
739
+ self._heal_restored(con, result, log)
740
  with _CATCH_UP_LOCK:
741
  _catch_up_source(rt, self.connector)["pass_pending"] = True
742
  self.cache[cache_key] = result
 
768
  self._log_restored(paths, total, rows, log)
769
  if not manifest.get("snapshot_id"):
770
  self._after_restore(con, result, log)
771
+ else:
772
+ # Trap: the healed local rows differ from the published snapshot until the
773
+ # next publish; each restore heals again and a second heal changes nothing.
774
+ self._heal_restored(con, result, log)
775
  table_states = {row[0]: row[1] for row in rows if row[0] in paths}
776
  current = (not refused and bool(paths) and set(paths) <= set(table_states)
777
  and all(row[1] in {"live", "done"} for row in rows))
platform/modules/opportunity_finder.py CHANGED
@@ -19,8 +19,11 @@ A rank is a position: a lower number is searched more, so a term rises when its
19
  The department filter and the rank comparison run in DuckDB over every row, with no cap (rule 1).
20
  """
21
  import contextlib
 
22
  import datetime as dt
23
  import hashlib
 
 
24
  from pathlib import Path
25
 
26
  APP_LABEL = "Opportunity finder"
@@ -45,6 +48,11 @@ TRIGGER = "brand_analytics"
45
 
46
  NO_MIRROR = "The Amazon mirror is not on this server yet. It appears after the first sync."
47
  NOT_SYNCED = "ut_amzn_brand_analytics has not synced yet, so no search term can be compared."
 
 
 
 
 
48
  NO_DEPARTMENTS = ("No brand analytics row shows one of your ASINs yet, so the finder has no department to "
49
  "search. It reads departments from ut_amzn_brand_analytics and your ASINs from "
50
  "ut_amzn_search_query_performance and ut_amzn_listings.")
@@ -58,6 +66,11 @@ OLDER_WEEK = ("Only the newest brand analytics week can be tied. The mirror keep
58
  LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3,000,000 search terms "
59
  "fit in one week. Split a week by department when a marketplace nears that size.")
60
 
 
 
 
 
 
61
  #: Accounts that sign in for a process, never a person who can own an item.
62
  SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
63
 
@@ -115,36 +128,126 @@ def _mirror_path(tenant_key, st):
115
  return amazon_store._tenant_mirror_path(tenant_key, rt)
116
 
117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  @contextlib.contextmanager
119
  def _reader(tenant_key, st):
120
- """A read-only cursor on the tenant mirror, or None when the mirror file does not exist."""
121
- path = Path(_mirror_path(tenant_key, st))
122
- if not path.exists():
123
- yield None
124
- return
125
  from harness import datastore
 
126
  try:
127
  shared = Path(datastore.DB_PATH).resolve() == path.resolve()
128
  except Exception: # noqa: BLE001
129
  shared = False
130
- # The process holds one writer on its own mirror: a separate read-only connect would refuse.
131
- if shared:
132
- cur = datastore.ro_cursor()
133
- try:
134
- yield cur
135
- finally:
136
- cur.close()
 
 
 
 
 
 
 
 
 
 
137
  return
138
- import duckdb
139
- con = duckdb.connect(str(path), read_only=True)
140
  try:
141
- yield con
142
  finally:
143
- con.close()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
 
145
 
146
  def _columns(cur, table):
147
- if cur is None:
148
  return set()
149
  try:
150
  return {r[0] for r in cur.execute(
@@ -153,6 +256,15 @@ def _columns(cur, table):
153
  return set()
154
 
155
 
 
 
 
 
 
 
 
 
 
156
  def _our_asins(cur):
157
  """SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
158
  ours = set()
@@ -165,10 +277,14 @@ def _our_asins(cur):
165
  return {a for a in ours if a}
166
 
167
 
168
- def _period(cur):
169
- """The newest brand analytics `_period_start` as text, or None."""
170
- row = cur.execute(f'SELECT MAX(CAST("_period_start" AS VARCHAR)) FROM {BA_TABLE}').fetchone()
171
- return str(row[0]) if row and row[0] is not None else None
 
 
 
 
172
 
173
 
174
  _TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))'
@@ -188,21 +304,46 @@ def _derived_departments(cur, ours):
188
  return sorted(_text(r[0]) for r in rows if _text(r[0]))
189
 
190
 
191
- def _source(cur, st):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
  """What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
193
  out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
194
  if cur is None:
195
  out["problems"].append(NO_MIRROR)
196
  return out
197
- if not set(BA_NEED) <= _columns(cur, BA_TABLE):
198
- out["problems"].append(NOT_SYNCED)
 
 
199
  return out
200
- out["period"] = _period(cur)
201
- if out["period"] is None:
202
  out["problems"].append(NOT_SYNCED)
203
  return out
204
- out["ours"] = _our_asins(cur)
205
- out["derived"] = _derived_departments(cur, out["ours"])
 
 
 
 
206
  chosen = _doc(st, KEY_SETTINGS).get("departments")
207
  out["departments"] = sorted({_text(d) for d in chosen if _text(d)}) if chosen else out["derived"]
208
  if not out["departments"]:
@@ -247,7 +388,10 @@ def _rising_rows(cur, period, departments, prior, min_pct):
247
  f'AND (prior.rank - cur.rank) * 100.0 / prior.rank >= ? '
248
  f'ORDER BY pct DESC, cur.term')
249
  params = [period, list(departments), [t for t, _r in pairs], [r for _t, r in pairs], *params_top, float(min_pct)]
250
- return cur.execute(sql, params).fetchall()
 
 
 
251
 
252
 
253
  def _top3_list(raw, ours):
@@ -371,7 +515,8 @@ def record_week(tenant_key, *, st=None, now=None):
371
  st = _rt(tenant_key, st)
372
  now = now or _now()
373
  with _reader(tenant_key, st) as cur:
374
- src = _source(cur, st)
 
375
  if src["period"] is None or not src["departments"]:
376
  return {"state": "nothing", "problems": src["problems"]}
377
  terms = week_ranks(cur, src["period"], src["departments"])
@@ -434,6 +579,7 @@ def find(tenant_key, *, st=None):
434
  error = _doc(st, KEY_INDEX).get("last_error")
435
  if error:
436
  problems.append(error)
 
437
  found = []
438
  for term, before, now_rank, rise, pct, top in rows:
439
  top3 = _top3_list(top, src["ours"])
@@ -441,10 +587,23 @@ def find(tenant_key, *, st=None):
441
  found.append({"id": item_id(term), "term": term, "title": term, "priorRank": int(before),
442
  "rank": int(now_rank), "rise": int(rise), "risePct": round(float(pct), 2),
443
  "ours": bool(mine), "asin": mine[0] if mine else None, "top3": top3, "rising": True,
 
 
444
  "reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
445
  return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
446
  "priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
447
- "rising": found, "problems": problems}
 
 
 
 
 
 
 
 
 
 
 
448
 
449
 
450
  def _stored(st):
@@ -499,7 +658,7 @@ def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMI
499
  page, nxt = _page(wanted, cursor, limit)
500
  return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
501
  "priorWeek": found["priorWeek"], "departments": found["departments"],
502
- "problems": found["problems"], "weeks": _index(st)}
503
 
504
 
505
  def _lookup(st, iid, found=None):
@@ -643,7 +802,7 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
643
  """Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
644
  st = _rt(tenant_key, st)
645
  with _reader(tenant_key, st) as cur:
646
- src = _source(cur, st)
647
  period = src["period"]
648
  if period is None:
649
  return {"ok": False, "app_total": None, "independent_total": None, "week": None,
 
19
  The department filter and the rank comparison run in DuckDB over every row, with no cap (rule 1).
20
  """
21
  import contextlib
22
+ import contextvars
23
  import datetime as dt
24
  import hashlib
25
+ import os
26
+ import threading
27
  from pathlib import Path
28
 
29
  APP_LABEL = "Opportunity finder"
 
48
 
49
  NO_MIRROR = "The Amazon mirror is not on this server yet. It appears after the first sync."
50
  NOT_SYNCED = "ut_amzn_brand_analytics has not synced yet, so no search term can be compared."
51
+ #: The mirror holds the table but this read failed or was busy. `{cause}` is the error class.
52
+ READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so no search term can be compared. "
53
+ "Try again in a minute.")
54
+ BAD_PERIOD = ("The newest brand analytics week start reads {value}, which is not an ISO date, so no week "
55
+ "can be compared.")
56
  NO_DEPARTMENTS = ("No brand analytics row shows one of your ASINs yet, so the finder has no department to "
57
  "search. It reads departments from ut_amzn_brand_analytics and your ASINs from "
58
  "ut_amzn_search_query_performance and ut_amzn_listings.")
 
66
  LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3,000,000 search terms "
67
  "fit in one week. Split a week by department when a marketplace nears that size.")
68
 
69
+ #: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
70
+ PROBE_SECONDS = 20.0
71
+ #: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
72
+ MEMO_KEEP = 16
73
+
74
  #: Accounts that sign in for a process, never a person who can own an item.
75
  SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
76
 
 
128
  return amazon_store._tenant_mirror_path(tenant_key, rt)
129
 
130
 
131
+ class _Unreadable:
132
+ """The mirror exists but no cursor could be opened on it. `cause` names the error class."""
133
+
134
+ def __init__(self, cause):
135
+ self.cause = cause
136
+
137
+
138
+ class _ReadFailed(Exception):
139
+ """A mirror probe failed or was interrupted. Never read as an absent table."""
140
+
141
+
142
+ #: The mirror mark of the cursor `_reader` yielded: (path, install generation, file stat).
143
+ _MARK = contextvars.ContextVar("opportunity_finder_mark", default=None)
144
+ _MEMO = {}
145
+ _MEMO_LOCK = threading.Lock()
146
+
147
+
148
+ def _remember(key, build):
149
+ """`build()` once per key. A failed build is not kept, so the next request reads again."""
150
+ if key is None:
151
+ return build()
152
+ with _MEMO_LOCK:
153
+ if key in _MEMO:
154
+ return _MEMO[key]
155
+ value = build()
156
+ with _MEMO_LOCK:
157
+ _MEMO[key] = value
158
+ while len(_MEMO) > MEMO_KEEP:
159
+ _MEMO.pop(next(iter(_MEMO)))
160
+ return value
161
+
162
+
163
+ def _mark(path, shared):
164
+ """Moves whenever the mirror rows can have changed: an install, a file swap or a rewrite."""
165
+ from harness import datastore
166
+ generation = None
167
+ if shared:
168
+ with contextlib.suppress(Exception):
169
+ generation = datastore.mirror_generation()
170
+ stamps = []
171
+ for part in (Path(path), Path(str(path) + ".wal")):
172
+ try:
173
+ stat = os.stat(part)
174
+ stamps.append((stat.st_ino, stat.st_mtime_ns, stat.st_size))
175
+ except OSError:
176
+ stamps.append(None)
177
+ return (str(path), generation, tuple(stamps))
178
+
179
+
180
  @contextlib.contextmanager
181
  def _reader(tenant_key, st):
182
+ """A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
183
+
184
+ The web process reads through the same door as the typed `ut_amzn_*` reads: assert the tenant, then
185
+ `datastore.ro_cursor()` on the serving instance, which follows the installed mirror version."""
 
186
  from harness import datastore
187
+ path = Path(_mirror_path(tenant_key, st))
188
  try:
189
  shared = Path(datastore.DB_PATH).resolve() == path.resolve()
190
  except Exception: # noqa: BLE001
191
  shared = False
192
+ if not path.exists():
193
+ yield None
194
+ return
195
+ cur = None
196
+ try:
197
+ if shared:
198
+ if hasattr(st, "assert_datastore_matches"):
199
+ st.assert_datastore_matches()
200
+ cur = datastore.ro_cursor()
201
+ else:
202
+ import duckdb
203
+ cur = duckdb.connect(str(path), read_only=True)
204
+ except Exception as exc: # noqa: BLE001
205
+ if cur is not None:
206
+ with contextlib.suppress(Exception):
207
+ cur.close()
208
+ yield _Unreadable(type(exc).__name__)
209
  return
210
+ token = _MARK.set(_mark(path, shared))
 
211
  try:
212
+ yield cur
213
  finally:
214
+ _MARK.reset(token)
215
+ cur.close()
216
+
217
+
218
+ def _interrupt(cur):
219
+ with contextlib.suppress(Exception):
220
+ cur.interrupt()
221
+
222
+
223
+ def _bounded(cur, sql, params=()):
224
+ """Every row of one probe query, interrupted past PROBE_SECONDS. Any failure raises `_ReadFailed`.
225
+
226
+ Pass a query without parameters: DuckDB ignores interrupt() on a parameterized execute."""
227
+ if cur is None or isinstance(cur, _Unreadable):
228
+ raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
229
+ timer = threading.Timer(PROBE_SECONDS, _interrupt, [cur])
230
+ timer.daemon = True
231
+ timer.start()
232
+ try:
233
+ return cur.execute(sql, list(params)).fetchall()
234
+ except Exception as exc: # noqa: BLE001
235
+ raise _ReadFailed(type(exc).__name__) from exc
236
+ finally:
237
+ timer.cancel()
238
+
239
+
240
+ def _types(cur, table):
241
+ """{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
242
+ if not table.replace("_", "").isalnum():
243
+ raise ValueError(f"{table!r} is not a table name.")
244
+ # A literal, not a parameter: DuckDB ignores interrupt() on a parameterized execute.
245
+ return {str(n): str(t) for n, t in _bounded(
246
+ cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
247
 
248
 
249
  def _columns(cur, table):
250
+ if cur is None or isinstance(cur, _Unreadable):
251
  return set()
252
  try:
253
  return {r[0] for r in cur.execute(
 
256
  return set()
257
 
258
 
259
+ def iso_period(value):
260
+ """True when a `_period_start` text starts with an ISO date (YYYY-MM-DD), as `stamp_week` writes it."""
261
+ try:
262
+ dt.date.fromisoformat(_text(value)[:10])
263
+ except ValueError:
264
+ return False
265
+ return len(_text(value)) >= 10
266
+
267
+
268
  def _our_asins(cur):
269
  """SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
270
  ours = set()
 
277
  return {a for a in ours if a}
278
 
279
 
280
+ def _period(cur, kind="VARCHAR"):
281
+ """The newest brand analytics `_period_start` as text, or None. One column scan, bounded.
282
+
283
+ A text column takes MAX as stored. A DATE or TIMESTAMP column casts only the one MAX value."""
284
+ newest = ('MAX("_period_start")' if str(kind).upper().startswith("VARCHAR")
285
+ else 'CAST(MAX("_period_start") AS VARCHAR)')
286
+ rows = _bounded(cur, f'SELECT {newest} FROM {BA_TABLE}')
287
+ return str(rows[0][0]) if rows and rows[0][0] is not None else None
288
 
289
 
290
  _TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))'
 
304
  return sorted(_text(r[0]) for r in rows if _text(r[0]))
305
 
306
 
307
+ def _facts(cur, fresh=False):
308
+ """The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived."""
309
+ def build():
310
+ types = _types(cur, BA_TABLE)
311
+ facts = {"columns": set(types), "period": None, "ours": set(), "derived": []}
312
+ if not set(BA_NEED) <= facts["columns"]:
313
+ return facts
314
+ facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
315
+ if facts["period"] is None or not iso_period(facts["period"]):
316
+ return facts
317
+ try:
318
+ facts["ours"] = _our_asins(cur)
319
+ facts["derived"] = _derived_departments(cur, facts["ours"])
320
+ except Exception as exc: # noqa: BLE001
321
+ raise _ReadFailed(type(exc).__name__) from exc
322
+ return facts
323
+ mark = _MARK.get()
324
+ return _remember(None if mark is None or fresh else ("facts", mark), build)
325
+
326
+
327
+ def _source(cur, st, fresh=False):
328
  """What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
329
  out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
330
  if cur is None:
331
  out["problems"].append(NO_MIRROR)
332
  return out
333
+ try:
334
+ facts = _facts(cur, fresh)
335
+ except _ReadFailed as exc:
336
+ out["problems"].append(READ_FAILED.format(cause=exc))
337
  return out
338
+ if not set(BA_NEED) <= facts["columns"] or facts["period"] is None:
 
339
  out["problems"].append(NOT_SYNCED)
340
  return out
341
+ if not iso_period(facts["period"]):
342
+ out["problems"].append(BAD_PERIOD.format(value=facts["period"]))
343
+ return out
344
+ out["period"] = facts["period"]
345
+ out["ours"] = set(facts["ours"])
346
+ out["derived"] = list(facts["derived"])
347
  chosen = _doc(st, KEY_SETTINGS).get("departments")
348
  out["departments"] = sorted({_text(d) for d in chosen if _text(d)}) if chosen else out["derived"]
349
  if not out["departments"]:
 
388
  f'AND (prior.rank - cur.rank) * 100.0 / prior.rank >= ? '
389
  f'ORDER BY pct DESC, cur.term')
390
  params = [period, list(departments), [t for t, _r in pairs], [r for _t, r in pairs], *params_top, float(min_pct)]
391
+ mark = _MARK.get()
392
+ key = (None if mark is None else
393
+ ("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
394
+ return _remember(key, lambda: cur.execute(sql, params).fetchall())
395
 
396
 
397
  def _top3_list(raw, ours):
 
515
  st = _rt(tenant_key, st)
516
  now = now or _now()
517
  with _reader(tenant_key, st) as cur:
518
+ # A sync just wrote the mirror, so the period is read again, never from the memo.
519
+ src = _source(cur, st, fresh=True)
520
  if src["period"] is None or not src["departments"]:
521
  return {"state": "nothing", "problems": src["problems"]}
522
  terms = week_ranks(cur, src["period"], src["departments"])
 
579
  error = _doc(st, KEY_INDEX).get("last_error")
580
  if error:
581
  problems.append(error)
582
+ windows = _windows(st, src["period"], prior_week)
583
  found = []
584
  for term, before, now_rank, rise, pct, top in rows:
585
  top3 = _top3_list(top, src["ours"])
 
587
  found.append({"id": item_id(term), "term": term, "title": term, "priorRank": int(before),
588
  "rank": int(now_rank), "rise": int(rise), "risePct": round(float(pct), 2),
589
  "ours": bool(mine), "asin": mine[0] if mine else None, "top3": top3, "rising": True,
590
+ "periodStart": src["period"], "priorPeriodStart": windows[1]["periodStart"] if len(windows) > 1
591
+ else None,
592
  "reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
593
  return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
594
  "priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
595
+ "windows": windows, "rising": found, "problems": problems}
596
+
597
+
598
+ def _windows(st, period, prior_week):
599
+ """The compared weeks, newest first: each window's week key and its `_period_start` text."""
600
+ out = []
601
+ if period:
602
+ out.append({"window": "current", "week": period[:10], "periodStart": period, "iso": iso_period(period)})
603
+ if prior_week:
604
+ text = _text(_doc(st, WEEK_PREFIX + prior_week).get("period")) or prior_week
605
+ out.append({"window": "prior", "week": prior_week, "periodStart": text, "iso": iso_period(text)})
606
+ return out
607
 
608
 
609
  def _stored(st):
 
658
  page, nxt = _page(wanted, cursor, limit)
659
  return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
660
  "priorWeek": found["priorWeek"], "departments": found["departments"],
661
+ "problems": found["problems"], "weeks": _index(st), "windows": found["windows"]}
662
 
663
 
664
  def _lookup(st, iid, found=None):
 
802
  """Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
803
  st = _rt(tenant_key, st)
804
  with _reader(tenant_key, st) as cur:
805
+ src = _source(cur, st, fresh=True)
806
  period = src["period"]
807
  if period is None:
808
  return {"ok": False, "app_total": None, "independent_total": None, "week": None,
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- b35a69b08faac6cbe97c8da0a68680946085ae1959a978723651ff4fa6f02e26
 
1
+ 7d1ba4682927d550a33ca880be112e51699b4a79459a302274e1e09cd13ef434
web/src/apps/catalog/CatalogApp.tsx CHANGED
@@ -24,11 +24,12 @@ export function CatalogApp({ definition, database, scope, onOpen, onData, editor
24
  useEffect(() => {
25
  if (!data.loading && !data.error) onData?.(data.fields, data.allRows);
26
  }, [data.loading, data.error, data.fields, data.allRows, onData]);
 
27
  if (data.loading) {
28
- return <div className="cg-app-launch-state" role="status" aria-label="Loading Catalog">
29
- <span className="cg-app-launch-line is-wide" />
30
- {data.progress > 0 && <span>Loading records, {data.progress.toLocaleString()} read</span>}
31
- </div>;
32
  }
33
  if (data.error) {
34
  return <div className="cg-app-launch-state is-error" role="alert">
 
24
  useEffect(() => {
25
  if (!data.loading && !data.error) onData?.(data.fields, data.allRows);
26
  }, [data.loading, data.error, data.fields, data.allRows, onData]);
27
+ // The Catalogs list reads its own records, so it paints while rows load. Only the editor waits.
28
  if (data.loading) {
29
+ return <section className="cg-app-catalog is-print-editor" aria-label={`${definition.name} catalog`}>
30
+ <CatalogApplicationView definition={definition} rows={data.rows} onOpen={onOpen}
31
+ editor={{ canEdit: editor.canEdit }} rowsPending={{ read: data.progress }} />
32
+ </section>;
33
  }
34
  if (data.error) {
35
  return <div className="cg-app-launch-state is-error" role="alert">
web/src/customer-grid/CatalogView.tsx CHANGED
@@ -221,17 +221,29 @@ export interface CatalogApplicationEditor {
221
  onSave: (catalogs: CatalogSpec[], catalogSpec: CatalogViewSpec) => Promise<void>;
222
  }
223
 
 
 
 
 
 
 
 
 
 
 
224
  /** Apps read C1 records. Sheets retain their read-only row projection. */
225
  export const CatalogApplicationView = memo(function CatalogApplicationView({
226
  definition,
227
  rows,
228
  onOpen,
229
  editor,
 
230
  }: {
231
  definition: CatalogApplicationDefinition;
232
  rows: Row[];
233
  onOpen: (pid: number) => void;
234
- editor?: CatalogApplicationEditor;
 
235
  }) {
236
  const items = useMemo(
237
  () => catalogApplicationItems(rows, definition.fieldMap),
@@ -243,7 +255,8 @@ export const CatalogApplicationView = memo(function CatalogApplicationView({
243
  key={definition.id}
244
  definition={definition}
245
  canEdit={editor.canEdit}
246
- rows={editor.poolRows}
 
247
  onOpen={onOpen}
248
  />
249
  );
@@ -316,10 +329,11 @@ function catalogHashId(next?: string | null): string | null {
316
  }
317
 
318
  /** Child content for the shared AppFrame. Only the editor owns Back to library. */
319
- export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
320
  definition: CatalogApplicationDefinition;
321
  canEdit: boolean;
322
  rows: Row[];
 
323
  onOpen: (pid: number) => void;
324
  }) {
325
  const [catalogs, setCatalogs] = useState<CatalogRecord[]>([]);
@@ -408,7 +422,7 @@ export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
408
  };
409
 
410
  if (openId) return <CatalogRecordOpener key={openId} appId={definition.id} id={openId}
411
- definition={definition} canEdit={canEdit} rows={rows} onOpen={onOpen}
412
  onBack={() => { setOpenId(null); catalogHashId(null); }} onSaved={replace} />;
413
 
414
  const actionTitle = action?.kind === "create" ? "New catalog" : action?.kind === "rename" ? "Rename catalog" :
@@ -480,8 +494,9 @@ export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
480
  </section>;
481
  }
482
 
483
- function CatalogRecordOpener({ appId, id, ...props }: {
484
  appId: string; id: string; definition: CatalogApplicationDefinition; canEdit: boolean; rows: Row[];
 
485
  onOpen: (pid: number) => void; onBack: () => void; onSaved: (catalog: CatalogRecord) => void;
486
  }) {
487
  const [record, setRecord] = useState<CatalogRecord | null>(null);
@@ -497,11 +512,16 @@ function CatalogRecordOpener({ appId, id, ...props }: {
497
  });
498
  return () => controller.abort();
499
  }, [appId, id, attempt]);
500
- if (record) return <CatalogRecordEditor key={record.id} record={record} {...props} />;
 
501
  return <section className="cg-catalog-record" aria-label="Catalog editor">
502
  <button type="button" className="cg-btn" onClick={props.onBack}>Back to library</button>
503
  {error ? <div className="cg-catalog-library-error" role="alert">{error}
504
  <button type="button" className="cg-btn" onClick={() => setAttempt((value) => value + 1)}>Retry</button></div> :
 
 
 
 
505
  <p role="status">Loading catalog</p>}
506
  </section>;
507
  }
 
221
  onSave: (catalogs: CatalogSpec[], catalogSpec: CatalogViewSpec) => Promise<void>;
222
  }
223
 
224
+ /** The library needs only the permission while rows load. The editor waits for the full editor data. */
225
+ export type CatalogPendingEditor = Pick<CatalogApplicationEditor, "canEdit">;
226
+
227
+ /** Rows the App has read so far, while its record walk is still running. */
228
+ export interface CatalogRowsPending {
229
+ read: number;
230
+ }
231
+
232
+ const NO_ROWS: Row[] = [];
233
+
234
  /** Apps read C1 records. Sheets retain their read-only row projection. */
235
  export const CatalogApplicationView = memo(function CatalogApplicationView({
236
  definition,
237
  rows,
238
  onOpen,
239
  editor,
240
+ rowsPending,
241
  }: {
242
  definition: CatalogApplicationDefinition;
243
  rows: Row[];
244
  onOpen: (pid: number) => void;
245
+ editor?: CatalogApplicationEditor | CatalogPendingEditor;
246
+ rowsPending?: CatalogRowsPending;
247
  }) {
248
  const items = useMemo(
249
  () => catalogApplicationItems(rows, definition.fieldMap),
 
255
  key={definition.id}
256
  definition={definition}
257
  canEdit={editor.canEdit}
258
+ rows={!rowsPending && "poolRows" in editor ? editor.poolRows : NO_ROWS}
259
+ rowsPending={rowsPending}
260
  onOpen={onOpen}
261
  />
262
  );
 
329
  }
330
 
331
  /** Child content for the shared AppFrame. Only the editor owns Back to library. */
332
+ export function CatalogLibrary({ definition, canEdit, rows, rowsPending, onOpen }: {
333
  definition: CatalogApplicationDefinition;
334
  canEdit: boolean;
335
  rows: Row[];
336
+ rowsPending?: CatalogRowsPending;
337
  onOpen: (pid: number) => void;
338
  }) {
339
  const [catalogs, setCatalogs] = useState<CatalogRecord[]>([]);
 
422
  };
423
 
424
  if (openId) return <CatalogRecordOpener key={openId} appId={definition.id} id={openId}
425
+ definition={definition} canEdit={canEdit} rows={rows} rowsPending={rowsPending} onOpen={onOpen}
426
  onBack={() => { setOpenId(null); catalogHashId(null); }} onSaved={replace} />;
427
 
428
  const actionTitle = action?.kind === "create" ? "New catalog" : action?.kind === "rename" ? "Rename catalog" :
 
494
  </section>;
495
  }
496
 
497
+ function CatalogRecordOpener({ appId, id, rowsPending, ...props }: {
498
  appId: string; id: string; definition: CatalogApplicationDefinition; canEdit: boolean; rows: Row[];
499
+ rowsPending?: CatalogRowsPending;
500
  onOpen: (pid: number) => void; onBack: () => void; onSaved: (catalog: CatalogRecord) => void;
501
  }) {
502
  const [record, setRecord] = useState<CatalogRecord | null>(null);
 
512
  });
513
  return () => controller.abort();
514
  }, [appId, id, attempt]);
515
+ // The editor places records, so it mounts only once the App's record walk is complete.
516
+ if (record && !rowsPending) return <CatalogRecordEditor key={record.id} record={record} {...props} />;
517
  return <section className="cg-catalog-record" aria-label="Catalog editor">
518
  <button type="button" className="cg-btn" onClick={props.onBack}>Back to library</button>
519
  {error ? <div className="cg-catalog-library-error" role="alert">{error}
520
  <button type="button" className="cg-btn" onClick={() => setAttempt((value) => value + 1)}>Retry</button></div> :
521
+ rowsPending ? <div className="cg-app-launch-state" role="status" aria-label="Loading catalog records">
522
+ <span className="cg-app-launch-line is-wide" />
523
+ {rowsPending.read > 0 && <span>Loading records, {rowsPending.read.toLocaleString()} read</span>}
524
+ </div> :
525
  <p role="status">Loading catalog</p>}
526
  </section>;
527
  }