fsanyoto commited on
Commit
82ff59c
·
verified ·
1 Parent(s): 3dbbea8

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "0085702694",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "99ba051e73",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 0085702694
 
1
+ 99ba051e73
api/amazon_relational.py CHANGED
@@ -872,6 +872,23 @@ def _missing_occurrence(source, partition, complete, predicate=''):
872
  f'WHERE "_identity_rowid" = {source}.rowid)')
873
 
874
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
875
  def _grid_identity(cur, key, names, prepared, is_view):
876
  table = key.removeprefix(PREFIX)
877
  identity_names = names | prepared
@@ -894,14 +911,13 @@ def _grid_identity(cur, key, names, prepared, is_view):
894
  complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
895
  fallback = sorted(set((*keys, *_fallback_columns(table))))
896
  partition = ", ".join(value(name, peer) for name in fallback)
 
897
  if is_view:
898
  if legacy:
899
  partition += f', ({peer}."_connection_id" = {legacy})'
900
- ranks.append(f'row_number() OVER (PARTITION BY {partition} '
901
- f'ORDER BY {peer}."_amazon_publication_order") - 1 AS {_quoted(label)}')
902
  occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
903
  else:
904
- peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
905
  predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
906
  occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
907
  if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
@@ -925,9 +941,7 @@ def _grid_identity(cur, key, names, prepared, is_view):
925
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
926
  spec = {'id': ident}
927
  if ranks:
928
- spec['from_sql'] = (f'(SELECT *, {", ".join(ranks)} FROM '
929
- f'(SELECT *, row_number() OVER () - 1 AS "_amazon_publication_order" '
930
- f'FROM {source}) AS {peer}) AS {source}')
931
  if not is_view and table not in amazon_store.DERIVED_TABLES:
932
  spec['page_key'] = f'{source}.rowid'
933
  return spec
 
872
  f'WHERE "_identity_rowid" = {source}.rowid)')
873
 
874
 
875
+ def _occurrence_view(source, peer, ranks):
876
+ """Number equal-key occurrences over incomplete-key rows only, in publication order.
877
+
878
+ A complete key never shares a partition with an incomplete one, so the narrow window over
879
+ incomplete rows yields the same occurrence a whole-table window would.
880
+ """
881
+ order = '"_amazon_publication_order"'
882
+ numbered = f'(SELECT *, row_number() OVER () - 1 AS {order} FROM {source})'
883
+ joins = "".join(
884
+ f' LEFT JOIN (SELECT {order}, row_number() OVER (PARTITION BY {partition} '
885
+ f'ORDER BY {peer}.{order}) - 1 AS {_quoted(label)} FROM {numbered} AS {peer} '
886
+ f'WHERE NOT ({complete})) AS "_amazon_rank_{index}" USING ({order})'
887
+ for index, (label, partition, complete) in enumerate(ranks))
888
+ labels = ", ".join(f'"_amazon_rank_{index}".{_quoted(label)}' for index, (label, _p, _c) in enumerate(ranks))
889
+ return f'(SELECT {peer}.*, {labels} FROM {numbered} AS {peer}{joins}) AS {source}'
890
+
891
+
892
  def _grid_identity(cur, key, names, prepared, is_view):
893
  table = key.removeprefix(PREFIX)
894
  identity_names = names | prepared
 
911
  complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
912
  fallback = sorted(set((*keys, *_fallback_columns(table))))
913
  partition = ", ".join(value(name, peer) for name in fallback)
914
+ peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
915
  if is_view:
916
  if legacy:
917
  partition += f', ({peer}."_connection_id" = {legacy})'
918
+ ranks.append((label, partition, peer_complete))
 
919
  occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
920
  else:
 
921
  predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
922
  occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
923
  if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
 
941
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
942
  spec = {'id': ident}
943
  if ranks:
944
+ spec['from_sql'] = _occurrence_view(source, peer, ranks)
 
 
945
  if not is_view and table not in amazon_store.DERIVED_TABLES:
946
  spec['page_key'] = f'{source}.rowid'
947
  return spec
api/registry_source.py CHANGED
@@ -558,10 +558,13 @@ class CanonicalStore(BatchStore):
558
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
559
  for key, meta in definitions.items() if isinstance(meta, dict)}}
560
  elif name == 'loopable_route_system':
561
- meta_key = ('route_definitions', name)
 
562
  if meta_key not in self.metadata or fresh:
563
  base = object_values(self._runtime, name, omit=('routes', 'zones'))
564
- base['routes'] = [(index, route) for batch in array_values(self._runtime, name,
 
 
565
  ['routes'], drop=('stops',)) for index, route in batch]
566
  base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
567
  'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
@@ -571,8 +574,7 @@ class CanonicalStore(BatchStore):
571
  value['routes'], value['zones'] = [], []
572
  for index, route in saved['routes']:
573
  if isinstance(route, dict):
574
- # A stop names a customer pid, so a product batch reads no stop membership.
575
- route = dict(route, stops=[] if self.table_key == 'product_data' else self._selected_json(
576
  name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
577
  value['routes'].append(route)
578
  value['zones'] = copy.deepcopy(saved['zones'])
 
558
  [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
559
  for key, meta in definitions.items() if isinstance(meta, dict)}}
560
  elif name == 'loopable_route_system':
561
+ # The product shape omits routes, so the cached definitions are keyed by that shape.
562
+ meta_key = ('route_definitions', name, self.table_key == 'product_data')
563
  if meta_key not in self.metadata or fresh:
564
  base = object_values(self._runtime, name, omit=('routes', 'zones'))
565
+ # A Route stops at customers, so a product batch reads only the Zone definitions.
566
+ base['routes'] = [] if self.table_key == 'product_data' else [
567
+ (index, route) for batch in array_values(self._runtime, name,
568
  ['routes'], drop=('stops',)) for index, route in batch]
569
  base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
570
  'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
 
574
  value['routes'], value['zones'] = [], []
575
  for index, route in saved['routes']:
576
  if isinstance(route, dict):
577
+ route = dict(route, stops=self._selected_json(
 
578
  name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
579
  value['routes'].append(route)
580
  value['zones'] = copy.deepcopy(saved['zones'])
platform/core/received_changes.py CHANGED
@@ -20,6 +20,8 @@ JOURNAL_DEADLINE_SECONDS = 120
20
  BASELINE_LOCK_SECONDS = 300
21
  BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
22
  PROJECTION_VERSION = 1
 
 
23
  _LISTENERS = []
24
  _PROJECTIONS = {}
25
  _LOCK = threading.RLock()
@@ -293,9 +295,8 @@ def _blob_path(identity, kind, digest):
293
  return f"{STORE_KEY}/observations/{identity}/{kind}/{digest}"
294
 
295
 
296
- def _read_blob(rt, path, digest, size):
297
  from core import blob_envelope as be
298
- raw = rt.download_bytes(path)
299
  if raw is None or len(raw) > BASELINE_CHUNK_BYTES:
300
  raise BaselineUnavailable("A required observation blob is missing or oversized.")
301
  data = be.open(rt.key, path, raw) if be.is_sealed(raw) else bytes(raw)
@@ -304,16 +305,22 @@ def _read_blob(rt, path, digest, size):
304
  return data
305
 
306
 
 
 
 
 
307
  def _write_blob(rt, path, data):
308
  """Reuse immutable content and verify persistence before publishing its reference."""
309
  from core import blob_envelope as be
310
  digest, size = _digest(data), len(data)
311
- if rt.download_bytes(path) is None:
 
312
  stored = be.seal(rt.key, path, data)
313
  if len(stored) > BASELINE_CHUNK_BYTES:
314
  raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
315
  rt.upload_bytes(path, stored)
316
- _read_blob(rt, path, digest, size)
 
317
  return {"sha256": digest, "bytes": size}
318
 
319
 
@@ -331,7 +338,19 @@ def _persist_baseline(rt, identity, binding, path, snapshot, stamp):
331
  return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
332
 
333
 
334
- def _restore_baseline(rt, identity, binding, reference, path, con):
 
 
 
 
 
 
 
 
 
 
 
 
335
  try:
336
  data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
337
  reference["sha256"], reference["bytes"])
@@ -342,15 +361,18 @@ def _restore_baseline(rt, identity, binding, reference, path, con):
342
  or not manifest.get("at")):
343
  raise BaselineUnavailable("The observation manifest has a different identity or shape.")
344
  _at(manifest["at"])
345
- content, size = hashlib.sha256(), 0
346
- with path.open("wb") as handle:
347
- for part in manifest["parts"]:
348
- chunk = _read_blob(rt, _blob_path(identity, "chunks", part["sha256"]),
349
- part["sha256"], part["bytes"])
350
- handle.write(chunk)
351
- content.update(chunk)
352
- size += len(chunk)
353
- if content.hexdigest() != manifest["sha256"] or size != manifest["bytes"]:
 
 
 
354
  raise BaselineUnavailable("The observation snapshot failed content verification.")
355
  _validate_projection(con, path)
356
  return manifest
@@ -360,17 +382,52 @@ def _restore_baseline(rt, identity, binding, reference, path, con):
360
  raise BaselineUnavailable("The required observation could not be verified.") from exc
361
 
362
 
363
- def _validate_projection(con, path):
 
 
 
 
 
 
 
 
 
 
 
364
  schema = con.execute("DESCRIBE SELECT * FROM read_parquet(?)", [str(path)]).fetchall()
365
  if [(row[0], row[1]) for row in schema] != [("row_id", "VARCHAR"), ("image", "VARCHAR")]:
366
  raise BaselineUnavailable("The observation projection has an invalid schema.")
367
- invalid = con.execute("SELECT count(*) != count(DISTINCT row_id) OR "
368
- "count(*) != count(row_id) OR count(*) != count(image) "
369
- "FROM read_parquet(?)", [str(path)]).fetchone()[0]
370
- if invalid:
 
 
 
 
 
371
  raise BaselineUnavailable("A received-change projection requires unique non-null row IDs and images.")
372
 
373
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
374
  def _observation_state(rt, identity):
375
  document = _document(rt, rt.get(STORE_KEY, fresh=True))
376
  states = document.get("observations", {})
@@ -392,7 +449,7 @@ def _observation_state(rt, identity):
392
  return state, document
393
 
394
 
395
- def _move_baseline(rt, identity, expected, updated):
396
  """Serialize replica transitions in the existing tenant journal's update boundary."""
397
  def move(raw):
398
  document = _document(rt, raw)
@@ -400,6 +457,8 @@ def _move_baseline(rt, identity, expected, updated):
400
  if states.get(identity) != expected:
401
  raise BaselineUnavailable("The observation changed on another replica. Retry publication.")
402
  states[identity] = updated
 
 
403
  return document
404
  result = rt.update(STORE_KEY, move, flush="sync")
405
  if result is None or result is False:
@@ -422,19 +481,24 @@ def _query_watchdog(con, deadline):
422
 
423
 
424
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
425
- diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
426
- "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
427
- count = con.execute("SELECT count(*) " + diff, [str(baseline), str(current)]).fetchone()[0]
 
 
428
  # A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
429
- if count > MAX_EVENTS:
 
430
  print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
431
  f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
432
  return
433
- cursor, done = con.cursor(), 0
434
  try:
435
  with _query_watchdog(cursor, deadline):
436
- cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
437
- + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
 
 
438
  while batch := cursor.fetchmany(256):
439
  for row_id, before_json, after_json in batch:
440
  # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
@@ -459,13 +523,17 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
459
  cursor.close()
460
 
461
 
 
 
 
 
462
  def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
463
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
464
  if not spec.get("cols"):
465
  return
466
  if _uncommitted():
467
  raise BaselineUnavailable("Connector observations require a completed publication transaction.")
468
- identity = hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
469
  binding = {"version": PROJECTION_VERSION, "tenant": _tenant(rt), "source": source,
470
  "database": database, "projection": _projection_shape(spec)}
471
  snapshot, stamp = str(manifest["snapshot_id"]), manifest.get("completed_at")
@@ -477,23 +545,28 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
477
  or any(event["source"] == source and event["database"] == database
478
  for event in document["events"])):
479
  raise BaselineUnavailable("A legacy observation requires verified migration before comparison.")
 
480
  with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
481
  baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
482
 
483
- def complete(pending):
484
- observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con)
 
 
 
 
 
485
  if pending["current"]:
486
- _restore_baseline(rt, identity, binding, pending["current"], baseline, con)
487
- _compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"],
488
- deadline)
489
  committed = {"current": pending["pending"], "pending": None}
490
- _move_baseline(rt, identity, pending, committed)
 
491
  return committed
492
 
493
  if state and state["pending"]:
494
  state = complete(state)
495
  if state:
496
- prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con)
497
  prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
498
  incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
499
  if prior["snapshot"] == snapshot or incoming_at < prior_at:
@@ -506,12 +579,41 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
506
  f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
507
  f"FROM {spec.get('from_sql') or spec['table']}{where}")
508
  current.unlink(missing_ok=True)
509
- con.execute(f"COPY ({query}) TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(current)])
510
- _validate_projection(con, current)
511
  reference = _persist_baseline(rt, identity, binding, current, snapshot, stamp)
512
  pending = {"current": state["current"] if state else None, "pending": reference}
513
  _move_baseline(rt, identity, state, pending)
514
- complete(pending)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
515
 
516
 
517
  def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
@@ -539,25 +641,24 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
539
  started = time.monotonic()
540
  specs = resolver(rt, con)
541
  resolution_seconds = time.monotonic() - started
542
- snapshots_started = time.monotonic()
543
- completed = deferred = skipped = 0
544
- for database, spec in specs.items():
545
- # A skipped database keeps its stored baseline for the next publication.
546
- if time.monotonic() >= deadline[0]:
547
- skipped += 1
548
- print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
549
- continue
550
- try:
551
- _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
552
- completed += 1
553
- except Exception as exc:
554
- deferred += 1
555
- _LOG.warning("Received-change projection deferred for %s: %s",
556
- database, type(exc).__name__)
557
- print(f'[received-changes] publication connector={connector} tables={len(specs)} '
558
- f'completed={completed} deferred={deferred} skipped={skipped} '
559
- f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
560
- flush=True)
561
  finally:
562
  con.close()
563
 
 
20
  BASELINE_LOCK_SECONDS = 300
21
  BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
22
  PROJECTION_VERSION = 1
23
+ # A 128-bit digest of each image: the compare join carries it, never the image itself.
24
+ IMAGE_DIGEST = "md5_number(image)"
25
  _LISTENERS = []
26
  _PROJECTIONS = {}
27
  _LOCK = threading.RLock()
 
295
  return f"{STORE_KEY}/observations/{identity}/{kind}/{digest}"
296
 
297
 
298
+ def _open_blob(rt, path, raw, digest, size):
299
  from core import blob_envelope as be
 
300
  if raw is None or len(raw) > BASELINE_CHUNK_BYTES:
301
  raise BaselineUnavailable("A required observation blob is missing or oversized.")
302
  data = be.open(rt.key, path, raw) if be.is_sealed(raw) else bytes(raw)
 
305
  return data
306
 
307
 
308
+ def _read_blob(rt, path, digest, size):
309
+ return _open_blob(rt, path, rt.download_bytes(path), digest, size)
310
+
311
+
312
  def _write_blob(rt, path, data):
313
  """Reuse immutable content and verify persistence before publishing its reference."""
314
  from core import blob_envelope as be
315
  digest, size = _digest(data), len(data)
316
+ raw = rt.download_bytes(path)
317
+ if raw is None:
318
  stored = be.seal(rt.key, path, data)
319
  if len(stored) > BASELINE_CHUNK_BYTES:
320
  raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
321
  rt.upload_bytes(path, stored)
322
+ raw = rt.download_bytes(path)
323
+ _open_blob(rt, path, raw, digest, size)
324
  return {"sha256": digest, "bytes": size}
325
 
326
 
 
338
  return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
339
 
340
 
341
+ def _cached_baseline(cache, identity, reference):
342
+ return cache / f"{identity}-{reference['sha256']}.parquet"
343
+
344
+
345
+ def _cache_baseline(cache, identity, reference, path):
346
+ """Keep the committed observation on local disk so the next pass skips its download."""
347
+ cache.mkdir(parents=True, exist_ok=True)
348
+ for stale in cache.glob(identity + "-*.parquet"):
349
+ stale.unlink(missing_ok=True)
350
+ os.replace(path, _cached_baseline(cache, identity, reference))
351
+
352
+
353
+ def _restore_baseline(rt, identity, binding, reference, path, con, cache):
354
  try:
355
  data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
356
  reference["sha256"], reference["bytes"])
 
361
  or not manifest.get("at")):
362
  raise BaselineUnavailable("The observation manifest has a different identity or shape.")
363
  _at(manifest["at"])
364
+ cached = _cached_baseline(cache, identity, reference)
365
+ if cached.exists():
366
+ with cached.open("rb") as source:
367
+ chunks = iter(lambda: source.read(BASELINE_CHUNK_BYTES), b"")
368
+ content, size = _write_chunks(path, chunks)
369
+ if content != manifest["sha256"] or size != manifest["bytes"]:
370
+ cached.unlink(missing_ok=True)
371
+ if not cached.exists():
372
+ content, size = _write_chunks(path, (
373
+ _read_blob(rt, _blob_path(identity, "chunks", part["sha256"]), part["sha256"], part["bytes"])
374
+ for part in manifest["parts"]))
375
+ if content != manifest["sha256"] or size != manifest["bytes"]:
376
  raise BaselineUnavailable("The observation snapshot failed content verification.")
377
  _validate_projection(con, path)
378
  return manifest
 
382
  raise BaselineUnavailable("The required observation could not be verified.") from exc
383
 
384
 
385
+ def _write_chunks(path, chunks):
386
+ content, size = hashlib.sha256(), 0
387
+ with path.open("wb") as handle:
388
+ for chunk in chunks:
389
+ handle.write(chunk)
390
+ content.update(chunk)
391
+ size += len(chunk)
392
+ return content.hexdigest(), size
393
+
394
+
395
+ def _projection_issues(con, path):
396
+ """Return (shared row IDs, rows sharing them, any null) for a projection of valid schema."""
397
  schema = con.execute("DESCRIBE SELECT * FROM read_parquet(?)", [str(path)]).fetchall()
398
  if [(row[0], row[1]) for row in schema] != [("row_id", "VARCHAR"), ("image", "VARCHAR")]:
399
  raise BaselineUnavailable("The observation projection has an invalid schema.")
400
+ return con.execute("SELECT count(*) FILTER (WHERE n > 1), coalesce(sum(n) FILTER (WHERE n > 1), 0), "
401
+ "coalesce(bool_or(row_id IS NULL OR missing > 0), false) FROM (SELECT row_id, "
402
+ "count(*) AS n, count(*) - count(image) AS missing FROM read_parquet(?) GROUP BY row_id)",
403
+ [str(path)]).fetchone()
404
+
405
+
406
+ def _validate_projection(con, path):
407
+ shared, _rows, missing = _projection_issues(con, path)
408
+ if shared or missing:
409
  raise BaselineUnavailable("A received-change projection requires unique non-null row IDs and images.")
410
 
411
 
412
+ def _project(con, query, path, database):
413
+ """Write the projection, leaving out row IDs the connector identity gives to several rows."""
414
+ raw = path.with_name("projection.parquet")
415
+ con.execute(f"COPY ({query}) TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(raw)])
416
+ shared, rows, missing = _projection_issues(con, raw)
417
+ if missing:
418
+ raise BaselineUnavailable("A received-change projection requires non-null row IDs and images.")
419
+ if not shared:
420
+ os.replace(raw, path)
421
+ return
422
+ # An ID shared by several rows names no single record, so it cannot carry a change event.
423
+ print(f"[received-changes] {database}: {rows} rows share {shared} row IDs; "
424
+ "those IDs are not observed until the connector identity separates them", flush=True)
425
+ con.execute("COPY (SELECT * FROM read_parquet($1) WHERE row_id NOT IN (SELECT row_id FROM read_parquet($1) "
426
+ "GROUP BY row_id HAVING count(*) > 1)) TO $2 (FORMAT PARQUET, COMPRESSION ZSTD)",
427
+ [str(raw), str(path)])
428
+ raw.unlink()
429
+
430
+
431
  def _observation_state(rt, identity):
432
  document = _document(rt, rt.get(STORE_KEY, fresh=True))
433
  states = document.get("observations", {})
 
449
  return state, document
450
 
451
 
452
+ def _move_baseline(rt, identity, expected, updated, observed_at=None):
453
  """Serialize replica transitions in the existing tenant journal's update boundary."""
454
  def move(raw):
455
  document = _document(rt, raw)
 
457
  if states.get(identity) != expected:
458
  raise BaselineUnavailable("The observation changed on another replica. Retry publication.")
459
  states[identity] = updated
460
+ if observed_at:
461
+ document.setdefault("observed", {})[identity] = observed_at
462
  return document
463
  result = rt.update(STORE_KEY, move, flush="sync")
464
  if result is None or result is False:
 
481
 
482
 
483
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
484
+ digests = f"(SELECT row_id, {IMAGE_DIGEST} AS image FROM read_parquet({{}}))"
485
+ changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
486
+ f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
487
+ files = [str(baseline), str(current)]
488
+ ids = [row[0] for row in con.execute(changed + f" LIMIT {MAX_EVENTS + 1}", files).fetchall()]
489
  # A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
490
+ if len(ids) > MAX_EVENTS:
491
+ count = con.execute(f"SELECT count(*) FROM ({changed})", files).fetchone()[0]
492
  print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
493
  f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
494
  return
495
+ count, cursor, done = len(ids), con.cursor(), 0
496
  try:
497
  with _query_watchdog(cursor, deadline):
498
+ picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
499
+ cursor.execute(f"SELECT coalesce(b.row_id, a.row_id), b.image, a.image FROM {picked.format('$1')} b "
500
+ f"FULL OUTER JOIN {picked.format('$2')} a USING (row_id) "
501
+ "ORDER BY coalesce(b.row_id, a.row_id)", [*files, ids])
502
  while batch := cursor.fetchmany(256):
503
  for row_id, before_json, after_json in batch:
504
  # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
 
523
  cursor.close()
524
 
525
 
526
+ def _observation_identity(rt, source, database):
527
+ return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
528
+
529
+
530
  def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
531
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
532
  if not spec.get("cols"):
533
  return
534
  if _uncommitted():
535
  raise BaselineUnavailable("Connector observations require a completed publication transaction.")
536
+ identity = _observation_identity(rt, source, database)
537
  binding = {"version": PROJECTION_VERSION, "tenant": _tenant(rt), "source": source,
538
  "database": database, "projection": _projection_shape(spec)}
539
  snapshot, stamp = str(manifest["snapshot_id"]), manifest.get("completed_at")
 
545
  or any(event["source"] == source and event["database"] == database
546
  for event in document["events"])):
547
  raise BaselineUnavailable("A legacy observation requires verified migration before comparison.")
548
+ cache = root / "baselines"
549
  with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
550
  baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
551
 
552
+ def complete(pending, observed=None):
553
+ # A pending reference this pass wrote is already on disk; one from an earlier pass is restored.
554
+ if observed is None:
555
+ observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con,
556
+ cache)["snapshot"]
557
+ if pending["current"]:
558
+ _restore_baseline(rt, identity, binding, pending["current"], baseline, con, cache)
559
  if pending["current"]:
560
+ _compare_baseline(rt, source, database, spec, con, baseline, current, observed, deadline)
 
 
561
  committed = {"current": pending["pending"], "pending": None}
562
+ _move_baseline(rt, identity, pending, committed, observed_at=_at())
563
+ _cache_baseline(cache, identity, pending["pending"], current)
564
  return committed
565
 
566
  if state and state["pending"]:
567
  state = complete(state)
568
  if state:
569
+ prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con, cache)
570
  prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
571
  incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
572
  if prior["snapshot"] == snapshot or incoming_at < prior_at:
 
579
  f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
580
  f"FROM {spec.get('from_sql') or spec['table']}{where}")
581
  current.unlink(missing_ok=True)
582
+ _project(con, query, current, database)
 
583
  reference = _persist_baseline(rt, identity, binding, current, snapshot, stamp)
584
  pending = {"current": state["current"] if state else None, "pending": reference}
585
  _move_baseline(rt, identity, state, pending)
586
+ complete(pending, snapshot)
587
+
588
+
589
+ def _publication_order(rt, connector, specs, paths):
590
+ """Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
591
+ observed = _document(rt, rt.get(STORE_KEY, fresh=True)).get("observed") or {}
592
+
593
+ def priority(item):
594
+ database, spec = item
595
+ files = paths.get(spec.get("table"), ()) if isinstance(spec, dict) else ()
596
+ return (str(observed.get(_observation_identity(rt, connector, str(database))) or ""),
597
+ sum(os.path.getsize(path) for path in files if os.path.exists(path)))
598
+ return sorted(specs.items(), key=priority)
599
+
600
+
601
+ def _project_database(rt, connector, database, spec, con, root, manifest, deadline):
602
+ """Return None when the database committed, else the named reason it was deferred."""
603
+ import duckdb
604
+ from core import request_timing
605
+ try:
606
+ with _query_watchdog(con, deadline):
607
+ _snapshot(rt, connector, database, spec, con, root, manifest, deadline)
608
+ except request_timing.RequestDeadlineExceeded as exc:
609
+ # The watchdog raises on exit after a clean body too; no cause means the database committed.
610
+ return None if exc.__cause__ is None else "journal deadline passed"
611
+ except BaselineUnavailable as exc:
612
+ return f"BaselineUnavailable: {exc}"
613
+ except duckdb.Error as exc:
614
+ # One database's failed query defers it alone; its stored baseline stays for the next pass.
615
+ return type(exc).__name__
616
+ return None
617
 
618
 
619
  def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
 
641
  started = time.monotonic()
642
  specs = resolver(rt, con)
643
  resolution_seconds = time.monotonic() - started
644
+ snapshots_started = time.monotonic()
645
+ completed = deferred = skipped = 0
646
+ for database, spec in _publication_order(rt, connector, specs, paths):
647
+ # A skipped database keeps its stored baseline and leads the next publication.
648
+ if time.monotonic() >= deadline[0]:
649
+ skipped += 1
650
+ print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
651
+ continue
652
+ reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline)
653
+ if reason is None:
654
+ completed += 1
655
+ continue
656
+ deferred += 1
657
+ _LOG.warning("Received-change projection deferred for %s: %s", database, reason)
658
+ print(f'[received-changes] publication connector={connector} tables={len(specs)} '
659
+ f'completed={completed} deferred={deferred} skipped={skipped} '
660
+ f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
661
+ flush=True)
 
662
  finally:
663
  con.close()
664
 
platform/core/shared_overlay.py CHANGED
@@ -861,6 +861,15 @@ def clear_row(table_key, pid, st=None):
861
  return hit[0]
862
 
863
 
 
 
 
 
 
 
 
 
 
864
  def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
865
  """Drop shared cells for rows that no longer exist. Returns how many rows were dropped.
866
 
@@ -878,6 +887,8 @@ def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
878
  keep = {_pid(p) for p in live_pids}
879
  if not keep:
880
  return 0
 
 
881
  dropped = []
882
  completed = False
883
 
 
861
  return hit[0]
862
 
863
 
864
+ def _stored_prune_pass(table_key, st):
865
+ """The committed prune marker read by path on a native store, else None (the write decides)."""
866
+ native = _cell_backend(table_key, st)
867
+ reader = getattr(native[0], 'get_json_paths', None) if native is not None else None
868
+ if not callable(reader):
869
+ return None
870
+ return (reader(native[1], [['_prune_pass']]) or [None])[0]
871
+
872
+
873
  def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
874
  """Drop shared cells for rows that no longer exist. Returns how many rows were dropped.
875
 
 
887
  keep = {_pid(p) for p in live_pids}
888
  if not keep:
889
  return 0
890
+ if pass_id is not None and _stored_prune_pass(table_key, st) == pass_id:
891
+ return 0
892
  dropped = []
893
  completed = False
894
 
platform/modules/competitor_watch.py CHANGED
@@ -46,6 +46,8 @@ READ_FAILED = ("The Amazon mirror could not be read ({cause}), so no search term
46
  "and the next Amazon sync will retry.")
47
  BA_DATES_MISSING = "Brand Analytics report dates are missing"
48
  BA_WEEKLY_MISSING = "Brand Analytics has no dated weekly reports"
 
 
49
  LIMIT = "Alerts and checks use separate store rows and server pages, with no row cap."
50
 
51
  KEY_SETTINGS = "competitor_watch_settings"
@@ -442,7 +444,13 @@ def _columns(cur, table):
442
 
443
  def _ba_columns(cur):
444
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
 
 
 
 
 
445
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
 
446
  if cols and not {"_period_start", "_period_end"} <= cols:
447
  try:
448
  populated = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE})").fetchone()[0]
@@ -459,7 +467,7 @@ def _ba_columns(cur):
459
  raise _ReadFailed(type(exc).__name__) from exc
460
  if populated and not weekly:
461
  raise SourceNotReady(BA_WEEKLY_MISSING)
462
- return cols
463
 
464
 
465
  def _sqp(cur, term=None):
@@ -781,7 +789,9 @@ def term_page(tenant_key, *, st=None, search="", offset=0, limit=50, selected=No
781
  source_error = None
782
  try:
783
  with _reader(tenant_key, st) as cur:
784
- _ba_columns(cur)
 
 
785
  except SourceNotReady as exc:
786
  source_error = exc
787
  cause = READ_FAILED.format(cause=exc)
@@ -1034,6 +1044,12 @@ def on_quotes(tenant, changed):
1034
  compare(tenant, ("daily_quotes",), st=st, quote_asins=changed)
1035
 
1036
 
 
 
 
 
 
 
1037
  def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
1038
  """Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
1039
  st = _rt(tenant_key, st)
@@ -1065,6 +1081,9 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
1065
  if daily and not retry:
1066
  return {"state": "already_compared", "period": newest}
1067
  if not daily and weeks.get("last_compared") == newest:
 
 
 
1068
  return {"state": "already_compared", "period": newest}
1069
  previous = periods[1] if len(periods) > 1 else None
1070
  conf = settings(tenant_key, st=st)
@@ -1160,8 +1179,7 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
1160
  "periodStart": newest, "previousPeriodStart": previous,
1161
  "comparedAt": _stamp(now), "alertsRaised": len(created),
1162
  "checks": len(checks)}
1163
- doc.pop("last_error", None)
1164
- return doc
1165
  st.update(KEY_WEEKS, _weeks)
1166
  return {"state": "compared" if ready else "waiting", "period": newest, "previous": previous,
1167
  "created": created, "checks": checks}
 
46
  "and the next Amazon sync will retry.")
47
  BA_DATES_MISSING = "Brand Analytics report dates are missing"
48
  BA_WEEKLY_MISSING = "Brand Analytics has no dated weekly reports"
49
+ #: Stored sync failures that a live probe finding a dated weekly report proves superseded.
50
+ SUPERSEDED_BY_WEEKLY = frozenset(READ_FAILED.format(cause=cause) for cause in (BA_DATES_MISSING, BA_WEEKLY_MISSING))
51
  LIMIT = "Alerts and checks use separate store rows and server pages, with no row cap."
52
 
53
  KEY_SETTINGS = "competitor_watch_settings"
 
444
 
445
  def _ba_columns(cur):
446
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
447
+ return _ba_source(cur)[0]
448
+
449
+
450
+ def _ba_source(cur):
451
+ """The Brand Analytics columns and whether a dated weekly report exists. Unready sources raise."""
452
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
453
+ weekly = False
454
  if cols and not {"_period_start", "_period_end"} <= cols:
455
  try:
456
  populated = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE})").fetchone()[0]
 
467
  raise _ReadFailed(type(exc).__name__) from exc
468
  if populated and not weekly:
469
  raise SourceNotReady(BA_WEEKLY_MISSING)
470
+ return cols, bool(weekly)
471
 
472
 
473
  def _sqp(cur, term=None):
 
789
  source_error = None
790
  try:
791
  with _reader(tenant_key, st) as cur:
792
+ weekly = _ba_source(cur)[1]
793
+ if weekly:
794
+ problems = [problem for problem in problems if problem not in SUPERSEDED_BY_WEEKLY]
795
  except SourceNotReady as exc:
796
  source_error = exc
797
  cause = READ_FAILED.format(cause=exc)
 
1044
  compare(tenant, ("daily_quotes",), st=st, quote_asins=changed)
1045
 
1046
 
1047
+ def _clear_error(doc):
1048
+ doc = doc if isinstance(doc, dict) else {}
1049
+ doc.pop("last_error", None)
1050
+ return doc
1051
+
1052
+
1053
  def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
1054
  """Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
1055
  st = _rt(tenant_key, st)
 
1081
  if daily and not retry:
1082
  return {"state": "already_compared", "period": newest}
1083
  if not daily and weeks.get("last_compared") == newest:
1084
+ # Every read this pass succeeded on a compared week, so a stored sync failure is superseded.
1085
+ if "last_error" in weeks:
1086
+ st.update(KEY_WEEKS, _clear_error)
1087
  return {"state": "already_compared", "period": newest}
1088
  previous = periods[1] if len(periods) > 1 else None
1089
  conf = settings(tenant_key, st=st)
 
1179
  "periodStart": newest, "previousPeriodStart": previous,
1180
  "comparedAt": _stamp(now), "alertsRaised": len(created),
1181
  "checks": len(checks)}
1182
+ return _clear_error(doc)
 
1183
  st.update(KEY_WEEKS, _weeks)
1184
  return {"state": "compared" if ready else "waiting", "period": newest, "previous": previous,
1185
  "created": created, "checks": checks}
platform/modules/product_data.py CHANGED
@@ -749,7 +749,8 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
749
  with bounded(cur, 'product', [code for _, code in codes], st):
750
  rows = ([{'pid': pid, 'code': code} for pid, code in codes]
751
  if needed is not None and not needed else
752
- pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
 
753
  yield rows
754
  rows.clear()
755
  codes.clear()
@@ -854,7 +855,7 @@ def _fba_by_code(cat):
854
  return out, report
855
 
856
 
857
- def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
858
  """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
859
 
860
  CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
@@ -865,6 +866,9 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
865
  passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
866
  driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
867
  catalogue has no team.
 
 
 
868
  """
869
  from harness import canonical_source, datastore
870
  source = canonical_source.active()
@@ -1191,7 +1195,8 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
1191
  # nothing sold".
1192
  rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
1193
  _assert_no_pid_collision(rows)
1194
- _prune_product_overlay(rows, source, prune_generation)
 
1195
  return rows
1196
 
1197
 
 
749
  with bounded(cur, 'product', [code for _, code in codes], st):
750
  rows = ([{'pid': pid, 'code': code} for pid, code in codes]
751
  if needed is not None and not needed else
752
+ pool(team_id=team_id, raw_fields=raw_fields, needed=needed,
753
+ prune_overlay=ids is None))
754
  yield rows
755
  rows.clear()
756
  codes.clear()
 
855
  return out, report
856
 
857
 
858
+ def pool(team_id=None, t=None, raw_fields=(), *, needed=None, prune_overlay=True):
859
  """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
860
 
861
  CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
 
866
  passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
867
  driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
868
  catalogue has no team.
869
+
870
+ `prune_overlay=False` is a page read: the whole-catalogue overlay prune is a store write that
871
+ belongs to whole-catalogue reads, never to a request that hydrates a few selected rows.
872
  """
873
  from harness import canonical_source, datastore
874
  source = canonical_source.active()
 
1195
  # nothing sold".
1196
  rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
1197
  _assert_no_pid_collision(rows)
1198
+ if prune_overlay:
1199
+ _prune_product_overlay(rows, source, prune_generation)
1200
  return rows
1201
 
1202
 
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- c62b077954cb16ddf668e8e5aee89d6e4cb852718d0b57b94aa3412a210747e6
 
1
+ ee4618c92d8cbec218349feb4f5f9fbc6cb19f8ba0f0e63314aaa24270742c00