fsanyoto commited on
Commit
c4a1388
·
verified ·
1 Parent(s): e048ba6

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "e2461aac6e",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "a1bdf2c861",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- e2461aac6e
 
1
+ a1bdf2c861
api/main.py CHANGED
@@ -1366,7 +1366,15 @@ def _restore_amazon_mirror(why):
1366
  print("[aios-api] amazon mirror boot restore already done in this process")
1367
  return True
1368
  from harness import runtime as _runtime
1369
- results = [_restore_amazon_mirror_once(why, key) for key in _runtime.known_tenants()]
 
 
 
 
 
 
 
 
1370
  errors = RESYNC_STATE.get("amazonRestoreErrors", {})
1371
  if errors:
1372
  RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
@@ -1420,6 +1428,8 @@ def _restore_amazon_mirror_once(why, tenant_key):
1420
  else TenantAmazonMirror().restore_mirror(tenant_key, rt))
1421
  # Each restore phase is timed, so a slow boot names the step that held it.
1422
  phases = {"store restore": _t.time() - started}
 
 
1423
  if result.get("state") == "refused":
1424
  raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
1425
  # Publish measured definitions before any Odoo read can block the resident writer.
 
1366
  print("[aios-api] amazon mirror boot restore already done in this process")
1367
  return True
1368
  from harness import runtime as _runtime
1369
+ restored = boot_done.setdefault("tenants", set())
1370
+ results = []
1371
+ for key in _runtime.known_tenants():
1372
+ good = why == "boot" and key in restored
1373
+ if not good:
1374
+ good = _restore_amazon_mirror_once(why, key)
1375
+ if why == "boot" and good:
1376
+ restored.add(key)
1377
+ results.append(good)
1378
  errors = RESYNC_STATE.get("amazonRestoreErrors", {})
1379
  if errors:
1380
  RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
 
1428
  else TenantAmazonMirror().restore_mirror(tenant_key, rt))
1429
  # Each restore phase is timed, so a slow boot names the step that held it.
1430
  phases = {"store restore": _t.time() - started}
1431
+ phases.update({f"store.{name}": seconds for name, seconds in (result.get("timings") or {}).items()
1432
+ if name != "total"})
1433
  if result.get("state") == "refused":
1434
  raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
1435
  # Publish measured definitions before any Odoo read can block the resident writer.
api/routes_sheet.py CHANGED
@@ -7194,7 +7194,18 @@ def _drill_page(session, database, record_ids, offset, limit, total):
7194
  import odoo_relational as rel
7195
  stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
7196
  and not database.startswith('ut_amzn_') and rot._materialised_table(database))
7197
- if ids and stored:
 
 
 
 
 
 
 
 
 
 
 
7198
  import registry_source
7199
  payload = registry_source.stored_page(session, database, ids)
7200
  else:
 
7194
  import odoo_relational as rel
7195
  stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
7196
  and not database.startswith('ut_amzn_') and rot._materialised_table(database))
7197
+ if (ids and database in {'customer_data', 'product_data'}
7198
+ and getattr(session.runtime, 'key', None) == 'royal-imports'):
7199
+ import registry_source
7200
+ source_cur, mismatch = rot._mirror_cur(session)
7201
+ if mismatch:
7202
+ raise err(409, 'cross_tenant_store', mismatch)
7203
+ if source_cur is None:
7204
+ raise err(503, 'canonical_source_unavailable', f'{database} requires its tenant mirror.')
7205
+ # The drill already selected these identities. Canonical hydration reapplies every current wall.
7206
+ with source_cur.cursor() as cur:
7207
+ payload = registry_source.canonical_page(session, cur, database, ids)
7208
+ elif ids and stored:
7209
  import registry_source
7210
  payload = registry_source.stored_page(session, database, ids)
7211
  else:
api/usage_activity.py CHANGED
@@ -87,11 +87,17 @@ def _security_record_mode(runtime):
87
  return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY)
88
 
89
 
90
- def _security(session, start, stop, now, source):
 
 
 
 
 
 
 
91
  rows = _unlimited_prefix(lambda limit: audit.read(
92
  session.tenant, limit=limit, now=now.timestamp(),
93
  kinds={"external.query", "external.write", "auth.login"}, viewer=session.user))
94
- source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS}
95
  if not _security_record_mode(session.runtime):
96
  stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
97
  for external in (True, False):
@@ -110,9 +116,15 @@ def _security(session, start, stop, now, source):
110
  yield _timestamp(row["ts"]), row.get("principal") or "", kind
111
 
112
 
113
- def _activity(session, start, stop, now, source):
114
  cursor = None
115
  source["retention"] = {"months": activity_log.RETENTION_MONTHS}
 
 
 
 
 
 
116
  with activity_log.read_scope(session.runtime):
117
  while True:
118
  page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
@@ -432,10 +444,13 @@ def read(session, *, from_=None, to=None, user=None, kind=None, now=None):
432
  "This source no longer holds the beginning of the requested range."})
433
  local = Counter()
434
  try:
435
- for at, actor, event_kind in reader(session, start, stop, now, source):
 
 
 
436
  if (start <= at < stop and at <= now and event_kind in kinds
437
  and (user is None or actor == user)):
438
- local[(at.date().isoformat(), actor, event_kind)] += 1
439
  except Exception as exc:
440
  source.update(status="error", error=_source_error(exc))
441
  for key in kinds:
 
87
  return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY)
88
 
89
 
90
+ def _security(session, start, stop, now, source, *, actor=None, kinds=None):
91
+ source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS}
92
+ counts = audit.usage_counts(session.runtime, session.tenant, since=start.timestamp(),
93
+ until=stop.timestamp(), now=now.timestamp(), kinds=kinds or SOURCE_KINDS['security_events'], actor=actor)
94
+ if counts is not None:
95
+ for day, who, kind, count in counts:
96
+ yield datetime.combine(date.fromisoformat(day), time.min, UTC), who or '', kind, count
97
+ return
98
  rows = _unlimited_prefix(lambda limit: audit.read(
99
  session.tenant, limit=limit, now=now.timestamp(),
100
  kinds={"external.query", "external.write", "auth.login"}, viewer=session.user))
 
101
  if not _security_record_mode(session.runtime):
102
  stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
103
  for external in (True, False):
 
116
  yield _timestamp(row["ts"]), row.get("principal") or "", kind
117
 
118
 
119
+ def _activity(session, start, stop, now, source, *, actor=None, kinds=None):
120
  cursor = None
121
  source["retention"] = {"months": activity_log.RETENTION_MONTHS}
122
+ counts = activity_log.usage_counts(session.runtime, since=start, until=stop, now=now,
123
+ kinds={kind.removesuffix('_edit') for kind in kinds or SOURCE_KINDS['activity_events']}, actor=actor)
124
+ if counts is not None:
125
+ for day, who, kind, count in counts:
126
+ yield datetime.combine(date.fromisoformat(day), time.min, UTC), who or '', kind + '_edit', count
127
+ return
128
  with activity_log.read_scope(session.runtime):
129
  while True:
130
  page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
 
444
  "This source no longer holds the beginning of the requested range."})
445
  local = Counter()
446
  try:
447
+ options = {'actor': user, 'kinds': kinds} if name in {'security_events', 'activity_events'} else {}
448
+ for entry in reader(session, start, stop, now, source, **options):
449
+ at, actor, event_kind = entry[:3]
450
+ amount = entry[3] if len(entry) == 4 else 1
451
  if (start <= at < stop and at <= now and event_kind in kinds
452
  and (user is None or actor == user)):
453
+ local[(at.date().isoformat(), actor, event_kind)] += amount
454
  except Exception as exc:
455
  source.update(status="error", error=_source_error(exc))
456
  for key in kinds:
platform/core/account_rows.py CHANGED
@@ -551,6 +551,39 @@ def append_events(st, items, *, retention_seconds):
551
  pg._bump_write_generation(key, slug)
552
 
553
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
554
  def read_events(st, tenant, *, since, limit, kinds=None):
555
  """Select retained events by tenant and time without draining the audit queue."""
556
  pg, slug = records_backend(st)
 
551
  pg._bump_write_generation(key, slug)
552
 
553
 
554
+ def event_counts(st, tenant, *, since, until, now, kinds, actor=None):
555
+ """Aggregate canonical security rows, or decline an unmigrated document without reading its payload."""
556
+ backend = records_backend(st)
557
+ if backend is None:
558
+ return None
559
+ pg, slug = backend
560
+ if slug != tenant:
561
+ raise ValueError('Security reader belongs to another tenant')
562
+ label = ("CASE WHEN value ->> 'kind' = 'auth.login' THEN 'sign_in' "
563
+ "WHEN value -> 'detail' ->> 'via' = 'mcp' THEN 'mcp' ELSE 'api' END")
564
+ stamp = "CAST(value ->> 'ts' AS double precision)"
565
+ selected = sorted(set(kinds) & {'sign_in', 'mcp', 'api'})
566
+ with read_snapshot(slug) as con:
567
+ if not _record_mode(con, pg, slug, EVENTS_KEY):
568
+ return None
569
+ if not selected:
570
+ return []
571
+ where = (f"key LIKE %s AND value ->> 'tenant' = %s AND {stamp} >= %s "
572
+ f"AND {stamp} < %s AND {stamp} <= %s "
573
+ "AND value ->> 'kind' IN ('auth.login', 'external.query', 'external.write') "
574
+ f"AND ({label}) IN ({', '.join('%s' for _ in selected)})")
575
+ params = [EVENT_PREFIX + '%', tenant, since, until, now, *selected]
576
+ if actor is not None:
577
+ where += " AND value ->> 'principal' = %s"
578
+ params.append(actor)
579
+ rows = con.execute(pg._q(
580
+ f"SELECT CAST(floor({stamp} / 86400) AS bigint), value ->> 'principal', "
581
+ f"{label}, count(*) FROM {{tbl}} WHERE {where} GROUP BY 1, 2, 3",
582
+ 'store_kv', slug), tuple(params)).fetchall()
583
+ return [(datetime.fromtimestamp(day * 86400, timezone.utc).date().isoformat(), who, kind, count)
584
+ for day, who, kind, count in rows]
585
+
586
+
587
  def read_events(st, tenant, *, since, limit, kinds=None):
588
  """Select retained events by tenant and time without draining the audit queue."""
589
  pg, slug = records_backend(st)
platform/core/activity_log.py CHANGED
@@ -374,6 +374,16 @@ def prune(rt, *, limit=PRUNE_BATCH, now=None):
374
  return _backend(rt).prune(retention_cutoff(now).isoformat(), limit)
375
 
376
 
 
 
 
 
 
 
 
 
 
 
377
  def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
378
  cursor=None, limit=100, scan_limit=MAX_SCAN):
379
  """Read an indexed page, authorizing each object before returning any event or change value."""
@@ -635,6 +645,24 @@ class PgActivityStore:
635
  'ORDER BY occurred_at DESC, id DESC')
636
  return sql, (limit, *args, *args, limit)
637
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
638
  def page(self, *, target, actor, since, until, before, limit):
639
  sql, args = self.page_query(target=target, actor=actor, since=since, until=until,
640
  before=before, limit=limit)
 
374
  return _backend(rt).prune(retention_cutoff(now).isoformat(), limit)
375
 
376
 
377
+ def usage_counts(rt, *, since, until, now, kinds, actor=None):
378
+ """Aggregate tenant-wide metadata, keeping the ordinary overlay reader for pending deliveries."""
379
+ backend = _backend(rt)
380
+ if not isinstance(backend, PgActivityStore) or _outbox(rt):
381
+ return None
382
+ now = _time(now)
383
+ return backend.usage_counts(since=max(_time(since), retention_cutoff(now)).isoformat(),
384
+ until=_time(until).isoformat(), now=now.isoformat(), kinds=kinds, actor=actor)
385
+
386
+
387
  def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
388
  cursor=None, limit=100, scan_limit=MAX_SCAN):
389
  """Read an indexed page, authorizing each object before returning any event or change value."""
 
645
  'ORDER BY occurred_at DESC, id DESC')
646
  return sql, (limit, *args, *args, limit)
647
 
648
+ def usage_counts(self, *, since, until, now, kinds, actor=None):
649
+ """Group exact UTC day, actor and kind counts without fetching event values."""
650
+ selected = sorted(set(kinds) & {'record', 'field'})
651
+ if not selected:
652
+ return []
653
+ where, args = self._where(target=None, actor=actor, since=since, until=now, before=None)
654
+ where += ' AND occurred_at < %s AND kind IN (' + ', '.join('%s' for _ in selected) + ')'
655
+ args += [until, *selected]
656
+ try:
657
+ with self._connection() as con:
658
+ return con.execute(
659
+ "SELECT to_char(occurred_at AT TIME ZONE 'UTC', 'YYYY-MM-DD'), actor, kind, count(*) "
660
+ f'FROM {self.table} WHERE {where} GROUP BY 1, 2, 3', args).fetchall()
661
+ except Exception as exc:
662
+ if getattr(exc, 'sqlstate', None) != '42P01':
663
+ raise
664
+ return []
665
+
666
  def page(self, *, target, actor, since, until, before, limit):
667
  sql, args = self.page_query(target=target, actor=actor, since=since, until=until,
668
  before=before, limit=limit)
platform/core/audit.py CHANGED
@@ -1,5 +1,7 @@
1
  """Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
2
  from collections.abc import Mapping
 
 
3
  import atexit
4
  import math
5
  import re
@@ -362,6 +364,29 @@ def record(kind, tenant, principal, detail=None) -> dict | None:
362
  return item
363
 
364
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
365
  def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
366
  """Read one tenant's retained events. Read failures propagate instead of claiming no history."""
367
  tenant = _attribution("tenant", tenant)
 
1
  """Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
2
  from collections.abc import Mapping
3
+ from collections import Counter
4
+ from datetime import datetime, timezone
5
  import atexit
6
  import math
7
  import re
 
364
  return item
365
 
366
 
367
+ def usage_counts(st, tenant, *, since, until, now, kinds, actor=None):
368
+ """Count retained metadata in SQL and include the same bounded local queue as audit reads."""
369
+ tenant = _attribution("tenant", tenant)
370
+ since = max(float(since), float(now) - RETENTION_SECONDS)
371
+ rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now,
372
+ kinds=kinds, actor=actor)
373
+ if rows is None:
374
+ return None
375
+ counts = Counter({(day, who, kind): count for day, who, kind, count in rows})
376
+ with _QUEUE_LOCK:
377
+ queued = list(_QUEUE.get(tenant, ()))
378
+ # Queue entries remain subject to the canonical validator and never become durable on a GET.
379
+ for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}):
380
+ if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor:
381
+ continue
382
+ kind = ('sign_in' if row['kind'] == 'auth.login' else
383
+ 'mcp' if row['detail'].get('via') == 'mcp' else 'api')
384
+ if kind in kinds:
385
+ day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat()
386
+ counts[(day, row['principal'], kind)] += 1
387
+ return [(*key, count) for key, count in sorted(counts.items())]
388
+
389
+
390
  def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
391
  """Read one tenant's retained events. Read failures propagate instead of claiming no history."""
392
  tenant = _attribution("tenant", tenant)
platform/harness/amazon_store.py CHANGED
@@ -3843,7 +3843,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
3843
  if not any(personal_columns(name) for name in result[2]):
3844
  return result
3845
  import duckdb
3846
- with closing(duckdb.connect(":memory:")) as con:
3847
  for name, files in result[2].items():
3848
  _assert_personal_storage(con, name, source=_parquet_source(files))
3849
  return result
@@ -3858,7 +3858,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
3858
 
3859
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
3860
  result = super()._install_snapshot(con, manifest, rows, paths, only=only)
3861
- refresh_sku_keywords(con)
 
3862
  return result
3863
 
3864
  def _after_publish(self, tenant_key, manifest):
 
3843
  if not any(personal_columns(name) for name in result[2]):
3844
  return result
3845
  import duckdb
3846
+ with _mirrors._restore_phase("personal_validation"), closing(duckdb.connect(":memory:")) as con:
3847
  for name, files in result[2].items():
3848
  _assert_personal_storage(con, name, source=_parquet_source(files))
3849
  return result
 
3858
 
3859
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
3860
  result = super()._install_snapshot(con, manifest, rows, paths, only=only)
3861
+ with _mirrors._restore_phase("derived_views"):
3862
+ refresh_sku_keywords(con)
3863
  return result
3864
 
3865
  def _after_publish(self, tenant_key, manifest):
platform/harness/mirror_store.py CHANGED
@@ -52,6 +52,18 @@ _PUBLISH_LISTENERS = []
52
  _RESTORE_FOLDER_MAX_AGE = 60 * 60
53
 
54
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  def _restore_folder_prefix(tenant_key, connector):
56
  """Bind scratch cleanup to one tenant and connector without exposing tenant names."""
57
  identity = json.dumps(_restore_key(tenant_key, connector)).encode("utf-8")
@@ -147,7 +159,7 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
147
  if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
148
  return False
149
  only = changed_tables(previous, manifest, store._tables(con), paths)
150
- store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
151
  # The web process serves the hand-off as installed, so it heals its own copy too.
152
  healed = {"dirty": set(), "problems": []}
153
  store._heal_published(con, healed, print)
@@ -582,7 +594,8 @@ class MirrorStore:
582
  self._heal_restored(con, result, log)
583
  con.execute("BEGIN TRANSACTION")
584
  try:
585
- summary_tables.rebuild(con, self.connector, datastore._publication(con, self.connector), self._tables(con))
 
586
  con.execute("COMMIT")
587
  except BaseException:
588
  con.execute("ROLLBACK")
@@ -644,9 +657,10 @@ class MirrorStore:
644
 
645
 
646
  def _checked_mirror_blob(self, rt, path, metadata):
647
- data = self._download(rt, path)
648
- if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
649
- raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
 
650
  return data
651
 
652
 
@@ -792,10 +806,12 @@ class MirrorStore:
792
  for index, (key, blob) in enumerate(self._entry_blob_keys(name, metadata)):
793
  data = self._checked_mirror_blob(rt, key, blob)
794
  path = Path(tmp) / f"{name}.{index:04d}.parquet"
795
- path.write_bytes(data)
 
796
  del data
797
  try:
798
- part_rows = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
 
799
  except Exception as exc: # noqa: BLE001
800
  raise ValueError(f"{self.label} mirror parquet is unreadable") from exc
801
  if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
@@ -815,6 +831,16 @@ class MirrorStore:
815
  return paths, sorted(refused)
816
 
817
 
 
 
 
 
 
 
 
 
 
 
818
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
819
  """Commit one source's tables, sync decisions and identity together.
820
 
@@ -827,15 +853,25 @@ class MirrorStore:
827
  for name, files in paths.items():
828
  if only is not None and name not in only:
829
  continue
830
- con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
831
- f"SELECT * FROM {_parquet_source(files)}")
832
- old_rows = self._state_rows(con)
833
- if old_rows:
834
- con.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in old_rows])
835
- if rows:
836
- con.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows)
 
 
 
 
 
837
  from harness import summary_tables
838
- summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
 
 
 
 
 
839
  datastore._write_publication(con, self.connector, manifest)
840
  con.execute("COMMIT")
841
  except BaseException:
@@ -888,6 +924,9 @@ class MirrorStore:
888
  _catch_up_source(rt, self.connector)["pass_pending"] = True
889
  return result
890
  previous_store = getattr(_RESTORE_PROGRESS, "store", None)
 
 
 
891
  _RESTORE_PROGRESS.store = self
892
  try:
893
  if self._has_local(con):
@@ -899,17 +938,21 @@ class MirrorStore:
899
  # Preserved local rows are not proof of the last durable publication.
900
  _RESTORE_PROGRESS.store = None
901
  with tempfile.TemporaryDirectory(prefix="mirror-local-check-") as tmp:
902
- saved, rows, paths, _total, _refused = self._validated_snapshot(
903
- tenant_key, rt, con, raw, tmp)
 
904
  local_id = datastore._publication(con, self.connector).get("snapshot_id")
905
  if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
906
- self._install_snapshot(con, saved, rows, paths)
907
- _notify_publish(self.connector, tenant_key, saved, rows, paths, log)
 
 
908
  result["manifest"] = saved
909
- if not result["manifest"].get("snapshot_id"):
910
- self._after_restore(con, result, log)
911
- else:
912
- self._heal_published(con, result, log)
 
913
  with _CATCH_UP_LOCK:
914
  _catch_up_source(rt, self.connector).update(pass_pending=True, refused=False)
915
  self.cache[cache_key] = result
@@ -930,21 +973,24 @@ class MirrorStore:
930
  progress["part_total"] = len(part_keys | progress["parts"])
931
  self._before_restore(tenant_key, rt)
932
  with _restore_directory(tenant_key, self.connector) as tmp:
933
- manifest, rows, paths, total, refused = self._validated_snapshot(
934
- tenant_key, rt, con, raw, tmp, allow_partial=True)
935
- self._install_snapshot(con, manifest, rows, paths)
936
- _notify_publish(self.connector, tenant_key, manifest, rows, paths, log)
 
 
 
937
  result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
938
  if refused:
939
  result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
940
  + ". Their sync markers were cleared; resync reloads these tables.")
941
  self._log_restored(paths, total, rows, log)
942
- if not manifest.get("snapshot_id"):
943
- self._after_restore(con, result, log)
944
- else:
945
- # Trap: the healed local rows differ from the published snapshot until the
946
- # next publish; each restore heals again and a second heal changes nothing.
947
- self._heal_published(con, result, log)
948
  current = not refused and self._restore_complete(paths, rows, con=con)
949
  newest = _record_newest(con, self.connector)
950
  with _CATCH_UP_LOCK:
@@ -965,7 +1011,12 @@ class MirrorStore:
965
  _catch_up_source(rt, self.connector).update(current=False, refused=True)
966
  finally:
967
  _RESTORE_PROGRESS.store = previous_store
 
 
968
  con.close()
 
 
 
969
  self.cache[cache_key] = result
970
  return result
971
 
 
52
  _RESTORE_FOLDER_MAX_AGE = 60 * 60
53
 
54
 
55
+ @contextmanager
56
+ def _restore_phase(name):
57
+ """Measure restore work without retaining source values or changing other callers."""
58
+ started = time.perf_counter()
59
+ try:
60
+ yield
61
+ finally:
62
+ timings = getattr(_RESTORE_PROGRESS, "timings", None)
63
+ if timings is not None:
64
+ timings[name] = timings.get(name, 0.0) + time.perf_counter() - started
65
+
66
+
67
  def _restore_folder_prefix(tenant_key, connector):
68
  """Bind scratch cleanup to one tenant and connector without exposing tenant names."""
69
  identity = json.dumps(_restore_key(tenant_key, connector)).encode("utf-8")
 
159
  if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
160
  return False
161
  only = changed_tables(previous, manifest, store._tables(con), paths)
162
+ store._install_restored_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
163
  # The web process serves the hand-off as installed, so it heals its own copy too.
164
  healed = {"dirty": set(), "problems": []}
165
  store._heal_published(con, healed, print)
 
594
  self._heal_restored(con, result, log)
595
  con.execute("BEGIN TRANSACTION")
596
  try:
597
+ with _restore_phase("summaries"):
598
+ summary_tables.rebuild(con, self.connector, datastore._publication(con, self.connector), self._tables(con))
599
  con.execute("COMMIT")
600
  except BaseException:
601
  con.execute("ROLLBACK")
 
657
 
658
 
659
  def _checked_mirror_blob(self, rt, path, metadata):
660
+ with _restore_phase("download_open_checksum"):
661
+ data = self._download(rt, path)
662
+ if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
663
+ raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
664
  return data
665
 
666
 
 
806
  for index, (key, blob) in enumerate(self._entry_blob_keys(name, metadata)):
807
  data = self._checked_mirror_blob(rt, key, blob)
808
  path = Path(tmp) / f"{name}.{index:04d}.parquet"
809
+ with _restore_phase("parquet_write"):
810
+ path.write_bytes(data)
811
  del data
812
  try:
813
+ with _restore_phase("parquet_validation"):
814
+ part_rows = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
815
  except Exception as exc: # noqa: BLE001
816
  raise ValueError(f"{self.label} mirror parquet is unreadable") from exc
817
  if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
 
831
  return paths, sorted(refused)
832
 
833
 
834
+ def _install_restored_snapshot(self, con, manifest, rows, paths, only=None):
835
+ """Defer published restore summaries until healing without changing connector override arguments."""
836
+ previous = getattr(_RESTORE_PROGRESS, 'summary_owner', None)
837
+ _RESTORE_PROGRESS.summary_owner = self if manifest.get('snapshot_id') else None
838
+ try:
839
+ return self._install_snapshot(con, manifest, rows, paths, only=only)
840
+ finally:
841
+ _RESTORE_PROGRESS.summary_owner = previous
842
+
843
+
844
  def _install_snapshot(self, con, manifest, rows, paths, only=None):
845
  """Commit one source's tables, sync decisions and identity together.
846
 
 
853
  for name, files in paths.items():
854
  if only is not None and name not in only:
855
  continue
856
+ with _restore_phase("table_install"):
857
+ con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
858
+ f"SELECT * FROM {_parquet_source(files)}")
859
+ with _restore_phase("state_install"):
860
+ old_rows = self._state_rows(con)
861
+ if old_rows:
862
+ con.execute("DELETE FROM _sync_state WHERE entity IN (SELECT unnest(?::VARCHAR[]))",
863
+ [[row[0] for row in old_rows]])
864
+ if rows:
865
+ con.execute("INSERT INTO _sync_state SELECT unnest(?::VARCHAR[]), unnest(?::VARCHAR[]), "
866
+ "unnest(?::BIGINT[]), unnest(?::VARCHAR[]), unnest(?::BIGINT[]), "
867
+ "unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
868
  from harness import summary_tables
869
+ with _restore_phase("summaries"):
870
+ if getattr(_RESTORE_PROGRESS, 'summary_owner', None) is not self:
871
+ summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
872
+ else:
873
+ # Healing can commit source repairs separately. Rebuild once after those repairs.
874
+ summary_tables.invalidate(con, self.connector)
875
  datastore._write_publication(con, self.connector, manifest)
876
  con.execute("COMMIT")
877
  except BaseException:
 
924
  _catch_up_source(rt, self.connector)["pass_pending"] = True
925
  return result
926
  previous_store = getattr(_RESTORE_PROGRESS, "store", None)
927
+ previous_timings = getattr(_RESTORE_PROGRESS, "timings", None)
928
+ timings = _RESTORE_PROGRESS.timings = {}
929
+ started = time.perf_counter()
930
  _RESTORE_PROGRESS.store = self
931
  try:
932
  if self._has_local(con):
 
938
  # Preserved local rows are not proof of the last durable publication.
939
  _RESTORE_PROGRESS.store = None
940
  with tempfile.TemporaryDirectory(prefix="mirror-local-check-") as tmp:
941
+ with _restore_phase("validation"):
942
+ saved, rows, paths, _total, _refused = self._validated_snapshot(
943
+ tenant_key, rt, con, raw, tmp)
944
  local_id = datastore._publication(con, self.connector).get("snapshot_id")
945
  if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
946
+ with _restore_phase("install"):
947
+ self._install_restored_snapshot(con, saved, rows, paths)
948
+ with _restore_phase("handoff"):
949
+ _notify_publish(self.connector, tenant_key, saved, rows, paths, log)
950
  result["manifest"] = saved
951
+ with _restore_phase("healing"):
952
+ if not result["manifest"].get("snapshot_id"):
953
+ self._after_restore(con, result, log)
954
+ else:
955
+ self._heal_published(con, result, log)
956
  with _CATCH_UP_LOCK:
957
  _catch_up_source(rt, self.connector).update(pass_pending=True, refused=False)
958
  self.cache[cache_key] = result
 
973
  progress["part_total"] = len(part_keys | progress["parts"])
974
  self._before_restore(tenant_key, rt)
975
  with _restore_directory(tenant_key, self.connector) as tmp:
976
+ with _restore_phase("validation"):
977
+ manifest, rows, paths, total, refused = self._validated_snapshot(
978
+ tenant_key, rt, con, raw, tmp, allow_partial=True)
979
+ with _restore_phase("install"):
980
+ self._install_restored_snapshot(con, manifest, rows, paths)
981
+ with _restore_phase("handoff"):
982
+ _notify_publish(self.connector, tenant_key, manifest, rows, paths, log)
983
  result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
984
  if refused:
985
  result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
986
  + ". Their sync markers were cleared; resync reloads these tables.")
987
  self._log_restored(paths, total, rows, log)
988
+ with _restore_phase("healing"):
989
+ if not manifest.get("snapshot_id"):
990
+ self._after_restore(con, result, log)
991
+ else:
992
+ # Restored source repairs remain idempotent until the next publication.
993
+ self._heal_published(con, result, log)
994
  current = not refused and self._restore_complete(paths, rows, con=con)
995
  newest = _record_newest(con, self.connector)
996
  with _CATCH_UP_LOCK:
 
1011
  _catch_up_source(rt, self.connector).update(current=False, refused=True)
1012
  finally:
1013
  _RESTORE_PROGRESS.store = previous_store
1014
+ _RESTORE_PROGRESS.timings = previous_timings
1015
+ result["timings"] = dict(timings, total=time.perf_counter() - started)
1016
  con.close()
1017
+ log(f"[mirror-restore] connector={self.connector} tenant={_tenant_key(tenant_key)} "
1018
+ f"state={result['state']} " + " ".join(
1019
+ f"{name}={seconds:.3f}s" for name, seconds in result["timings"].items()))
1020
  self.cache[cache_key] = result
1021
  return result
1022
 
platform/harness/mirror_tokens.py CHANGED
@@ -46,9 +46,9 @@ def _store_identity(st):
46
  def _revision(cur, marks, generation, sources):
47
  """Decline reuse during writes or transactions whose snapshot can outlive file marks."""
48
  from harness import datastore
49
- if marks is None or getattr(cur, '_in_transaction', False) or datastore._ACTIVE_WRITES[0]:
50
  return None
51
- return fingerprint([marks, generation, datastore._MIRROR_WRITES[0], sources])
52
 
53
 
54
  def mirror_change_token(st, database):
@@ -81,7 +81,7 @@ def mirror_change_token(st, database):
81
  generation = datastore.mirror_generation(cur)
82
  # An in-memory mirror cannot show an in-place write, so it is never cached.
83
  marks = _file_marks(cur)
84
- slot = (_store_identity(st), database)
85
  revision = _revision(cur, marks, generation, [sources, sorted(optional)])
86
  with _TABLE_ROWS_LOCK:
87
  cached = _TABLE_ROWS.get(slot)
@@ -115,7 +115,7 @@ def mirror_change_token(st, database):
115
  sizes = ("SELECT 'size', table_name, CAST(estimated_size AS VARCHAR), '' FROM duckdb_tables() "
116
  f"WHERE table_name IN ({','.join('?' for _ in hashed)})")
117
  key = None if marks is None else fingerprint([
118
- marks, list(generation), datastore._MIRROR_WRITES[0], hashed, sorted(optional - present),
119
  cur.execute(" UNION ALL ".join([*meta, sizes]) + " ORDER BY 1,2,3,4",
120
  [*meta_params, *hashed]).fetchall()])
121
  with _TABLE_ROWS_LOCK:
@@ -140,6 +140,8 @@ def mirror_change_token(st, database):
140
  # Absent names stay in the token, so a table that appears later changes it.
141
  token = fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows)
142
  after = _revision(cur, _file_marks(cur), datastore.mirror_generation(cur), [sources, sorted(optional)])
 
 
143
  if revision is not None and after == revision:
144
  with _TABLE_ROWS_LOCK:
145
  if slot not in _TABLE_ROWS and len(_TABLE_ROWS) >= _TABLE_ROWS_MAX:
 
46
  def _revision(cur, marks, generation, sources):
47
  """Decline reuse during writes or transactions whose snapshot can outlive file marks."""
48
  from harness import datastore
49
+ if marks is None or getattr(cur, '_in_transaction', False) or datastore.mirror_active_writes(cur):
50
  return None
51
+ return fingerprint([marks, generation, datastore.mirror_writes(cur), sources])
52
 
53
 
54
  def mirror_change_token(st, database):
 
81
  generation = datastore.mirror_generation(cur)
82
  # An in-memory mirror cannot show an in-place write, so it is never cached.
83
  marks = _file_marks(cur)
84
+ slot = (_store_identity(st), fingerprint([sources, sorted(optional)]))
85
  revision = _revision(cur, marks, generation, [sources, sorted(optional)])
86
  with _TABLE_ROWS_LOCK:
87
  cached = _TABLE_ROWS.get(slot)
 
115
  sizes = ("SELECT 'size', table_name, CAST(estimated_size AS VARCHAR), '' FROM duckdb_tables() "
116
  f"WHERE table_name IN ({','.join('?' for _ in hashed)})")
117
  key = None if marks is None else fingerprint([
118
+ marks, list(generation), datastore.mirror_writes(cur), hashed, sorted(optional - present),
119
  cur.execute(" UNION ALL ".join([*meta, sizes]) + " ORDER BY 1,2,3,4",
120
  [*meta_params, *hashed]).fetchall()])
121
  with _TABLE_ROWS_LOCK:
 
140
  # Absent names stay in the token, so a table that appears later changes it.
141
  token = fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows)
142
  after = _revision(cur, _file_marks(cur), datastore.mirror_generation(cur), [sources, sorted(optional)])
143
+ if revision is not None and after != revision:
144
+ raise RuntimeError("The mirror changed while reading its change token.")
145
  if revision is not None and after == revision:
146
  with _TABLE_ROWS_LOCK:
147
  if slot not in _TABLE_ROWS and len(_TABLE_ROWS) >= _TABLE_ROWS_MAX:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- b708e0c713d9440532f58f1153efa962a05abccb1b9d42f5c62e15111fd5f798
 
1
+ b4c8cbaebc5ccbde2b3f83cb6c84b606c16ce361e5f5a31816633a233a008f92