Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +11 -1
- api/routes_sheet.py +12 -1
- api/usage_activity.py +20 -5
- platform/core/account_rows.py +33 -0
- platform/core/activity_log.py +28 -0
- platform/core/audit.py +25 -0
- platform/harness/amazon_store.py +3 -2
- platform/harness/mirror_store.py +84 -33
- platform/harness/mirror_tokens.py +6 -4
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "a1bdf2c861",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
a1bdf2c861
|
api/main.py
CHANGED
|
@@ -1366,7 +1366,15 @@ def _restore_amazon_mirror(why):
|
|
| 1366 |
print("[aios-api] amazon mirror boot restore already done in this process")
|
| 1367 |
return True
|
| 1368 |
from harness import runtime as _runtime
|
| 1369 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1370 |
errors = RESYNC_STATE.get("amazonRestoreErrors", {})
|
| 1371 |
if errors:
|
| 1372 |
RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
|
|
@@ -1420,6 +1428,8 @@ def _restore_amazon_mirror_once(why, tenant_key):
|
|
| 1420 |
else TenantAmazonMirror().restore_mirror(tenant_key, rt))
|
| 1421 |
# Each restore phase is timed, so a slow boot names the step that held it.
|
| 1422 |
phases = {"store restore": _t.time() - started}
|
|
|
|
|
|
|
| 1423 |
if result.get("state") == "refused":
|
| 1424 |
raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
|
| 1425 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
|
|
|
| 1366 |
print("[aios-api] amazon mirror boot restore already done in this process")
|
| 1367 |
return True
|
| 1368 |
from harness import runtime as _runtime
|
| 1369 |
+
restored = boot_done.setdefault("tenants", set())
|
| 1370 |
+
results = []
|
| 1371 |
+
for key in _runtime.known_tenants():
|
| 1372 |
+
good = why == "boot" and key in restored
|
| 1373 |
+
if not good:
|
| 1374 |
+
good = _restore_amazon_mirror_once(why, key)
|
| 1375 |
+
if why == "boot" and good:
|
| 1376 |
+
restored.add(key)
|
| 1377 |
+
results.append(good)
|
| 1378 |
errors = RESYNC_STATE.get("amazonRestoreErrors", {})
|
| 1379 |
if errors:
|
| 1380 |
RESYNC_STATE["amazonRestoreError"] = next(iter(errors.values()))
|
|
|
|
| 1428 |
else TenantAmazonMirror().restore_mirror(tenant_key, rt))
|
| 1429 |
# Each restore phase is timed, so a slow boot names the step that held it.
|
| 1430 |
phases = {"store restore": _t.time() - started}
|
| 1431 |
+
phases.update({f"store.{name}": seconds for name, seconds in (result.get("timings") or {}).items()
|
| 1432 |
+
if name != "total"})
|
| 1433 |
if result.get("state") == "refused":
|
| 1434 |
raise RuntimeError("; ".join(result.get("problems") or ["Amazon mirror restore refused."]))
|
| 1435 |
# Publish measured definitions before any Odoo read can block the resident writer.
|
api/routes_sheet.py
CHANGED
|
@@ -7194,7 +7194,18 @@ def _drill_page(session, database, record_ids, offset, limit, total):
|
|
| 7194 |
import odoo_relational as rel
|
| 7195 |
stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
|
| 7196 |
and not database.startswith('ut_amzn_') and rot._materialised_table(database))
|
| 7197 |
-
if ids and
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7198 |
import registry_source
|
| 7199 |
payload = registry_source.stored_page(session, database, ids)
|
| 7200 |
else:
|
|
|
|
| 7194 |
import odoo_relational as rel
|
| 7195 |
stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
|
| 7196 |
and not database.startswith('ut_amzn_') and rot._materialised_table(database))
|
| 7197 |
+
if (ids and database in {'customer_data', 'product_data'}
|
| 7198 |
+
and getattr(session.runtime, 'key', None) == 'royal-imports'):
|
| 7199 |
+
import registry_source
|
| 7200 |
+
source_cur, mismatch = rot._mirror_cur(session)
|
| 7201 |
+
if mismatch:
|
| 7202 |
+
raise err(409, 'cross_tenant_store', mismatch)
|
| 7203 |
+
if source_cur is None:
|
| 7204 |
+
raise err(503, 'canonical_source_unavailable', f'{database} requires its tenant mirror.')
|
| 7205 |
+
# The drill already selected these identities. Canonical hydration reapplies every current wall.
|
| 7206 |
+
with source_cur.cursor() as cur:
|
| 7207 |
+
payload = registry_source.canonical_page(session, cur, database, ids)
|
| 7208 |
+
elif ids and stored:
|
| 7209 |
import registry_source
|
| 7210 |
payload = registry_source.stored_page(session, database, ids)
|
| 7211 |
else:
|
api/usage_activity.py
CHANGED
|
@@ -87,11 +87,17 @@ def _security_record_mode(runtime):
|
|
| 87 |
return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY)
|
| 88 |
|
| 89 |
|
| 90 |
-
def _security(session, start, stop, now, source):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
rows = _unlimited_prefix(lambda limit: audit.read(
|
| 92 |
session.tenant, limit=limit, now=now.timestamp(),
|
| 93 |
kinds={"external.query", "external.write", "auth.login"}, viewer=session.user))
|
| 94 |
-
source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS}
|
| 95 |
if not _security_record_mode(session.runtime):
|
| 96 |
stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
|
| 97 |
for external in (True, False):
|
|
@@ -110,9 +116,15 @@ def _security(session, start, stop, now, source):
|
|
| 110 |
yield _timestamp(row["ts"]), row.get("principal") or "", kind
|
| 111 |
|
| 112 |
|
| 113 |
-
def _activity(session, start, stop, now, source):
|
| 114 |
cursor = None
|
| 115 |
source["retention"] = {"months": activity_log.RETENTION_MONTHS}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
with activity_log.read_scope(session.runtime):
|
| 117 |
while True:
|
| 118 |
page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
|
|
@@ -432,10 +444,13 @@ def read(session, *, from_=None, to=None, user=None, kind=None, now=None):
|
|
| 432 |
"This source no longer holds the beginning of the requested range."})
|
| 433 |
local = Counter()
|
| 434 |
try:
|
| 435 |
-
|
|
|
|
|
|
|
|
|
|
| 436 |
if (start <= at < stop and at <= now and event_kind in kinds
|
| 437 |
and (user is None or actor == user)):
|
| 438 |
-
local[(at.date().isoformat(), actor, event_kind)] +=
|
| 439 |
except Exception as exc:
|
| 440 |
source.update(status="error", error=_source_error(exc))
|
| 441 |
for key in kinds:
|
|
|
|
| 87 |
return account_rows._record_mode(con, pg, slug, account_rows.EVENTS_KEY)
|
| 88 |
|
| 89 |
|
| 90 |
+
def _security(session, start, stop, now, source, *, actor=None, kinds=None):
|
| 91 |
+
source["retention"] = {"days": 400, "legacyRowsPerPartition": audit.MAX_EVENTS}
|
| 92 |
+
counts = audit.usage_counts(session.runtime, session.tenant, since=start.timestamp(),
|
| 93 |
+
until=stop.timestamp(), now=now.timestamp(), kinds=kinds or SOURCE_KINDS['security_events'], actor=actor)
|
| 94 |
+
if counts is not None:
|
| 95 |
+
for day, who, kind, count in counts:
|
| 96 |
+
yield datetime.combine(date.fromisoformat(day), time.min, UTC), who or '', kind, count
|
| 97 |
+
return
|
| 98 |
rows = _unlimited_prefix(lambda limit: audit.read(
|
| 99 |
session.tenant, limit=limit, now=now.timestamp(),
|
| 100 |
kinds={"external.query", "external.write", "auth.login"}, viewer=session.user))
|
|
|
|
| 101 |
if not _security_record_mode(session.runtime):
|
| 102 |
stored = (session.runtime.get(audit.BUCKET) or {}).get("events", [])
|
| 103 |
for external in (True, False):
|
|
|
|
| 116 |
yield _timestamp(row["ts"]), row.get("principal") or "", kind
|
| 117 |
|
| 118 |
|
| 119 |
+
def _activity(session, start, stop, now, source, *, actor=None, kinds=None):
|
| 120 |
cursor = None
|
| 121 |
source["retention"] = {"months": activity_log.RETENTION_MONTHS}
|
| 122 |
+
counts = activity_log.usage_counts(session.runtime, since=start, until=stop, now=now,
|
| 123 |
+
kinds={kind.removesuffix('_edit') for kind in kinds or SOURCE_KINDS['activity_events']}, actor=actor)
|
| 124 |
+
if counts is not None:
|
| 125 |
+
for day, who, kind, count in counts:
|
| 126 |
+
yield datetime.combine(date.fromisoformat(day), time.min, UTC), who or '', kind + '_edit', count
|
| 127 |
+
return
|
| 128 |
with activity_log.read_scope(session.runtime):
|
| 129 |
while True:
|
| 130 |
page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
|
|
|
|
| 444 |
"This source no longer holds the beginning of the requested range."})
|
| 445 |
local = Counter()
|
| 446 |
try:
|
| 447 |
+
options = {'actor': user, 'kinds': kinds} if name in {'security_events', 'activity_events'} else {}
|
| 448 |
+
for entry in reader(session, start, stop, now, source, **options):
|
| 449 |
+
at, actor, event_kind = entry[:3]
|
| 450 |
+
amount = entry[3] if len(entry) == 4 else 1
|
| 451 |
if (start <= at < stop and at <= now and event_kind in kinds
|
| 452 |
and (user is None or actor == user)):
|
| 453 |
+
local[(at.date().isoformat(), actor, event_kind)] += amount
|
| 454 |
except Exception as exc:
|
| 455 |
source.update(status="error", error=_source_error(exc))
|
| 456 |
for key in kinds:
|
platform/core/account_rows.py
CHANGED
|
@@ -551,6 +551,39 @@ def append_events(st, items, *, retention_seconds):
|
|
| 551 |
pg._bump_write_generation(key, slug)
|
| 552 |
|
| 553 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 554 |
def read_events(st, tenant, *, since, limit, kinds=None):
|
| 555 |
"""Select retained events by tenant and time without draining the audit queue."""
|
| 556 |
pg, slug = records_backend(st)
|
|
|
|
| 551 |
pg._bump_write_generation(key, slug)
|
| 552 |
|
| 553 |
|
| 554 |
+
def event_counts(st, tenant, *, since, until, now, kinds, actor=None):
|
| 555 |
+
"""Aggregate canonical security rows, or decline an unmigrated document without reading its payload."""
|
| 556 |
+
backend = records_backend(st)
|
| 557 |
+
if backend is None:
|
| 558 |
+
return None
|
| 559 |
+
pg, slug = backend
|
| 560 |
+
if slug != tenant:
|
| 561 |
+
raise ValueError('Security reader belongs to another tenant')
|
| 562 |
+
label = ("CASE WHEN value ->> 'kind' = 'auth.login' THEN 'sign_in' "
|
| 563 |
+
"WHEN value -> 'detail' ->> 'via' = 'mcp' THEN 'mcp' ELSE 'api' END")
|
| 564 |
+
stamp = "CAST(value ->> 'ts' AS double precision)"
|
| 565 |
+
selected = sorted(set(kinds) & {'sign_in', 'mcp', 'api'})
|
| 566 |
+
with read_snapshot(slug) as con:
|
| 567 |
+
if not _record_mode(con, pg, slug, EVENTS_KEY):
|
| 568 |
+
return None
|
| 569 |
+
if not selected:
|
| 570 |
+
return []
|
| 571 |
+
where = (f"key LIKE %s AND value ->> 'tenant' = %s AND {stamp} >= %s "
|
| 572 |
+
f"AND {stamp} < %s AND {stamp} <= %s "
|
| 573 |
+
"AND value ->> 'kind' IN ('auth.login', 'external.query', 'external.write') "
|
| 574 |
+
f"AND ({label}) IN ({', '.join('%s' for _ in selected)})")
|
| 575 |
+
params = [EVENT_PREFIX + '%', tenant, since, until, now, *selected]
|
| 576 |
+
if actor is not None:
|
| 577 |
+
where += " AND value ->> 'principal' = %s"
|
| 578 |
+
params.append(actor)
|
| 579 |
+
rows = con.execute(pg._q(
|
| 580 |
+
f"SELECT CAST(floor({stamp} / 86400) AS bigint), value ->> 'principal', "
|
| 581 |
+
f"{label}, count(*) FROM {{tbl}} WHERE {where} GROUP BY 1, 2, 3",
|
| 582 |
+
'store_kv', slug), tuple(params)).fetchall()
|
| 583 |
+
return [(datetime.fromtimestamp(day * 86400, timezone.utc).date().isoformat(), who, kind, count)
|
| 584 |
+
for day, who, kind, count in rows]
|
| 585 |
+
|
| 586 |
+
|
| 587 |
def read_events(st, tenant, *, since, limit, kinds=None):
|
| 588 |
"""Select retained events by tenant and time without draining the audit queue."""
|
| 589 |
pg, slug = records_backend(st)
|
platform/core/activity_log.py
CHANGED
|
@@ -374,6 +374,16 @@ def prune(rt, *, limit=PRUNE_BATCH, now=None):
|
|
| 374 |
return _backend(rt).prune(retention_cutoff(now).isoformat(), limit)
|
| 375 |
|
| 376 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
|
| 378 |
cursor=None, limit=100, scan_limit=MAX_SCAN):
|
| 379 |
"""Read an indexed page, authorizing each object before returning any event or change value."""
|
|
@@ -635,6 +645,24 @@ class PgActivityStore:
|
|
| 635 |
'ORDER BY occurred_at DESC, id DESC')
|
| 636 |
return sql, (limit, *args, *args, limit)
|
| 637 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 638 |
def page(self, *, target, actor, since, until, before, limit):
|
| 639 |
sql, args = self.page_query(target=target, actor=actor, since=since, until=until,
|
| 640 |
before=before, limit=limit)
|
|
|
|
| 374 |
return _backend(rt).prune(retention_cutoff(now).isoformat(), limit)
|
| 375 |
|
| 376 |
|
| 377 |
+
def usage_counts(rt, *, since, until, now, kinds, actor=None):
|
| 378 |
+
"""Aggregate tenant-wide metadata, keeping the ordinary overlay reader for pending deliveries."""
|
| 379 |
+
backend = _backend(rt)
|
| 380 |
+
if not isinstance(backend, PgActivityStore) or _outbox(rt):
|
| 381 |
+
return None
|
| 382 |
+
now = _time(now)
|
| 383 |
+
return backend.usage_counts(since=max(_time(since), retention_cutoff(now)).isoformat(),
|
| 384 |
+
until=_time(until).isoformat(), now=now.isoformat(), kinds=kinds, actor=actor)
|
| 385 |
+
|
| 386 |
+
|
| 387 |
def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
|
| 388 |
cursor=None, limit=100, scan_limit=MAX_SCAN):
|
| 389 |
"""Read an indexed page, authorizing each object before returning any event or change value."""
|
|
|
|
| 645 |
'ORDER BY occurred_at DESC, id DESC')
|
| 646 |
return sql, (limit, *args, *args, limit)
|
| 647 |
|
| 648 |
+
def usage_counts(self, *, since, until, now, kinds, actor=None):
|
| 649 |
+
"""Group exact UTC day, actor and kind counts without fetching event values."""
|
| 650 |
+
selected = sorted(set(kinds) & {'record', 'field'})
|
| 651 |
+
if not selected:
|
| 652 |
+
return []
|
| 653 |
+
where, args = self._where(target=None, actor=actor, since=since, until=now, before=None)
|
| 654 |
+
where += ' AND occurred_at < %s AND kind IN (' + ', '.join('%s' for _ in selected) + ')'
|
| 655 |
+
args += [until, *selected]
|
| 656 |
+
try:
|
| 657 |
+
with self._connection() as con:
|
| 658 |
+
return con.execute(
|
| 659 |
+
"SELECT to_char(occurred_at AT TIME ZONE 'UTC', 'YYYY-MM-DD'), actor, kind, count(*) "
|
| 660 |
+
f'FROM {self.table} WHERE {where} GROUP BY 1, 2, 3', args).fetchall()
|
| 661 |
+
except Exception as exc:
|
| 662 |
+
if getattr(exc, 'sqlstate', None) != '42P01':
|
| 663 |
+
raise
|
| 664 |
+
return []
|
| 665 |
+
|
| 666 |
def page(self, *, target, actor, since, until, before, limit):
|
| 667 |
sql, args = self.page_query(target=target, actor=actor, since=since, until=until,
|
| 668 |
before=before, limit=limit)
|
platform/core/audit.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
| 1 |
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
|
| 2 |
from collections.abc import Mapping
|
|
|
|
|
|
|
| 3 |
import atexit
|
| 4 |
import math
|
| 5 |
import re
|
|
@@ -362,6 +364,29 @@ def record(kind, tenant, principal, detail=None) -> dict | None:
|
|
| 362 |
return item
|
| 363 |
|
| 364 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 365 |
def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
|
| 366 |
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
|
| 367 |
tenant = _attribution("tenant", tenant)
|
|
|
|
| 1 |
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
|
| 2 |
from collections.abc import Mapping
|
| 3 |
+
from collections import Counter
|
| 4 |
+
from datetime import datetime, timezone
|
| 5 |
import atexit
|
| 6 |
import math
|
| 7 |
import re
|
|
|
|
| 364 |
return item
|
| 365 |
|
| 366 |
|
| 367 |
+
def usage_counts(st, tenant, *, since, until, now, kinds, actor=None):
|
| 368 |
+
"""Count retained metadata in SQL and include the same bounded local queue as audit reads."""
|
| 369 |
+
tenant = _attribution("tenant", tenant)
|
| 370 |
+
since = max(float(since), float(now) - RETENTION_SECONDS)
|
| 371 |
+
rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now,
|
| 372 |
+
kinds=kinds, actor=actor)
|
| 373 |
+
if rows is None:
|
| 374 |
+
return None
|
| 375 |
+
counts = Counter({(day, who, kind): count for day, who, kind, count in rows})
|
| 376 |
+
with _QUEUE_LOCK:
|
| 377 |
+
queued = list(_QUEUE.get(tenant, ()))
|
| 378 |
+
# Queue entries remain subject to the canonical validator and never become durable on a GET.
|
| 379 |
+
for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}):
|
| 380 |
+
if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor:
|
| 381 |
+
continue
|
| 382 |
+
kind = ('sign_in' if row['kind'] == 'auth.login' else
|
| 383 |
+
'mcp' if row['detail'].get('via') == 'mcp' else 'api')
|
| 384 |
+
if kind in kinds:
|
| 385 |
+
day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat()
|
| 386 |
+
counts[(day, row['principal'], kind)] += 1
|
| 387 |
+
return [(*key, count) for key, count in sorted(counts.items())]
|
| 388 |
+
|
| 389 |
+
|
| 390 |
def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
|
| 391 |
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
|
| 392 |
tenant = _attribution("tenant", tenant)
|
platform/harness/amazon_store.py
CHANGED
|
@@ -3843,7 +3843,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 3843 |
if not any(personal_columns(name) for name in result[2]):
|
| 3844 |
return result
|
| 3845 |
import duckdb
|
| 3846 |
-
with closing(duckdb.connect(":memory:")) as con:
|
| 3847 |
for name, files in result[2].items():
|
| 3848 |
_assert_personal_storage(con, name, source=_parquet_source(files))
|
| 3849 |
return result
|
|
@@ -3858,7 +3858,8 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 3858 |
|
| 3859 |
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 3860 |
result = super()._install_snapshot(con, manifest, rows, paths, only=only)
|
| 3861 |
-
|
|
|
|
| 3862 |
return result
|
| 3863 |
|
| 3864 |
def _after_publish(self, tenant_key, manifest):
|
|
|
|
| 3843 |
if not any(personal_columns(name) for name in result[2]):
|
| 3844 |
return result
|
| 3845 |
import duckdb
|
| 3846 |
+
with _mirrors._restore_phase("personal_validation"), closing(duckdb.connect(":memory:")) as con:
|
| 3847 |
for name, files in result[2].items():
|
| 3848 |
_assert_personal_storage(con, name, source=_parquet_source(files))
|
| 3849 |
return result
|
|
|
|
| 3858 |
|
| 3859 |
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 3860 |
result = super()._install_snapshot(con, manifest, rows, paths, only=only)
|
| 3861 |
+
with _mirrors._restore_phase("derived_views"):
|
| 3862 |
+
refresh_sku_keywords(con)
|
| 3863 |
return result
|
| 3864 |
|
| 3865 |
def _after_publish(self, tenant_key, manifest):
|
platform/harness/mirror_store.py
CHANGED
|
@@ -52,6 +52,18 @@ _PUBLISH_LISTENERS = []
|
|
| 52 |
_RESTORE_FOLDER_MAX_AGE = 60 * 60
|
| 53 |
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
def _restore_folder_prefix(tenant_key, connector):
|
| 56 |
"""Bind scratch cleanup to one tenant and connector without exposing tenant names."""
|
| 57 |
identity = json.dumps(_restore_key(tenant_key, connector)).encode("utf-8")
|
|
@@ -147,7 +159,7 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
|
| 147 |
if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
|
| 148 |
return False
|
| 149 |
only = changed_tables(previous, manifest, store._tables(con), paths)
|
| 150 |
-
store.
|
| 151 |
# The web process serves the hand-off as installed, so it heals its own copy too.
|
| 152 |
healed = {"dirty": set(), "problems": []}
|
| 153 |
store._heal_published(con, healed, print)
|
|
@@ -582,7 +594,8 @@ class MirrorStore:
|
|
| 582 |
self._heal_restored(con, result, log)
|
| 583 |
con.execute("BEGIN TRANSACTION")
|
| 584 |
try:
|
| 585 |
-
|
|
|
|
| 586 |
con.execute("COMMIT")
|
| 587 |
except BaseException:
|
| 588 |
con.execute("ROLLBACK")
|
|
@@ -644,9 +657,10 @@ class MirrorStore:
|
|
| 644 |
|
| 645 |
|
| 646 |
def _checked_mirror_blob(self, rt, path, metadata):
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
|
|
|
| 650 |
return data
|
| 651 |
|
| 652 |
|
|
@@ -792,10 +806,12 @@ class MirrorStore:
|
|
| 792 |
for index, (key, blob) in enumerate(self._entry_blob_keys(name, metadata)):
|
| 793 |
data = self._checked_mirror_blob(rt, key, blob)
|
| 794 |
path = Path(tmp) / f"{name}.{index:04d}.parquet"
|
| 795 |
-
|
|
|
|
| 796 |
del data
|
| 797 |
try:
|
| 798 |
-
|
|
|
|
| 799 |
except Exception as exc: # noqa: BLE001
|
| 800 |
raise ValueError(f"{self.label} mirror parquet is unreadable") from exc
|
| 801 |
if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
|
|
@@ -815,6 +831,16 @@ class MirrorStore:
|
|
| 815 |
return paths, sorted(refused)
|
| 816 |
|
| 817 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 818 |
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 819 |
"""Commit one source's tables, sync decisions and identity together.
|
| 820 |
|
|
@@ -827,15 +853,25 @@ class MirrorStore:
|
|
| 827 |
for name, files in paths.items():
|
| 828 |
if only is not None and name not in only:
|
| 829 |
continue
|
| 830 |
-
|
| 831 |
-
|
| 832 |
-
|
| 833 |
-
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 837 |
from harness import summary_tables
|
| 838 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 839 |
datastore._write_publication(con, self.connector, manifest)
|
| 840 |
con.execute("COMMIT")
|
| 841 |
except BaseException:
|
|
@@ -888,6 +924,9 @@ class MirrorStore:
|
|
| 888 |
_catch_up_source(rt, self.connector)["pass_pending"] = True
|
| 889 |
return result
|
| 890 |
previous_store = getattr(_RESTORE_PROGRESS, "store", None)
|
|
|
|
|
|
|
|
|
|
| 891 |
_RESTORE_PROGRESS.store = self
|
| 892 |
try:
|
| 893 |
if self._has_local(con):
|
|
@@ -899,17 +938,21 @@ class MirrorStore:
|
|
| 899 |
# Preserved local rows are not proof of the last durable publication.
|
| 900 |
_RESTORE_PROGRESS.store = None
|
| 901 |
with tempfile.TemporaryDirectory(prefix="mirror-local-check-") as tmp:
|
| 902 |
-
|
| 903 |
-
|
|
|
|
| 904 |
local_id = datastore._publication(con, self.connector).get("snapshot_id")
|
| 905 |
if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
|
| 906 |
-
|
| 907 |
-
|
|
|
|
|
|
|
| 908 |
result["manifest"] = saved
|
| 909 |
-
|
| 910 |
-
|
| 911 |
-
|
| 912 |
-
|
|
|
|
| 913 |
with _CATCH_UP_LOCK:
|
| 914 |
_catch_up_source(rt, self.connector).update(pass_pending=True, refused=False)
|
| 915 |
self.cache[cache_key] = result
|
|
@@ -930,21 +973,24 @@ class MirrorStore:
|
|
| 930 |
progress["part_total"] = len(part_keys | progress["parts"])
|
| 931 |
self._before_restore(tenant_key, rt)
|
| 932 |
with _restore_directory(tenant_key, self.connector) as tmp:
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
|
| 936 |
-
|
|
|
|
|
|
|
|
|
|
| 937 |
result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
|
| 938 |
if refused:
|
| 939 |
result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
|
| 940 |
+ ". Their sync markers were cleared; resync reloads these tables.")
|
| 941 |
self._log_restored(paths, total, rows, log)
|
| 942 |
-
|
| 943 |
-
|
| 944 |
-
|
| 945 |
-
|
| 946 |
-
|
| 947 |
-
|
| 948 |
current = not refused and self._restore_complete(paths, rows, con=con)
|
| 949 |
newest = _record_newest(con, self.connector)
|
| 950 |
with _CATCH_UP_LOCK:
|
|
@@ -965,7 +1011,12 @@ class MirrorStore:
|
|
| 965 |
_catch_up_source(rt, self.connector).update(current=False, refused=True)
|
| 966 |
finally:
|
| 967 |
_RESTORE_PROGRESS.store = previous_store
|
|
|
|
|
|
|
| 968 |
con.close()
|
|
|
|
|
|
|
|
|
|
| 969 |
self.cache[cache_key] = result
|
| 970 |
return result
|
| 971 |
|
|
|
|
| 52 |
_RESTORE_FOLDER_MAX_AGE = 60 * 60
|
| 53 |
|
| 54 |
|
| 55 |
+
@contextmanager
|
| 56 |
+
def _restore_phase(name):
|
| 57 |
+
"""Measure restore work without retaining source values or changing other callers."""
|
| 58 |
+
started = time.perf_counter()
|
| 59 |
+
try:
|
| 60 |
+
yield
|
| 61 |
+
finally:
|
| 62 |
+
timings = getattr(_RESTORE_PROGRESS, "timings", None)
|
| 63 |
+
if timings is not None:
|
| 64 |
+
timings[name] = timings.get(name, 0.0) + time.perf_counter() - started
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def _restore_folder_prefix(tenant_key, connector):
|
| 68 |
"""Bind scratch cleanup to one tenant and connector without exposing tenant names."""
|
| 69 |
identity = json.dumps(_restore_key(tenant_key, connector)).encode("utf-8")
|
|
|
|
| 159 |
if manifest.get("snapshot_id") and previous.get("snapshot_id") == manifest["snapshot_id"]:
|
| 160 |
return False
|
| 161 |
only = changed_tables(previous, manifest, store._tables(con), paths)
|
| 162 |
+
store._install_restored_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
|
| 163 |
# The web process serves the hand-off as installed, so it heals its own copy too.
|
| 164 |
healed = {"dirty": set(), "problems": []}
|
| 165 |
store._heal_published(con, healed, print)
|
|
|
|
| 594 |
self._heal_restored(con, result, log)
|
| 595 |
con.execute("BEGIN TRANSACTION")
|
| 596 |
try:
|
| 597 |
+
with _restore_phase("summaries"):
|
| 598 |
+
summary_tables.rebuild(con, self.connector, datastore._publication(con, self.connector), self._tables(con))
|
| 599 |
con.execute("COMMIT")
|
| 600 |
except BaseException:
|
| 601 |
con.execute("ROLLBACK")
|
|
|
|
| 657 |
|
| 658 |
|
| 659 |
def _checked_mirror_blob(self, rt, path, metadata):
|
| 660 |
+
with _restore_phase("download_open_checksum"):
|
| 661 |
+
data = self._download(rt, path)
|
| 662 |
+
if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
|
| 663 |
+
raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
|
| 664 |
return data
|
| 665 |
|
| 666 |
|
|
|
|
| 806 |
for index, (key, blob) in enumerate(self._entry_blob_keys(name, metadata)):
|
| 807 |
data = self._checked_mirror_blob(rt, key, blob)
|
| 808 |
path = Path(tmp) / f"{name}.{index:04d}.parquet"
|
| 809 |
+
with _restore_phase("parquet_write"):
|
| 810 |
+
path.write_bytes(data)
|
| 811 |
del data
|
| 812 |
try:
|
| 813 |
+
with _restore_phase("parquet_validation"):
|
| 814 |
+
part_rows = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
|
| 815 |
except Exception as exc: # noqa: BLE001
|
| 816 |
raise ValueError(f"{self.label} mirror parquet is unreadable") from exc
|
| 817 |
if "parts" in metadata and (type(blob.get("rows")) is not int or part_rows != blob["rows"]):
|
|
|
|
| 831 |
return paths, sorted(refused)
|
| 832 |
|
| 833 |
|
| 834 |
+
def _install_restored_snapshot(self, con, manifest, rows, paths, only=None):
|
| 835 |
+
"""Defer published restore summaries until healing without changing connector override arguments."""
|
| 836 |
+
previous = getattr(_RESTORE_PROGRESS, 'summary_owner', None)
|
| 837 |
+
_RESTORE_PROGRESS.summary_owner = self if manifest.get('snapshot_id') else None
|
| 838 |
+
try:
|
| 839 |
+
return self._install_snapshot(con, manifest, rows, paths, only=only)
|
| 840 |
+
finally:
|
| 841 |
+
_RESTORE_PROGRESS.summary_owner = previous
|
| 842 |
+
|
| 843 |
+
|
| 844 |
def _install_snapshot(self, con, manifest, rows, paths, only=None):
|
| 845 |
"""Commit one source's tables, sync decisions and identity together.
|
| 846 |
|
|
|
|
| 853 |
for name, files in paths.items():
|
| 854 |
if only is not None and name not in only:
|
| 855 |
continue
|
| 856 |
+
with _restore_phase("table_install"):
|
| 857 |
+
con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
|
| 858 |
+
f"SELECT * FROM {_parquet_source(files)}")
|
| 859 |
+
with _restore_phase("state_install"):
|
| 860 |
+
old_rows = self._state_rows(con)
|
| 861 |
+
if old_rows:
|
| 862 |
+
con.execute("DELETE FROM _sync_state WHERE entity IN (SELECT unnest(?::VARCHAR[]))",
|
| 863 |
+
[[row[0] for row in old_rows]])
|
| 864 |
+
if rows:
|
| 865 |
+
con.execute("INSERT INTO _sync_state SELECT unnest(?::VARCHAR[]), unnest(?::VARCHAR[]), "
|
| 866 |
+
"unnest(?::BIGINT[]), unnest(?::VARCHAR[]), unnest(?::BIGINT[]), "
|
| 867 |
+
"unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
|
| 868 |
from harness import summary_tables
|
| 869 |
+
with _restore_phase("summaries"):
|
| 870 |
+
if getattr(_RESTORE_PROGRESS, 'summary_owner', None) is not self:
|
| 871 |
+
summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
|
| 872 |
+
else:
|
| 873 |
+
# Healing can commit source repairs separately. Rebuild once after those repairs.
|
| 874 |
+
summary_tables.invalidate(con, self.connector)
|
| 875 |
datastore._write_publication(con, self.connector, manifest)
|
| 876 |
con.execute("COMMIT")
|
| 877 |
except BaseException:
|
|
|
|
| 924 |
_catch_up_source(rt, self.connector)["pass_pending"] = True
|
| 925 |
return result
|
| 926 |
previous_store = getattr(_RESTORE_PROGRESS, "store", None)
|
| 927 |
+
previous_timings = getattr(_RESTORE_PROGRESS, "timings", None)
|
| 928 |
+
timings = _RESTORE_PROGRESS.timings = {}
|
| 929 |
+
started = time.perf_counter()
|
| 930 |
_RESTORE_PROGRESS.store = self
|
| 931 |
try:
|
| 932 |
if self._has_local(con):
|
|
|
|
| 938 |
# Preserved local rows are not proof of the last durable publication.
|
| 939 |
_RESTORE_PROGRESS.store = None
|
| 940 |
with tempfile.TemporaryDirectory(prefix="mirror-local-check-") as tmp:
|
| 941 |
+
with _restore_phase("validation"):
|
| 942 |
+
saved, rows, paths, _total, _refused = self._validated_snapshot(
|
| 943 |
+
tenant_key, rt, con, raw, tmp)
|
| 944 |
local_id = datastore._publication(con, self.connector).get("snapshot_id")
|
| 945 |
if saved.get("snapshot_id") and saved["snapshot_id"] != local_id:
|
| 946 |
+
with _restore_phase("install"):
|
| 947 |
+
self._install_restored_snapshot(con, saved, rows, paths)
|
| 948 |
+
with _restore_phase("handoff"):
|
| 949 |
+
_notify_publish(self.connector, tenant_key, saved, rows, paths, log)
|
| 950 |
result["manifest"] = saved
|
| 951 |
+
with _restore_phase("healing"):
|
| 952 |
+
if not result["manifest"].get("snapshot_id"):
|
| 953 |
+
self._after_restore(con, result, log)
|
| 954 |
+
else:
|
| 955 |
+
self._heal_published(con, result, log)
|
| 956 |
with _CATCH_UP_LOCK:
|
| 957 |
_catch_up_source(rt, self.connector).update(pass_pending=True, refused=False)
|
| 958 |
self.cache[cache_key] = result
|
|
|
|
| 973 |
progress["part_total"] = len(part_keys | progress["parts"])
|
| 974 |
self._before_restore(tenant_key, rt)
|
| 975 |
with _restore_directory(tenant_key, self.connector) as tmp:
|
| 976 |
+
with _restore_phase("validation"):
|
| 977 |
+
manifest, rows, paths, total, refused = self._validated_snapshot(
|
| 978 |
+
tenant_key, rt, con, raw, tmp, allow_partial=True)
|
| 979 |
+
with _restore_phase("install"):
|
| 980 |
+
self._install_restored_snapshot(con, manifest, rows, paths)
|
| 981 |
+
with _restore_phase("handoff"):
|
| 982 |
+
_notify_publish(self.connector, tenant_key, manifest, rows, paths, log)
|
| 983 |
result.update(state="partial" if refused else "restored", manifest=manifest, refused_tables=refused)
|
| 984 |
if refused:
|
| 985 |
result["problems"].append(f"{self.label} mirror tables refused: " + ", ".join(refused)
|
| 986 |
+ ". Their sync markers were cleared; resync reloads these tables.")
|
| 987 |
self._log_restored(paths, total, rows, log)
|
| 988 |
+
with _restore_phase("healing"):
|
| 989 |
+
if not manifest.get("snapshot_id"):
|
| 990 |
+
self._after_restore(con, result, log)
|
| 991 |
+
else:
|
| 992 |
+
# Restored source repairs remain idempotent until the next publication.
|
| 993 |
+
self._heal_published(con, result, log)
|
| 994 |
current = not refused and self._restore_complete(paths, rows, con=con)
|
| 995 |
newest = _record_newest(con, self.connector)
|
| 996 |
with _CATCH_UP_LOCK:
|
|
|
|
| 1011 |
_catch_up_source(rt, self.connector).update(current=False, refused=True)
|
| 1012 |
finally:
|
| 1013 |
_RESTORE_PROGRESS.store = previous_store
|
| 1014 |
+
_RESTORE_PROGRESS.timings = previous_timings
|
| 1015 |
+
result["timings"] = dict(timings, total=time.perf_counter() - started)
|
| 1016 |
con.close()
|
| 1017 |
+
log(f"[mirror-restore] connector={self.connector} tenant={_tenant_key(tenant_key)} "
|
| 1018 |
+
f"state={result['state']} " + " ".join(
|
| 1019 |
+
f"{name}={seconds:.3f}s" for name, seconds in result["timings"].items()))
|
| 1020 |
self.cache[cache_key] = result
|
| 1021 |
return result
|
| 1022 |
|
platform/harness/mirror_tokens.py
CHANGED
|
@@ -46,9 +46,9 @@ def _store_identity(st):
|
|
| 46 |
def _revision(cur, marks, generation, sources):
|
| 47 |
"""Decline reuse during writes or transactions whose snapshot can outlive file marks."""
|
| 48 |
from harness import datastore
|
| 49 |
-
if marks is None or getattr(cur, '_in_transaction', False) or datastore.
|
| 50 |
return None
|
| 51 |
-
return fingerprint([marks, generation, datastore.
|
| 52 |
|
| 53 |
|
| 54 |
def mirror_change_token(st, database):
|
|
@@ -81,7 +81,7 @@ def mirror_change_token(st, database):
|
|
| 81 |
generation = datastore.mirror_generation(cur)
|
| 82 |
# An in-memory mirror cannot show an in-place write, so it is never cached.
|
| 83 |
marks = _file_marks(cur)
|
| 84 |
-
slot = (_store_identity(st),
|
| 85 |
revision = _revision(cur, marks, generation, [sources, sorted(optional)])
|
| 86 |
with _TABLE_ROWS_LOCK:
|
| 87 |
cached = _TABLE_ROWS.get(slot)
|
|
@@ -115,7 +115,7 @@ def mirror_change_token(st, database):
|
|
| 115 |
sizes = ("SELECT 'size', table_name, CAST(estimated_size AS VARCHAR), '' FROM duckdb_tables() "
|
| 116 |
f"WHERE table_name IN ({','.join('?' for _ in hashed)})")
|
| 117 |
key = None if marks is None else fingerprint([
|
| 118 |
-
marks, list(generation), datastore.
|
| 119 |
cur.execute(" UNION ALL ".join([*meta, sizes]) + " ORDER BY 1,2,3,4",
|
| 120 |
[*meta_params, *hashed]).fetchall()])
|
| 121 |
with _TABLE_ROWS_LOCK:
|
|
@@ -140,6 +140,8 @@ def mirror_change_token(st, database):
|
|
| 140 |
# Absent names stay in the token, so a table that appears later changes it.
|
| 141 |
token = fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows)
|
| 142 |
after = _revision(cur, _file_marks(cur), datastore.mirror_generation(cur), [sources, sorted(optional)])
|
|
|
|
|
|
|
| 143 |
if revision is not None and after == revision:
|
| 144 |
with _TABLE_ROWS_LOCK:
|
| 145 |
if slot not in _TABLE_ROWS and len(_TABLE_ROWS) >= _TABLE_ROWS_MAX:
|
|
|
|
| 46 |
def _revision(cur, marks, generation, sources):
|
| 47 |
"""Decline reuse during writes or transactions whose snapshot can outlive file marks."""
|
| 48 |
from harness import datastore
|
| 49 |
+
if marks is None or getattr(cur, '_in_transaction', False) or datastore.mirror_active_writes(cur):
|
| 50 |
return None
|
| 51 |
+
return fingerprint([marks, generation, datastore.mirror_writes(cur), sources])
|
| 52 |
|
| 53 |
|
| 54 |
def mirror_change_token(st, database):
|
|
|
|
| 81 |
generation = datastore.mirror_generation(cur)
|
| 82 |
# An in-memory mirror cannot show an in-place write, so it is never cached.
|
| 83 |
marks = _file_marks(cur)
|
| 84 |
+
slot = (_store_identity(st), fingerprint([sources, sorted(optional)]))
|
| 85 |
revision = _revision(cur, marks, generation, [sources, sorted(optional)])
|
| 86 |
with _TABLE_ROWS_LOCK:
|
| 87 |
cached = _TABLE_ROWS.get(slot)
|
|
|
|
| 115 |
sizes = ("SELECT 'size', table_name, CAST(estimated_size AS VARCHAR), '' FROM duckdb_tables() "
|
| 116 |
f"WHERE table_name IN ({','.join('?' for _ in hashed)})")
|
| 117 |
key = None if marks is None else fingerprint([
|
| 118 |
+
marks, list(generation), datastore.mirror_writes(cur), hashed, sorted(optional - present),
|
| 119 |
cur.execute(" UNION ALL ".join([*meta, sizes]) + " ORDER BY 1,2,3,4",
|
| 120 |
[*meta_params, *hashed]).fetchall()])
|
| 121 |
with _TABLE_ROWS_LOCK:
|
|
|
|
| 140 |
# Absent names stay in the token, so a table that appears later changes it.
|
| 141 |
token = fingerprint([rows, {"absent": absent}]) if absent else fingerprint(rows)
|
| 142 |
after = _revision(cur, _file_marks(cur), datastore.mirror_generation(cur), [sources, sorted(optional)])
|
| 143 |
+
if revision is not None and after != revision:
|
| 144 |
+
raise RuntimeError("The mirror changed while reading its change token.")
|
| 145 |
if revision is not None and after == revision:
|
| 146 |
with _TABLE_ROWS_LOCK:
|
| 147 |
if slot not in _TABLE_ROWS and len(_TABLE_ROWS) >= _TABLE_ROWS_MAX:
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b4c8cbaebc5ccbde2b3f83cb6c84b606c16ce361e5f5a31816633a233a008f92
|