fsanyoto commited on
Commit
1f7b5d2
·
verified ·
1 Parent(s): 023301a

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "627b4a3783d5 (627b4a3783d5c)",
3
  "releases": [
4
  {
5
  "version": "v86",
 
1
  {
2
+ "current": "8165ce5cac (8165ce5cac1ed)",
3
  "releases": [
4
  {
5
  "version": "v86",
VERSION CHANGED
@@ -1 +1 @@
1
- 627b4a3783d5 (627b4a3783d5c)
 
1
+ 8165ce5cac (8165ce5cac1ed)
api/main.py CHANGED
@@ -2393,6 +2393,10 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
2393
  failure = (routes_connectors._odoo_failure(failed, "materialization")
2394
  if failed else None)
2395
  tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
 
 
 
 
2396
  database_receipts = report.get("databases")
2397
  database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
2398
  canonical_receipts = report.get("canonicalDatabases")
 
2393
  failure = (routes_connectors._odoo_failure(failed, "materialization")
2394
  if failed else None)
2395
  tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
2396
+ # A discovered model's clean pass is a per-database success too, or its old failure never clears.
2397
+ discovered = report.get("discovered") if isinstance(report.get("discovered"), dict) else {}
2398
+ if isinstance(discovered.get("tables"), dict):
2399
+ tables = {**discovered["tables"], **tables}
2400
  database_receipts = report.get("databases")
2401
  database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
2402
  canonical_receipts = report.get("canonicalDatabases")
api/registry_source.py CHANGED
@@ -301,6 +301,13 @@ class BatchStore:
301
 
302
  put = upload_bytes = delete_path = update
303
 
 
 
 
 
 
 
 
304
  def _workspace(self, name):
305
  reader = backend(self._runtime)
306
  project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
@@ -577,6 +584,21 @@ class CanonicalStore(BatchStore):
577
  return super().get_entry(name, entry)
578
 
579
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
580
  def canonical_metadata(session, table_key, *, metadata=None, cur=None):
581
  """Read the canonical permitted definitions without constructing source rows."""
582
  import routes_customers as customers
@@ -815,7 +837,9 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
815
  # A caller's request snapshot carries the generation taken before its first read.
816
  metadata = {'_scan': ids is None} if metadata is None else metadata
817
  before = metadata.get('_generation') or generation(session, table_key)
818
- definition = metadata.get('_definition') or canonical_metadata(session, table_key, metadata=metadata, cur=cur)
 
 
819
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
820
  tree = perm_scope.entry(session.user, table_key) or {}
821
  wall = tree.get('filter') or {}
@@ -846,7 +870,8 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
846
  if required is not None and {'zone', 'routes'} & required:
847
  required.update({'lat', 'lon'})
848
  runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
849
- all_keys = {f['key'] for f in definition['fields']} if required is None else required
 
850
  raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
851
  if required is not None:
852
  raw = [field for field in raw if field.get('key') in required]
@@ -1018,14 +1043,18 @@ def canonical_page(session, cur, table_key, ids, *, related=None):
1018
  """Hydrate a selected page through canonical projections without constructing a full payload."""
1019
  ids = list(dict.fromkeys(int(pid) for pid in ids))
1020
  found, docs, payload = {}, {}, None
1021
- with closing(canonical_batches(session, cur, table_key, ids=ids, related=related)) as batches:
 
 
 
 
1022
  for batch in batches:
1023
  payload = {key: value for key, value in batch.items()
1024
  if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
1025
  found.update((int(row['pid']), row) for row in batch['rows'])
1026
  docs.update(batch.get('docs') or {})
1027
  if payload is None:
1028
- g = canonical_metadata(session, table_key)
1029
  payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
1030
  return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
1031
 
 
301
 
302
  put = upload_bytes = delete_path = update
303
 
304
+ def exists(self, name):
305
+ """One existence probe per key per request. This reader cannot write, so the answer holds."""
306
+ key = ('exists', name)
307
+ if key not in self.metadata:
308
+ self.metadata[key] = bool(self._runtime.exists(name))
309
+ return self.metadata[key]
310
+
311
  def _workspace(self, name):
312
  reader = backend(self._runtime)
313
  project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
 
584
  return super().get_entry(name, entry)
585
 
586
 
587
+ class _EveryField:
588
+ """`needed` for a whole-field read: every served field, and no condition sets."""
589
+ def __contains__(self, key):
590
+ return True
591
+
592
+ def __bool__(self):
593
+ return True
594
+
595
+ def __iter__(self):
596
+ raise TypeError('EVERY_FIELD names no keys. Test membership only.')
597
+
598
+
599
+ EVERY_FIELD = _EveryField()
600
+
601
+
602
  def canonical_metadata(session, table_key, *, metadata=None, cur=None):
603
  """Read the canonical permitted definitions without constructing source rows."""
604
  import routes_customers as customers
 
837
  # A caller's request snapshot carries the generation taken before its first read.
838
  metadata = {'_scan': ids is None} if metadata is None else metadata
839
  before = metadata.get('_generation') or generation(session, table_key)
840
+ # A whole-field read needs no schema pass: its one row assembly serves every visible field.
841
+ definition = metadata.get('_definition') or (None if needed is None else
842
+ canonical_metadata(session, table_key, metadata=metadata, cur=cur))
843
  # Every permanent wall operand is populated before apply_row_scope judges the batch.
844
  tree = perm_scope.entry(session.user, table_key) or {}
845
  wall = tree.get('filter') or {}
 
870
  if required is not None and {'zone', 'routes'} & required:
871
  required.update({'lat', 'lon'})
872
  runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
873
+ all_keys = required if required is not None else (
874
+ EVERY_FIELD if definition is None else {f['key'] for f in definition['fields']})
875
  raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
876
  if required is not None:
877
  raw = [field for field in raw if field.get('key') in required]
 
1043
  """Hydrate a selected page through canonical projections without constructing a full payload."""
1044
  ids = list(dict.fromkeys(int(pid) for pid in ids))
1045
  found, docs, payload = {}, {}, None
1046
+ # One request memo serves every batch and the empty-page fallback. `_canonical_batches`
1047
+ # takes the generation once, before its first read, and rechecks that same snapshot.
1048
+ shared = {'_scan': False}
1049
+ with closing(canonical_batches(session, cur, table_key, ids=ids, related=related,
1050
+ metadata=shared)) as batches:
1051
  for batch in batches:
1052
  payload = {key: value for key, value in batch.items()
1053
  if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
1054
  found.update((int(row['pid']), row) for row in batch['rows'])
1055
  docs.update(batch.get('docs') or {})
1056
  if payload is None:
1057
+ g = canonical_metadata(session, table_key, metadata=shared)
1058
  payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
1059
  return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
1060
 
api/routes_connectors.py CHANGED
@@ -2162,7 +2162,81 @@ def _connection_jobs(rt, provider, connection_id):
2162
  cid = str(connection_id or "").strip() or "default"
2163
  if provider == "odoo" and cid == "default":
2164
  cid = _odoo_connection_slot(rt, connection_id)
2165
- return connector_sync.jobs(rt, provider, cid)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2166
 
2167
 
2168
  _PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
@@ -3272,6 +3346,8 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
3272
  jobs = _connection_jobs(rt, connector_key, resolved_connection)
3273
  row = (connector_sync.job_summary(jobs)
3274
  or connector_sync.read(rt, connector_key, resolved_connection))
 
 
3275
  # Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
3276
  answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
3277
  if connector_key == "odoo":
@@ -3400,6 +3476,7 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi
3400
  payload = {"provider": connector_key, "jobs": connector_sync.jobs(
3401
  session.runtime, connector_key, connection_id)}
3402
  if connector_key == "odoo":
 
3403
  payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
3404
  return _safe_connector_status(connector_key, payload)
3405
 
 
2162
  cid = str(connection_id or "").strip() or "default"
2163
  if provider == "odoo" and cid == "default":
2164
  cid = _odoo_connection_slot(rt, connection_id)
2165
+ receipts = connector_sync.jobs(rt, provider, cid)
2166
+ return _odoo_current_jobs(rt, receipts) if provider == "odoo" else receipts
2167
+
2168
+
2169
+ def _odoo_offered_labels(rt, cid):
2170
+ """Enabled database keys this connection offers, with labels, or None when unreadable."""
2171
+ import odoo_relational as rel
2172
+ try:
2173
+ default = _odoo_connection_slot(rt, None)
2174
+ choices = rel.grid_choices(rt, None if cid in ("", "default", default) else cid)
2175
+ except Exception: # noqa: BLE001
2176
+ return None
2177
+ labels = {}
2178
+ for row in choices or ():
2179
+ if isinstance(row, dict) and row.get("enabled"):
2180
+ for key in (row.get("key"), row.get("baseKey")):
2181
+ if key:
2182
+ labels[str(key)] = str(row.get("label") or row.get("key"))
2183
+ return labels or None
2184
+
2185
+
2186
+ def _odoo_current_jobs(rt, jobs):
2187
+ """Odoo receipts as the newest pass reads them, for every door that summarizes them.
2188
+
2189
+ A failure older than its own database's newest clean sync reads ready. A failure for a
2190
+ database the connection no longer offers drops. Every other failure stays failed, labelled.
2191
+ """
2192
+ # A cached read keeps the route at one fresh health read; an older document only keeps a failure longer.
2193
+ try:
2194
+ document = rt.get(_ODOO_HEALTH_STATUS) or {}
2195
+ except Exception: # noqa: BLE001
2196
+ document = {}
2197
+ connections = document.get("connections") if isinstance(document, dict) else None
2198
+ connections = connections if isinstance(connections, dict) else {}
2199
+ offered, current = {}, []
2200
+ for job in jobs:
2201
+ if job.get("state") != "failed":
2202
+ current.append(job)
2203
+ continue
2204
+ cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
2205
+ if cid not in offered:
2206
+ offered[cid] = _odoo_offered_labels(rt, cid)
2207
+ if offered[cid] is not None and key not in offered[cid]:
2208
+ continue
2209
+ database = ((connections.get(cid) or {}).get("databases") or {}).get(key)
2210
+ success = _odoo_receipt_success(database) if isinstance(database, dict) else None
2211
+ if _odoo_failure_is_current(job.get("at"), success):
2212
+ current.append({**job, "label": (offered[cid] or {}).get(key) or key})
2213
+ continue
2214
+ current.append({**job, "state": "ready", "error": "", "retryable": False,
2215
+ "at": _odoo_timestamp(success), "supersededFailureAt": job.get("at")})
2216
+ return current
2217
+
2218
+
2219
+ def _odoo_named_failures(row, jobs):
2220
+ """Name each database that still fails, with its own time, on a failed sync row."""
2221
+ failed = [job for job in jobs if job.get("state") == "failed"]
2222
+ if not row or row.get("state") != "failed" or not failed:
2223
+ return row
2224
+ named = []
2225
+ for job in failed:
2226
+ stamp = _odoo_timestamp(job.get("at"))
2227
+ named.append({"databaseKey": job.get("databaseKey"), "label": job.get("label") or job.get("databaseKey"),
2228
+ "at": (datetime.fromtimestamp(stamp, timezone.utc).isoformat()
2229
+ if stamp is not None else None),
2230
+ "error": job.get("error") or ""})
2231
+ named.sort(key=lambda item: item["at"] or "", reverse=True)
2232
+ lines = []
2233
+ for item in named:
2234
+ cause = re.sub(r"^[a-z][a-z0-9_]*: ", "", item["error"])
2235
+ lines.append(f"{item['label']}: {cause}" if cause else f"{item['label']} did not sync.")
2236
+ others = [error for error in row.get("errors") or ()
2237
+ if error not in {item["error"] for item in named}]
2238
+ errors = lines + others
2239
+ return {**row, "failed": named, "errors": errors, "errorsTotal": len(errors)}
2240
 
2241
 
2242
  _PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
 
3346
  jobs = _connection_jobs(rt, connector_key, resolved_connection)
3347
  row = (connector_sync.job_summary(jobs)
3348
  or connector_sync.read(rt, connector_key, resolved_connection))
3349
+ if connector_key == "odoo":
3350
+ row = _odoo_named_failures(row, jobs)
3351
  # Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
3352
  answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
3353
  if connector_key == "odoo":
 
3476
  payload = {"provider": connector_key, "jobs": connector_sync.jobs(
3477
  session.runtime, connector_key, connection_id)}
3478
  if connector_key == "odoo":
3479
+ payload["jobs"] = _odoo_current_jobs(session.runtime, payload["jobs"])
3480
  payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
3481
  return _safe_connector_status(connector_key, payload)
3482
 
api/routes_products.py CHANGED
@@ -913,6 +913,23 @@ def _seed_image(rows, fields):
913
  return rows
914
 
915
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
916
  def pd_fields(consolidated=True, st=None):
917
  """The product field contract, minus `product_data.CONSOLIDATED_ONLY` when the caller is scoped.
918
 
@@ -937,9 +954,7 @@ def pd_fields(consolidated=True, st=None):
937
  if st is not None:
938
  pd.shared_fields(st=st)
939
 
940
- doc = json.loads((Path(aios_grid.__file__).resolve().parent /
941
- "aios_grid_fields.json").read_text(encoding="utf-8"))
942
- fields = list((doc.get("product_data") or {}).get("fields") or [])
943
  # R7: the Image column is BUILT IN on this table, injected here rather than added to
944
  # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
945
  # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
 
913
  return rows
914
 
915
 
916
+ #: The parsed product contract per file, keyed by its mtime and size. A request reads a deep copy.
917
+ _CONTRACT_MEMO = {}
918
+
919
+
920
+ def _contract_fields(path):
921
+ """The product contract fields from `path`, parsed once per file version."""
922
+ import copy
923
+ import os
924
+ stat = os.stat(path)
925
+ stamp = (stat.st_mtime_ns, stat.st_size)
926
+ hit = _CONTRACT_MEMO.get(str(path))
927
+ if hit is None or hit[0] != stamp:
928
+ doc = json.loads(path.read_text(encoding="utf-8"))
929
+ hit = _CONTRACT_MEMO[str(path)] = (stamp, list((doc.get("product_data") or {}).get("fields") or []))
930
+ return copy.deepcopy(hit[1])
931
+
932
+
933
  def pd_fields(consolidated=True, st=None):
934
  """The product field contract, minus `product_data.CONSOLIDATED_ONLY` when the caller is scoped.
935
 
 
954
  if st is not None:
955
  pd.shared_fields(st=st)
956
 
957
+ fields = _contract_fields(Path(aios_grid.__file__).resolve().parent / "aios_grid_fields.json")
 
 
958
  # R7: the Image column is BUILT IN on this table, injected here rather than added to
959
  # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
960
  # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
api/routes_sheet.py CHANGED
@@ -7083,8 +7083,12 @@ def _drill_tokens(session, databases):
7083
 
7084
 
7085
  @contextlib.contextmanager
7086
- def _drill_mirror(session, database, window, needed, databases):
7087
- """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard."""
 
 
 
 
7088
  from core import sheet_binding, perm_scope
7089
  from core.request_timing import raise_if_resource_error
7090
  import datetime
@@ -7096,7 +7100,7 @@ def _drill_mirror(session, database, window, needed, databases):
7096
  # across requests; the request's ExitStack closes it after the page is produced.
7097
  yield _mirror(session)(database, metric_window=window, needed=needed), None
7098
  return
7099
- sources = _drill_tokens(session, databases)
7100
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7101
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7102
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
@@ -7466,7 +7470,7 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
7466
  databases.update(_binding_related_entry(session, database, via)["target"]
7467
  for via in sheet_mod.related_vias(scope_nodes))
7468
  sources = _drill_tokens(session, databases)
7469
- with _drill_mirror(session, database, window, needed, databases) as (made, token):
7470
  cur, spec, cols = made
7471
  cols = dict(cols)
7472
  # Apply the aggregate's field wall before compiling either the count or the IDs.
 
7083
 
7084
 
7085
  @contextlib.contextmanager
7086
+ def _drill_mirror(session, database, window, needed, databases, sources=None):
7087
+ """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.
7088
+
7089
+ `sources` is the caller's opening token pass. The lease key reuses it, so one request reads
7090
+ the tokens once before the build. The build-miss and end-of-request rechecks still read anew.
7091
+ """
7092
  from core import sheet_binding, perm_scope
7093
  from core.request_timing import raise_if_resource_error
7094
  import datetime
 
7100
  # across requests; the request's ExitStack closes it after the page is produced.
7101
  yield _mirror(session)(database, metric_window=window, needed=needed), None
7102
  return
7103
+ sources = _drill_tokens(session, databases) if sources is None else dict(sources)
7104
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7105
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7106
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
 
7470
  databases.update(_binding_related_entry(session, database, via)["target"]
7471
  for via in sheet_mod.related_vias(scope_nodes))
7472
  sources = _drill_tokens(session, databases)
7473
+ with _drill_mirror(session, database, window, needed, databases, sources=sources) as (made, token):
7474
  cur, spec, cols = made
7475
  cols = dict(cols)
7476
  # Apply the aggregate's field wall before compiling either the count or the IDs.
platform/core/blob_envelope.py CHANGED
@@ -72,7 +72,7 @@ def open(tenant, path, data):
72
  if key is None:
73
  raise BlobEnvelopeError("Opening this blob requires a tenant data key.")
74
  try:
75
- return tenant_keys.aead_open(key, data[len(HEADER):], aad)
76
  except Exception:
77
  raise BlobEnvelopeError("The blob envelope did not authenticate.") from None
78
 
 
72
  if key is None:
73
  raise BlobEnvelopeError("Opening this blob requires a tenant data key.")
74
  try:
75
+ return tenant_keys.aead_open(key, memoryview(data)[len(HEADER):], aad)
76
  except Exception:
77
  raise BlobEnvelopeError("The blob envelope did not authenticate.") from None
78
 
platform/core/tenant_keys.py CHANGED
@@ -302,11 +302,15 @@ def subkey(tenant, purpose):
302
  """HKDF-SHA256 subkey of the tenant data key for one purpose, or None on the legacy path."""
303
  if purpose not in PURPOSES:
304
  raise ValueError(f"unknown data key purpose {purpose!r}")
305
- key = data_key(tenant)
306
- if key is None:
307
- return None
308
- return HKDF(algorithm=hashes.SHA256(), length=32, salt=None,
309
- info=f"aios/tenant/{purpose}/v1".encode("ascii")).derive(key)
 
 
 
 
310
 
311
 
312
  def aead_seal(key, plaintext, aad):
@@ -321,10 +325,17 @@ def aead_open(key, blob, aad):
321
  """Open an `aead_seal` blob. Raises on a wrong key, a wrong AAD or a changed byte."""
322
  if not isinstance(key, bytes) or len(key) != 32:
323
  raise ValueError("an AES-256 key is 32 bytes")
324
- blob = bytes(blob)
 
 
325
  if len(blob) < 28:
326
  raise ValueError("the sealed value is too short")
327
- return AESGCM(key).decrypt(blob[:12], blob[12:], bytes(aad))
 
 
 
 
 
328
 
329
 
330
  def _active_tenants():
 
302
  """HKDF-SHA256 subkey of the tenant data key for one purpose, or None on the legacy path."""
303
  if purpose not in PURPOSES:
304
  raise ValueError(f"unknown data key purpose {purpose!r}")
305
+ with _LOCK:
306
+ key = data_key(tenant)
307
+ if key is None:
308
+ return None
309
+ slot = (context(tenant)["tenant"], _fingerprint(backend_name()), purpose)
310
+ if slot not in _CACHE:
311
+ _CACHE[slot] = HKDF(algorithm=hashes.SHA256(), length=32, salt=None,
312
+ info=f"aios/tenant/{purpose}/v1".encode("ascii")).derive(key)
313
+ return _CACHE[slot]
314
 
315
 
316
  def aead_seal(key, plaintext, aad):
 
325
  """Open an `aead_seal` blob. Raises on a wrong key, a wrong AAD or a changed byte."""
326
  if not isinstance(key, bytes) or len(key) != 32:
327
  raise ValueError("an AES-256 key is 32 bytes")
328
+ if not (isinstance(blob, memoryview) and isinstance(blob.obj, bytes)
329
+ and blob.c_contiguous and blob.format == "B"):
330
+ blob = memoryview(bytes(blob))
331
  if len(blob) < 28:
332
  raise ValueError("the sealed value is too short")
333
+ cipher, nonce, aad = AESGCM(key), bytes(blob[:12]), bytes(aad)
334
+ try:
335
+ return cipher.decrypt(nonce, blob[12:], aad)
336
+ except TypeError:
337
+ # Older supported cryptography releases require bytes for the ciphertext.
338
+ return cipher.decrypt(nonce, bytes(blob[12:]), aad)
339
 
340
 
341
  def _active_tenants():
platform/harness/datastore.py CHANGED
@@ -165,7 +165,13 @@ def apply_limits(con):
165
  con.execute(f"SET memory_limit='{memory}'")
166
  if threads.isdigit() and int(threads) > 0:
167
  con.execute(f"SET threads={int(threads)}")
168
- con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'")
 
 
 
 
 
 
169
  con.execute(f"SET max_temp_directory_size='{spill}'")
170
  return con
171
 
 
165
  con.execute(f"SET memory_limit='{memory}'")
166
  if threads.isdigit() and int(threads) > 0:
167
  con.execute(f"SET threads={int(threads)}")
168
+ # DuckDB refuses any temp_directory SET, the same path included, once the instance has spilled.
169
+ if con.execute("SELECT current_setting('temp_directory')").fetchone()[0] != str(temp):
170
+ try:
171
+ con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'")
172
+ except duckdb.NotImplementedException as exc:
173
+ if "Cannot switch temporary directory" not in str(exc):
174
+ raise
175
  con.execute(f"SET max_temp_directory_size='{spill}'")
176
  return con
177
 
platform/harness/mirror_store.py CHANGED
@@ -526,8 +526,10 @@ class MirrorStore:
526
  return MIRROR_PART_MAX_BYTES
527
 
528
  def _download(self, rt, key):
529
- raw = rt.download_bytes(self.prefix + key)
530
- return None if raw is None else be.open(rt.key, self.prefix + key, raw)
 
 
531
 
532
  def _upload(self, rt, key, data):
533
  stored = be.seal(rt.key, self.prefix + key, data)
@@ -584,6 +586,12 @@ class MirrorStore:
584
  def _heal_published(self, con, result, log):
585
  """Disable summaries across repairs whose source transactions can commit separately."""
586
  from harness import summary_tables
 
 
 
 
 
 
587
  con.execute("BEGIN TRANSACTION")
588
  try:
589
  summary_tables.invalidate(con, self.connector)
@@ -592,6 +600,8 @@ class MirrorStore:
592
  con.execute("ROLLBACK")
593
  raise
594
  self._heal_restored(con, result, log)
 
 
595
  con.execute("BEGIN TRANSACTION")
596
  try:
597
  with _restore_phase("summaries"):
@@ -659,8 +669,9 @@ class MirrorStore:
659
  def _checked_mirror_blob(self, rt, path, metadata):
660
  with _restore_phase("download_open_checksum"):
661
  data = self._download(rt, path)
662
- if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
663
- raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
 
664
  return data
665
 
666
 
@@ -867,10 +878,11 @@ class MirrorStore:
867
  "unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
868
  from harness import summary_tables
869
  with _restore_phase("summaries"):
870
- if getattr(_RESTORE_PROGRESS, 'summary_owner', None) is not self:
 
871
  summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
872
  else:
873
- # Healing can commit source repairs separately. Rebuild once after those repairs.
874
  summary_tables.invalidate(con, self.connector)
875
  datastore._write_publication(con, self.connector, manifest)
876
  con.execute("COMMIT")
 
526
  return MIRROR_PART_MAX_BYTES
527
 
528
  def _download(self, rt, key):
529
+ with _restore_phase("download"):
530
+ raw = rt.download_bytes(self.prefix + key)
531
+ with _restore_phase("blob_open"):
532
+ return None if raw is None else be.open(rt.key, self.prefix + key, raw)
533
 
534
  def _upload(self, rt, key, data):
535
  stored = be.seal(rt.key, self.prefix + key, data)
 
586
  def _heal_published(self, con, result, log):
587
  """Disable summaries across repairs whose source transactions can commit separately."""
588
  from harness import summary_tables
589
+ publication = datastore._publication(con, self.connector)
590
+ with _restore_phase("summaries"):
591
+ # A current local catalog proves this publication already completed its source repairs.
592
+ if result.get("state") == "local" and summary_tables.is_current(
593
+ con, self.connector, publication, self._tables(con)):
594
+ return
595
  con.execute("BEGIN TRANSACTION")
596
  try:
597
  summary_tables.invalidate(con, self.connector)
 
600
  con.execute("ROLLBACK")
601
  raise
602
  self._heal_restored(con, result, log)
603
+ if os.environ.get("AIOS_PROCESS_ROLE") == "worker":
604
+ return
605
  con.execute("BEGIN TRANSACTION")
606
  try:
607
  with _restore_phase("summaries"):
 
669
  def _checked_mirror_blob(self, rt, path, metadata):
670
  with _restore_phase("download_open_checksum"):
671
  data = self._download(rt, path)
672
+ with _restore_phase("checksum"):
673
+ if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
674
+ raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
675
  return data
676
 
677
 
 
878
  "unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
879
  from harness import summary_tables
880
  with _restore_phase("summaries"):
881
+ if (os.environ.get("AIOS_PROCESS_ROLE") != "worker"
882
+ and getattr(_RESTORE_PROGRESS, 'summary_owner', None) is not self):
883
  summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
884
  else:
885
+ # Workers serve no aggregate reads. Other roles rebuild after source repairs.
886
  summary_tables.invalidate(con, self.connector)
887
  datastore._write_publication(con, self.connector, manifest)
888
  con.execute("COMMIT")
platform/harness/summary_tables.py CHANGED
@@ -107,6 +107,33 @@ def rebuild(con, source, manifest, tables):
107
  [source, table, group, tenant, snapshot, name, definition])
108
 
109
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
  def read(con, *, tenant, table, group_column, group_expression, plan, where, params,
111
  group_offset, group_limit):
112
  """Read an exact unfiltered grouping from the publication on this request's own cursor."""
 
107
  [source, table, group, tenant, snapshot, name, definition])
108
 
109
 
110
+ def is_current(con, source, manifest, tables):
111
+ """Require every declared summary to match the tenant, snapshot and current SQL definition."""
112
+ tenant, snapshot = manifest.get("tenant"), manifest.get("snapshot_id")
113
+ if not tenant or not snapshot:
114
+ return False
115
+ resident = {row[0] for row in con.execute(
116
+ "SELECT table_name FROM information_schema.tables WHERE table_schema='main' "
117
+ "AND table_type='BASE TABLE'").fetchall()}
118
+ if CATALOG not in resident:
119
+ return False
120
+ expected = {}
121
+ for table in sorted(set(tables) | set(manifest.get("tables") or {})):
122
+ if not connector_spec.mirror_summary_groups(table):
123
+ continue
124
+ if table not in resident:
125
+ return False
126
+ for group, expression, totals in _definitions(con, table):
127
+ name = _name(source, table, group)
128
+ if name not in resident:
129
+ return False
130
+ expected[(table, group)] = (tenant, snapshot, name,
131
+ json.dumps({"group": expression, "totals": totals}, sort_keys=True))
132
+ rows = con.execute(f"SELECT source_table, group_column, tenant, snapshot_id, summary_table, definition "
133
+ f"FROM {CATALOG} WHERE source=?", [source]).fetchall()
134
+ return bool(expected) and {(row[0], row[1]): row[2:] for row in rows} == expected
135
+
136
+
137
  def read(con, *, tenant, table, group_column, group_expression, plan, where, params,
138
  group_offset, group_limit):
139
  """Read an exact unfiltered grouping from the publication on this request's own cursor."""
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 8b2b09688a388b71d7a2128cafb505b0550507bbe37005f26e2ada2528eb598a
 
1
+ b3436af160188b7d0b91b7654aad92979aab79eb3488ced1a43912bae20410d7