Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +4 -0
- api/registry_source.py +33 -4
- api/routes_connectors.py +78 -1
- api/routes_products.py +18 -3
- api/routes_sheet.py +8 -4
- platform/core/blob_envelope.py +1 -1
- platform/core/tenant_keys.py +18 -7
- platform/harness/datastore.py +7 -1
- platform/harness/mirror_store.py +18 -6
- platform/harness/summary_tables.py +27 -0
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "8165ce5cac (8165ce5cac1ed)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
8165ce5cac (8165ce5cac1ed)
|
api/main.py
CHANGED
|
@@ -2393,6 +2393,10 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2393 |
failure = (routes_connectors._odoo_failure(failed, "materialization")
|
| 2394 |
if failed else None)
|
| 2395 |
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2396 |
database_receipts = report.get("databases")
|
| 2397 |
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 2398 |
canonical_receipts = report.get("canonicalDatabases")
|
|
|
|
| 2393 |
failure = (routes_connectors._odoo_failure(failed, "materialization")
|
| 2394 |
if failed else None)
|
| 2395 |
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
| 2396 |
+
# A discovered model's clean pass is a per-database success too, or its old failure never clears.
|
| 2397 |
+
discovered = report.get("discovered") if isinstance(report.get("discovered"), dict) else {}
|
| 2398 |
+
if isinstance(discovered.get("tables"), dict):
|
| 2399 |
+
tables = {**discovered["tables"], **tables}
|
| 2400 |
database_receipts = report.get("databases")
|
| 2401 |
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
|
| 2402 |
canonical_receipts = report.get("canonicalDatabases")
|
api/registry_source.py
CHANGED
|
@@ -301,6 +301,13 @@ class BatchStore:
|
|
| 301 |
|
| 302 |
put = upload_bytes = delete_path = update
|
| 303 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
def _workspace(self, name):
|
| 305 |
reader = backend(self._runtime)
|
| 306 |
project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
|
|
@@ -577,6 +584,21 @@ class CanonicalStore(BatchStore):
|
|
| 577 |
return super().get_entry(name, entry)
|
| 578 |
|
| 579 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
def canonical_metadata(session, table_key, *, metadata=None, cur=None):
|
| 581 |
"""Read the canonical permitted definitions without constructing source rows."""
|
| 582 |
import routes_customers as customers
|
|
@@ -815,7 +837,9 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 815 |
# A caller's request snapshot carries the generation taken before its first read.
|
| 816 |
metadata = {'_scan': ids is None} if metadata is None else metadata
|
| 817 |
before = metadata.get('_generation') or generation(session, table_key)
|
| 818 |
-
|
|
|
|
|
|
|
| 819 |
# Every permanent wall operand is populated before apply_row_scope judges the batch.
|
| 820 |
tree = perm_scope.entry(session.user, table_key) or {}
|
| 821 |
wall = tree.get('filter') or {}
|
|
@@ -846,7 +870,8 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
|
|
| 846 |
if required is not None and {'zone', 'routes'} & required:
|
| 847 |
required.update({'lat', 'lon'})
|
| 848 |
runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
|
| 849 |
-
all_keys =
|
|
|
|
| 850 |
raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
|
| 851 |
if required is not None:
|
| 852 |
raw = [field for field in raw if field.get('key') in required]
|
|
@@ -1018,14 +1043,18 @@ def canonical_page(session, cur, table_key, ids, *, related=None):
|
|
| 1018 |
"""Hydrate a selected page through canonical projections without constructing a full payload."""
|
| 1019 |
ids = list(dict.fromkeys(int(pid) for pid in ids))
|
| 1020 |
found, docs, payload = {}, {}, None
|
| 1021 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1022 |
for batch in batches:
|
| 1023 |
payload = {key: value for key, value in batch.items()
|
| 1024 |
if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
|
| 1025 |
found.update((int(row['pid']), row) for row in batch['rows'])
|
| 1026 |
docs.update(batch.get('docs') or {})
|
| 1027 |
if payload is None:
|
| 1028 |
-
g = canonical_metadata(session, table_key)
|
| 1029 |
payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
|
| 1030 |
return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
|
| 1031 |
|
|
|
|
| 301 |
|
| 302 |
put = upload_bytes = delete_path = update
|
| 303 |
|
| 304 |
+
def exists(self, name):
|
| 305 |
+
"""One existence probe per key per request. This reader cannot write, so the answer holds."""
|
| 306 |
+
key = ('exists', name)
|
| 307 |
+
if key not in self.metadata:
|
| 308 |
+
self.metadata[key] = bool(self._runtime.exists(name))
|
| 309 |
+
return self.metadata[key]
|
| 310 |
+
|
| 311 |
def _workspace(self, name):
|
| 312 |
reader = backend(self._runtime)
|
| 313 |
project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
|
|
|
|
| 584 |
return super().get_entry(name, entry)
|
| 585 |
|
| 586 |
|
| 587 |
+
class _EveryField:
|
| 588 |
+
"""`needed` for a whole-field read: every served field, and no condition sets."""
|
| 589 |
+
def __contains__(self, key):
|
| 590 |
+
return True
|
| 591 |
+
|
| 592 |
+
def __bool__(self):
|
| 593 |
+
return True
|
| 594 |
+
|
| 595 |
+
def __iter__(self):
|
| 596 |
+
raise TypeError('EVERY_FIELD names no keys. Test membership only.')
|
| 597 |
+
|
| 598 |
+
|
| 599 |
+
EVERY_FIELD = _EveryField()
|
| 600 |
+
|
| 601 |
+
|
| 602 |
def canonical_metadata(session, table_key, *, metadata=None, cur=None):
|
| 603 |
"""Read the canonical permitted definitions without constructing source rows."""
|
| 604 |
import routes_customers as customers
|
|
|
|
| 837 |
# A caller's request snapshot carries the generation taken before its first read.
|
| 838 |
metadata = {'_scan': ids is None} if metadata is None else metadata
|
| 839 |
before = metadata.get('_generation') or generation(session, table_key)
|
| 840 |
+
# A whole-field read needs no schema pass: its one row assembly serves every visible field.
|
| 841 |
+
definition = metadata.get('_definition') or (None if needed is None else
|
| 842 |
+
canonical_metadata(session, table_key, metadata=metadata, cur=cur))
|
| 843 |
# Every permanent wall operand is populated before apply_row_scope judges the batch.
|
| 844 |
tree = perm_scope.entry(session.user, table_key) or {}
|
| 845 |
wall = tree.get('filter') or {}
|
|
|
|
| 870 |
if required is not None and {'zone', 'routes'} & required:
|
| 871 |
required.update({'lat', 'lon'})
|
| 872 |
runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
|
| 873 |
+
all_keys = required if required is not None else (
|
| 874 |
+
EVERY_FIELD if definition is None else {f['key'] for f in definition['fields']})
|
| 875 |
raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
|
| 876 |
if required is not None:
|
| 877 |
raw = [field for field in raw if field.get('key') in required]
|
|
|
|
| 1043 |
"""Hydrate a selected page through canonical projections without constructing a full payload."""
|
| 1044 |
ids = list(dict.fromkeys(int(pid) for pid in ids))
|
| 1045 |
found, docs, payload = {}, {}, None
|
| 1046 |
+
# One request memo serves every batch and the empty-page fallback. `_canonical_batches`
|
| 1047 |
+
# takes the generation once, before its first read, and rechecks that same snapshot.
|
| 1048 |
+
shared = {'_scan': False}
|
| 1049 |
+
with closing(canonical_batches(session, cur, table_key, ids=ids, related=related,
|
| 1050 |
+
metadata=shared)) as batches:
|
| 1051 |
for batch in batches:
|
| 1052 |
payload = {key: value for key, value in batch.items()
|
| 1053 |
if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
|
| 1054 |
found.update((int(row['pid']), row) for row in batch['rows'])
|
| 1055 |
docs.update(batch.get('docs') or {})
|
| 1056 |
if payload is None:
|
| 1057 |
+
g = canonical_metadata(session, table_key, metadata=shared)
|
| 1058 |
payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
|
| 1059 |
return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
|
| 1060 |
|
api/routes_connectors.py
CHANGED
|
@@ -2162,7 +2162,81 @@ def _connection_jobs(rt, provider, connection_id):
|
|
| 2162 |
cid = str(connection_id or "").strip() or "default"
|
| 2163 |
if provider == "odoo" and cid == "default":
|
| 2164 |
cid = _odoo_connection_slot(rt, connection_id)
|
| 2165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2166 |
|
| 2167 |
|
| 2168 |
_PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
|
|
@@ -3272,6 +3346,8 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
|
|
| 3272 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3273 |
row = (connector_sync.job_summary(jobs)
|
| 3274 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
|
|
|
|
|
|
| 3275 |
# Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
|
| 3276 |
answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
|
| 3277 |
if connector_key == "odoo":
|
|
@@ -3400,6 +3476,7 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi
|
|
| 3400 |
payload = {"provider": connector_key, "jobs": connector_sync.jobs(
|
| 3401 |
session.runtime, connector_key, connection_id)}
|
| 3402 |
if connector_key == "odoo":
|
|
|
|
| 3403 |
payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
|
| 3404 |
return _safe_connector_status(connector_key, payload)
|
| 3405 |
|
|
|
|
| 2162 |
cid = str(connection_id or "").strip() or "default"
|
| 2163 |
if provider == "odoo" and cid == "default":
|
| 2164 |
cid = _odoo_connection_slot(rt, connection_id)
|
| 2165 |
+
receipts = connector_sync.jobs(rt, provider, cid)
|
| 2166 |
+
return _odoo_current_jobs(rt, receipts) if provider == "odoo" else receipts
|
| 2167 |
+
|
| 2168 |
+
|
| 2169 |
+
def _odoo_offered_labels(rt, cid):
|
| 2170 |
+
"""Enabled database keys this connection offers, with labels, or None when unreadable."""
|
| 2171 |
+
import odoo_relational as rel
|
| 2172 |
+
try:
|
| 2173 |
+
default = _odoo_connection_slot(rt, None)
|
| 2174 |
+
choices = rel.grid_choices(rt, None if cid in ("", "default", default) else cid)
|
| 2175 |
+
except Exception: # noqa: BLE001
|
| 2176 |
+
return None
|
| 2177 |
+
labels = {}
|
| 2178 |
+
for row in choices or ():
|
| 2179 |
+
if isinstance(row, dict) and row.get("enabled"):
|
| 2180 |
+
for key in (row.get("key"), row.get("baseKey")):
|
| 2181 |
+
if key:
|
| 2182 |
+
labels[str(key)] = str(row.get("label") or row.get("key"))
|
| 2183 |
+
return labels or None
|
| 2184 |
+
|
| 2185 |
+
|
| 2186 |
+
def _odoo_current_jobs(rt, jobs):
|
| 2187 |
+
"""Odoo receipts as the newest pass reads them, for every door that summarizes them.
|
| 2188 |
+
|
| 2189 |
+
A failure older than its own database's newest clean sync reads ready. A failure for a
|
| 2190 |
+
database the connection no longer offers drops. Every other failure stays failed, labelled.
|
| 2191 |
+
"""
|
| 2192 |
+
# A cached read keeps the route at one fresh health read; an older document only keeps a failure longer.
|
| 2193 |
+
try:
|
| 2194 |
+
document = rt.get(_ODOO_HEALTH_STATUS) or {}
|
| 2195 |
+
except Exception: # noqa: BLE001
|
| 2196 |
+
document = {}
|
| 2197 |
+
connections = document.get("connections") if isinstance(document, dict) else None
|
| 2198 |
+
connections = connections if isinstance(connections, dict) else {}
|
| 2199 |
+
offered, current = {}, []
|
| 2200 |
+
for job in jobs:
|
| 2201 |
+
if job.get("state") != "failed":
|
| 2202 |
+
current.append(job)
|
| 2203 |
+
continue
|
| 2204 |
+
cid, key = str(job.get("connectionId") or ""), str(job.get("databaseKey") or "")
|
| 2205 |
+
if cid not in offered:
|
| 2206 |
+
offered[cid] = _odoo_offered_labels(rt, cid)
|
| 2207 |
+
if offered[cid] is not None and key not in offered[cid]:
|
| 2208 |
+
continue
|
| 2209 |
+
database = ((connections.get(cid) or {}).get("databases") or {}).get(key)
|
| 2210 |
+
success = _odoo_receipt_success(database) if isinstance(database, dict) else None
|
| 2211 |
+
if _odoo_failure_is_current(job.get("at"), success):
|
| 2212 |
+
current.append({**job, "label": (offered[cid] or {}).get(key) or key})
|
| 2213 |
+
continue
|
| 2214 |
+
current.append({**job, "state": "ready", "error": "", "retryable": False,
|
| 2215 |
+
"at": _odoo_timestamp(success), "supersededFailureAt": job.get("at")})
|
| 2216 |
+
return current
|
| 2217 |
+
|
| 2218 |
+
|
| 2219 |
+
def _odoo_named_failures(row, jobs):
|
| 2220 |
+
"""Name each database that still fails, with its own time, on a failed sync row."""
|
| 2221 |
+
failed = [job for job in jobs if job.get("state") == "failed"]
|
| 2222 |
+
if not row or row.get("state") != "failed" or not failed:
|
| 2223 |
+
return row
|
| 2224 |
+
named = []
|
| 2225 |
+
for job in failed:
|
| 2226 |
+
stamp = _odoo_timestamp(job.get("at"))
|
| 2227 |
+
named.append({"databaseKey": job.get("databaseKey"), "label": job.get("label") or job.get("databaseKey"),
|
| 2228 |
+
"at": (datetime.fromtimestamp(stamp, timezone.utc).isoformat()
|
| 2229 |
+
if stamp is not None else None),
|
| 2230 |
+
"error": job.get("error") or ""})
|
| 2231 |
+
named.sort(key=lambda item: item["at"] or "", reverse=True)
|
| 2232 |
+
lines = []
|
| 2233 |
+
for item in named:
|
| 2234 |
+
cause = re.sub(r"^[a-z][a-z0-9_]*: ", "", item["error"])
|
| 2235 |
+
lines.append(f"{item['label']}: {cause}" if cause else f"{item['label']} did not sync.")
|
| 2236 |
+
others = [error for error in row.get("errors") or ()
|
| 2237 |
+
if error not in {item["error"] for item in named}]
|
| 2238 |
+
errors = lines + others
|
| 2239 |
+
return {**row, "failed": named, "errors": errors, "errorsTotal": len(errors)}
|
| 2240 |
|
| 2241 |
|
| 2242 |
_PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
|
|
|
|
| 3346 |
jobs = _connection_jobs(rt, connector_key, resolved_connection)
|
| 3347 |
row = (connector_sync.job_summary(jobs)
|
| 3348 |
or connector_sync.read(rt, connector_key, resolved_connection))
|
| 3349 |
+
if connector_key == "odoo":
|
| 3350 |
+
row = _odoo_named_failures(row, jobs)
|
| 3351 |
# Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
|
| 3352 |
answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
|
| 3353 |
if connector_key == "odoo":
|
|
|
|
| 3476 |
payload = {"provider": connector_key, "jobs": connector_sync.jobs(
|
| 3477 |
session.runtime, connector_key, connection_id)}
|
| 3478 |
if connector_key == "odoo":
|
| 3479 |
+
payload["jobs"] = _odoo_current_jobs(session.runtime, payload["jobs"])
|
| 3480 |
payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
|
| 3481 |
return _safe_connector_status(connector_key, payload)
|
| 3482 |
|
api/routes_products.py
CHANGED
|
@@ -913,6 +913,23 @@ def _seed_image(rows, fields):
|
|
| 913 |
return rows
|
| 914 |
|
| 915 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 916 |
def pd_fields(consolidated=True, st=None):
|
| 917 |
"""The product field contract, minus `product_data.CONSOLIDATED_ONLY` when the caller is scoped.
|
| 918 |
|
|
@@ -937,9 +954,7 @@ def pd_fields(consolidated=True, st=None):
|
|
| 937 |
if st is not None:
|
| 938 |
pd.shared_fields(st=st)
|
| 939 |
|
| 940 |
-
|
| 941 |
-
"aios_grid_fields.json").read_text(encoding="utf-8"))
|
| 942 |
-
fields = list((doc.get("product_data") or {}).get("fields") or [])
|
| 943 |
# R7: the Image column is BUILT IN on this table, injected here rather than added to
|
| 944 |
# `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
|
| 945 |
# column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
|
|
|
|
| 913 |
return rows
|
| 914 |
|
| 915 |
|
| 916 |
+
#: The parsed product contract per file, keyed by its mtime and size. A request reads a deep copy.
|
| 917 |
+
_CONTRACT_MEMO = {}
|
| 918 |
+
|
| 919 |
+
|
| 920 |
+
def _contract_fields(path):
|
| 921 |
+
"""The product contract fields from `path`, parsed once per file version."""
|
| 922 |
+
import copy
|
| 923 |
+
import os
|
| 924 |
+
stat = os.stat(path)
|
| 925 |
+
stamp = (stat.st_mtime_ns, stat.st_size)
|
| 926 |
+
hit = _CONTRACT_MEMO.get(str(path))
|
| 927 |
+
if hit is None or hit[0] != stamp:
|
| 928 |
+
doc = json.loads(path.read_text(encoding="utf-8"))
|
| 929 |
+
hit = _CONTRACT_MEMO[str(path)] = (stamp, list((doc.get("product_data") or {}).get("fields") or []))
|
| 930 |
+
return copy.deepcopy(hit[1])
|
| 931 |
+
|
| 932 |
+
|
| 933 |
def pd_fields(consolidated=True, st=None):
|
| 934 |
"""The product field contract, minus `product_data.CONSOLIDATED_ONLY` when the caller is scoped.
|
| 935 |
|
|
|
|
| 954 |
if st is not None:
|
| 955 |
pd.shared_fields(st=st)
|
| 956 |
|
| 957 |
+
fields = _contract_fields(Path(aios_grid.__file__).resolve().parent / "aios_grid_fields.json")
|
|
|
|
|
|
|
| 958 |
# R7: the Image column is BUILT IN on this table, injected here rather than added to
|
| 959 |
# `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
|
| 960 |
# column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
|
api/routes_sheet.py
CHANGED
|
@@ -7083,8 +7083,12 @@ def _drill_tokens(session, databases):
|
|
| 7083 |
|
| 7084 |
|
| 7085 |
@contextlib.contextmanager
|
| 7086 |
-
def _drill_mirror(session, database, window, needed, databases):
|
| 7087 |
-
"""A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7088 |
from core import sheet_binding, perm_scope
|
| 7089 |
from core.request_timing import raise_if_resource_error
|
| 7090 |
import datetime
|
|
@@ -7096,7 +7100,7 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 7096 |
# across requests; the request's ExitStack closes it after the page is produced.
|
| 7097 |
yield _mirror(session)(database, metric_window=window, needed=needed), None
|
| 7098 |
return
|
| 7099 |
-
sources = _drill_tokens(session, databases)
|
| 7100 |
token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
|
| 7101 |
# C1 also fingerprints derived wall sources, which can change without the primary table.
|
| 7102 |
# An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
|
|
@@ -7466,7 +7470,7 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
|
| 7466 |
databases.update(_binding_related_entry(session, database, via)["target"]
|
| 7467 |
for via in sheet_mod.related_vias(scope_nodes))
|
| 7468 |
sources = _drill_tokens(session, databases)
|
| 7469 |
-
with _drill_mirror(session, database, window, needed, databases) as (made, token):
|
| 7470 |
cur, spec, cols = made
|
| 7471 |
cols = dict(cols)
|
| 7472 |
# Apply the aggregate's field wall before compiling either the count or the IDs.
|
|
|
|
| 7083 |
|
| 7084 |
|
| 7085 |
@contextlib.contextmanager
|
| 7086 |
+
def _drill_mirror(session, database, window, needed, databases, sources=None):
|
| 7087 |
+
"""A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.
|
| 7088 |
+
|
| 7089 |
+
`sources` is the caller's opening token pass. The lease key reuses it, so one request reads
|
| 7090 |
+
the tokens once before the build. The build-miss and end-of-request rechecks still read anew.
|
| 7091 |
+
"""
|
| 7092 |
from core import sheet_binding, perm_scope
|
| 7093 |
from core.request_timing import raise_if_resource_error
|
| 7094 |
import datetime
|
|
|
|
| 7100 |
# across requests; the request's ExitStack closes it after the page is produced.
|
| 7101 |
yield _mirror(session)(database, metric_window=window, needed=needed), None
|
| 7102 |
return
|
| 7103 |
+
sources = _drill_tokens(session, databases) if sources is None else dict(sources)
|
| 7104 |
token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
|
| 7105 |
# C1 also fingerprints derived wall sources, which can change without the primary table.
|
| 7106 |
# An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
|
|
|
|
| 7470 |
databases.update(_binding_related_entry(session, database, via)["target"]
|
| 7471 |
for via in sheet_mod.related_vias(scope_nodes))
|
| 7472 |
sources = _drill_tokens(session, databases)
|
| 7473 |
+
with _drill_mirror(session, database, window, needed, databases, sources=sources) as (made, token):
|
| 7474 |
cur, spec, cols = made
|
| 7475 |
cols = dict(cols)
|
| 7476 |
# Apply the aggregate's field wall before compiling either the count or the IDs.
|
platform/core/blob_envelope.py
CHANGED
|
@@ -72,7 +72,7 @@ def open(tenant, path, data):
|
|
| 72 |
if key is None:
|
| 73 |
raise BlobEnvelopeError("Opening this blob requires a tenant data key.")
|
| 74 |
try:
|
| 75 |
-
return tenant_keys.aead_open(key, data[len(HEADER):], aad)
|
| 76 |
except Exception:
|
| 77 |
raise BlobEnvelopeError("The blob envelope did not authenticate.") from None
|
| 78 |
|
|
|
|
| 72 |
if key is None:
|
| 73 |
raise BlobEnvelopeError("Opening this blob requires a tenant data key.")
|
| 74 |
try:
|
| 75 |
+
return tenant_keys.aead_open(key, memoryview(data)[len(HEADER):], aad)
|
| 76 |
except Exception:
|
| 77 |
raise BlobEnvelopeError("The blob envelope did not authenticate.") from None
|
| 78 |
|
platform/core/tenant_keys.py
CHANGED
|
@@ -302,11 +302,15 @@ def subkey(tenant, purpose):
|
|
| 302 |
"""HKDF-SHA256 subkey of the tenant data key for one purpose, or None on the legacy path."""
|
| 303 |
if purpose not in PURPOSES:
|
| 304 |
raise ValueError(f"unknown data key purpose {purpose!r}")
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
|
| 311 |
|
| 312 |
def aead_seal(key, plaintext, aad):
|
|
@@ -321,10 +325,17 @@ def aead_open(key, blob, aad):
|
|
| 321 |
"""Open an `aead_seal` blob. Raises on a wrong key, a wrong AAD or a changed byte."""
|
| 322 |
if not isinstance(key, bytes) or len(key) != 32:
|
| 323 |
raise ValueError("an AES-256 key is 32 bytes")
|
| 324 |
-
blob
|
|
|
|
|
|
|
| 325 |
if len(blob) < 28:
|
| 326 |
raise ValueError("the sealed value is too short")
|
| 327 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
|
| 329 |
|
| 330 |
def _active_tenants():
|
|
|
|
| 302 |
"""HKDF-SHA256 subkey of the tenant data key for one purpose, or None on the legacy path."""
|
| 303 |
if purpose not in PURPOSES:
|
| 304 |
raise ValueError(f"unknown data key purpose {purpose!r}")
|
| 305 |
+
with _LOCK:
|
| 306 |
+
key = data_key(tenant)
|
| 307 |
+
if key is None:
|
| 308 |
+
return None
|
| 309 |
+
slot = (context(tenant)["tenant"], _fingerprint(backend_name()), purpose)
|
| 310 |
+
if slot not in _CACHE:
|
| 311 |
+
_CACHE[slot] = HKDF(algorithm=hashes.SHA256(), length=32, salt=None,
|
| 312 |
+
info=f"aios/tenant/{purpose}/v1".encode("ascii")).derive(key)
|
| 313 |
+
return _CACHE[slot]
|
| 314 |
|
| 315 |
|
| 316 |
def aead_seal(key, plaintext, aad):
|
|
|
|
| 325 |
"""Open an `aead_seal` blob. Raises on a wrong key, a wrong AAD or a changed byte."""
|
| 326 |
if not isinstance(key, bytes) or len(key) != 32:
|
| 327 |
raise ValueError("an AES-256 key is 32 bytes")
|
| 328 |
+
if not (isinstance(blob, memoryview) and isinstance(blob.obj, bytes)
|
| 329 |
+
and blob.c_contiguous and blob.format == "B"):
|
| 330 |
+
blob = memoryview(bytes(blob))
|
| 331 |
if len(blob) < 28:
|
| 332 |
raise ValueError("the sealed value is too short")
|
| 333 |
+
cipher, nonce, aad = AESGCM(key), bytes(blob[:12]), bytes(aad)
|
| 334 |
+
try:
|
| 335 |
+
return cipher.decrypt(nonce, blob[12:], aad)
|
| 336 |
+
except TypeError:
|
| 337 |
+
# Older supported cryptography releases require bytes for the ciphertext.
|
| 338 |
+
return cipher.decrypt(nonce, bytes(blob[12:]), aad)
|
| 339 |
|
| 340 |
|
| 341 |
def _active_tenants():
|
platform/harness/datastore.py
CHANGED
|
@@ -165,7 +165,13 @@ def apply_limits(con):
|
|
| 165 |
con.execute(f"SET memory_limit='{memory}'")
|
| 166 |
if threads.isdigit() and int(threads) > 0:
|
| 167 |
con.execute(f"SET threads={int(threads)}")
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
con.execute(f"SET max_temp_directory_size='{spill}'")
|
| 170 |
return con
|
| 171 |
|
|
|
|
| 165 |
con.execute(f"SET memory_limit='{memory}'")
|
| 166 |
if threads.isdigit() and int(threads) > 0:
|
| 167 |
con.execute(f"SET threads={int(threads)}")
|
| 168 |
+
# DuckDB refuses any temp_directory SET, the same path included, once the instance has spilled.
|
| 169 |
+
if con.execute("SELECT current_setting('temp_directory')").fetchone()[0] != str(temp):
|
| 170 |
+
try:
|
| 171 |
+
con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'")
|
| 172 |
+
except duckdb.NotImplementedException as exc:
|
| 173 |
+
if "Cannot switch temporary directory" not in str(exc):
|
| 174 |
+
raise
|
| 175 |
con.execute(f"SET max_temp_directory_size='{spill}'")
|
| 176 |
return con
|
| 177 |
|
platform/harness/mirror_store.py
CHANGED
|
@@ -526,8 +526,10 @@ class MirrorStore:
|
|
| 526 |
return MIRROR_PART_MAX_BYTES
|
| 527 |
|
| 528 |
def _download(self, rt, key):
|
| 529 |
-
|
| 530 |
-
|
|
|
|
|
|
|
| 531 |
|
| 532 |
def _upload(self, rt, key, data):
|
| 533 |
stored = be.seal(rt.key, self.prefix + key, data)
|
|
@@ -584,6 +586,12 @@ class MirrorStore:
|
|
| 584 |
def _heal_published(self, con, result, log):
|
| 585 |
"""Disable summaries across repairs whose source transactions can commit separately."""
|
| 586 |
from harness import summary_tables
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 587 |
con.execute("BEGIN TRANSACTION")
|
| 588 |
try:
|
| 589 |
summary_tables.invalidate(con, self.connector)
|
|
@@ -592,6 +600,8 @@ class MirrorStore:
|
|
| 592 |
con.execute("ROLLBACK")
|
| 593 |
raise
|
| 594 |
self._heal_restored(con, result, log)
|
|
|
|
|
|
|
| 595 |
con.execute("BEGIN TRANSACTION")
|
| 596 |
try:
|
| 597 |
with _restore_phase("summaries"):
|
|
@@ -659,8 +669,9 @@ class MirrorStore:
|
|
| 659 |
def _checked_mirror_blob(self, rt, path, metadata):
|
| 660 |
with _restore_phase("download_open_checksum"):
|
| 661 |
data = self._download(rt, path)
|
| 662 |
-
|
| 663 |
-
|
|
|
|
| 664 |
return data
|
| 665 |
|
| 666 |
|
|
@@ -867,10 +878,11 @@ class MirrorStore:
|
|
| 867 |
"unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
|
| 868 |
from harness import summary_tables
|
| 869 |
with _restore_phase("summaries"):
|
| 870 |
-
if
|
|
|
|
| 871 |
summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
|
| 872 |
else:
|
| 873 |
-
#
|
| 874 |
summary_tables.invalidate(con, self.connector)
|
| 875 |
datastore._write_publication(con, self.connector, manifest)
|
| 876 |
con.execute("COMMIT")
|
|
|
|
| 526 |
return MIRROR_PART_MAX_BYTES
|
| 527 |
|
| 528 |
def _download(self, rt, key):
|
| 529 |
+
with _restore_phase("download"):
|
| 530 |
+
raw = rt.download_bytes(self.prefix + key)
|
| 531 |
+
with _restore_phase("blob_open"):
|
| 532 |
+
return None if raw is None else be.open(rt.key, self.prefix + key, raw)
|
| 533 |
|
| 534 |
def _upload(self, rt, key, data):
|
| 535 |
stored = be.seal(rt.key, self.prefix + key, data)
|
|
|
|
| 586 |
def _heal_published(self, con, result, log):
|
| 587 |
"""Disable summaries across repairs whose source transactions can commit separately."""
|
| 588 |
from harness import summary_tables
|
| 589 |
+
publication = datastore._publication(con, self.connector)
|
| 590 |
+
with _restore_phase("summaries"):
|
| 591 |
+
# A current local catalog proves this publication already completed its source repairs.
|
| 592 |
+
if result.get("state") == "local" and summary_tables.is_current(
|
| 593 |
+
con, self.connector, publication, self._tables(con)):
|
| 594 |
+
return
|
| 595 |
con.execute("BEGIN TRANSACTION")
|
| 596 |
try:
|
| 597 |
summary_tables.invalidate(con, self.connector)
|
|
|
|
| 600 |
con.execute("ROLLBACK")
|
| 601 |
raise
|
| 602 |
self._heal_restored(con, result, log)
|
| 603 |
+
if os.environ.get("AIOS_PROCESS_ROLE") == "worker":
|
| 604 |
+
return
|
| 605 |
con.execute("BEGIN TRANSACTION")
|
| 606 |
try:
|
| 607 |
with _restore_phase("summaries"):
|
|
|
|
| 669 |
def _checked_mirror_blob(self, rt, path, metadata):
|
| 670 |
with _restore_phase("download_open_checksum"):
|
| 671 |
data = self._download(rt, path)
|
| 672 |
+
with _restore_phase("checksum"):
|
| 673 |
+
if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
|
| 674 |
+
raise ValueError(f"{self.label} mirror blob checksum or byte count mismatch")
|
| 675 |
return data
|
| 676 |
|
| 677 |
|
|
|
|
| 878 |
"unnest(?::VARCHAR[])", list(map(list, zip(*rows))))
|
| 879 |
from harness import summary_tables
|
| 880 |
with _restore_phase("summaries"):
|
| 881 |
+
if (os.environ.get("AIOS_PROCESS_ROLE") != "worker"
|
| 882 |
+
and getattr(_RESTORE_PROGRESS, 'summary_owner', None) is not self):
|
| 883 |
summary_tables.rebuild(con, self.connector, manifest, self._tables(con))
|
| 884 |
else:
|
| 885 |
+
# Workers serve no aggregate reads. Other roles rebuild after source repairs.
|
| 886 |
summary_tables.invalidate(con, self.connector)
|
| 887 |
datastore._write_publication(con, self.connector, manifest)
|
| 888 |
con.execute("COMMIT")
|
platform/harness/summary_tables.py
CHANGED
|
@@ -107,6 +107,33 @@ def rebuild(con, source, manifest, tables):
|
|
| 107 |
[source, table, group, tenant, snapshot, name, definition])
|
| 108 |
|
| 109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
def read(con, *, tenant, table, group_column, group_expression, plan, where, params,
|
| 111 |
group_offset, group_limit):
|
| 112 |
"""Read an exact unfiltered grouping from the publication on this request's own cursor."""
|
|
|
|
| 107 |
[source, table, group, tenant, snapshot, name, definition])
|
| 108 |
|
| 109 |
|
| 110 |
+
def is_current(con, source, manifest, tables):
|
| 111 |
+
"""Require every declared summary to match the tenant, snapshot and current SQL definition."""
|
| 112 |
+
tenant, snapshot = manifest.get("tenant"), manifest.get("snapshot_id")
|
| 113 |
+
if not tenant or not snapshot:
|
| 114 |
+
return False
|
| 115 |
+
resident = {row[0] for row in con.execute(
|
| 116 |
+
"SELECT table_name FROM information_schema.tables WHERE table_schema='main' "
|
| 117 |
+
"AND table_type='BASE TABLE'").fetchall()}
|
| 118 |
+
if CATALOG not in resident:
|
| 119 |
+
return False
|
| 120 |
+
expected = {}
|
| 121 |
+
for table in sorted(set(tables) | set(manifest.get("tables") or {})):
|
| 122 |
+
if not connector_spec.mirror_summary_groups(table):
|
| 123 |
+
continue
|
| 124 |
+
if table not in resident:
|
| 125 |
+
return False
|
| 126 |
+
for group, expression, totals in _definitions(con, table):
|
| 127 |
+
name = _name(source, table, group)
|
| 128 |
+
if name not in resident:
|
| 129 |
+
return False
|
| 130 |
+
expected[(table, group)] = (tenant, snapshot, name,
|
| 131 |
+
json.dumps({"group": expression, "totals": totals}, sort_keys=True))
|
| 132 |
+
rows = con.execute(f"SELECT source_table, group_column, tenant, snapshot_id, summary_table, definition "
|
| 133 |
+
f"FROM {CATALOG} WHERE source=?", [source]).fetchall()
|
| 134 |
+
return bool(expected) and {(row[0], row[1]): row[2:] for row in rows} == expected
|
| 135 |
+
|
| 136 |
+
|
| 137 |
def read(con, *, tenant, table, group_column, group_expression, plan, where, params,
|
| 138 |
group_offset, group_limit):
|
| 139 |
"""Read an exact unfiltered grouping from the publication on this request's own cursor."""
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b3436af160188b7d0b91b7654aad92979aab79eb3488ced1a43912bae20410d7
|