diff --git a/RELEASES.json b/RELEASES.json index ce661ddb8a31876998f0f07dbb6942f22dc3bfab..5926a16cb1ada7e019d9fed89372480570906b33 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,5 +1,5 @@ { - "current": "5a226b6c1f70 (5a226b6c1f70)", + "current": "bb3a8714cd79044c8a7f99ddd29afbf5a891cdd3 (bb3a8714cd79)", "releases": [ { "version": "v83", diff --git a/VERSION b/VERSION index b759dc86d96042aa62b6821adddedc7df8630211..0ecae12a4d748ec69b4c7587685e412c88c8f482 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5a226b6c1f70 (5a226b6c1f70) +bb3a8714cd79044c8a7f99ddd29afbf5a891cdd3 (bb3a8714cd79) diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 486dad5302595e53dbccfa506f96b7f254f98a58..3c14f9b6def4601c7b4eb783192efa6c98725079 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -238,8 +238,14 @@ def _format(col, duck_type): return None -def mirror_columns(cur, table): +def mirror_columns(cur, table, *, strict=False): """[(name, type)] for a mirror table, or [] when it has never been synced.""" + if strict: + # A successful empty metadata query proves absence on this cursor only. + return cur.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_catalog = current_database() AND table_schema = current_schema() " + "AND table_name = ? ORDER BY ordinal_position", [table]).fetchall() try: return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()] except Exception: # noqa: BLE001 @@ -369,7 +375,7 @@ def _parent_lookup_sql(cur): return resolved -def lookups(cur, key, names=None): +def lookups(cur, key, names=None, *, allowed=True): """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains. `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name @@ -385,6 +391,23 @@ def lookups(cur, key, names=None): sku = sku if sku in names else None if not asin and not sku: return [] + if not allowed: + # Keep declared cells blank without inspecting a denied lookup dependency. + keys = [] + if asin: + if "parent_asin" not in names: + keys.append((LOOKUP_PARENT, asin, asin_via)) + if not sku: + keys.append((LOOKUP_SKU, asin, asin_via)) + elif sku: + if not names & _NATIVE_ASIN: + keys.append((LOOKUP_ASIN, sku, sku_via)) + if "parent_asin" not in names: + keys.append((LOOKUP_PARENT, sku, sku_via)) + return [{"key": name, "label": _LOOKUP_LABELS[name], "after": after, + "expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None, + "description": _LOOKUP_DESCRIPTIONS[name]} + for name, after, via in keys if name not in names] listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])} parents = None if "parent_asin" in names else _parent_lookup_sql(cur) latest = _latest_listings_sql(listing_names) @@ -529,14 +552,15 @@ def _quoted(name): AMAZON_STORED_BYTES = 4 * 1024 * 1024 -def _read_through(cur, key, count): +def _read_through(cur, key, count, *, columns=None): """Choose storage before reading rows; estimate wide snapshots with a bounded sample.""" ut = _odoo._ut() if key in READ_THROUGH or count > ut.MAX_ROWS: return True if not count: return False - names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])] + names = [name for name, _dtype in (columns if columns is not None + else mirror_columns(cur, SOURCE[key]))] sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall() # Include the stored id, refreshed stamp and row-object punctuation, with the same # ASCII escaping as the document writer. The largest sampled row is conservative. @@ -590,9 +614,9 @@ def _integer(value): return None if value in (None, "") else int(value) -def grid_source(cur, key): +def grid_source(cur, key, *, columns=None, allow_lookups=True): """One binding from the current mirror schema, or an explicit unloaded-mirror reason.""" - columns = mirror_columns(cur, SOURCE[key]) + columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns if not columns: return missing_source(key) names = {name for name, _dtype in columns} @@ -610,11 +634,15 @@ def grid_source(cur, key): complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity) peer = '"_amazon_identity_peer"' - equal = " AND ".join(f"{value(name, peer)} IS NOT DISTINCT FROM {value(name)}" - for name in fallback if name in names) or "TRUE" - # Equal incomplete keys retain source occurrence order, independent of mutable cells. - occurrence = (f"(SELECT CAST(count(*) AS VARCHAR) FROM {source} AS {peer} " - f"WHERE {equal} AND {peer}.rowid < {source}.rowid)") + peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity) + partition = ", ".join(value(name, peer) for name in fallback) + # Rank only incomplete identities; earlier equal peers are exactly row_number minus one. + occurrence = ( + '(SELECT CAST("_identity_occurrence" AS VARCHAR) FROM ' + f'(SELECT {peer}.rowid AS "_identity_rowid", ' + f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 ' + f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) ' + f'AS "_amazon_missing_identity" WHERE "_identity_rowid" = {source}.rowid)') fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), occurrence]) digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} " @@ -641,10 +669,26 @@ def grid_source(cur, key): cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text)) # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a # group (count by parent) and a sort compile against them in SQL with no per-row Python. - for lookup in lookups(cur, key, names): + for lookup in lookups(cur, key, names, allowed=allow_lookups): cols[lookup["key"]] = (lookup["expr"], _text) - return {"table": SOURCE[key], "where": "", "id": ident, + spec = {"table": SOURCE[key], "where": "", "id": ident, "needs_excluded": False, "cols": cols} + if table not in amazon_store.DERIVED_TABLES: + spec["page_key"] = f"{source}.rowid" + return spec + + +def page_source(cur, key, definition=None, *, allow_lookups=True): + """Resolve one target's storage policy on the admitted cursor without provider discovery.""" + columns = mirror_columns(cur, SOURCE[key], strict=True) + mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough")) + if not columns: + return missing_source(key) if mandatory else None + if not mandatory: + count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] + if not _read_through(cur, key, count, columns=columns): + return None + return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups) def grid_sources(cur=None): @@ -766,7 +810,7 @@ def wall_link_expression(source, predicate, identity): f"WHERE {predicate}) AS _wall_links)") -def fill_link_cells(rows, fields, read_targets): +def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refusal=None): """Populate derived Amazon links with one read per distinct target on this page. The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped @@ -775,17 +819,42 @@ def fill_link_cells(rows, fields, read_targets): Neither path stores history rows or caps the matching target set. """ edges = link_edges(fields) - requests = {} + requests, weights = {}, {} for _field, target, frm, on in edges: values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")} requests.setdefault(target, {}).setdefault(on, set()).update(values) + for row in rows: + if row.get(frm) not in (None, ""): + slot = (target, on, str(row[frm])) + weights[slot] = weights.get(slot, 0) + 1 indexes = {} + remaining = max_bytes for target, matches in requests.items(): indexes[target] = {column: {} for column in matches} - for row in read_targets(target, matches) if any(matches.values()) else (): - for column in matches: - if row.get(column) not in (None, ""): - indexes[target][column].setdefault(str(row[column]), set()).add(int(row["pid"])) + iterator = iter(read_targets(target, matches) if any(matches.values()) else ()) + try: + for row in iterator: + pid = int(row["pid"]) + for column, wanted in matches.items(): + value = str(row[column]) if row.get(column) not in (None, "") else None + if value not in wanted: + continue + bucket = indexes[target][column].get(value) + if bucket is not None and pid in bucket: + continue + if remaining is not None: + # Charge every output occurrence before allocating its distinct ID. + cost = (len(str(pid)) + (1 if bucket else 2)) * weights[target, column, value] + if cost > remaining: + raise capacity_refusal() + remaining -= cost + if bucket is None: + bucket = indexes[target][column][value] = set() + bucket.add(pid) + finally: + close = getattr(iterator, "close", None) + if close is not None: + close() for field, target, frm, on in edges: for row in rows: row[field["key"]] = ",".join(str(pid) for pid in sorted( diff --git a/api/application_registry.py b/api/application_registry.py index edf5482ef6d03aaaea4044f2aeb0fb2045ed5bf9..5358e1b3fa962b2efdf2a5f38aea0070f3484345 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -6,6 +6,7 @@ import copy import hashlib import json from core import shares, write_execution +from core.request_timing import raise_if_resource_error import app_rules from modules.finance_apps import (finance_application as _finance_application, @@ -114,10 +115,11 @@ def _brand_analytics_refused(runtime): if path.resolve() == Path(datastore.DB_PATH).resolve(): row = datastore.ro_cursor().execute(query, key).fetchone() else: - with duckdb.connect(str(path), read_only=True) as cursor: + with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor: row = cursor.execute(query, key).fetchone() return bool(row) and row[0] == "refused" - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) return False @@ -133,7 +135,8 @@ def brand_analytics_offered(session): and (enablement.get("databases") or {}).get(BRAND_ANALYTICS_DATABASE) is True) refused = _brand_analytics_refused(session.runtime) if connected and enabled else None return brand_analytics_holds(connected, enabled, refused) - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) return False diff --git a/api/assistant_stream.py b/api/assistant_stream.py index de9f61d85ef9eae052f6125745ebe6faf2fbdf23..a421bdd0ee562743a1533c901d4e933bfc90f1de 100644 --- a/api/assistant_stream.py +++ b/api/assistant_stream.py @@ -1,9 +1,13 @@ """Expose one Query turn as server-sent events with same-turn error recovery.""" import asyncio import contextvars +import gzip +import io import json import threading +import anyio + from fastapi import APIRouter, Body, Depends from starlette.responses import StreamingResponse @@ -16,16 +20,65 @@ router = APIRouter(prefix="/api/v1", dependencies=[Depends(module_gate("analyst" class StreamGZipMiddleware: - """Bypass compression for the streaming chat path only.""" + """Compress response chunks in the bounded anyio pool while passing SSE through.""" def __init__(self, app, **kwargs): - from starlette.middleware.gzip import GZipMiddleware self.app = app - self.compressed = GZipMiddleware(app, **kwargs) + self.minimum_size = kwargs.get("minimum_size", 500) + self.compresslevel = kwargs.get("compresslevel", 9) async def __call__(self, scope, receive, send): - app = self.app if scope.get("path") == PATH else self.compressed - await app(scope, receive, send) + if scope.get("type") != "http" or scope.get("path") == PATH: + return await self.app(scope, receive, send) + from starlette.datastructures import Headers + from starlette.middleware.gzip import IdentityResponder + if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""): + responder = _OffLoopGZipResponder(self.app, self.minimum_size, self.compresslevel) + else: + responder = IdentityResponder(self.app, self.minimum_size) + await responder(scope, receive, send) + + +from starlette.middleware.gzip import GZipResponder, IdentityResponder + + +class _OffLoopGZipResponder(GZipResponder): + """Retain Starlette's header and chunk behavior with compression outside the event loop.""" + + def __init__(self, app, minimum_size, compresslevel): + IdentityResponder.__init__(self, app, minimum_size) + self.compresslevel = compresslevel + self.gzip_file = None + + async def __call__(self, scope, receive, send): + try: + await IdentityResponder.__call__(self, scope, receive, send) + finally: + if self.gzip_file is not None: + def close(): + self.gzip_file.close() + self.gzip_buffer.close() + with anyio.CancelScope(shield=True): + await anyio.to_thread.run_sync(close) + + def _compress(self, body, more): + if self.gzip_file is None: + self.gzip_buffer = io.BytesIO() + self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer, + compresslevel=self.compresslevel) + return GZipResponder.apply_compression(self, body, more_body=more) + + async def send_with_compression(self, message): + if (message["type"] == "http.response.body" + and not self.content_encoding_set and not self.content_type_is_excluded): + body, more = message.get("body", b""), message.get("more_body", False) + if self.started or more or len(body) >= self.minimum_size: + self._compressed_body = await anyio.to_thread.run_sync( + lambda: self._compress(body, more)) + await super().send_with_compression(message) + + def apply_compression(self, body, *, more_body): + return self._compressed_body def _event(name, data): @@ -88,6 +141,12 @@ async def events(body, session, *, submit=None): put("text_delta", {"text": content[len(text):]}) put("done", result) except Exception as exc: + from request_timing import resource_cause + cause = resource_cause(exc) + if cause is not None: + put("error", {"code": "server_busy", "cause": cause, + "message": "The server is busy. Try again shortly.", "retry": False}) + return detail = getattr(exc, "detail", None) error = (detail or {}).get("error") if isinstance(detail, dict) else None put("error", {"code": (error or {}).get("code", "stream_failed"), diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 75c1d35780a268085517d4df42541ce7dc2a04a5..9518f992896635a90b478eb76acc9cd2d38e2e48 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -29,6 +29,8 @@ _PLATFORM = Path(__file__).resolve().parents[2] / "platform" if str(_PLATFORM) not in sys.path: sys.path.insert(0, str(_PLATFORM)) +from core.request_timing import raise_if_resource_error + # ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report # map and the transport; this module adds the probe and the registration and nothing else. from harness.amazon_client import ( # noqa: F401,E402 @@ -189,7 +191,8 @@ def _mirror_counts(cur, az): try: counts[table] = int(cur.execute( f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0]) - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) continue return counts @@ -337,7 +340,8 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): try: purge_plan = az.scheduled_purge(tenant_key, rt, now=now) purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else "" - except Exception: + except Exception as exc: + raise_if_resource_error(exc) purge_plan, purge_message, purge_unknown = None, "", True def finish(status): @@ -407,10 +411,11 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): entities = _sync_entities(_cur, cadence) mirrored = _mirror_counts(_cur, az) else: - with duckdb.connect(str(path), read_only=True) as cur: + with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur: entities = _sync_entities(cur, cadence) mirrored = _mirror_counts(cur, az) - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) # No exception text: paths, provider payloads and credentials never become status copy. return finish(unavailable) if entities is None: @@ -642,7 +647,8 @@ def read_database_enablement(rt): keys = database_keys() try: current = rt.get(ENABLEMENT_KEY) if rt is not None else None - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) return {"available": False, "databases": {key: False for key in keys}} # Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are # the migration default rather than malformed state. diff --git a/api/deps.py b/api/deps.py index 7eaa85d4510a2ec84192dde899f488490955db5c..bc9e9c720cd9b2677ccc75e7233edd1dc7c4ecd5 100644 --- a/api/deps.py +++ b/api/deps.py @@ -16,8 +16,10 @@ import difflib import os import sys import time +import threading from collections import OrderedDict from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from fastapi import Depends, HTTPException, Request, Response @@ -1429,47 +1431,10 @@ def assistant_source_status(session: Session, databases=None): return out -#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every -#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page. -#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28 -#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only -#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres -#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next -#: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still -#: answers 503 when the store is unreadable); a cached registry is never served in its place. -#: The key carries the identity of the reader and the store it reads, so a swapped store or a -#: patched `users.registry` / `store.get` misses instead of replaying another registry. -_SESSION_REGISTRY = {"key": None, "reg": None} -_SESSION_REGISTRY_LOCK = __import__("threading").Lock() - - def _session_registry(): - """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed. - - ⛔ POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`), - and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a - fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the - memo on HF), and its `get()` already holds the bucket for the life of the process, so the - re-read this memo removes does not exist there: it reads straight through. - """ - key = None - try: - backend = store.backend() - token = (store.revision("users") or {}).get("token") if backend == "pg" else None - if token not in (None, ""): - where = os.environ.get("AIOS_TENANT") or "royal-imports" - key = (backend, where, id(users.registry), id(store.get), str(token)) - except Exception: # noqa: BLE001 -- fail-closed below - key = None - if key is not None: - with _SESSION_REGISTRY_LOCK: - if _SESSION_REGISTRY["key"] == key: - return _SESSION_REGISTRY["reg"] - reg = users.registry() or {} - if key is not None: - with _SESSION_REGISTRY_LOCK: - _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg) - return reg + """Read the checked request registry through the store's revision-and-timestamp text cache.""" + # The same request snapshot serves roster hashing and people helpers through users.registry. + return users.registry() or {} def _user_for(claims): @@ -1531,26 +1496,16 @@ def _user_for(claims): return None -# --- wave 19 (R4): the LAST-ACTIVE stamp, throttled in this process ------------------------------ -# "Is anyone actually using this account?" is one of the two questions the Loopable admin plane -# answers, and the only place that can honestly answer it is the session resolver — every -# authenticated request passes through here. -# -# ⛔ THEREFORE IT MUST COST ALMOST NOTHING. A store write per request would put the HF store's -# 256-commits/hour budget on the critical path of the entire API, and a store READ per request to -# decide whether to write would be little better. So the staleness decision is made from a -# PROCESS-LOCAL map: no I/O at all on the 99.97% of requests inside the window, and the write -# itself (once an hour per account) happens on `core.users`' own background thread, so no request -# ever waits for a hub round-trip. `core.users` explains why that write is SYNC and not the -# coalesced async path — `users.json` is shared with the Streamlit host and an async flush rebases -# on this process's cache, which would let a stamp revert somebody else's deactivation. -# -# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each -# write once per window (still bounded); a restart re-arms every user's first request; and the -# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane -# renders it as. Bounded independently so it cannot grow into a leak on a deployment -# with many accounts. +# --- R4 activity: read observations and persisted-write throttling are separate ---------------- +# GET/HEAD/OPTIONS only observe admitted activity in bounded process memory. They never start a +# store stamp, even after restart or window expiry. Admin projections may show these observations; +# they disappear on restart and are not flushed. Passive notification polling remains excluded. +# Login already persists both stamps. Other admitted mutating requests may launch core.users' +# fresh-read background merge once per account/window/process. Reads cannot prime that throttle. +# Both maps evict oldest entries at the bound; eviction or restart re-arms the next mutation. _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict() +_ACTIVE_OBSERVED: "OrderedDict[tuple[str, str], float]" = OrderedDict() +_ACTIVE_LOCK = threading.Lock() _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600) _ACTIVE_MAX_TRACKED = 4096 @@ -1572,30 +1527,59 @@ def _principal(tenant, username): def note_active(uname, when=None): - """Record that `uname` was just seen — WITHOUT writing. The login path calls this because it - has already stamped `last_active` itself; without it the very next request would find no entry - and stamp again immediately, making the "at most once an hour" rule false by one write per - sign-in.""" + """Prime the persisted-write throttle after login's stamp; read observations never call this.""" key = (uname or "").strip().lower() if not key: return - _ACTIVE_SEEN[key] = when if when is not None else time.time() - _ACTIVE_SEEN.move_to_end(key) - while len(_ACTIVE_SEEN) > _ACTIVE_MAX_TRACKED: - _ACTIVE_SEEN.popitem(last=False) + with _ACTIVE_LOCK: + _remember_active(_ACTIVE_SEEN, key, when if when is not None else time.time()) + + +def _remember_active(target, key, when): + """Bound an activity map while its caller holds _ACTIVE_LOCK.""" + target[key] = when + target.move_to_end(key) + while len(target) > _ACTIVE_MAX_TRACKED: + target.popitem(last=False) + + +def _observe_active(tenant, uname): + """Observe admitted activity without I/O or changing the persisted-write throttle.""" + key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower()) + if all(key): + with _ACTIVE_LOCK: + _remember_active(_ACTIVE_OBSERVED, key, time.time()) + + +def observed_last_active(tenant, uname, persisted=None): + """Project the newer persisted or process-local observation for this exact tenant/account.""" + key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower()) + with _ACTIVE_LOCK: + observed = _ACTIVE_OBSERVED.get(key) + if observed is None: + return persisted + if persisted: + try: + saved = datetime.fromisoformat(str(persisted).replace("Z", "+00:00")) + if saved.tzinfo is not None and saved.timestamp() >= observed: + return persisted + except (ValueError, OverflowError): + pass + return datetime.fromtimestamp(observed, timezone.utc).isoformat(timespec="seconds") def _touch_active(uname): - """Stamp `last_active` at most once per window per account, per process. Never raises.""" + """Throttle an admitted mutation's persisted stamp per account/process. Never raises.""" try: key = (uname or "").strip().lower() if not key: return now = time.time() - last = _ACTIVE_SEEN.get(key) - if last is not None and (now - last) < _ACTIVE_EVERY: - return - note_active(key, now) + with _ACTIVE_LOCK: + last = _ACTIVE_SEEN.get(key) + if last is not None and (now - last) < _ACTIVE_EVERY: + return + _remember_active(_ACTIVE_SEEN, key, now) users.touch_active(key) except Exception: # noqa: BLE001 — telemetry may never break a request pass @@ -1640,13 +1624,15 @@ def require_session(request: Request, response: Response) -> Session: # `session.tenant` read back from something the request could influence. response.headers[_TENANT_HEADER] = str(claims["t"]) response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"]) - # R4: last AFTER the session is fully resolved — a stamp is only true of a request that was - # actually admitted, and putting it here means no failed-auth path can ever write one. + # Observe only AFTER authentication admission. Read methods stay store-read-only; only an + # admitted mutation can launch the persisted stamp, without moving it outside request timing. # qa-runner has no global account record. Never let the normal activity telemetry create or # touch one through the default tenant-#0 users store. if (not user.get("qa_runner") and not (request.method == "GET" and request.url.path == "/api/v1/notifications")): - _touch_active(user.get("username")) + _observe_active(claims["t"], user.get("username")) + if request.method not in {"GET", "HEAD", "OPTIONS"}: + _touch_active(user.get("username")) return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt) diff --git a/api/main.py b/api/main.py index 829658e1a434d12b24d98ade601e7d4ffb0c331e..69526a65b1a29749f775cb829472441569194b05 100644 --- a/api/main.py +++ b/api/main.py @@ -140,7 +140,16 @@ _PASSTHROUGH_KEYS = {f["key"] for f in FIELDS if f["source"] == "odoo" and f["type"] in ("text", "status", "select", "date")} -app = FastAPI(title="AIOS web API") +from request_timing import RequestTiming # noqa: E402 + + +class _TimedFastAPI(FastAPI): + def build_middleware_stack(self): + # Include error responses, freeze refusals, admission and compressed body sends. + return RequestTiming(super().build_middleware_stack()) + + +app = _TimedFastAPI(title="AIOS web API") # The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with # the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the @@ -159,6 +168,10 @@ def _error_shape(request: Request, exc: StarletteHTTPException): 422 from a malformed path param) is wrapped here so a client never has to branch on two different error bodies. """ + from request_timing import resource_response + busy = resource_response(exc) + if busy is not None: + return busy detail = exc.detail if isinstance(detail, dict) and "error" in detail: body = detail @@ -183,12 +196,51 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): adapter, which has no durable session dict to degrade into. A 200 over a write that evaporated is the failure the whole rule exists to prevent. """ + from request_timing import resource_response + busy = resource_response(exc) + if busy is not None: + return busy return JSONResponse( {"error": {"code": "store_unavailable", "message": "the tenant store is unavailable. No change was saved"}}, status_code=503) +async def _resource_unavailable(request: Request, exc: Exception): + from request_timing import resource_response + return resource_response(exc) + + +@app.exception_handler(Exception) +async def _unexpected_error(request: Request, exc: Exception): + from request_timing import resource_response + busy = resource_response(exc) + if busy is not None: + return busy + return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}}, + status_code=500) + + +from core.odoo import OdooPoolTimeout +from core.store_pg import SharedReadTimeout +from core.request_timing import RequestDeadlineExceeded +from request_timing import DeadlineCapacityExceeded +from duckdb import OutOfMemoryException +for _resource_error in (OdooPoolTimeout, SharedReadTimeout, RequestDeadlineExceeded, + DeadlineCapacityExceeded, OutOfMemoryException): + app.add_exception_handler(_resource_error, _resource_unavailable) +try: + from psycopg_pool import PoolTimeout, TooManyRequests + from psycopg import OperationalError, InterfaceError + from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout +except ImportError: + pass +else: + for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable, + OperationalError, InterfaceError, IdleInTransactionSessionTimeout): + app.add_exception_handler(_resource_error, _resource_unavailable) + + @app.exception_handler(freeze_notice.DataInputFrozen) def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen): """Use the existing readable error envelope for every maintenance refusal.""" @@ -306,7 +358,7 @@ def public_freeze_notice(): @app.get("/api/health") -def health(challenge: str = ""): +async def health(challenge: str = ""): """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not reveal anything about the deployment beyond "the process is up". No version, no tenant list, no config: a health endpoint is the one URL every scanner finds first. diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 073793aa64cb9660a0c614e5d6bbe1efb9c34667..503f62d114ae56d3e7b141bb86b1ca6dbc1945d6 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -3534,17 +3534,35 @@ def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}' for k in keys) cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})') - values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys) - for r in (rows or []) if isinstance(r, dict)] - if values: - if bulk: - # Column arrays avoid 40,000 Python/DuckDB execute round trips for stored shipments. - selects = ", ".join( - f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys) - cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)]) - else: - placeholders = ", ".join(["?"] * len(keys)) - cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values) + import itertools + source = iter(rows if rows is not None else ()) + try: + while True: + batch = list(itertools.islice(source, 2048)) + if not batch: + break + values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys) + for r in batch if isinstance(r, dict)] + # Serialized values no longer need the dictionaries held by this batch. + batch.clear() + del batch + if not values: + del values + continue + if bulk: + # Every insertion holds at most 2,048 rows, including column-array conversion. + selects = ", ".join( + f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys) + cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)]) + else: + placeholders = ", ".join(["?"] * len(keys)) + cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values) + values.clear() + del values + finally: + close = getattr(source, "close", None) + if callable(close): + close() return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}} diff --git a/api/process_role.py b/api/process_role.py index 32c6bc70fa77bdef570aea2c0774d1164bcdd046..5cb322e5cbea916d2d6a1aa406726ce9c0b540ca 100644 --- a/api/process_role.py +++ b/api/process_role.py @@ -46,10 +46,8 @@ os.environ["AIOS_PROCESS_ROLE"] = ROLE if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web": print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True) if ROLE == "web": - # The web process keeps a smaller DuckDB budget. apply_limits reads this at connect time. - os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "3GB") - if os.environ.get("AIOS_WEB_DUCKDB_THREADS"): - os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ["AIOS_WEB_DUCKDB_THREADS"]) + os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB") + os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2") _LOCK = threading.Lock() _STARTED = {} diff --git a/api/registry_source.py b/api/registry_source.py new file mode 100644 index 0000000000000000000000000000000000000000..19f707a001cbf992383c9c88c91baa45ec652521 --- /dev/null +++ b/api/registry_source.py @@ -0,0 +1,768 @@ +"""Bounded stored-source reads before canonical grid projection.""" +import copy +from contextlib import closing +from dataclasses import is_dataclass, replace + +from deps import err + +BATCH_ROWS = 1024 + + +def with_runtime(session, runtime): + """Retain ordinary lightweight host sessions beside the API dataclass.""" + if is_dataclass(session): + return replace(session, runtime=runtime) + actor = copy.copy(session) + actor.runtime = runtime + return actor + + +def backend(runtime): + """Resolve the admitted runtime's physical store through its tenant guard.""" + while hasattr(runtime, '_runtime'): + runtime = runtime._runtime + document_store = getattr(runtime, '_document_store', None) + if callable(document_store): + return document_store() + reader = runtime._store() if callable(getattr(runtime, '_store', None)) else runtime + from core import store + return store._d() if reader is store else reader + + +class MetadataStore: + """Keep per-record enrichment states outside the definition vocabulary.""" + schema_only = True + + def __init__(self, runtime): + self._runtime = runtime + + def __getattr__(self, name): + return getattr(self._runtime, name) + + def get_projection(self, name, drop=()): + if name == 'user_tables': + drop = tuple(dict.fromkeys([*drop, 'aiEnrich'])) + return self._runtime.get_projection(name, drop=drop) + + +def paths(runtime, name, selected): + """Read explicit values through the paged store door, or a borrowed offline document.""" + if not selected: + return [] + reader = backend(runtime) + method = getattr(reader, 'get_json_paths', None) + if callable(method): + return method(getattr(runtime, 'store_key', str)(name), selected) + doc = runtime.get(name) or {} + out = [] + for path in selected: + value = doc + for part in path: + value = value.get(str(part)) if isinstance(value, dict) else None + out.append(copy.deepcopy(value)) + return out + + +def items(runtime, name, path=(), *, selected=None): + """Yield bounded batches through a streaming store door or the legacy offline reader.""" + reader = backend(runtime) + if selected is not None: + selected = iter(selected) + import itertools + while True: + keys = list(itertools.islice(selected, BATCH_ROWS)) + if not keys: + return + values = paths(runtime, name, [[*path, str(key)] for key in keys]) + yield [(str(key), value) for key, value in zip(keys, values) if value is not None] + else: + stream = getattr(reader, 'iter_json_items', None) + if callable(stream): + try: + yield from stream(getattr(runtime, 'store_key', str)(name), path, batch_size=BATCH_ROWS) + except RuntimeError as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + code = str(exc).split(':', 1)[0] + if code in {'canonical_source_unavailable', 'canonical_source_invalid'}: + raise err(503, code, str(exc).partition(':')[2].strip()) from exc + raise + return + else: + # Offline compatibility retains its existing whole-document read. + import itertools + doc = runtime.get(name) or {} + for part in path: + doc = doc.get(str(part)) if isinstance(doc, dict) else None + if doc is None: + return + if not isinstance(doc, dict): + raise err(409, 'canonical_source_invalid', f'{name} does not contain a row object.') + source = iter(doc.items()) + while True: + batch = list(itertools.islice(source, BATCH_ROWS)) + if not batch: + return + yield batch + + +def array_values(runtime, name, path=(), *, drop=()): + """Stream array entries through the bound store with an ordinary offline fallback.""" + reader = backend(runtime) + method = getattr(reader, 'iter_json_values', None) + if callable(method): + yield from method(getattr(runtime, 'store_key', str)(name), path, drop=drop, batch_size=BATCH_ROWS) + return + value = runtime.get(name) or {} + for part in path: + value = value[int(part)] if isinstance(value, list) else value.get(str(part), {}) + import itertools + source = enumerate(value if isinstance(value, list) else []) + while True: + batch = list(itertools.islice(source, BATCH_ROWS)) + if not batch: + return + yield [(str(index), {k: v for k, v in entry.items() if k not in drop} + if isinstance(entry, dict) else entry) for index, entry in batch] + + +def object_values(runtime, name, path=(), *, drop=(), omit=()): + """Read nested definitions with record operands excluded at the backing store.""" + reader = backend(runtime) + method = getattr(reader, 'get_json_object', None) + if callable(method): + return method(getattr(runtime, 'store_key', str)(name), path, drop=drop, omit=omit) + value = runtime.get(name) or {} + for part in path: + value = value.get(str(part), {}) if isinstance(value, dict) else {} + return {key: ({k: v for k, v in entry.items() if k not in drop} if isinstance(entry, dict) else entry) + for key, entry in value.items() if key not in omit} if isinstance(value, dict) else {} + + +def members(runtime, name, path, ids, *, id_key=None, source_limit=None): + """Read complete selected membership through the bound store's SQL predicate.""" + reader = backend(runtime) + method = getattr(reader, 'get_json_members', None) + if callable(method): + return method(getattr(runtime, 'store_key', str)(name), path, ids, + id_key=id_key, source_limit=source_limit) + value = runtime.get(name) or {} + for part in path: + value = value[int(part)] if isinstance(value, list) else value.get(str(part), {}) + selected, seen, out = {int(pid) for pid in ids}, set(), [] + for index, item in enumerate(value if isinstance(value, list) else []): + if source_limit is not None and index >= source_limit: + break + raw = item.get(id_key) if id_key is not None and isinstance(item, dict) else item + try: + pid = int(raw) + except (TypeError, ValueError): + continue + if pid in selected and pid not in seen: + out.append(item) + seen.add(pid) + return out + + +def generation(session, table_key): + """Check population and projection buckets without reading their record values.""" + from core import schema_overlay, shares, store_pg, user_tables + runtime, reader = session.runtime, backend(session.runtime) + names = [user_tables.STORE_KEY, table_key + '_table_workspace', table_key + '__shared', + table_key + '_docs', user_tables.ZONE_STORE_KEY, schema_overlay.STORE_KEY, + shares.SHARES_KEY, table_key + '_cohorts'] + if table_key in {'customer_data', 'product_data'}: + workspace = 'customer_table_workspace' if table_key == 'customer_data' else 'product_table_workspace' + names += [workspace, workspace + '__shared', 'customer_ledger', 'ar_days_to_pay', + 'geocoords', 'loopable_route_system', 'customer_docs', 'product_docs', + 'customer_cohorts', 'product_cohorts'] + if isinstance(reader, store_pg.PgStore): + key = getattr(runtime, 'store_key', str) + # A post-read check must cross the backing validation boundary, including updated_at. + tokens = store_pg._read_revisions([key(name) for name in names], reader.tenant_slug) + else: + revision = getattr(runtime, 'revision', None) + if not callable(revision): + return None # Ordinary offline readers retain their original snapshot semantics. + tokens = {name: revision(name) for name in names} + return copy.deepcopy((session.user, tokens)) + + +def require_generation(session, table_key, before): + if before is not None and generation(session, table_key) != before: + raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.') + + +class BatchStore: + """Lend canonical readers one row batch and its overlays, with every write refused.""" + + def __init__(self, runtime, table_key, definition, rows, *, metadata=None, uname=None, needed=None): + self._runtime, self.table_key = runtime, table_key + self.definition = dict(definition, rows=dict(rows)) + self.row_ids = [str(key) for key, _ in rows] + self.docs = {} + self.metadata = metadata if metadata is not None else {} + self.uname = uname + self._canonical_needed = needed + self.measure_memo, self.mset_memo, self.pool_cache = {}, {}, {} + from core import user_tables + for field in definition.get('fields') or []: + if field.get('type') != 'ai_enrich': + continue + col = str(field['key']) + if needed is not None and col not in needed: + continue + values = paths(runtime, user_tables.STORE_KEY, + [[table_key, user_tables.AI_ENRICH_MARK_KEY, col, pid] for pid in self.row_ids]) + self.definition.setdefault(user_tables.AI_ENRICH_MARK_KEY, {})[col] = { + pid: mark for pid, mark in zip(self.row_ids, values) if isinstance(mark, dict)} + + def __getattr__(self, name): + return getattr(self._runtime, name) + + def selected_values(self, name, path, ids): + ids = [str(pid) for pid in ids] + return {pid: value for pid, value in zip(ids, paths(self._runtime, name, + [[*path, pid] for pid in ids])) if value is not None} + + def update(self, *args, **kwargs): + raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.') + + put = upload_bytes = delete_path = update + + def _workspace(self, name): + reader = backend(self._runtime) + project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None) + if name not in self.metadata: + self.metadata[name] = (project(getattr(self._runtime, 'store_key', str)(name)) + if callable(project) else self._runtime.get_projection(name, drop=('overlays',))) or {} + meta = copy.deepcopy(self.metadata[name]) + reader = backend(self._runtime) + stream = getattr(reader, 'iter_view_members', None) or getattr(reader, '_iter_view_members', None) + selected = set(map(int, self.row_ids)) + def invalid_membership(): + return err(503, 'canonical_view_membership_invalid', + 'A saved view contains membership entries but no valid record IDs. ' + 'Repair its membership before reading this database.') + for owner, ws in meta.items(): + if not isinstance(ws, dict) or owner.startswith('_'): + continue + for vid, view in (ws.get('views') or {}).items(): + cfg = view.get('config') if isinstance(view, dict) else None + if not isinstance(cfg, dict): + continue + if 'memberPids' in cfg: + authored = cfg.get('memberPids') or () + if authored: + for raw in authored: + try: + int(raw) + except (TypeError, ValueError): + continue + break + else: + raise invalid_membership() + continue + if not callable(stream): + if callable(project): + raise err(503, 'canonical_source_unavailable', 'Saved view membership cannot be read safely.') + continue # An ordinary offline document did not strip memberships. + first = [None] + saw_any = [False] + def values(): + parts = stream(getattr(self._runtime, 'store_key', str)(name), + [owner, 'views', vid, 'config', 'memberPids'], + getattr(self.metadata[name], 'source_revision', None), batch_size=BATCH_ROWS) + try: + with closing(parts): + for batch in parts: + for _, raw in batch: + saw_any[0] = True + try: + pid = int(raw) + except (TypeError, ValueError): + continue + if first[0] is None: + first[0] = pid + yield pid + except RuntimeError as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc + cur = getattr(self, '_canonical_source_cursor', None) + if cur is not None and selected: + from routes_odoo_tables import _membership_relation + cache_key = ('view_members', name, owner, vid) + if cache_key not in self.metadata: + operand = _membership_relation(cur, values()) + self.metadata[cache_key] = (operand, first[0], saw_any[0]) + operand, first[0], saw_any[0] = self.metadata[cache_key] + if saw_any[0] and first[0] is None: + raise invalid_membership() + kept = [int(row[0]) for row in cur.execute( + f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) ' + f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid', + [*operand.params, *selected]).fetchall()] + else: + # Metadata consumes no row memberships; retain only the curated witness. + with closing(values()) as source: + next(source, None) + kept = [] + if saw_any[0] and first[0] is None: + # Config sanitization drops invalid entries; [] would mean all rows. + # Refuse instead of inventing an ID to stand for empty curated intent. + raise invalid_membership() + # Retain an actual authored witness when this batch has no member. An empty + # list would turn a curated view into all rows. Full SQL view resolution uses + # the original revision-bound backend, never this batch-local projection. + cfg['memberPids'] = kept if kept or first[0] is None else [first[0]] + for owner, ws in meta.items(): + if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname: + continue + values = paths(self._runtime, name, [[owner, 'overlays', pid] for pid in self.row_ids]) + # Only this owner's selected cells have been read, so admit a new bounded document. + meta[owner] = dict(ws, overlays={pid: value for pid, value in zip(self.row_ids, values) + if isinstance(value, dict)}) + return meta + + def _definitions(self): + from core import user_tables + if user_tables.STORE_KEY not in self.metadata: + self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection( + user_tables.STORE_KEY, drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {} + return {**self.metadata[user_tables.STORE_KEY], self.table_key: self.definition} + + def get(self, name, fresh=False): + from core import user_tables + if name == user_tables.STORE_KEY: + return self._definitions() + if name not in self.docs or fresh: + if name.endswith('_table_workspace'): + value = self._workspace(name) + elif name.endswith('__shared'): + selected = paths(self._runtime, name, [['fields'], + *[['cells', pid] for pid in self.row_ids]]) + fields, values = selected[0] or {}, selected[1:] + value = {'fields': fields, 'cells': {pid: cell for pid, cell in zip(self.row_ids, values) + if isinstance(cell, dict)}} + elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}: + values = (paths(self._runtime, name, [[pid] for pid in self.row_ids]) + if self._canonical_needed is None else [None] * len(self.row_ids)) + value = {pid: docs for pid, docs in zip(self.row_ids, values) if isinstance(docs, list)} + else: + if name not in self.metadata or fresh: + self.metadata[name] = self._runtime.get(name, fresh=fresh) if fresh else self._runtime.get(name) + value = self.metadata[name] + self.docs[name] = value + return copy.deepcopy(self.docs[name]) + + def _read_strict(self, name): + return self.get(name) + + def get_entry(self, name, entry): + from core import user_tables + if name == user_tables.STORE_KEY: + return self.definition if str(entry) == self.table_key else self._runtime.get_entry(name, entry) + return (self.get(name) or {}).get(str(entry)) + + def get_projection(self, name, drop=()): + if name == 'user_tables': + from core import store + return store._project(self._definitions(), frozenset(drop)) + return self._runtime.get_projection(name, drop=drop) + + +class CanonicalStore(BatchStore): + """Lend the existing customer/product assemblies one canonical source batch.""" + schema_only = False + _canonical_read_only = True + + def __init__(self, session, table_key, rows, *, cur, metadata, needed): + super().__init__(session.runtime, table_key, {'fields': []}, + [(str(row['pid']), {}) for row in rows], metadata=metadata, + uname=session.uname, needed=needed) + self._canonical_source_cursor = cur + self._canonical_rows = rows if table_key == 'customer_data' else None + self._canonical_product_rows = rows if table_key == 'product_data' else None + from core import perm_scope + team, agent = perm_scope.derive_pool_scope(session.user, table_key) + import time + key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent) + self.pool_cache[key] = (time.time(), rows) + + def get_projection(self, name, drop=()): + if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'): + return self.get(name) + if name.endswith('_table_workspace') or name.endswith('__shared'): + from core import store + return store._project(self.get(name), frozenset(drop)) + return super().get_projection(name, drop=drop) + + def get(self, name, fresh=False): + if name in self.docs and not fresh: + return copy.deepcopy(self.docs[name]) + if name.endswith('_cohorts'): + from modules import cohort + definitions = object_values(self._runtime, name, [self.uname], drop=('members',)) + value = {self.uname: {key: dict(meta, members=members(self._runtime, name, + [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2)) + for key, meta in definitions.items() if isinstance(meta, dict)}} + elif name == 'loopable_route_system': + value = object_values(self._runtime, name, omit=('routes', 'zones')) + value['routes'], value['zones'] = [], [] + for batch in array_values(self._runtime, name, ['routes'], drop=('stops',)): + for index, route in batch: + if isinstance(route, dict): + route = dict(route, stops=members(self._runtime, name, ['routes', index, 'stops'], + self.row_ids, id_key='customerPid')) + value['routes'].append(route) + for batch in array_values(self._runtime, name, ['zones'], drop=( + 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')): + value['zones'].extend(meta for _, meta in batch) + elif name == 'table_zones': + value = {self.table_key: []} + for batch in array_values(self._runtime, name, [self.table_key], drop=( + 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')): + value[self.table_key].extend(meta for _, meta in batch) + else: + return super().get(name, fresh=fresh) + self.docs[name] = value + return copy.deepcopy(value) + + def _definitions(self): + from core import user_tables + if user_tables.STORE_KEY not in self.metadata: + self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection(user_tables.STORE_KEY, + drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {} + return self.metadata[user_tables.STORE_KEY] + + def get_entry(self, name, entry): + if name == 'user_tables': + return self._definitions().get(str(entry)) + return super().get_entry(name, entry) + + +def canonical_metadata(session, table_key): + """Read the canonical permitted definitions without constructing source rows.""" + import routes_customers as customers + import routes_products as products + session.require(table_key) + lent = CanonicalStore(session, table_key, [], cur=None, metadata={}, needed=frozenset()) + actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True)) + build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly + return build(actor, scope='product' if table_key == 'product_data' else 'customer', + consume_corrections=False, with_rows=False, needed=frozenset()) + + +def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None): + """Refuse a failed compiled source by name without publishing an empty population.""" + from fastapi import HTTPException + from core.request_timing import raise_if_resource_error + try: + with closing(_canonical_batches(session, cur, table_key, ids=ids, needed=needed, + metric_window=metric_window, related=related)) as batches: + yield from batches + except HTTPException: + raise + except Exception as exc: + raise_if_resource_error(exc) + raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc + + +def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None): + """Run canonical family, permission, overlay and measure projections over bounded identities.""" + import aios_grid + import routes_customers as customers + import routes_products as products + import routes_sheet as sheet + from core import perm_scope, user_tables, route_system + from modules import customer_data, product_data + if getattr(session.runtime, 'key', None) != 'royal-imports': + raise err(503, 'canonical_source_unavailable', f'{table_key} has no compiled source for this tenant.') + session.require(table_key) + team, agent = perm_scope.derive_pool_scope(session.user, table_key) + before = generation(session, table_key) + metadata = {} + definition = canonical_metadata(session, table_key) + # Every permanent wall operand is populated before apply_row_scope judges the batch. + tree = perm_scope.entry(session.user, table_key) or {} + wall = tree.get('filter') or {} + def keys(node): + if isinstance(node, dict): + if node.get('colId'): + yield str(node['colId']) + for group in ('nodes', 'children', 'filters'): + for child in node.get(group) or []: + yield from keys(child) + required = None if needed is None else set(needed) | set(keys(wall)) + if required is not None: + from core.if_field import references + vocabulary = {f['key']: f for f in [*definition.get('fields', []), + *(definition.get('_permission_scope') or {}).get('fields', [])] if isinstance(f, dict) and f.get('key')} + changed = True + while changed: + previous = set(required) + for key in list(required): + field = vocabulary.get(key) or {} + required.update(match.strip() for match in aios_grid._FORMULA_REF.findall(str(field.get('formula') or ''))) + if field.get('type') == 'if': + required.update(references(field.get('if') or {})) + if field.get('type') == 'created_time': + required.add('_created') + changed = previous != required + # Zone and Route projections consume location and identity operands before display projection. + if required is not None and {'zone', 'routes'} & required: + required.update({'lat', 'lon'}) + runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required) + all_keys = {f['key'] for f in definition['fields']} if required is None else required + raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else [] + if required is not None: + raw = [field for field in raw if field.get('key') in required] + import routes_keychain + import time + paused = routes_keychain.odoo_paused(session.runtime) + pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent) + cached = session.runtime.pool_cache.get(pool_key) + if cached and (paused or time.time() - cached[0] < 900): + import itertools + def borrowed(): + source = (r for r in cached[1] if ids is None or r['pid'] in ids) + while True: + batch = list(itertools.islice(source, BATCH_ROWS)) + if not batch: + return + yield batch + batches = borrowed() + elif paused: + import itertools + key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent) + hit = session.runtime.pool_cache.get(key) + if hit: + def frozen(): + source = (r for r in hit[1] if ids is None or r['pid'] in ids) + while True: + batch = list(itertools.islice(source, BATCH_ROWS)) + if not batch: + return + yield batch + batches = frozen() + elif table_key == 'customer_data': + path = ['odoo_pool', routes_keychain._snap_scope_key(team, agent)] + stamp = paths(session.runtime, routes_keychain.SNAPSHOT_KEY, [[*path, 'ts']])[0] + if stamp is None: + raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.') + def frozen(): + for batch in array_values(session.runtime, routes_keychain.SNAPSHOT_KEY, [*path, 'rows']): + yield [r for _, r in batch if isinstance(r, dict) and (ids is None or r.get('pid') in ids)] + batches = frozen() + else: + raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.') + elif table_key == 'product_data': + batches = product_data.source_batches(cur, st=runtime, team_id=team, ids=ids, needed=required, + raw_fields=[field['rawName'] for field in raw]) + else: + batches = customer_data.source_batches(cur, st=runtime, agent_name=agent, team_id=team, + ids=ids, needed=required, ledger_batches=items(session.runtime, customer_data.LEDGER_KEY, + selected=ids) if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ()) + with closing(batches): + for rows in batches: + runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required) + actor = with_runtime(session, runtime) + build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly + g = build(actor, scope='product' if table_key == 'product_data' else 'customer', + consume_corrections=False, with_rows=True, needed=all_keys) + kept = g['fields'] if needed is None else [f for f in g['fields'] if f['key'] in needed] + derived, bounded, limits = sheet._metric_derived(actor, table_key, + dict(g, fields=kept), metric_window) + if table_key == 'product_data': + projected = products.product_rows(dict(g, fields=kept, derived=derived)) if kept else [ + {'pid': r['pid']} for r in g['rows_src']] + projected = user_tables.zone_cells(table_key, projected, allowed_row_ids=g['pids'], st=runtime) + else: + projected = aios_grid.rows_from_pool(g['rows_src'], kept, g['ws'].get('overlays'), derived=derived) \ + if kept else [{'pid': r['pid']} for r in g['rows_src']] + projected = route_system.apply_customer_projections(projected, g['routes'], g['zones']) + if related: + from routes_odoo_tables import narrow_related_rows + projected = narrow_related_rows(actor, table_key, projected, related) + from routes_tables import scope_linked_metrics + projected, _fields, _refusals = scope_linked_metrics(actor, table_key, projected, + kept, g['measures'], related) + from core import grid_events + docs = grid_events.docs_for(g['pids'], scope_key='customer', uname=session.uname, + admin=session.admin, st=runtime) if table_key == 'customer_data' and ids is not None and needed is None else {} + yield {'rows': projected, 'fields': g['fields'], 'today': g['today'], 'docs': docs, + 'bounded': bounded, 'unbounded': limits, + 'limits': list(g['workspace'].get('limits') or [])} + projected.clear() + runtime.pool_cache.clear() + runtime.measure_memo.clear() + require_generation(session, table_key, before) + + +def canonical_grid(session, cur, table_key, *, needed=None, metric_window=None): + """Materialize only required canonical source and wall columns into the isolated cursor.""" + import odoo_relational as rel + import routes_odoo_tables as rot + from modules import sheet + g = canonical_metadata(session, table_key) + fields = g['fields'] + agent_field = {'key': 'agent_id', 'type': 'int', 'label': 'Current agent ID'} + join_agent = False + if table_key == 'customer_data' and (needed is None or 'agent_id' in needed): + from core import perm_scope + hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime) + if not {'agent', 'agent_id'} & set(hidden): + if 'agent_id' in rel.columns(cur, 'res_partner'): + fields = [f for f in fields if f['key'] != 'agent_id'] + [agent_field] + join_agent = True + elif needed is not None and 'agent_id' in needed: + raise err(409, 'related_invalid', 'customer_data.agent_id is unavailable in the partner mirror.') + kept = fields if needed is None else [f for f in fields if f['key'] in needed] + unbounded, bounded = {}, set() + def rows(): + seen = False + empty_limits = {} + with closing(canonical_batches(session, cur, table_key, needed=needed, metric_window=metric_window)) as batches: + for batch in batches: + if batch['rows']: + seen = True + unbounded.update(batch['unbounded']) + bounded.update(batch['bounded']) + else: + empty_limits.update(batch['unbounded']) + yield from batch['rows'] + batch['rows'].clear() + if not seen: + unbounded.update(empty_limits) + typed = sheet.mirror_fields(kept) + import hashlib + import json + slot = hashlib.sha1(json.dumps([metric_window], + sort_keys=True, default=str).encode()).hexdigest()[:12] + spec = rel.registry_grid_spec(cur, f'{table_key}__at_{slot}', typed, rows(), bulk=True) + if join_agent: + cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id=partner.agent_id FROM res_partner partner WHERE grid.pid=partner.id') + scope_fields = (g.get('_permission_scope') or {}).get('fields', fields) + return cur, dict(spec, fields=fields, scope_applied=True, + scope_fields=tuple(f['key'] for f in scope_fields), + bounded=tuple(bounded), unbounded=unbounded), rot._filter_columns(spec, typed) + + +def canonical_page(session, cur, table_key, ids, *, related=None): + """Hydrate a selected page through canonical projections without constructing a full payload.""" + ids = list(dict.fromkeys(int(pid) for pid in ids)) + found, docs, payload = {}, {}, None + with closing(canonical_batches(session, cur, table_key, ids=ids, related=related)) as batches: + for batch in batches: + payload = {key: value for key, value in batch.items() + if key not in {'rows', 'docs', 'bounded', 'unbounded'}} + found.update((int(row['pid']), row) for row in batch['rows']) + docs.update(batch.get('docs') or {}) + if payload is None: + g = canonical_metadata(session, table_key) + payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])} + return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs) + + +def stored_batches(session, table_key, *, ids=None, related=None, needed=None, cur=None): + """Project bounded stored batches through the table's existing permission and overlay reader.""" + import routes_tables as tables + from core import user_tables + metadata_actor = with_runtime(session, MetadataStore(session.runtime)) + definition = tables._defn_or_refuse(metadata_actor, table_key, defs_only=True, scope_applied=True) + if definition.get('readThrough'): + raise err(409, 'canonical_source_unavailable', f'{table_key} requires its declared SQL source.') + metadata = {} + with closing(items(session.runtime, user_tables.STORE_KEY, [table_key, 'rows'], selected=ids)) as batches: + for rows in batches: + runtime = BatchStore(session.runtime, table_key, definition, rows, + metadata=metadata, uname=session.uname, needed=needed) + runtime._canonical_source_cursor = cur + actor = with_runtime(session, runtime) + # table_rows remains the canonical evaluator. Its document contains this batch only. + payload = tables.table_rows(table_key, session=actor, related=related) + if needed is not None: + payload = dict(payload, rows=[{k: v for k, v in row.items() if k == 'pid' or k in needed} + for row in payload['rows']]) + yield payload + + +def metadata_limits(session, table_key, metadata): + """Retain the editable capacity receipt without asking for a population document.""" + from core import user_tables + limits = list((metadata.get('workspace') or {}).get('limits') or []) + report = user_tables.limit_report(table_key, st=user_tables.lend( + session.runtime, user_tables={table_key: metadata['defn']})) + if report and report not in limits: + limits.append(report) + return limits + + +def stored_grid(session, cur, table_key, *, needed=None): + """Build an exact SQL population through bounded canonical table projections.""" + import odoo_relational as rel + import routes_odoo_tables as rot + import routes_tables as tables + from modules import sheet + + before = generation(session, table_key) + metadata_actor = with_runtime(session, MetadataStore(session.runtime)) + metadata = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True, + consume_corrections=False) + fields = metadata['fields'] + kept = fields if needed is None else [f for f in fields if f['key'] in needed] + + def rows(): + with closing(stored_batches(session, table_key, needed=needed)) as batches: + for payload in batches: + yield from payload['rows'] + payload['rows'].clear() + + spec_fields = sheet.mirror_fields(kept) + spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows(), bulk=True) + require_generation(session, table_key, before) + spec = dict(spec, fields=fields, scope_applied=True, + limits=metadata_limits(session, table_key, metadata)) + return cur, spec, rot._filter_columns(spec, spec_fields) + + +def stored_page(session, table_key, ids, *, related=None): + """Hydrate only selected identities through the canonical stored-row projection.""" + before = generation(session, table_key) + ids = list(dict.fromkeys(int(pid) for pid in ids)) + payload, found, docs, zones = None, {}, {}, {} + with closing(stored_batches(session, table_key, ids=ids, related=related)) as batches: + for batch in batches: + payload = {k: v for k, v in batch.items() if k != 'rows'} + found.update((int(row['pid']), row) for row in batch['rows']) + docs.update(batch.get('docs') or {}) + for zone in batch.get('zones') or []: + key = str(zone.get('id') or zone.get('name')) + if key not in zones: + zones[key] = dict(zone, memberships=[]) + zones[key]['memberships'].extend(zone.get('memberships') or []) + if payload is None: + import routes_tables as tables + metadata_actor = with_runtime(session, MetadataStore(session.runtime)) + g = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True, + consume_corrections=False) + payload = {'fields': g['fields'], 'today': g['today'], + 'limits': metadata_limits(session, table_key, g)} + rows = [found[pid] for pid in ids if pid in found] + require_generation(session, table_key, before) + return dict(payload, rows=rows, docs=docs, zones=list(zones.values()), + scope={**payload.get('scope', {}), 'rowCount': len(rows)}) + + +def page_cells(runtime, table_key, pids): + """Read a page's shared cells through the canonical parser and coherent row-bounded snapshots.""" + from core import shared_overlay + import itertools + source, out = iter(pids), {} + while True: + batch = list(itertools.islice(source, BATCH_ROWS)) + if not batch: + return out + reader = BatchStore(runtime, table_key, {'fields': []}, [(str(pid), {}) for pid in batch]) + out.update(shared_overlay.cells(table_key, batch, st=reader)) diff --git a/api/request_class.py b/api/request_class.py index f7479d003ba14bdb9e22757e724628db3951a5ac..084377b25e21283e96345511b1b9a9f793cd6393 100644 --- a/api/request_class.py +++ b/api/request_class.py @@ -10,7 +10,9 @@ import asyncio import json import os import re +import sys import threading +import time from urllib.parse import parse_qs BUSY_CODE = "server_busy" @@ -120,65 +122,6 @@ class _Counters: _COUNTERS = _Counters() -class _StackSampler: - """Opt-in (AIOS_HEAVY_PROFILE=1): samples app frames while heavy requests run, logs the hot ones.""" - - INTERVAL = 0.2 - - def __init__(self): - self.lock = threading.Lock() - self.active = 0 - self.counts = {} - self.thread = None - - def begin(self): - with self.lock: - self.active += 1 - if self.active == 1: - self.counts = {} - if self.thread is None or not self.thread.is_alive(): - import process_role - self.thread = process_role.thread(target=self._run, name="heavy-profile", kind="light") - self.thread.start() - - def end(self, label, seconds): - with self.lock: - self.active -= 1 - counts = dict(self.counts) - if seconds < 3 or not counts: - return - top = sorted(counts.items(), key=lambda kv: -kv[1])[:18] - print(f"[heavy-profile] {label} {seconds:.1f}s " + " | ".join( - f"{name} {n * self.INTERVAL:.1f}s" for name, n in top), flush=True) - - def _run(self): - import sys - import time - me = threading.get_ident() - while True: - time.sleep(self.INTERVAL) - with self.lock: - if self.active <= 0: - self.thread = None - return - for tid, frame in sys._current_frames().items(): - if tid == me: - continue - seen = set() - while frame is not None: - code = frame.f_code - path = code.co_filename.replace("\\", "/") - if ("/api/" in path or "/platform/" in path) and "site-packages" not in path: - name = f"{code.co_name}@{path.rsplit('/', 1)[-1]}" - if name not in seen: - seen.add(name) - self.counts[name] = self.counts.get(name, 0) + 1 - frame = frame.f_back - - -_SAMPLER = _StackSampler() if os.environ.get("AIOS_HEAVY_PROFILE") == "1" else None - - def status(): """Snapshot of heavy request admission for a settings or health surface.""" c = _COUNTERS @@ -204,8 +147,8 @@ def _env_float(name, default): return default -async def _send_busy(send): - body = json.dumps({"error": {"code": BUSY_CODE, "message": BUSY_MESSAGE}}).encode() +async def _send_busy(send, cause="heavy_queue_timeout"): + body = json.dumps({"error": {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}}).encode() await send({"type": "http.response.start", "status": 503, "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(body)).encode()), @@ -253,22 +196,53 @@ class HeavyGate: await self._run_heavy(scope, body_msgs, receive, send) async def _run_heavy(self, scope, body_msgs, receive, send): + queued_at = time.perf_counter() sem = self._semaphore() c = self.counters c.add("queued", 1) + timing = sys.modules.get("core.request_timing") + budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError) acquire = asyncio.ensure_future(sem.acquire()) gone = asyncio.ensure_future(_wait_disconnect(receive)) try: - done, _ = await asyncio.wait({acquire, gone}, timeout=self.queue_seconds, + timeout = timing.remaining_budget(self.queue_seconds) if timing else self.queue_seconds + done, _ = await asyncio.wait({acquire, gone}, timeout=timeout, return_when=asyncio.FIRST_COMPLETED) + if timing: + timing.remaining_budget() + except budget_error: + if acquire.done() and not acquire.cancelled() and acquire.exception() is None: + sem.release() + c.add("busy_refusals", 1) + await _send_busy(send, "request_deadline") + return + except BaseException: + # Cancellation can race a successful acquire before the handler owns the permit. + if acquire.done() and not acquire.cancelled() and acquire.exception() is None: + sem.release() + raise finally: + # The classifier can also run alone, without the app or an active timing scope. + timing = sys.modules.get("core.request_timing") + if timing is not None: + timing.record_queue((time.perf_counter() - queued_at) * 1000) c.add("queued", -1) if not acquire.done(): acquire.cancel() if not gone.done(): gone.cancel() + await asyncio.gather(acquire, gone, return_exceptions=True) # A permit can land in the same tick as a timeout or disconnect. Hold it only if used. got = acquire.done() and not acquire.cancelled() and acquire.exception() is None + try: + if timing: + timing.remaining_budget() + except budget_error: + if got: + sem.release() + c.add("busy_refusals", 1) + await _send_busy(send, "request_deadline") + return if acquire not in done or gone in done: if got: sem.release() @@ -292,17 +266,10 @@ class HeavyGate: c.add("in_flight", 1) c.add("admitted", 1) - started = asyncio.get_running_loop().time() - if _SAMPLER is not None: - _SAMPLER.begin() try: await self.app(scope, _replay(body_msgs, receive), send_wrapped) finally: release() - if _SAMPLER is not None: - q = scope.get("query_string", b"").decode("latin-1")[:60] - _SAMPLER.end(f"{scope.get('method')} {scope.get('path')}?{q}", - asyncio.get_running_loop().time() - started) async def _read_body(receive, limit): diff --git a/api/request_timing.py b/api/request_timing.py new file mode 100644 index 0000000000000000000000000000000000000000..a99e4c507e6e93c0a9cc9cc03c550693820bae9c --- /dev/null +++ b/api/request_timing.py @@ -0,0 +1,421 @@ +"""Outermost ASGI timing, bounded route statistics and request-associated stack sampling.""" +from __future__ import annotations + +import asyncio +import contextvars +import json +import logging +import math +import os +import sys +import threading +import time +from collections import OrderedDict, deque + +from core import request_timing as timing + +LOG = logging.getLogger("aios.request_timing") +_STATS_LOCK = threading.Lock() +_STATS = OrderedDict() +_ROUTE_LIMIT = 512 + + +def request_timeout_s(): + try: + value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", "45")) + return max(.01, value) if math.isfinite(value) else 45.0 + except ValueError: + return 45.0 + + +_BUSY_CAUSES = { + "PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full", + "QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout", + "SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout", + "RequestDeadlineExceeded": "request_deadline", "DeadlineCapacityExceeded": "deadline_capacity", + "OperationalError": "postgres_connection_unavailable", "InterfaceError": "postgres_connection_unavailable", + "ConnectionTimeout": "postgres_connection_timeout", + "IdleInTransactionSessionTimeout": "postgres_idle_transaction_timeout", + "OutOfMemoryException": "duckdb_memory_limit", + "PostgresOperationTimeout": "postgres_operation_timeout", + "OdooRequestTimeout": "odoo_rpc_timeout", +} + + +def resource_cause(error): + """Find a resource refusal through route wrappers without exposing database messages.""" + seen = set() + while error is not None and id(error) not in seen: + seen.add(id(error)) + cause = _BUSY_CAUSES.get(type(error).__name__) + if cause: + return cause + error = error.__cause__ or error.__context__ + return None + + +def resource_response(error): + from starlette.responses import JSONResponse + cause = resource_cause(error) + if cause is None: + return None + return JSONResponse({"error": {"code": "server_busy", "cause": cause, + "message": "The server is busy. Try again shortly."}}, + status_code=503, headers={"Retry-After": "1"}) + + +class DeadlineCapacityExceeded(RuntimeError): + """The bounded statement watchdog has no free registration slot.""" + + +class _StatementAlarm: + def __init__(self, owner, connection, acc): + self.owner, self.connection, self.acc = owner, connection, acc + self.lock = threading.Lock() + self.active, self.expired = True, False + + def fire(self): + with self.lock: + if self.active and (self.acc.cancelled or time.monotonic() >= self.acc.deadline): + self.expired = True + self.active = False + self.connection.interrupt() + + def close(self): + # Serialize disarm with interrupt before the cursor can execute another statement. + with self.lock: + self.active = False + expired = self.expired + with self.owner.lock: + self.owner.active.discard(self) + return expired + + +class _DeadlineWatchdog: + INTERVAL = .01 + CAPACITY = 128 + + def __init__(self): + self.lock = threading.Lock() + self.active = set() + self.thread = None + + def arm(self, connection, acc): + import process_role + with self.lock: + if len(self.active) >= self.CAPACITY: + raise DeadlineCapacityExceeded("request statement watchdog is full") + alarm = _StatementAlarm(self, connection, acc) + self.active.add(alarm) + if self.thread is None: + job = process_role.thread(target=self._run, name="request-deadline", kind="light") + if not process_role.admitted(job): + self.active.remove(alarm) + raise DeadlineCapacityExceeded("request statement watchdog is unavailable") + self.thread = job + try: + job.start() + except BaseException: + self.thread = None + self.active.remove(alarm) + raise + return alarm + + def _run(self): + while True: + time.sleep(self.INTERVAL) + with self.lock: + if not self.active: + self.thread = None + return + active = tuple(self.active) + for alarm in active: + try: + alarm.fire() + except Exception: + LOG.warning("request query interrupt failed", exc_info=False) + + +_DEADLINES = _DeadlineWatchdog() +timing.configure_deadlines(_DEADLINES.arm) + +_LOCK_PROBE_SLOTS = threading.BoundedSemaphore(8) + + +class _LockProbe: + def __init__(self, pid, diagnose): + self.pid, self.diagnose = pid, diagnose + self.lock = threading.Lock() + self.active, self.fired = True, False + self.job = None + + def run(self): + with self.lock: + if not self.active: + return + self.fired = True + try: + self.diagnose(self.pid) + finally: + _LOCK_PROBE_SLOTS.release() + + def close(self): + with self.lock: + if not self.active: + return + self.active = False + self.job.cancel() + if not self.fired: + _LOCK_PROBE_SLOTS.release() + + +def _arm_lock_probe(pid, wait_s, diagnose): + import process_role + if not _LOCK_PROBE_SLOTS.acquire(blocking=False): + return None + probe = _LockProbe(pid, diagnose) + try: + probe.job = process_role.thread(target=probe.run, name="store-lock-diagnosis", + kind="light", delay=max(.01, wait_s * .75)) + probe.job.start() + except Exception: + _LOCK_PROBE_SLOTS.release() + LOG.warning("store lock diagnosis could not start", exc_info=False) + return None + return probe + + +timing.configure_lock_probes(_arm_lock_probe) + + +def slow_threshold_ms(): + try: + value = float(os.environ.get("AIOS_SLOW_REQUEST_MS", "1000")) + return max(0.0, value) if math.isfinite(value) else 1000.0 + except ValueError: + return 1000.0 + + +def route_template(scope): + route = scope.get("route") + return getattr(route, "path", None) or scope.get("_timing_route", "") + + +def _resolve_route(scope): + # Refusals can finish before routing. Match registered templates without invoking a handler. + from starlette.routing import Match + for route in getattr(scope.get("app"), "routes", ()): + match, child = route.matches(scope) + if match == Match.FULL: + scope["_timing_route"] = getattr(route, "path", None) or "" + scope["_timing_endpoint"] = child.get("endpoint") + return + + +def _save_stats(scope, status, milliseconds): + key = (scope.get("method", ""), route_template(scope)) + with _STATS_LOCK: + values = _STATS.setdefault(key, deque(maxlen=500)) + _STATS.move_to_end(key) + values.append((milliseconds, int(status is not None and status >= 500))) + while len(_STATS) > _ROUTE_LIMIT: + _STATS.popitem(last=False) + + +def route_stats(): + """Return process-wide route timings without tenant, user or request identifiers.""" + with _STATS_LOCK: + snapshot = [(key, list(values)) for key, values in _STATS.items()] + rows = [] + for (method, route), values in snapshot: + ordered = sorted(value[0] for value in values) + rows.append({"method": method, "route": route, "count": len(values), + "5xx": sum(value[1] for value in values), + "p50": round(ordered[math.ceil(len(ordered) * .50) - 1], 2), + "p95": round(ordered[math.ceil(len(ordered) * .95) - 1], 2), + "max": round(ordered[-1], 2)}) + return {"window": 500, "unit": "ms", "routes": rows} + + +def _app_frames(frame): + names = set() + while frame is not None: + code = frame.f_code + path = code.co_filename.replace("\\", "/") + if ("/api/" in path or "/platform/" in path) and "site-packages" not in path: + names.add(f"{code.co_name}@{path.rsplit('/', 1)[-1]}") + frame = frame.f_back + return names + + +def _coroutine_frames(task): + frames, seen = [], set() + coro = task.get_coro() + while coro is not None and id(coro) not in seen: + seen.add(id(coro)) + frame = getattr(coro, "cr_frame", None) or getattr(coro, "gi_frame", None) + if frame is not None: + frames.append(frame) + coro = getattr(coro, "cr_await", None) or getattr(coro, "gi_yieldfrom", None) + return frames + + +class _StackSampler: + """Sample only matched worker contexts and coroutine scopes, including light requests.""" + + INTERVAL = .2 + + def __init__(self): + self.lock = threading.Lock() + self.active = {} + self.thread = None + + def begin(self, acc, scope): + import process_role + with self.lock: + self.active[acc] = (scope, asyncio.get_running_loop()) + if self.thread is None: + job = process_role.thread(target=self._run, name="request-profile", kind="light") + if process_role.admitted(job): + self.thread = job + job.start() + + def end(self, acc): + with self.lock: + self.active.pop(acc, None) + + def sample(self): + with self.lock: + active = dict(self.active) + matched = {acc: set() for acc in active} + scopes = {id(scope): acc for acc, (scope, _loop) in active.items()} + threads = timing.associated_threads() + for tid, frame in sys._current_frames().items(): + acc = threads.get(tid) + probe = frame + while probe is not None and acc not in active: + candidate = probe.f_locals.get("scope") + owner = scopes.get(id(candidate)) + if owner is not None and active[owner][0] is candidate: + acc = owner + path = probe.f_code.co_filename.replace("\\", "/") + if "/anyio/" in path and probe.f_code.co_name == "run": + context = probe.f_locals.get("context") + if isinstance(context, contextvars.Context): + acc = context.get(timing.CURRENT) + probe = probe.f_back + if acc in active: + matched[acc].update(_app_frames(frame)) + for loop in {loop for _scope, loop in active.values()}: + try: + tasks = asyncio.all_tasks(loop) + except RuntimeError: + continue + for task in tasks: + frames = _coroutine_frames(task) + owners = set() + for frame in frames: + candidate = frame.f_locals.get("scope") + owner = scopes.get(id(candidate)) + if owner is not None and active[owner][0] is candidate: + owners.add(owner) + for acc in owners: + for frame in frames: + matched[acc].update(_app_frames(frame)) + for acc, names in matched.items(): + with acc.lock: + if not acc.closed: + acc.samples.update(names) + + def _run(self): + while True: + time.sleep(self.INTERVAL) + with self.lock: + if not self.active: + self.thread = None + return + try: + self.sample() + except Exception: + LOG.debug("request sampler skipped a changing stack", exc_info=True) + + +_SAMPLER = _StackSampler() + + +class RequestTiming: + """Measure final body sends without buffering or consuming the receive channel.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope.get("type") != "http": + return await self.app(scope, receive, send) + with timing.request_scope() as acc: + acc.deadline = time.monotonic() + request_timeout_s() + status, failed, disconnected, complete = None, False, False, False + threshold = slow_threshold_ms() + _resolve_route(scope) + _SAMPLER.begin(acc, scope) + + async def receive_timed(): + nonlocal disconnected + # Response streaming remains open after headers, including SSE disconnect waits. + if status is None: + remaining = timing.remaining_budget() + try: + message = await asyncio.wait_for(receive(), remaining) + except asyncio.TimeoutError as exc: + raise timing.RequestDeadlineExceeded("request body deadline exceeded") from exc + timing.remaining_budget() + else: + message = await receive() + if message["type"] == "http.disconnect": + disconnected = True + acc.cancelled = True + return message + + async def send_timed(message): + nonlocal status, complete + if message["type"] == "http.response.start": + values = acc.snapshot() + elapsed = (time.perf_counter() - acc.started) * 1000 + app_ms = max(0., elapsed - values["queue_ms"] - values["store_ms"] - values["duckdb_ms"]) + # Headers describe time until headers. Final streaming time is logged after completion. + header = (f'queue;dur={values["queue_ms"]:.2f}, store;dur={values["store_ms"]:.2f}, ' + f'duckdb;dur={values["duckdb_ms"]:.2f}, app;dur={app_ms:.2f}') + message = dict(message) + message["headers"] = [(key, value) for key, value in message.get("headers", []) + if key.lower() != b"server-timing"] + [(b"server-timing", header.encode("ascii"))] + await send(message) + if message["type"] == "http.response.start": + status = message["status"] + elif message["type"] == "http.response.body": + with acc.lock: + acc.response_bytes += len(message.get("body", b"")) + complete = not message.get("more_body", False) + + try: + await self.app(scope, receive_timed, send_timed) + except BaseException: + failed = True + raise + finally: + elapsed = (time.perf_counter() - acc.started) * 1000 + _SAMPLER.end(acc) + with acc.lock: + acc.closed = True + acc.cancelled = True + samples = sorted(acc.samples.items(), key=lambda item: -item[1])[:18] + _save_stats(scope, status, elapsed) + if elapsed >= threshold: + endpoint = scope.get("endpoint") or scope.get("_timing_endpoint") + values = acc.snapshot() + values["app_ms"] = round(max(0., elapsed - values["queue_ms"] - values["store_ms"] + - values["duckdb_ms"]), 2) + values.update(method=scope.get("method", ""), route=route_template(scope), status=status, + total_ms=round(elapsed, 2), failed=failed, disconnected=disconnected, + complete=complete, endpoint=getattr(endpoint, "__name__", ""), + samples={name: round(count * _SAMPLER.INTERVAL * 1000, 2) for name, count in samples}) + LOG.warning("[slow-request] %s", json.dumps(values, separators=(",", ":"), ensure_ascii=True)) diff --git a/api/routes_admin.py b/api/routes_admin.py index 65d79545406ad2c6fca2f6c5fd8dc699a3a679ca..6f72740a3b75f86f7d5ae20c33156a1d9f5d430c 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -40,7 +40,7 @@ import core.registry as registry import core.schema_overlay as schema_overlay # W46-T36 / C1 - the overlay merge + R12 wall import core.store as store -from deps import Session, err, perms, require_session, users +from deps import Session, err, observed_last_active, perms, require_session, users from routes_auth import _public_user router = APIRouter(prefix="/api/v1") @@ -112,6 +112,7 @@ def _registry(): def _view(uname, rec, governed=None, st=None, surface_keys=None): """What an administrator may see about an account. NEVER `salt` or `hash` — this function is the only projection these routes use, so there is one place that can leak and it does not. + `last_active` includes this tenant/account's temporary read observations without saving them. ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, resolved once for the whole request; the single-record routes pass neither, because their @@ -127,7 +128,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None): "email": rec.get("email") or None, "tenant": _tenant_of(rec), "active": bool(rec.get("active", True)), - "last_active": rec.get("last_active"), + "last_active": observed_last_active(_tenant_of(rec), uname, rec.get("last_active")), # The session-revocation handle. An admin needs it: it is the only visible evidence # that "sign them out everywhere" actually happened. "epoch": int(rec.get("epoch") or 0), @@ -2805,6 +2806,7 @@ def settings(session: Session = Depends(require_session)): modules = perms.allowed_modules(session.user) from core import freeze_notice import process_role + from request_timing import route_stats roles = process_role.status() worker = roles["worker"] if not perms.is_admin(session.user): @@ -2813,6 +2815,7 @@ def settings(session: Session = Depends(require_session)): return { # Which process role answered, and whether its sync worker is alive and beating. "processRole": roles["role"], + "requestTiming": route_stats() if perms.is_admin(session.user) else None, "worker": worker, "freezeNotice": freeze_notice.notice(), # The build this Space is running. Behind the session on purpose — see `main.py::health`, diff --git a/api/routes_connectors.py b/api/routes_connectors.py index aff35be98ac7f511a0b7cdc37a724ea31f54df9f..4eb1adb97a7b5de963db1f1becc24fb9526cb6dc 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -51,6 +51,7 @@ from deps import Session, err, require_session, running_main #: connector is in. from core import connector_directory from core import connector_sync +from core.request_timing import raise_if_resource_error from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD from harness import amazon_client #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, @@ -995,6 +996,7 @@ def _entry_types(rt, uname=""): scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE) return types, scopes, None except Exception as e: # noqa: BLE001 + raise_if_resource_error(e) return set(), {}, f"{type(e).__name__}: {e}" @@ -2902,9 +2904,10 @@ def _amazon_completed_sync(rt): return None, False if path.resolve() == Path(datastore.DB_PATH).resolve(): return read(datastore.ro_cursor()) - with duckdb.connect(str(path), read_only=True) as cursor: + with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor: return read(cursor) - except Exception: + except Exception as exc: + raise_if_resource_error(exc) return None, False @@ -2931,9 +2934,10 @@ def _daily_completed_sync(rt, connector_key): return None, False if path.resolve() == Path(datastore.DB_PATH).resolve(): return read(datastore.ro_cursor()) - with duckdb.connect(str(path), read_only=True) as cursor: + with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor: return read(cursor) - except Exception: + except Exception as exc: + raise_if_resource_error(exc) return None, False return None, False @@ -4642,12 +4646,13 @@ def _amazon_databases(rt, is_admin, spec=None): # trims an over-long cause itself; a credential shape still refuses, and the refusal carries # the guarded rows that passed, so only the refused tables read "Sync status is unavailable # for this database." (a missing status is exactly what `_amazon_database_sync_health` - # degrades). Any other failure reading status degrades every row, never the whole page. + # degrades). Resource failures propagate; other unreadable status degrades every row. try: states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None except cx.AmazonStatusSafetyError as refused: states = dict(getattr(refused, "safe_status", None) or {}) - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + raise_if_resource_error(exc) states = {} # Which OTHER declared tables are filled from the SAME report document. Derived from the @@ -5401,6 +5406,7 @@ def _plain_read_failure(exc, code, sentence): `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log, and no exception text (which can echo a payload) reaches the wire. """ + raise_if_resource_error(exc) import logging # noqa: PLC0415 logging.getLogger(__name__).exception("connector read failed: %s", code) name = type(exc).__name__ diff --git a/api/routes_customers.py b/api/routes_customers.py index 0582e685b8e5a2d7a40b4585e61c80a962e3be53..cf3931819a8979f6ec68103b645c242c4022a4e9 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -121,15 +121,7 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r found = {} if requested: try: - with store_pg._pool().connection() as con: - rows = con.execute(store_pg._q( - "SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])", - "store_kv", handle.tenant_slug), (sorted(requested),)).fetchall() - for name, raw_rev, stamp in rows: - rev = int(raw_rev) if raw_rev is not None else 0 - found[name] = {"rev": rev, - "updated_at": stamp.timestamp() if hasattr(stamp, "timestamp") else None, - "token": f"pg:{rev}"} + found = store_pg._revision_many(sorted(requested), tenant_slug=handle.tenant_slug) except Exception as exc: if type(exc).__name__ in {'PoolTimeout', 'OperationalError', 'InterfaceError', 'ConnectionTimeout'}: from core.grid_events import StoreUnavailable @@ -189,6 +181,9 @@ def _pool_for(rt, team_id, agent): # capability result; empty is safe, another tenant's customer book is not. if rt.key != "royal-imports": return [] + bounded = getattr(rt, '_canonical_rows', None) + if bounded is not None: + return bounded import modules.customer_data as cl import routes_keychain @@ -460,18 +455,44 @@ class _AssemblyStore: self.tokens[name] = (self._validated_tokens[name] if name in self._validated_tokens else _assembly_revision(self._runtime, name)) except AttributeError as exc: + if self.schema_only and callable(getattr(self._runtime, 'revision', None)): + from core.shared_overlay import SchemaUnavailable + raise SchemaUnavailable('table definition revisions are unavailable') from exc self.cacheable = False if os.environ.get("AIOS_HEAVY_PROFILE") == "1": print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True) + except Exception as exc: + if self.schema_only: + from core.request_timing import raise_if_resource_error + from core.shared_overlay import SchemaUnavailable + raise_if_resource_error(exc) + raise SchemaUnavailable('table definition revisions are unavailable') from exc + raise key = (name, shape) if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs: - self.docs[key] = read() + try: + self.docs[key] = read() + except Exception as exc: + if self.schema_only: + from core.request_timing import raise_if_resource_error + from core.shared_overlay import SchemaUnavailable + raise_if_resource_error(exc) + raise SchemaUnavailable('table definitions are unavailable') from exc + raise # TableStore readers can mutate nested projections; lend an independent read each time. return deepcopy(self.docs[key]) def get(self, name, fresh=False): + if fresh: + from core.request_timing import clear_read_memo + clear_read_memo() if self.schema_only and fresh: self.docs = {key: value for key, value in self.docs.items() if key[0] != name} + if self.schema_only and name == "user_tables": + from core.user_tables import DEF_ONLY_DROPS + drops = (*DEF_ONLY_DROPS, 'aiEnrich') + return self._read(name, ("projection", drops), + lambda: self._runtime.get_projection(name, drop=drops), fresh=fresh) if self.schema_only and name.endswith("_table_workspace"): return self._read(name, ("projection", ("overlays",)), lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh) @@ -482,10 +503,14 @@ class _AssemblyStore: self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh) def get_projection(self, name, drop=()): + if self.schema_only and name == 'user_tables': + drop = tuple(dict.fromkeys([*drop, 'aiEnrich'])) return self._read(name, ("projection", tuple(drop)), lambda: self._runtime.get_projection(name, drop=drop)) def get_entry(self, name, entry): + if self.schema_only and name == 'user_tables': + return (self.get(name) or {}).get(str(entry)) if self.schema_only: backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime if not callable(getattr(backend, "get_entry", None)): @@ -504,6 +529,9 @@ class _AssemblyStore: return self._read(name, "exists", lambda: self._runtime.exists(name)) def revision(self, name): + # Revisionless schema reads cannot supply a token for a permission cache. + if self.schema_only and not callable(getattr(self._runtime, "revision", None)): + return None token = _assembly_revision(self._runtime, name) self.tokens.setdefault(name, token) return token @@ -600,20 +628,30 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = result = ((*prefix, json.dumps(None)), {}) return result names = {name for name, _token in revisions} + if (not with_rows and (scope.startswith('ut_') or scope in {'customer', 'product'}) + and _assembly_pg_store(assembly_rt if assembly_rt is not None else rt) is not None): + # Check known metadata together before any borrowed read asks for its own token. + names.update({'user_tables', 'object_shares', 'automations', 'loopable_route_system', + 'table_zones', 'ut_schema_overlay', 'database_setup', scope + '_cohorts', + scope + '_table_workspace', scope + '_table_workspace__shared'}) + if scope.startswith('ut_'): + names.add(scope + '__shared') with _grid_assembly_lock: for memo_key in list(rt.pool_cache): if memo_key[:-2] == prefix: names.update(name for name, _token in memo_key[-1]) - tokens, users_stamp = _assembly_revisions(rt, names, uses_roster, + tokens, users_stamp = _assembly_revisions(assembly_rt if assembly_rt is not None else rt, names, uses_roster, getattr(session, "tenant", None), roster_memo) result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens) return result def own_copy(result): - # The pool is already shared by scope_cache and rows_from_pool never mutates it. - # Copy the rest together to preserve internal aliases without lending mutable state. - out = deepcopy({k: v for k, v in result.items() if k != "rows_src"}) - out["rows_src"] = result["rows_src"] + # Both aliases lend immutable source rows. Copy the request-shaped metadata together. + borrowed = {id(result['rows_src']): result['rows_src']} + wall_rows = (result.get('_permission_scope') or {}).get('rows') + if wall_rows is not None: + borrowed[id(wall_rows)] = wall_rows + out = deepcopy(result, borrowed) out["workspace"]["storageKey"] = storage_key return out @@ -653,7 +691,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = def valid(revisions, tokens): if any(name not in tokens for name, _token in revisions): - tokens, _users = _assembly_revisions(rt, [name for name, _token in revisions]) + tokens, _users = _assembly_revisions(assembly_rt if assembly_rt is not None else rt, + [name for name, _token in revisions]) result = all(json.dumps(tokens[name], sort_keys=True, default=str) == token for name, token in revisions) return result diff --git a/api/routes_grid.py b/api/routes_grid.py index 5918779d115f9517a2fd8466628849bed1855c29..bc0fda74833bdc8c8e0288702f7e06586b3d7c39 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -51,8 +51,9 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw): if scope_key == "product": import modules.product_data as pd from routes_products import MODULE as _PMOD, pd_fields + from registry_source import MetadataStore - module, canonical = _PMOD, pd_fields(consolidated=True, st=session.runtime) + module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(session.runtime)) kw.setdefault("table", pd.table_ops(session.runtime)) hidden = perm_scope.hidden_keys(session.user, module, canonical if permission_fields is None else permission_fields, @@ -1224,6 +1225,98 @@ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "custome return out +def _builtin_activity_metadata(session, scope, *, st): + """Return detached visible fields, complete permission fields, hidden keys and no row identities for one Activity page.""" + from copy import copy, deepcopy + import aios_grid + import odoo_relational + from core import perm_scope, route_system, schema_overlay + from modules import cohort + from routes_customers import _AssemblyStore, _merge_shared_fields, shared_fields + + if scope not in ("customer", "product"): + raise err(400, "bad_scope", "Activity requires a customer or product database.") + reader = st if isinstance(st, _AssemblyStore) and st.schema_only else _AssemblyStore(st, schema_only=True) + borrowed = copy(session) + borrowed.runtime = reader + module = "product_data" if scope == "product" else MODULE + borrowed.require(module) + if scope == "product": + from routes_products import scoped_pool + import modules.product_data as pd + _, _, _, contract = scoped_pool(borrowed, with_rows=False) + definitions = pd.shared_fields(st=reader) + table_key = pd.TABLE_KEY + else: + contract = list(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE) + definitions = shared_fields(st=reader) + table_key = "customer_table_workspace" + + if not reader.available(): + from core.grid_events import StoreUnavailable + raise StoreUnavailable("Activity source fields require an available workspace.") + document = reader.get(table_key) or {} + personal = (document.get(session.uname) or {}).get("fields") or {} + has_cohorts = bool(cohort.scoped(scope, st=reader).all_for(session.uname)) + fields = aios_grid.fields_from_workspace({"fields": personal}, cohorts=has_cohorts, + scope_key=scope, fields_base=contract) + if scope == "customer": + # Shared definitions supersede personal collisions before the dependent-field wall. + fields = _merge_shared_fields(fields, definitions, session=borrowed) + else: + fields = aios_grid.apply_choice_overrides(fields, definitions) + from core import shares + for field in fields: + saved = definitions.get(field.get("key")) + if isinstance(saved, dict) and saved.get("granted") is True: + field["granted"] = True + if isinstance(saved.get("createdBy"), str): + field["createdBy"] = saved["createdBy"] + have = {field["key"] for field in fields} + fields += [dict(field, source="overlay", shared=True) + for key, field in definitions.items() if key not in have] + for field in fields: + if field.get("granted") is True: + role = shares.role_for("field", shares.field_oid(module, field["key"]), + session.uname, is_admin=session.admin, st=reader) + if role: + field["sharedRole"] = role + + # Keep projection field identities without loading Route/Zone objects or membership cells. + projection = route_system.customer_projection_fields([], []) + if scope == "product": + projection = [field for field in projection if field["key"] == route_system.ZONE_FIELD_KEY] + projected = {field["key"] for field in projection} + fields = [field for field in fields if field["key"] not in projected] + projection + corpus = deepcopy(fields) + hidden = perm_scope.hidden_keys(session.user, module, corpus, st=reader) + fields = [field for field in fields if field["key"] not in hidden] + wall_contract = contract if scope == "product" else aios_grid.FIELDS + fields, excluded = schema_overlay.for_grid(module, fields, st=reader, contract=wall_contract) + return {"fields": deepcopy(fields), "pids": frozenset(), + "defn": {"fields": corpus}, "hidden": hidden | excluded, + "_permission_scope": {"row_fields": deepcopy(wall_contract)}} + + +def _builtin_activity_pids(session, database, wanted, metadata, *, st): + """Authorize only requested builtin identities through the ordinary pool and permanent row walls.""" + from core import perm_scope + team_id, agent = perm_scope.derive_pool_scope(session.user, database) + if database == "customer_data": + from routes_customers import _pool_for + rows = _pool_for(st, team_id, agent) + elif database == "product_data": + from routes_products import _pool_for + rows = _pool_for(st, team_id) + else: + raise err(400, "bad_scope", "Activity requires a customer or product database.") + rows = [row for row in rows if str(row.get("pid")) in wanted] + # The row wall needs real shared cells, never a schema projection with missing values. + rows = perm_scope.apply_row_scope(rows, session.user, database, + metadata["_permission_scope"]["row_fields"], st=session.runtime) + return frozenset(row["pid"] for row in rows) + + def _builtin_history_context(session, scope, *, restoring=False): """Resolve history through the same assembly and write context as grid events.""" from routes_customers import grid_assembly diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index c22d07ae658a9a749cc4abede7b4de06e3bb68a3..53c1634bc375c7206acceda87dd8d2a342249aae 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -289,8 +289,7 @@ def odoo_tables_status(brief: bool = Query(False), from harness import datastore as _ds frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}) - n = _ds.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + n = _ds.count_rows(where=spec.get("where") or "", cur=cur, **frm) else: n = -1 # unknown, and never reported as zero mirror_refreshed = mirror_stamp( @@ -652,7 +651,9 @@ def _sync_entities(cur, cadence_seconds): rows = cur.execute( "SELECT entity, phase, last_id, cursor_wd, rows, updated_at " "FROM _sync_state ORDER BY entity").fetchall() - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) return None out = {} for entity, phase, last_id, cursor_wd, nrows, updated_at in rows: @@ -777,25 +778,19 @@ GRID_SOURCES = {} _SOURCE_PROVIDERS = [] _SOURCE_ANALYSES = {} -#: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every -#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's -#: keys to the shared dict as it went, and returned the dict to any caller that found it -#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered -#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read -#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under -#: this lock and publishes it in one `dict.update`; a reader waits on the lock until -#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the -#: Amazon fallback) happens only after a build has been published, so it can never stand in -#: for one either. +# Provider SQL runs outside the publication lock. Invalidation rejects an older build. _SOURCES_LOCK = threading.RLock() _SOURCES_BUILT = False -_SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse +_SOURCES_BUILDER = None # compatibility for callers that reset legacy build state +_SOURCES_BUILDING = threading.local() +_SOURCES_EPOCH = 0 def invalidate_sources(): """Forget measured bindings after a connector publishes a new mirror schema.""" - global _SOURCES_BUILT + global _SOURCES_BUILT, _SOURCES_EPOCH with _SOURCES_LOCK: + _SOURCES_EPOCH += 1 _SOURCES_BUILT = False GRID_SOURCES.clear() @@ -847,54 +842,54 @@ def _sources(): importing that module at file-import time would drag the Odoo layer into every process that mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. - ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the - lock's comment). An empty registry still means "build" (gates clear it to force a rebuild), - and so does one that holds only admissions made since the last invalidation. + Providers build a local dictionary outside the publication lock. """ - global _SOURCES_BUILT, _SOURCES_BUILDER + global _SOURCES_BUILT if _SOURCES_BUILT and GRID_SOURCES: return GRID_SOURCES with _SOURCES_LOCK: - if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident(): + if (_SOURCES_BUILT and GRID_SOURCES) or getattr(_SOURCES_BUILDING, "active", False): return GRID_SOURCES - _SOURCES_BUILDER = threading.get_ident() - try: - built = {} - for build in list(_SOURCE_PROVIDERS): - try: - extra = build() or {} - except Exception: # noqa: BLE001 - # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo - # grids must not go dark because a newer connector's module is unhappy, and - # vice versa. - continue - if extra: - built.update(extra) - _ut().register_connected(*extra) - # A key admitted since the last invalidation was measured at least as recently as - # this build, so it stands; everything else comes from the build. ONE update, so a - # lock-free reader of the dict sees the old entry or the new one, never a gap. + epoch = _SOURCES_EPOCH + from harness import datastore + generation = datastore.mirror_generation() + _SOURCES_BUILDING.active = True + try: + built = {} + for build in list(_SOURCE_PROVIDERS): + try: + extra = build() or {} + except Exception: # noqa: BLE001 + continue + built.update(extra) + with _SOURCES_LOCK: + if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation(): + return GRID_SOURCES GRID_SOURCES.update({**built, **GRID_SOURCES}) _SOURCES_BUILT = True - finally: - _SOURCES_BUILDER = None + if built: + _ut().register_connected(*built) + finally: + _SOURCES_BUILDING.active = False return GRID_SOURCES -def _learn_amazon_sources(cur=None): - """Publish the complete current Amazon binding set, including restored missing entries.""" - _sources() +def _learn_amazon_sources(cur=None, *, tenant=None): + """Refresh discovery for non-page callers without retaining negative results.""" + with _SOURCES_LOCK: + epoch = _SOURCES_EPOCH + from harness import datastore + generation = datastore.mirror_generation() if cur is None: extra = _amazon_sources() else: import amazon_relational extra = amazon_relational.grid_sources(cur) - if not extra: - return with _SOURCES_LOCK: - # One known key cannot certify every history table. Replace old missing bindings as - # one publication, so the first rows AND aggregates request sees the restored schema. + if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation(): + return GRID_SOURCES.update(extra) + if extra: _ut().register_connected(*extra) @@ -1297,13 +1292,7 @@ def _amazon_sources(): def require_amazon_mirror(table_key, session, definition=None, st=None): - """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry. - - The provider contributes no keys on an Odoo-only mirror. A persisted read-through - definition still must refuse there instead of falling through to its empty stored rows. - S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the - window route stops paying a second definitions projection for this one check. - """ + """Admit one Amazon grid and return its binding, or None for a stored snapshot.""" if not str(table_key).startswith("ut_amzn_"): return import amazon_relational as amazon @@ -1321,26 +1310,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None): if mismatch: raise err(409, "cross_tenant_store", mismatch) try: - spec = _sources().get(table_key) - if (spec is not None or definition.get("readThrough") - or table_key in amazon.READ_THROUGH): - # Check current table presence even if a binding was cached before a restart. - current = amazon.grid_source(cur, table_key) - if current.get("missing"): - raise err(503, "amazon_mirror_not_loaded", current["missing"]) - # A dynamically oversized snapshot may have a durable stub before this process - # has measured it. Admit its binding without changing another source's registry. - # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon - # key. Admitted into an empty or Amazon-less registry, this one key used to make - # `_sources()` skip its build, so the grid resolved while its link targets - # (orders, read-through) were dispatched as stored tables: a first-page 500. - if spec is None or spec.get("missing"): - _learn_amazon_sources(cur) - with _SOURCES_LOCK: - _sources() - held = GRID_SOURCES.get(table_key) - if held is None or held.get("missing"): - GRID_SOURCES[table_key] = current + return _amazon_page_source(cur, table_key, definition, allow_lookups=False) finally: cur.close() @@ -1512,8 +1482,7 @@ def _unmaterialisable(cur=None): try: frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}) - total = datastore.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + total = datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm) except Exception: # noqa: BLE001 total = None if total is not None and total > max_rows: @@ -1557,7 +1526,27 @@ def _degrade(expr, have_by_alias, default_alias=""): return expr if bare.lower() in have else "NULL" -def _source_for(cur, table_key): +def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_lookups=True): + """Resolve one admitted Amazon target and keep query failures distinct from stored tables.""" + import amazon_relational as amazon + + try: + if binding: + columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True) + spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups) + else: + spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups) + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, "amazon_source_unavailable", + "This Amazon database could not be resolved. Retry the request.") from exc + if spec is not None and spec.get("missing"): + raise err(503, "amazon_mirror_not_loaded", spec["missing"]) + return spec + + +def _source_for(cur, table_key, *, allow_lookups=True): """The resolved spec for one connected grid, or None when this grid is not read-through yet. ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists @@ -1584,28 +1573,15 @@ def _source_for(cur, table_key): # table and turns Products ordered into the wrong 404. if module_cache_source(key) is not None: return None + if key.startswith("ut_amzn_"): + import amazon_relational as amazon + if key in amazon.SOURCE: + return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups) spec = _sources().get(key) - if (not spec or spec.get("missing")) and key.startswith("ut_amzn_"): - # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no - # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync - # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every - # read-through Amazon grid answered "no read-through binding". S70-T12 re-asks the - # complete Amazon provider on this restored cursor, including cached missing entries. - _learn_amazon_sources(cur) - spec = GRID_SOURCES.get(key) if not spec: return None if spec.get("missing"): raise err(503, "amazon_mirror_not_loaded", spec["missing"]) - if key.startswith("ut_amzn_"): - import amazon_relational as amazon - if key in amazon.SOURCE: - # The registry survives syncs. Resolved parent data embedded in its old - # expressions must follow this cursor's revision rather than the boot revision. - derived = {lookup["key"]: (lookup["expr"], amazon._text) - for lookup in amazon.lookups(cur, key)} - spec = {**spec, "cols": {**{k: v for k, v in spec["cols"].items() - if k not in amazon.LOOKUP_KEYS}, **derived}} from harness import datastore if _rel().base_table_key(table_key) == _rel().BILLS_KEY: try: @@ -1658,9 +1634,21 @@ def _source_for(cur, table_key): for key, (expr, cast) in spec["cols"].items(): cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, have_by_alias), cast) + if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"): + spec = {**spec, "page_key": spec["id"]} return {**spec, "cols": cols} +def _page_source_for(cur, table_key, st=None, *, session=None): + """Plan admitted lookups afresh because an autocommit cursor can change snapshots.""" + if str(table_key).startswith("ut_amzn_") and session is not None: + import amazon_relational as amazon + from routes_grid import _may_open + allowed = _may_open(session, amazon.LISTINGS, st=st) + return _source_for(cur, table_key, allow_lookups=allowed) + return _source_for(cur, table_key) + + #: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in #: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of #: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of @@ -1874,7 +1862,8 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=( if table_key in trail: raise ValueError("a relation has a circular access rule") trail = (*trail, table_key) - handle = _ut().lend_defs(st) + handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False) + else _ut().lend_defs(st, schema_only=True)) declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")} columns = _filter_columns(spec, fields) guards, resolved, targets = [], {}, {} @@ -1895,13 +1884,14 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=( targets[key] = None return None definition = _ut().get(key, st=handle) - target_spec = _source_for(cur, key) + target_spec = _page_source_for(cur, key, handle, session=session) if target_spec is None and str(key).startswith("ut_amzn_"): # Small Amazon targets (notably listings) are stored snapshots, but # their raw mirror still answers the same automatic link in SQL. import amazon_relational if key in amazon_relational.SOURCE: - target_spec = amazon_relational.grid_source(cur, key) + target_spec = amazon_relational.grid_source(cur, key, + allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle)) if not definition or target_spec is None or target_spec.get("missing"): refuse(field) target_fields = definition.get("fields") or [] @@ -2375,7 +2365,12 @@ def _related_source_value(session, entry, record): if not source: return None try: - rows = perm_scope.scoped_table(session.user, source, st=session.runtime) or [] + cursor = getattr(session.runtime, '_canonical_source_cursor', None) + if cursor is not None and source in {'customer_data', 'product_data'}: + import registry_source + rows = registry_source.canonical_page(session, cursor, source, [record])['rows'] + else: + rows = perm_scope.scoped_table(session.user, source, st=session.runtime) or [] except Exception: # noqa: BLE001 return None want = str(record) @@ -2386,21 +2381,25 @@ def _related_source_value(session, entry, record): return None -def _materialised_table(table_key): - """Storage dispatch, independent of the browser's fallible status roster. - - ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first - (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before - the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one. - """ +def _materialised_table(table_key, *, cur=None, st=None): + """Dispatch one target against its cursor without treating registry misses as evidence.""" if module_cache_source(table_key) is not None: return True - if table_key in _sources(): - return False if str(table_key).startswith("ut_amzn_"): - _learn_amazon_sources() - return table_key not in GRID_SOURCES - return True + from harness import datastore + import amazon_relational as amazon + if table_key not in amazon.SOURCE: + return True + own_cursor = cur is None + cur = datastore.ro_cursor() if own_cursor else cur + try: + definition = _ut().get(table_key, st=st) if st is not None else None + spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False) + return spec is None + finally: + if own_cursor: + cur.close() + return table_key not in _sources() #: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's @@ -2425,11 +2424,17 @@ _UNREAD = ... def _user_tables_token(rt): - """The durable change token of this tenant's `user_tables` bucket, or None when unreadable.""" - try: - return rt.revision(_ut().STORE_KEY)["token"] - except Exception: # noqa: BLE001 + """Return the durable revision and timestamp, or decline a revisionless reader.""" + reader = getattr(rt, "revision", None) + if not callable(reader): return None + try: + revision = reader(_ut().STORE_KEY) + return (revision["token"], revision.get("updated_at")) if revision else None + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): @@ -2463,17 +2468,10 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): def _page_defs(session): - """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's. - - Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked - against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link - target does not pay a second revision round trip. - The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per - `user_tables` revision instead of once per question. Kept only when every definition really is - a projection (`store.is_projected`), because `all_defs` silently falls back to the whole - 28 MB document on a failed read and that must never be held, let alone copied per request. - ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it - lends lazily live for this one request and are never carried to the next. + """Return one schema-only `(snapshot, revision)` for a page and its scoped targets. + + Shared metadata and source plans belong to the fresh lend, never to the revision memo. + Projection and revision failures refuse instead of falling back to a whole document. """ import copy import core.store as store @@ -2483,15 +2481,23 @@ def _page_defs(session): token = _user_tables_token(rt) def keep(v): - tables = [d for d in (v or {}).values() if isinstance(d, dict)] + # Activity is a generated descriptor, not a stored row-bearing definition. + tables = [d for key, d in (v or {}).items() if key != "ut_activity" and isinstance(d, dict)] return bool(tables) and all(store.is_projected(d) for d in tables) - defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt), - keep=keep, clone=copy.deepcopy, token=token) - return ut.lend(rt, **{ut.STORE_KEY: defs}), token + try: + defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True), + keep=keep, clone=copy.deepcopy, token=token) + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, "schema_unavailable", "table definitions are unavailable") from exc + lent = ut.lend(rt, **{ut.STORE_KEY: defs}) + lent.schema_only = True + return lent, token -def _stored_link_rows(session, table, token=_UNREAD): +def _stored_link_rows(session, table, token=_UNREAD, *, materialised=None): """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision. ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module @@ -2505,7 +2511,8 @@ def _stored_link_rows(session, table, token=_UNREAD): rt = session.runtime if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES - or module_cache_source(table) is not None or not _materialised_table(table)): + or module_cache_source(table) is not None + or not (materialised if materialised is not None else _materialised_table(table))): return perm_scope._read(table, session.user, rt) return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt), keep=lambda v: bool(v[1]), @@ -2528,8 +2535,17 @@ def _traversal_defn_or_refuse(session, table_key, st=None): return defn +class _RelationRows: + def __init__(self, read): + self.read = read + + def __iter__(self): + return self.read() + + def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None, - traversal_of=None, st=None, rev=..., normalize_match=False): # `...` IS `_UNREAD` + traversal_of=None, st=None, rev=..., normalize_match=False, + compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD` """Internal join values, after the ordinary row wall, with no row-count ceiling. Join columns may be hidden on the wire. Only identities leave the resolver; the native @@ -2547,9 +2563,13 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non import core.relations as relations columns = set(columns) | {relations.IDENTITY_COL} - if not _materialised_table(table): - read = (_compiled_read(table, session, traversal_intermediate=True, st=st) - if traversal_of is not None else _compiled_read(table, session, st=st)) + if traversal_of is None and not perm_scope.may_read( + session.user, table, st=st if st is not None else session.runtime): + raise err(403, "forbidden_table", "This account cannot open the linked database.") + if not (materialised if materialised is not None else _materialised_table(table)): + read = compiled if compiled is not None else ( + _compiled_read(table, session, traversal_intermediate=True, st=st) + if traversal_of is not None else _compiled_read(table, session, st=st)) spec = read["spec"] bindings = dict(spec["cols"], pid=(spec["id"], int)) if columns - set(bindings): @@ -2560,10 +2580,17 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non if values is None: continue expr, cast = bindings[key] - values = list(values) if normalize_match and key != "pid": expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))" cast = lambda value: str(value).strip().lower() + if isinstance(values, _fs().SqlMembership): + operand = _membership_relation(read['cur'], _membership_values(values), + normalize=cast, sql_type='HUGEINT' if cast is int else 'DOUBLE' if cast is float else 'VARCHAR', + skip_invalid=False) + parts.append(f'{expr} IN ({operand.sql})') + params.extend(operand.params) + continue + values = list(values) parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0") params.extend(cast(value) for value in values) if matches is not None: @@ -2571,6 +2598,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non for key, values in matches.items(): expr, cast = bindings[key] values = sorted(values) + if normalize_match: + expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))" + cast = lambda value: str(value).strip().lower() if values: alternatives.append(f"{expr} IN ({','.join('?' for _ in values)})") params.extend(cast(value) for value in values) @@ -2579,14 +2609,30 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non projection = ", ".join(bindings[key][0] for key in names) source = spec.get("from_sql") or spec["table"] predicate = " AND ".join(f"({part})" for part in parts) or "1=1" - return [dict(zip(names, row)) for row in read["cur"].execute( - f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()] + if spec.get("page_key") and not spec.get("from_sql"): + key = spec["page_key"] + sql = (f'WITH _relation_keys AS MATERIALIZED (SELECT {key} AS "_target_key" ' + f'FROM {source} WHERE {predicate}) SELECT {projection} FROM {source} ' + f'SEMI JOIN _relation_keys ON {key} = _relation_keys."_target_key"') + else: + sql = f"SELECT {projection} FROM {source} WHERE {predicate}" + def batches(): + try: + cursor = read["cur"].execute(sql, params) + while True: + batch = cursor.fetchmany(2048) + if not batch: + return + for row in batch: + yield dict(zip(names, row)) + batch.clear() + finally: + if stream and compiled is None: + read["cur"].close() + return _RelationRows(batches) if stream else list(batches()) # This is an internal join, never a cell response. Apply the same permanent wall on the # complete contract before taking any join value, including a join on a hidden column. - if traversal_of is None and not perm_scope.may_read( - session.user, table, st=st if st is not None else session.runtime): - raise err(403, "forbidden_table", "This account cannot open the linked database.") if module_cache_source(table) is not None: session.require(table) # The revision memo serves a caller that handed this request's token (the page's link fills, @@ -2597,7 +2643,7 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition) if definition is not None else perm_scope._read(table, session.user, session.runtime)) else: - fields, rows = _stored_link_rows(session, table, token=rev) + fields, rows = _stored_link_rows(session, table, token=rev, materialised=True) if ids is not None: wanted = {str(value) for value in ids} rows = [row for row in rows if str(row.get("pid")) in wanted] @@ -2615,8 +2661,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non wanted = {normalize(value) for value in values} rows = [row for row in rows if normalize(row.get(key)) in wanted] if matches is not None: - wanted = {key: {str(value) for value in values} for key, values in matches.items()} - rows = [row for row in rows if any(str(row.get(key)) in values + normalize = (lambda value: str(value).strip().lower()) if normalize_match else str + wanted = {key: {normalize(value) for value in values} for key, values in matches.items()} + rows = [row for row in rows if any(normalize(row.get(key)) in values for key, values in wanted.items())] return [{key: row.get(key) for key in columns} for row in rows] @@ -2627,7 +2674,48 @@ def _relation_cell_ids(value): return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0} -def _relation_leg_ids(session, source, source_ids, entry): +def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_col, + target_col, manual, inverse, reverse): + """Follow a scoped leg with SQL operands at every population-sized boundary.""" + import core.relations as relations + target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {} + observed = {'rows': 0, 'complete': True, 'available': False} + def source_values(): + for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True): + observed['rows'] += 1 + observed['complete'] &= source_col in row + if row.get(source_col) not in (None, ''): + observed['available'] = True + if (manual and not reverse) or (inverse and reverse): + yield from _relation_cell_ids(row[source_col]) + else: + yield str(row[source_col]) + values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR') + if source_col != relations.IDENTITY_COL and not observed['available']: + if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_') + and observed['rows'] and observed['complete']): + return values + raise err(400, 'unknown_relation', relations.unresolvable_source_refusal(source, entry)) + if (manual and not reverse) or (inverse and reverse): + target_col = 'pid' + through = relations.declared_through(link) if not manual else None + if through: + left = str(link['throughTo'] if reverse else link['throughOn']) + right = str(link['throughOn'] if reverse else link['throughTo']) + middle = _relation_rows(session, str(through), [left, right], match=(left, values), + traversal_of=str(entry.get('key') or via.get('key') or ''), stream=True) + values = _membership_relation(cur, + (str(row[right]) for row in middle if row.get(right) not in (None, '')), + normalize=str, sql_type='VARCHAR') + if (manual and reverse) or (inverse and not reverse): + rows = _relation_rows(session, target, [target_col], stream=True) + return _relation_cell_membership(cur, rows, target_col, values) + rows = _relation_rows(session, target, [target_col], match=(target_col, values), + where=entry.get('where'), stream=True) + return _membership_relation(cur, (row['pid'] for row in rows)) + + +def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None): """Follow one declared leg, including manual, inverse and through links.""" import core.relations as relations @@ -2645,6 +2733,9 @@ def _relation_leg_ids(session, source, source_ids, entry): source_col, target_col = ("pid", field) if reverse else (field, "pid") elif inverse: source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse) + if membership_cur is not None: + return _relation_leg_membership(session, source, source_ids, entry, membership_cur, + source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse) source_rows = _relation_rows(session, source, [source_col], ids=source_ids) values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")} # A column-keyed edge cannot fall back to the wire record id. In particular, @@ -2669,11 +2760,11 @@ def _relation_leg_ids(session, source, source_ids, entry): traversal_of=str(entry.get("key") or via.get("key") or "")) values = {str(row[right]) for row in middle if row.get(right) not in (None, "")} if (manual and reverse) or (inverse and not reverse): - rows = _relation_rows(session, target, [target_col]) - rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values] + rows = _relation_rows(session, target, [target_col], stream=True) + rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values) else: rows = _relation_rows(session, target, [target_col], match=(target_col, values), - where=entry.get("where")) + where=entry.get("where"), stream=True) return {int(row["pid"]) for row in rows} @@ -2686,7 +2777,7 @@ def _relation_label(key): if table == key and label), str(key)) -def _relation_target_ids(session, target, related): +def _relation_target_ids(session, target, related, membership_cur=None): """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list.""" import core.relations as relations from routes_grid import _may_open, related_path_entries @@ -2721,11 +2812,13 @@ def _relation_target_ids(session, target, related): if not _may_open(session, leg["target"]): raise err(403, "forbidden_table", f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open") - if not ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"): + empty_ids = (not ids.params if membership_cur is not None + and isinstance(ids, _fs().SqlMembership) else not ids) + if empty_ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"): # An earlier, authorised Amazon hop matched nothing. There is no next # source record to read, and that is an empty relation, not a missing key. continue - ids = _relation_leg_ids(session, table, ids, leg) + ids = _relation_leg_ids(session, table, ids, leg, membership_cur) return ids @@ -2961,7 +3054,9 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related related = _json_arg(related, "related") if related in (None, "", {}): return None, [], [] - ids = _relation_target_ids(session, table_key, related) + ids = _relation_target_ids(session, table_key, related, membership_cur=cur) + if isinstance(ids, _fs().SqlMembership): + return f'COALESCE({spec["id"]} IN ({ids.sql}), FALSE)', list(ids.params), [] sql, params = relations.pin_sql(spec["id"], sorted(ids)) notes = [] if len(ids) > _PIN_REPORT_AT: @@ -3020,6 +3115,119 @@ def _active_cohort_leaves(nodes): return out +class _RelationLease(int): + """A bound scalar keeps its cursor-local relations alive through copied query params. + + Every leaf gets its own lease. Releasing one query cannot invalidate another leaf or query + on a retained cursor. No process/session map holds leases after their parameters disappear. + """ + def __new__(cls, cur): + value = super().__new__(cls, 1) + value.cur, value.names = cur, [] + return value + + def table(self, suffix=""): + import uuid + name = '_request_members_' + uuid.uuid4().hex + suffix + self.names.append(name) + return name + + def __reduce_ex__(self, protocol): + raise TypeError('Request-owned membership cannot be persisted or cached') + + def close(self): + # Cleanup remains possible after the request's query deadline has fired. + cur = getattr(self.cur, '_cursor', self.cur) + while self.names: + name = self.names[-1] + cur.execute(f'DROP TABLE IF EXISTS "{name}"') + self.names.pop() + + def __del__(self): + try: + self.close() + except Exception: + pass # A closed cursor has already destroyed its TEMP relations. + + +def _relation_cell_membership(cur, rows, column, values): + """Match complete manual/inverse cells in SQL without retaining the target population.""" + import csv + import tempfile + from pathlib import Path + from core.request_timing import remaining_budget + lease = _RelationLease(cur) + name = lease.table('_cells') + count = 0 + with tempfile.TemporaryDirectory(prefix='aios-related-') as directory: + path = Path(directory) / 'cells.csv' + with path.open('w', encoding='utf-8', newline='') as output: + writer = csv.writer(output) + for index, row in enumerate(rows): + if index % 2048 == 0: + remaining_budget() + for token in _relation_cell_ids(row.get(column)): + writer.writerow([int(row['pid']), token]) + count += 1 + if not count: + return _fs().SqlMembership('SELECT NULL WHERE FALSE', []) + cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT * FROM read_csv(?, header=false, ' + "columns={'pid':'BIGINT','token':'VARCHAR'})", [str(path)]) + return _fs().SqlMembership(f'SELECT pid FROM "{name}" WHERE token IN ({values.sql}) AND ?=1', + [*values.params, lease]) + + +def _membership_values(membership): + """Replay SQL membership in bounded batches on its owning cursor.""" + if not membership.params: + return + cur = membership.params[-1].cur + cur.execute(membership.sql, membership.params) + while batch := cur.fetchmany(2048): + for row in batch: + yield row[0] + batch.clear() + + +def _membership_relation(cur, values, *, normalize=int, sql_type='HUGEINT', skip_invalid=True): + """Normalize a complete operand with bounded Python memory, then own its SQL lifetime.""" + import tempfile + lease = _RelationLease(cur) + name = lease.table() + count = 0 + # Spooling also permits the producer to read this same cursor without an INSERT replacing + # its active result. No population-sized list or Python set is constructed. + import csv + from pathlib import Path + from core.request_timing import remaining_budget + with tempfile.TemporaryDirectory(prefix='aios-members-') as directory: + path = Path(directory) / 'members.csv' + source = iter(values) + try: + with path.open('w', encoding='utf-8', newline='') as output: + writer = csv.writer(output) + for index, raw in enumerate(source): + if index % 2048 == 0: + remaining_budget() + try: + value = normalize(raw) + except (TypeError, ValueError): + if not skip_invalid or normalize is not int: + raise + continue + writer.writerow([str(value)]) + count += 1 + finally: + close = getattr(source, 'close', None) + if close: + close() + if count: + cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT pid FROM read_csv(?, ' + "header=false, columns={'pid':'VARCHAR'}, force_not_null=['pid'])", [str(path)]) + return _fs().SqlMembership(f'SELECT TRY_CAST(pid AS {sql_type}) FROM "{name}" WHERE ?=1', + [lease]) if count else _fs().SqlMembership('SELECT NULL WHERE FALSE', []) + + def _view_store(session, table_key): """The workspace document a saved view of THIS database lives in. @@ -3028,7 +3236,43 @@ def _view_store(session, table_key): to fail it: the client reads the views `/workspace` served out of this same document. """ import core.table_store as table_store - return table_store.make(f"{table_key}_table_workspace", st=session.runtime) + from routes_customers import _AssemblyStore + runtime = session.runtime + from registry_source import backend as document_backend + backend = document_backend(runtime) + project = getattr(backend, 'get_view_workspace', None) or getattr(backend, '_get_view_workspace', None) + stream_values = getattr(backend, 'iter_json_values', None) or getattr(backend, '_iter_json_values', None) + class ViewStore(_AssemblyStore): + def get(self, name, fresh=False): + if callable(project) and name == f'{table_key}_table_workspace': + key = getattr(runtime, 'store_key', str)(name) + return self._read(name, 'view_operands', lambda: project(key), fresh=fresh) + return super().get(name, fresh=fresh) + + def _read_strict(self, name): + return self.get(name) + result = table_store.make(f"{table_key}_table_workspace", st=ViewStore(runtime, schema_only=True)) + if callable(project): + def members(owner, vid): + name = f'{table_key}_table_workspace' + key = getattr(runtime, 'store_key', str)(name) + path = [owner, 'views', vid, 'config', 'memberPids'] + document = result.st.get(name) + stable = getattr(backend, 'iter_view_members', None) or getattr(backend, '_iter_view_members', None) + parts = (stable(key, path, getattr(document, 'source_revision', None), batch_size=2048) + if callable(stable) else stream_values(key, path, batch_size=2048)) + from contextlib import closing + try: + with closing(parts): + for batch in parts: + for _, value in batch: + yield value + except Exception as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, 'schema_unavailable', 'Saved view membership is unavailable. Try again.') from exc + result.member_values = members + return result def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits): @@ -3052,8 +3296,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par * no filters and a curated `memberPids` is the client's `else if (curated)` branch: the membership IS the view; * no filters and no members means the view holds every row in scope; - * the ids are materialised in ONE `SELECT DISTINCT` under `base_where`, uncapped, the same - way `_related_pin` does it, so the wall rides along and standing rule 1 is kept. + * membership stays a `SELECT DISTINCT` subquery under the wall, without a Python ID list. THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY: resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets` @@ -3082,6 +3325,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par store = _view_store(session, table_key) sets, memo, unresolved = {}, {}, [] + cyclic = set() counter = [0] def key_for(): @@ -3092,10 +3336,11 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par return "__view_%d__" % counter[0] def resolve(vid, seen): - """`('ids', [...])` / `('all', None)` / `(None, reason)` for one view id.""" + """Return SQL membership, authored IDs, all rows, or an unresolved reason.""" if vid in memo: return memo[vid] if vid in seen: + cyclic.update(seen[seen.index(vid):]) # `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left # ABSENT, so a stored loop degrades to an explained empty table, not a hung request. return None, "it refers to itself through another saved view" @@ -3115,12 +3360,24 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par "own records rather than from this database's live connection") memo[vid] = got return got - members = [] - for raw in (cfg.get("memberPids") or ()): + source = (store.member_values(owner, vid) if 'memberPids' not in cfg + and callable(getattr(store, 'member_values', None)) + else (cfg.get('memberPids') or ())) + has_members = False + def authored_values(): + nonlocal has_members + values = iter(source) try: - members.append(int(raw)) - except (TypeError, ValueError): # noqa: PERF203 - continue + for raw in values: + # A nonempty authored operand stays curated even if normalization + # rejects every value. An actually empty operand means all rows. + has_members = True + yield raw + finally: + close = getattr(values, 'close', None) + if close: + close() + members = _membership_relation(cur, authored_values()) raw_nodes = cfg.get("filters") or [] if not isinstance(raw_nodes, list): raw_nodes = [] @@ -3129,10 +3386,14 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par # "anything active": with no nodes at all the curated set IS the view, and with nodes # that all happen to be half-built the view holds everything. Two different answers # from one list, so the emptiness is checked before the compiler sees it. - got = ("ids", members) if members else ("all", None) + got = ("ids", members) if has_members else ("all", None) memo[vid] = got return got nodes = rewrite(raw_nodes, seen + (vid,)) + if vid in cyclic: + got = (None, "it refers to itself through another saved view") + memo[vid] = got + return got unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols)) if unknown: # ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it @@ -3146,7 +3407,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par try: pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or" else "and"), - columns=cols, member_ids=(members or None), + columns=cols, member_ids=(members if has_members else None), id_sql=spec["id"], today=_today(), cohort_sets=sets) except ValueError as exc: got = (None, "it uses a condition this database's live connection cannot evaluate (" @@ -3167,9 +3428,9 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par return got frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql - ids = [int(r[0]) for r in cur.execute( + ids = _fs().SqlMembership( f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}', - list(wall_params) + list(pred.params)).fetchall() if r and r[0] is not None] + list(wall_params) + list(pred.params)) got = ("ids", ids) memo[vid] = got return got @@ -3200,7 +3461,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par if kind == "ids": sets[key] = got op = "noneOf" if not_in else "anyOf" - if len(got) >= _PIN_REPORT_AT: + if not isinstance(got, _fs().SqlMembership) and len(got) >= _PIN_REPORT_AT: limits.append({ "subject": "view", "effect": "slow", "cause": f"the saved view this condition names holds {len(got):,} rows, " @@ -3220,7 +3481,12 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par out.append({"colId": "__cohort__", "op": op, "value": key}) return out - tree = rewrite(tree, ()) + try: + tree = rewrite(tree, ()) + finally: + # Recursive closures must not postpone relation release until cyclic GC. + memo.clear() + resolve = rewrite = None if unresolved: limits.append({ "subject": ", ".join(vid for vid, _why in unresolved), @@ -3240,9 +3506,9 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho `__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`. A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered - here the way the client answers it: the residue (every non-rank condition) under the wall is - the domain, and `filter_eval.resolve_rank_sets` (the Python port of the client's set pass, - ties by ascending pid) picks the members. Only the id and the ranked columns are read, uncapped. + here as SQL over the non-rank residue under the wall. Finite values, group scopes and + ascending-ID ties follow `filter_eval.resolve_rank_sets`. Averages stream sorted values + through runtime Python sum with bounded batches. Other ranks remain entirely in SQL. """ fe = _fe() leaves = fe._rank_leaves(tree or [], cols) @@ -3270,30 +3536,8 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho params.extend(pred.params) frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] answers = {} - python_leaves = [] for n in leaves: - got = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur) - if got is None: - python_leaves.append(n) - else: - answers[id(n)] = got - if python_leaves: - # A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and - # the ranked columns once, uncapped, and let `resolve_rank_sets` answer it. - ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols}) - select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked]) - sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "") - rows = [] - for r in cur.execute(sql, params).fetchall(): - if not r or r[0] is None: - continue - row = {"pid": int(r[0])} - for i, c in enumerate(ranked, start=1): - row[c] = r[i] - rows.append(row) - resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols) - for n in python_leaves: - answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ()) + answers[id(n)] = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur) sets = dict(cohort_sets or {}) counter = [0] wanted = {id(n) for n in leaves} @@ -3322,66 +3566,101 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho out.append(n) return out - return rewrite(tree), sets + try: + return rewrite(tree), sets + finally: + rewrite = None def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur): - """One GLOBAL rank leaf as SQL: a `SqlMembership` selecting its ids, or None for the resolver. - - Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a - paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds, - and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line - for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values - are ABSENT, ties break by ascending id in both directions, a bound outside the builder's - whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than - buckets is empty. The domain's count and mean are read once here and inlined as literals. - """ + """Resolve global and explicit group ranks as SQL membership over the walled residue.""" fe = _fe() Sql = _fs().SqlMembership empty = Sql("SELECT NULL WHERE FALSE", []) - scope, _field = fe._rank_scope(leaf, None) - if scope != "global": - return None + scope, field = fe._rank_scope(leaf, None) + if scope == "group" and field not in cols: + return empty col = cols.get(leaf.get("colId")) or {} if col.get("type") not in fe.NUMERIC_TYPES: return empty v = f"TRY_CAST({col['sql']} AS DOUBLE)" - dom = (f"SELECT {id_sql} AS pid, {v} AS v FROM {frm_body} WHERE " - + (f"({where}) AND " if where else "") + f"{v} IS NOT NULL AND isfinite({v})") + partition = cols[field]["sql"] if scope == "group" else "NULL" + dom = (f"SELECT {id_sql} AS pid, {v} AS v, {partition} AS bucket FROM {frm_body} WHERE " + + (f"({where}) AND " if where else "") + + f"{id_sql} IS NOT NULL AND {v} IS NOT NULL AND isfinite({v})") params = list(params) op = leaf.get("op") + if op in ("aboveAvg", "belowAvg"): + import itertools + import tempfile + from pathlib import Path + + # Python 3.12 sum uses compensation. SQL sum/fsum cannot replace its sorted oracle. + domain = f"SELECT *, dense_rank() OVER (ORDER BY bucket) AS bucket_id FROM ({dom}) d" + lease = _RelationLease(cur) + name = lease.table('_mean') + frozen = lease.table('_domain') + # The mean and selected IDs must share one source snapshot across both statements. + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{frozen}" AS {domain}', params) + cur.execute(f'SELECT bucket_id, v FROM "{frozen}" ORDER BY bucket_id, v, pid') + + def values(): + while True: + batch = cur.fetchmany(4096) + if not batch: + return + yield from batch + + # Spill scalar group means, never a Python map of groups or a list of population rows. + with tempfile.TemporaryDirectory(prefix="aios-rank-") as directory: + path = Path(directory) / "means.csv" + groups = 0 + with path.open("w", encoding="ascii") as output: + for bucket, rows in itertools.groupby(values(), key=lambda row: row[0]): + count = 0 + + def numbers(): + nonlocal count + for _, number in rows: + count += 1 + yield number + + mean = sum(numbers()) / count + output.write(f"{bucket},{mean!r}\n") + groups += 1 + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{name}" (bucket_id BIGINT, mean DOUBLE)') + if groups: + cur.execute(f'INSERT INTO "{name}" SELECT * FROM read_csv(?, header=false, ' + "columns={'bucket_id':'BIGINT', 'mean':'DOUBLE'})", [str(path)]) + cmp = ">" if op == "aboveAvg" else "<" + return Sql(f'SELECT d.pid FROM "{frozen}" d JOIN "{name}" m USING (bucket_id) ' + f"WHERE NOT isnan(m.mean) AND d.v {cmp} m.mean AND ?=1", [lease]) + direction = "DESC" if op in ("topN", "inTopPct") else "ASC" + ranked = (f"SELECT pid, v, row_number() OVER (PARTITION BY bucket ORDER BY v {direction}, " + "pid ASC) AS rn, count(*) OVER (PARTITION BY bucket) AS n " + f"FROM ({dom}) d") if op in ("topN", "bottomN"): bound = fe._rank_bound(leaf.get("value"), 1, 10000) if bound is None: return empty - direction = "DESC" if op == "topN" else "ASC" - return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {bound}", - params) - count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone() - count = int(count or 0) - if not count: - return empty - if op in ("inTopPct", "inBottomPct"): + predicate = f"rn <= {bound}" + elif op in ("inTopPct", "inBottomPct"): pct = fe._rank_bound(leaf.get("value"), 1, 100) if pct is None: return empty - take = min(count, max(1, math.ceil(count * pct / 100))) - direction = "DESC" if op == "inTopPct" else "ASC" - return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}", - params) - if op in ("aboveAvg", "belowAvg"): - mean = float(total) / count - cmp = ">" if op == "aboveAvg" else "<" - return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params) - if op in ("inQuartile", "inDecile"): + predicate = f"rn <= greatest(1, ceil(n * {pct} / 100.0))" + elif op in ("inQuartile", "inDecile"): buckets = 4 if op == "inQuartile" else 10 wanted = fe._rank_bound(leaf.get("value"), 1, buckets) - if wanted is None or count < buckets: + if wanted is None: return empty - return Sql(f"SELECT pid FROM (SELECT pid, row_number() OVER (ORDER BY v ASC, pid ASC) - 1" - f" AS idx FROM ({dom}) d) r WHERE least({buckets}, (idx * {buckets}) // {count}" - f" + 1) = {int(wanted)}", params) - return None + # The Python resolver clears every group if any finite-value group is too small. + ranked = f"SELECT *, min(n) OVER () AS smallest_group FROM ({ranked}) ranked_groups" + predicate = (f"smallest_group >= {buckets} AND " + f"least({buckets}, ((rn - 1) * {buckets}) // n + 1) = {wanted}") + else: + return empty + return Sql(f"SELECT pid FROM ({ranked}) r WHERE {predicate}", params) def _may_see_view(session, vid, store): """May this caller open that saved view? The same predicate the view rail asks. @@ -3404,7 +3683,12 @@ def resolved_fields(table_key, defn): fields = rel.namespaced_fields(mk_fields(), connection) break if fields is None: - if table_key not in _sources(): + if str(table_key).startswith("ut_amzn_"): + import amazon_relational as amazon + connected = table_key in amazon.SOURCE + else: + connected = table_key in _sources() + if not connected: raise err(404, "not_connected", "that database is not a connected grid") fields = (defn or {}).get("fields") or [] if not fields: @@ -3509,7 +3793,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= if mismatch: raise err(409, "cross_tenant_store", mismatch) - spec = _source_for(cur, table_key) + spec = _page_source_for(cur, table_key, st, session=session) if spec is None: from core import perm_scope rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter") @@ -3771,36 +4055,96 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= @contextmanager -def _materialised_read(table_key, session, query, related, ids=None): +def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None): """One compiler for native rows and their aggregate sibling, over the walled payload. The temporary connection contains only this request's already stripped, pinned rows and is closed before returning. It never registers data on the tenant's shared mirror connection. """ import duckdb + import itertools + from harness import datastore - if table_key == "customer_data": - from routes_customers import _payload + compiled = payload is None and table_key in {'customer_data', 'product_data'} + streamed = payload is None and table_key != 'ut_activity' + source_generation = None + if streamed: + import registry_source + source_generation = registry_source.generation(session, table_key) + if payload is not None: + pass + elif compiled: + import registry_source session.require(table_key) - payload = _payload(session, related=related) - elif table_key == "product_data": - from routes_products import products - session.require(table_key) - payload = products(session=session, related=related) + metadata = registry_source.canonical_metadata(session, table_key) + payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [], + 'limits': list(metadata['workspace'].get('limits') or [])} + elif streamed: + import routes_tables + from registry_source import MetadataStore, with_runtime, metadata_limits + metadata_actor = with_runtime(session, MetadataStore(session.runtime)) + metadata = routes_tables.ut_assembly(metadata_actor, table_key, with_rows=False, + fields_only=True, consume_corrections=False) + payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [], + 'limits': metadata_limits(session, table_key, metadata)} else: from routes_tables import table_rows payload = table_rows(table_key, session=session, related=related) - if ids is not None: - wanted = set(ids) - payload = {**payload, "rows": [row for row in payload["rows"] - if int(row["pid"]) in wanted]} fields = payload["fields"] - with duckdb.connect(":memory:") as cur: - cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)", - [json.dumps(payload["rows"], default=str)]) + if compiled: + source_cur, mismatch = _mirror_cur(session) + if mismatch: + raise err(409, 'cross_tenant_store', mismatch) + if source_cur is None: + raise err(503, 'canonical_source_unavailable', f'{table_key} requires its tenant mirror.') + connection = source_cur.cursor() + else: + connection = datastore.bounded_connection(duckdb.connect(":memory:")) + with connection as cur: + cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)") + wanted = set(ids) if ids is not None else None + if streamed: + import registry_source + def source_rows(): + from contextlib import closing + factory = registry_source.canonical_batches if compiled else registry_source.stored_batches + options = {'ids': ids, 'related': related, 'needed': needed} + if compiled: + parts = factory(session, cur, table_key, **options) + else: + parts = factory(session, table_key, **options) + with closing(parts) as parts: + for part in parts: + if compiled: + payload.setdefault('docs', {}).update(part.get('docs') or {}) + yield from part['rows'] + part['rows'].clear() + rows = source_rows() + else: + rows = payload['rows'] + source = ((index, row) for index, row in enumerate(rows) + if wanted is None or int(row["pid"]) in wanted) + source_count = 0 + try: + while True: + batch = list(itertools.islice(source, 2048)) + if not batch: + break + cur.execute("INSERT INTO pinned_rows SELECT unnest(?::BIGINT[]), unnest(?::JSON[])", + [[index for index, _ in batch], + [json.dumps(row, default=str) for _, row in batch]]) + source_count += len(batch) + batch.clear() + finally: + source.close() + close = getattr(rows, "close", None) + if callable(close): + close() bindings = {} for field in fields: key = str(field["key"]) + if streamed and needed is not None and key not in needed: + continue pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''") expr = f"json_extract_string(payload, '{pointer}')" numeric = field.get("type") in ("int", "currency", "pct", "float", "number") @@ -3833,7 +4177,10 @@ def _materialised_read(table_key, session, query, related, ids=None): where = f"({where}) AND {search.sql}" if where else search.sql params.extend(search.params) yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params, - "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload} + "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload, + "source_count": source_count, 'streamed': streamed} + if streamed: + registry_source.require_generation(session, table_key, source_generation) def _group_pin(cols, group_by, group_key): @@ -3901,7 +4248,30 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj, query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, "sorts": _json_arg(sorts, "sorts"), "search": search}) - with _materialised_read(table_key, session, query, related, ids=ids) as read: + payload = None + window_offset = offset + if table_key == 'ut_activity': + payload, offset, limit = _activity_window_payload(session, offset, limit, related) + if ids is not None: + wanted = set(ids) + payload = dict(payload, rows=[row for row in payload['rows'] if int(row['pid']) in wanted]) + if not (query['filters'] or query['sorts'] or query['search'] + or (isinstance(groupBy, str) and groupBy.strip())): + return {**payload, 'ok': True, 'total': len(payload['rows']), + 'totalUnfiltered': len(payload['rows']), 'offset': offset, 'limit': limit, + 'scope': {'table': table_key, 'readThrough': True}} + window_offset = 0 + needed = _leaf_cols(query['filters']) - _PSEUDO_COLS + needed.update(sort['colId'] for sort in query['sorts']) + if isinstance(groupBy, str) and groupBy.strip(): + needed.add(groupBy.strip()) + for leaf in _fe()._rank_leaves(query['filters'], {key: {} for key in needed}): + _mode, group_field = _fe()._rank_scope(leaf, None) + if group_field: + needed.add(group_field) + if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS: + needed = None + with _materialised_read(table_key, session, query, related, ids=ids, payload=payload, needed=needed) as read: spec = read["spec"] order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"]) where, params = read["where"], list(read["params"]) @@ -3909,19 +4279,73 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj, if gsql: where = f"({where}) AND {gsql}" if where else gsql params.extend(gparams) - win = datastore.window(select=f'{spec["id"]} AS pid', where=where, + win = datastore.window(select='_row_index', where=where, params=tuple(params), order_by=order or spec["id"], - offset=offset, limit=limit or None, cur=read["cur"], **read["frm"]) + offset=window_offset, limit=limit or None, cur=read["cur"], **read["frm"]) payload = read["payload"] - by_id = {int(row["pid"]): row for row in payload["rows"]} - rows = [by_id[int(row[0])] for row in win["rows"]] + indexes = [int(row[0]) for row in win['rows']] + value_sql = read['spec']['id'] if read['streamed'] else 'payload' + values = (read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows ' + 'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall() + if indexes else []) + if read['streamed']: + import registry_source + by_index = {int(index): int(value) for index, value in values} + page_ids = [by_index[index] for index in indexes] + if table_key in {'customer_data', 'product_data'}: + payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related) + else: + payload = registry_source.stored_page(session, table_key, page_ids, related=related) + rows = payload['rows'] + else: + by_index = {int(index): json.loads(value) for index, value in values} + rows = [by_index[index] for index in indexes] if win["clamped"]: read["limits"].append({"subject": "limit", "effect": "window_clamped", "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.", "recommendation": "Continue paging with offset. Every matching row remains available."}) + limits = list(payload.get('limits') or []) + limits.extend(report for report in read['limits'] if report not in limits) return {**payload, "ok": True, "rows": rows, "total": win["total"], - "totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"], - "limits": read["limits"], "scope": {"table": table_key, "readThrough": True}} + "totalUnfiltered": read["source_count"], + "offset": offset if table_key == 'ut_activity' else win["offset"], "limit": win["limit"], + "limits": limits, "scope": {"table": table_key, "readThrough": True}} + + +def _activity_window_payload(session, offset, limit, related): + """Walk native cursors to the requested offset without retaining skipped event values.""" + from core import activity_log + from harness import datastore + from routes_tables import table_rows + + offset = max(0, int(offset or 0)) + requested = int(limit or datastore.WINDOW_DEFAULT) + limit = min(max(requested, 1), datastore.WINDOW_MAX, activity_log.MAX_PAGE) + remaining, cursor = offset, None + for _ in range(16): + size = min(remaining, activity_log.MAX_PAGE) if remaining else limit + payload = table_rows('ut_activity', session=session, related=related, cursor=cursor, limit=size) + next_cursor = payload.get('next_cursor') + if next_cursor is not None and next_cursor == cursor: + raise err(503, 'activity_cursor_stalled', 'Reload Activity to continue reading events.') + if not remaining: + break + remaining -= len(payload['rows']) + if next_cursor is None: + payload = dict(payload, rows=[]) + break + cursor = next_cursor + else: + raise HTTPException(status_code=409, detail={'error': { + 'code': 'activity_cursor_required', + 'message': 'Continue this deep Activity window through the native cursor route.'}, + 'next_cursor': cursor}) + if requested != limit: + payload = dict(payload, limits=[*(payload.get('limits') or []), { + 'subject': 'limit', 'effect': 'window_clamped', + 'cause': f'One response carries at most {limit} Activity rows.', + 'recommendation': 'Continue with the returned Activity cursor or the next offset.'}]) + return payload, offset, limit @router.get("/odoo-tables/{table_key}/rows") @@ -3950,7 +4374,7 @@ def odoo_table_rows(table_key: str, that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in `limits` with its cause and a recommendation — R6's second sentence — and never silently dropped, because an ignored condition widens. - + ⭐⭐ `related` (W51-T05 / C6) IS A **JSON QUERY STRING**, copying the `filters` precedent one argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this window to the rows linked to that ONE record, ahead of `filters`, and `total` / @@ -3976,13 +4400,10 @@ def odoo_table_rows(table_key: str, "limit, filters, sorts and search in the query string.") ids = _record_ids(body) - # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check - # asks it instead of reading its own definitions projection (one store round trip fewer on - # every Amazon page). Not reordered past it: the check may register this grid's binding, - # which is what `_materialised_table` below then reads. + # Admission resolves this Amazon database without publishing a process-wide binding. page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None - require_amazon_mirror(table_key, session, st=page[0] if page else None) - if _materialised_table(table_key): + admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) + if (admitted is None if page is not None else _materialised_table(table_key)): return _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related, groupBy, groupKey, ids=ids) @@ -4016,7 +4437,8 @@ def odoo_table_rows(table_key: str, select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where, params=tuple(params), order_by=order, - offset=offset, limit=(limit or None), cur=cur, page_first=True, **frm) + offset=offset, limit=(limit or None), cur=cur, page_first=True, + page_key=spec.get("page_key"), **frm) if win["clamped"]: limits.append({ "subject": "limit", "effect": "window_clamped", @@ -4058,12 +4480,15 @@ def odoo_table_rows(table_key: str, if not _may_open(session, target, st=page_st): return [] return _relation_rows(session, target, list(matches), matches=matches, - st=page_st, rev=page_rev) + st=page_st, rev=page_rev, + stream=True, + materialised=_materialised_table(target, cur=cur, st=page_st)) # Fields are the current reader's contract; target rows cross the same wall # as /related pins. No unscoped SQL subquery can leak a hidden target's id. - amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets) - overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src} + amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets, + max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal) + overlays = {} # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. # @@ -4078,20 +4503,16 @@ def odoo_table_rows(table_key: str, # the wall and the predicate, so the argument it demands is the list we just fetched. # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already # answered "may this session open this surface". - # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request - # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two. - # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row). + # Definitions use the projected reader. Values use coherent page-bound snapshots. shared_defs = _so().fields(table_key, st=page_st) if shared_defs: declared = {field["key"]: field for field in fields} declared.update({key: dict(field, source="overlay") for key, field in shared_defs.items()}) fields = list(declared.values()) - for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], - st=page_st).items(): + from registry_source import page_cells + for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items(): overlays.setdefault(pid, {}).update(cells) if_keys = {field["key"] for field in fields if field.get("type") == "if"} - for row in rows_src: - overlays[str(row["pid"])].update({key: row.get(key, "") for key in if_keys}) # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula # in an overlay column that reads a hidden mirror column was outside its reach - and a formula @@ -4108,11 +4529,16 @@ def odoo_table_rows(table_key: str, rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src] overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden} for pid, cells in overlays.items()} - rows = aios_grid.rows_from_pool(rows_src, fields, overlays) + rows = [] + for row in rows_src: + # Physical payloads stay separate even when business identities share a pid. + cells = {key: value for key, value in row.items() if key != "pid"} + cells.update(overlays.get(str(row["pid"]), {})) + cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden}) + rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells})) - unfiltered = win["total"] if where == base_where else datastore.window( - select="1", where=base_where, params=tuple(wall_params), order_by=spec["id"], - limit=1, cur=cur, **frm)["total"] + unfiltered = win["total"] if where == base_where else datastore.count_rows( + where=base_where, params=tuple(wall_params), cur=cur, **frm) from core import connector_spec rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime) @@ -4135,7 +4561,7 @@ def odoo_table_rows(table_key: str, # ══════════════════════════════════════════════════════════ #: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers -#: that omit groupLimit receive the complete grouping; card modes page columns explicitly. +#: that omit groupLimit receive one page and exact traversal metadata. MAX_GROUPS = 1_000 #: One request asks for at most this many summaries. Each is one more aggregate expression in a @@ -4143,10 +4569,52 @@ MAX_GROUPS = 1_000 MAX_AGGREGATES = 200 +LINK_VALUE_MAX_BYTES = 1_048_576 + + +def _link_capacity_refusal(): + """Name the complete-value capacity and the action that reduces it.""" + return err(503, 'relation_value_capacity', + f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. ' + 'Reduce the page size or open the related database with filters.') + + +def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None): + """Reduce scoped target identities in SQL; refuse bytes before string aggregation.""" + read = read if read is not None else _compiled_read(target, session, st=st) + spec, cur = read['spec'], read['cur'] + bindings = dict(spec['cols'], pid=(spec['id'], int)) + if on not in bindings: + raise err(409, 'relation_target_unavailable', 'A required relation column is unavailable.') + expr = f'LOWER(TRIM(CAST({bindings[on][0]} AS VARCHAR)))' + wanted = _membership_relation(cur, values, normalize=str, sql_type='VARCHAR') + lease = _RelationLease(cur) + name = lease.table('_link_values') + source = spec.get('from_sql') or spec['table'] + where = read['where'] or '1=1' + try: + cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT DISTINCT {expr} AS k, ' + f'{spec["id"]} AS pid FROM {source} WHERE ({where}) AND {expr} IN ({wanted.sql})', + [*read['params'], *wanted.params]) + population = f'(SELECT k, min(pid) AS pid FROM "{name}" GROUP BY k)' if single else f'"{name}"' + required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT ' + f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0]) + if required > remaining: + raise _link_capacity_refusal() + # The byte check bounds both the aggregate and its Python result. IDs are never a + # Python population; DISTINCT and numeric ordering preserve the existing wire value. + cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) " + f'FROM {population} GROUP BY k').fetchall()) + return cells, required + finally: + lease.close() + + def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...): - """Resolve each page's Odoo link cells through the scoped relationship reader.""" + """Resolve page link cells through scoped SQL and a complete-value byte budget.""" from routes_grid import _may_open declared = {field["key"] for field in fields} + edges, requested = [], {} for field in fields: bag = field.get("link") or {} if field.get("type") != "link" or not bag.get("on") or not bag.get("from"): @@ -4154,19 +4622,36 @@ def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...): target, on, source = bag["table"], bag["on"], bag["from"] if _rel().base_table_key(target) not in _rel().READ_THROUGH_KEYS: continue - matches = list({row.get(source) for row in rows if row.get(source) not in (None, "")}) - targets = [] - if source in declared and matches and _may_open(session, target, st=st): - targets = _relation_rows(session, target, [on], match=(on, matches), st=st, rev=rev, - normalize_match=True) - index = {} - for row in targets: - index.setdefault(str(row.get(on) or "").strip().lower(), []).append(str(row["pid"])) + edges.append((field, target, on, source, bag)) + if source in declared: + values = {str(row[source]).strip().lower() for row in rows if row.get(source) not in (None, "")} + if values: + requested.setdefault(target, {}).setdefault(on, set()).update(values) + remaining = LINK_VALUE_MAX_BYTES + target_reads, resolved = {}, {} + for field, target, on, source, bag in edges: + if source not in declared: + for row in rows: + row[field["key"]] = "" + continue + matches = requested.get(target, {}).get(on, ()) + if not matches or not _may_open(session, target, st=st): + cells = {} + else: + if target not in target_reads: + target_reads[target] = _compiled_read(target, session, st=st) + cache_key = (target, on, bool(bag.get('single'))) + if cache_key not in resolved: + resolved[cache_key], _required = _sql_link_cells(session, target, on, matches, st=st, + single=cache_key[2], remaining=remaining, read=target_reads[target]) + cells = resolved[cache_key] for row in rows: - ids = sorted(set(index.get(str(row.get(source) or "").strip().lower(), [])), key=int) - if bag.get("single"): - ids = ids[:1] - row[field["key"]] = ",".join(ids[:_ut().LINK_MAX_IDS]) + value = cells.get(str(row.get(source) or '').strip().lower(), '') + # Count repeated cells as well: several page rows may share one target key. + remaining -= len(value) + (2 if value else 0) + if remaining < 0: + raise _link_capacity_refusal() + row[field['key']] = value def _agg_number(value): @@ -4358,15 +4843,10 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li f"THEN 1 ELSE 0 END"), where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0) projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj) - if group_limit is not None: - rows = datastore.window( - select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC", - group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"] - else: - source = frm.get("from_sql") or frm["table"] - tail = f" WHERE {where}" if where else "" - rows = cur.execute(f"SELECT {projection} FROM {source}{tail} GROUP BY {gexpr} " - "ORDER BY 2 DESC, 1 ASC", tuple(params)).fetchall() + group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS) + rows = datastore.window( + select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC", + group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"] groups = [] for r in rows: vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None) @@ -4375,7 +4855,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li "count": int(r[1] or 0), "aggregates": vals}) return {"total_rows": total_rows, "aggregates": answers, "groups": groups, "total_groups": distinct, "group_offset": group_offset, - "group_limit": group_limit, "limits": limits} + "group_limit": group_limit, "has_more": group_offset + len(groups) < distinct, + "limits": limits} @router.post("/odoo-tables/{table_key}/aggregates") @@ -4424,13 +4905,11 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), raise err(400, "bad_selection", "groupOffset must be a nonnegative integer") if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS): raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}") - if group_offset and group_limit is None: - raise err(400, "bad_selection", "Send groupLimit when paging groups") page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None - require_amazon_mirror(table_key, session, st=page[0] if page else None) + admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) - if _materialised_table(table_key): + if (admitted is None if page is not None else _materialised_table(table_key)): with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read: import odoo_default_scope if (odoo_default_scope.applies(table_key) @@ -4481,7 +4960,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), "limits": out["limits"], "scope": {"table": table_key, "readThrough": True}} if out["groups"] is not None: body_out["groups"] = out["groups"] - body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit")}) + body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit", "has_more")}) return body_out @@ -4611,8 +5090,7 @@ def population(table_key, cur=None, rt=None): if spec is None: return None frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"], - limit=1, cur=cur, **frm)["total"] + return datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm) def whole_pool(table_key, cur=None, rt=None): diff --git a/api/routes_platform_admin.py b/api/routes_platform_admin.py index 72c80b352971965dbbba7bc9b771a507304b0bf2..2271a61efaf35eefe5cf233f6fd503f54761cba7 100644 --- a/api/routes_platform_admin.py +++ b/api/routes_platform_admin.py @@ -47,7 +47,7 @@ import core.do_usage as do_usage import core.store as store import operator_control import voice_control_plane as voice -from deps import Session, err, require_session, users +from deps import Session, err, observed_last_active, require_session, users @asynccontextmanager async def _neon_lifespan(app): @@ -329,7 +329,7 @@ def _user_rows(tenant=None): `routes_admin._view`'s discipline: one function that can leak, and it does not. R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record, - rendered as "never" rather than as a fabricated date. + rendered as "never" when neither a persisted stamp nor a process-local observation exists. """ try: reg = users.registry() or {} @@ -351,7 +351,7 @@ def _user_rows(tenant=None): "role": rec.get("role", "user"), "active": bool(rec.get("active", True)), "lastLogin": rec.get("last_login") or "", - "lastActive": rec.get("last_active") or "", + "lastActive": observed_last_active(slug, uname, rec.get("last_active")) or "", "platformAdmin": rec.get("platform_admin") is True, }) return rows @@ -699,14 +699,14 @@ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)): """Every account on the platform, or one tenant's — the drill behind the Users count. R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by - `routes_auth.login`, `lastActive` by `deps.require_session` (throttled to once an hour per - account per process). Absent means never seen, and the pane renders it as "never". + `routes_auth.login`. `lastActive` includes bounded process-local read observations; persisted + activity comes from login and throttled admitted mutations. Absent means no recorded activity. """ rows = _user_rows(tenant) return {"users": rows, "count": len(rows), - "stampsNote": ("Login and activity stamps started with this release; accounts that " - "have not signed in since show no date rather than an invented one. " - "Activity is recorded at most once an hour per account.")} + "stampsNote": ("Activity dates include recent visits observed by this server. " + "Visits are tracked temporarily. Saved dates come from sign-ins " + "and changes, and can be older after a server restart.")} @router.get("/databases") diff --git a/api/routes_products.py b/api/routes_products.py index 13579a55585ed5f3a2219c7a7fbde7ef7a0d1758..ac57cd091650cdbed7bfe83671a7b19638a60a36 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -53,6 +53,9 @@ def _pool_for(rt, team_id): # fail-closed seam rather than widening it by accident. if rt.key != "royal-imports": return [] + bounded = getattr(rt, '_canonical_product_rows', None) + if bounded is not None: + return bounded import modules.product_data as pd from core import user_tables @@ -128,13 +131,14 @@ def scoped_pool(session: Session, *, with_rows: bool = True): except Exception as e: raise err(503, "pool_unavailable", f"the product catalogue could not be built; {str(e)[:160]}") - # Image is a tenant-wide created field. Seed/repair its definition before - # the field wall runs so every reader receives the same metadata and the - # Leadership account owns the permission editor from the first request. + # Project detached definitions without seeding fields or replacing caller grants. + # Keep the original runtime for row/value hydration and authoritative mutations. import modules.product_data as pd + from registry_source import MetadataStore + metadata = MetadataStore(session.runtime) if with_rows: - pd.image_field_definition(st=session.runtime) - fields_base = pd_fields(consolidated=team_id is None, st=session.runtime) + pd.image_field_definition(st=metadata) + fields_base = pd_fields(consolidated=team_id is None, st=metadata) # The SAME wall the customer assembly applies, in the same order: rows first (before pids # are taken, so an out-of-filter row never enters allowed_pids), then the field closure, # then the values stripped from the rows as well as the field list. @@ -217,7 +221,7 @@ def product_measures(session): return [] -def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): +def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer, *, cur=None): """`{pid: {field key: value}}` for every measure column on this grid. ⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's @@ -236,6 +240,11 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries. """ import aios_grid + if cur is None: + from harness.canonical_source import active + source = active() + if source is not None: + cur = source.cur mfields = aios_grid.measure_fields_of(fields) if not mfields or not offer: @@ -263,7 +272,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): out = {} for (dfrom, dto), group in by_window.items(): keys = tuple(sorted({f["measure"]["key"] for f in group})) - mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys) + mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys, + tuple(sorted(codes.values(), key=str))) if mkey not in memo: try: memo[mkey] = sem.entity_measure_values( @@ -272,7 +282,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): # here. The service filter exists to stop Delivery Charges polluting a SKU # RANKING; on a per-row column the row IS the service product, and excluding # it would print a FALSE 0 — which under C1's rule asserts "sold nothing". - exclude_services=False, offer=offer) + exclude_services=False, offer=offer, group_keys=list(codes.values()), + **({'cur': cur} if cur is not None else {})) except Exception as e: # noqa: BLE001 _measure_err("product-measure-column", e) # Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure @@ -441,8 +452,9 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_ import aios_grid import core.perm_scope as perm_scope import modules.product_data as pd + from registry_source import MetadataStore pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows) - shared_defs = pd.shared_fields(st=session.runtime) + shared_defs = pd.shared_fields(st=MetadataStore(session.runtime)) ctx = grid_events.EventCtx( uname=session.uname, allowed_pids=pids, fields=[], @@ -841,6 +853,12 @@ def pd_fields(consolidated=True, st=None): # (D-400); `platform/verify_product_pool.py::section_product_field_contract` does. if st is not None: image = pd.image_field_definition(st=st) + # An explicit grant still governs Image when its stored definition is missing + # or predates the grant marker. Let the existing permission closure decide access. + from core import shares + record = shares.grants("field", shares.field_oid(MODULE, PRODUCT_IMAGE_KEY), st=st) + if record.get("owner") or record.get("entries"): + image = dict(image, granted=True) else: # Contract checks and offline tooling have no tenant runtime. Keep a # truthful local projection with the same created-field semantics. diff --git a/api/routes_sheet.py b/api/routes_sheet.py index b89cd0d6ee13e2a51f2ca8268fc9f2fa25c36b18..f173fb848cc694ad95c4ae7dd26d267325763137 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -259,7 +259,7 @@ def _mirror(session: Session): if connector_defn is None: native = _native_table_definition(session, table_key) if native is not None: - cur, spec, cols = _native_table_mirror(session, table_key) + cur, spec, cols = _native_table_mirror(session, table_key, needed=needed) return cur, {**spec, "declared_fields": native.get("fields") or []}, cols raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.") fields = list(connector_defn.get("fields") or []) @@ -301,12 +301,12 @@ def _mirror(session: Session): stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {} # The store keeps records as {row id: row}. Iterating that dict as a list yields only # the ids, which materialises an EMPTY table and answers every cell blank. - rows = [row for row in (stored.values() if isinstance(stored, dict) else stored) - if isinstance(row, dict)] + rows = (row for row in (stored.values() if isinstance(stored, dict) else stored) + if isinstance(row, dict)) fields = rel.shipment_fields() hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime) fields = [field for field in fields if field["key"] not in hidden] - if not rows: + if not stored: # S75-T59: a full pass stores a rowless definition that reads through the mirror. spec = rot._source_for(cur, table_key) if spec is None: @@ -419,23 +419,40 @@ def _native_table_definition(session, table_key): return defn -def _native_table_mirror(session, table_key): +def _native_table_mirror(session, table_key, *, needed=None): """Reuse Data's scoped grid/spec reader, with a cursor owned by this Sheet request. The payload has already applied permanent predicates, allowed IDs and field/BU walls. Authored Sheet filters still compile below, against only the visible column vocabulary. """ - import routes_odoo_tables as rot + import duckdb + from harness import datastore + import registry_source memo = getattr(session, "_sheet_read_memo", None) if not isinstance(memo, dict) or "sources" not in memo: raise RuntimeError("A native Sheet source needs a request scope.") slot = ("native_source", table_key) - if slot not in memo: - read = memo["sources"].enter_context(rot._materialised_read( - table_key, session, {"filters": [], "filterConj": "and", "search": ""}, None)) - spec = {**read["spec"], "fields": read["payload"]["fields"], - "scope_applied": True, "limits": list(read["limits"])} - memo[slot] = (read["cur"], spec, read["cols"]) + if table_key == 'ut_activity': + if slot not in memo: + import routes_odoo_tables as rot + read = memo['sources'].enter_context(rot._materialised_read(table_key, session, + {'filters': [], 'filterConj': 'and', 'search': ''}, None)) + spec = dict(read['spec'], fields=read['payload']['fields'], + scope_applied=True, limits=list(read['limits'])) + memo[slot] = (read['cur'], spec, read['cols']) + return memo[slot] + shape = ('native_columns', table_key) + built = memo.get(shape) + if slot not in memo or (built is not None and (needed is None or not set(needed) <= built)): + wanted = None if needed is None else set(needed) | (built or set()) + owned = getattr(session, '_sheet_drill_cursor', None) + if owned is None: + cur = memo['sources'].enter_context(datastore.bounded_connection(duckdb.connect(':memory:'))) + session._sheet_drill_cursor = (cur, None) + else: + cur = owned[0] + memo[slot] = registry_source.stored_grid(session, cur, table_key, needed=wanted) + memo[shape] = wanted return memo[slot] @@ -535,10 +552,18 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr with_rows=False, fields_only=not with_rows, needed=needed) return memo[slot] if table_key == "product_data": + if not with_rows: + from registry_source import canonical_metadata + memo[slot] = canonical_metadata(session, table_key) + return memo[slot] from routes_products import product_assembly memo[slot] = product_assembly(session, scope="product", consume_corrections=False, with_rows=with_rows, needed=needed) return memo[slot] + if not with_rows: + from registry_source import canonical_metadata + memo[slot] = canonical_metadata(session, table_key) + return memo[slot] from routes_customers import grid_assembly memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False, with_rows=with_rows, needed=needed) @@ -565,6 +590,9 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr def _sheet_permission_reader(session): """Lend a captured schema/grant snapshot only inside the current Sheet request.""" memo = getattr(session, "_sheet_read_memo", None) + if (isinstance(memo, dict) and 'permission_reader' not in memo + and callable(getattr(session.runtime, 'get_projection', None))): + _capture_sheet_permissions(session) return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime @@ -798,6 +826,22 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits): spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0]) total_rows = int(cur.execute(count_sql, params).fetchone()[0]) tail = count_sql[len("SELECT COUNT(*)"):] + # Count both dense axes before fetching any grouped records or allocating the matrix. + def axis_count(expressions): + if not expressions: + return 1 + axis = "SELECT " + ", ".join(expressions) + tail + axis += " GROUP BY " + ", ".join(str(i + 1) for i in range(len(expressions))) + return int(cur.execute(f"SELECT count(*) FROM ({axis}) axis", params).fetchone()[0]) + + row_count = axis_count(groups[:len(row_fields)]) + column_count = axis_count(groups[len(row_fields):]) + cells_needed = row_count * column_count * len(pivot["values"]) + if cells_needed > _PIVOT_MAX_CELLS: + raise err(409, "pivot_too_large", + f"This pivot needs {row_count:,} rows by {column_count:,} columns, " + f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. " + "Move a field to Filters, or use a field with fewer distinct values.") sql = "SELECT " + ", ".join([*groups, *expressions]) + tail if groups: positions = ", ".join(str(index + 1) for index in range(len(groups))) @@ -816,14 +860,6 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits): row_keys.setdefault(row_key, len(row_keys)) column_keys.setdefault(column_key, len(column_keys)) values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]] - # The matrix below is dense, so its size is rows x columns x values whatever the data holds. - # Standing rule 1: a bound that cannot be lifted is reported with its cause and its fix. - cells_needed = len(row_keys) * len(column_keys) * len(pivot["values"]) - if cells_needed > _PIVOT_MAX_CELLS: - raise err(409, "pivot_too_large", - f"This pivot needs {len(row_keys):,} rows by {len(column_keys):,} columns, " - f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. " - "Move a field to Filters, or use a field with fewer distinct values.") # SQL orders the entire tuple, but a column tuple may first occur in a later row group. # Sort each axis independently using its JSON spelling for deterministic mixed/null keys. ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False)) @@ -1014,6 +1050,32 @@ def _linked_filter_fields(database, session): return {"fields": offered, "databases": databases, "unavailable": unavailable} +def _registry_rows(source, fields, overlays, derived): + """Project borrowed scoped inputs through the grid's converter in bounded batches.""" + import itertools + import aios_grid + source = iter(source) + while True: + batch = list(itertools.islice(source, 2048)) + if not batch: + return + try: + if fields: + projected = aios_grid.rows_from_pool(batch, fields, overlays, derived=derived) + # Release source rows before yielding their independent projections. + batch.clear() + try: + yield from projected + finally: + projected.clear() + del projected + else: + yield from ({"pid": row["pid"]} for row in batch) + finally: + batch.clear() + del batch + + def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None): """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period. @@ -1046,7 +1108,9 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede column just because a filter named it would invent an empty column for a name this database does not have, and the wall would then compile against nothing and quietly stop restricting. """ - import aios_grid + if table_key in {'customer_data', 'product_data'}: + from registry_source import canonical_grid + return canonical_grid(session, cur, table_key, needed=needed, metric_window=metric_window) import odoo_relational as rel import routes_odoo_tables as rot from modules import sheet as sheet_mod @@ -1058,8 +1122,8 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window) # Authorization has already scoped rows using the full wall vocabulary. Restrict only # cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default. - rows = (aios_grid.rows_from_pool(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"), - derived=derived) if kept else [{"pid": row["pid"]} for row in g["rows_src"]]) + rows = _registry_rows(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"), derived) + join_agent = False if table_key == "customer_data" and (needed is None or "agent_id" in needed): from core import perm_scope agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"} @@ -1068,8 +1132,7 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede # Join only to already-scoped customer rows. pid is the actual res.partner id. have = rel.columns(cur, "res_partner") if "agent_id" in have: - agents = dict(cur.execute("SELECT id, agent_id FROM res_partner").fetchall()) - rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows] + join_agent = True fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field] elif needed is not None and "agent_id" in needed: raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.") @@ -1086,35 +1149,14 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede spec_fields = sheet_mod.mirror_fields(kept) # The related customer side can contain thousands of permitted rows. Insert its column # arrays in one call; per-row execute round trips delayed the first drill page by seconds. - spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows, bulk=True) - # ⛔⛔ THE MATERIALISED TABLE IS GIVEN THE WINDOW'S OWN NAME, AND WITHOUT THIS THE MEMO ABOVE - # SERVES ONE WINDOW'S ROWS UNDER ANOTHER WINDOW'S ANSWER (found by W53-T14, live). - # `odoo_relational.registry_grid_spec` names its temp table `_registry_grid_` and - # `CREATE OR REPLACE`s it, so the name carries the DATABASE and not the PERIOD, while - # `_mirror`'s memo is keyed by `(table_key, window)`. Resolve one customer_data cell at no - # period and a second at `{month, 2026-08}` inside ONE request and the second build replaces - # the first one's table underneath the spec the memo is still holding; a third cell reading - # the first slot then runs its SQL against August's rows. - # ⚠ MEASURED, and it is silent: `avg` over `measure_sales_the_last_12_months_6hlkf` answered - # **138.3942281513749** in a batch that had already resolved that month (508,322.0 / 3,673) - # and **1710.679825755513** on its own, which is the all-time number the preview door and the - # grid both give. No refusal, no limits row, a plausible number under the wrong label. - # ⛔ IT IS FIXED HERE RATHER THAN IN `registry_grid_spec` BECAUSE THAT FILE BELONGS TO ANOTHER - # LANE THIS WAVE. The copy is one `CREATE TABLE AS SELECT` over rows already in memory, next - # to an assembly that costs 20 to 130 seconds, so it is not a cost worth conditioning on. - # ⚠ ALWAYS, INCLUDING THE `None` WINDOW: the unqualified name is REPLACED by every later - # build, so leaving the no-period case pointing at it is exactly the case measured above. - # ⛔⛔ W54-T36 — THE COLUMN SET IS PART OF THAT NAME FOR THE SAME REASON THE WINDOW IS. Two - # cells on one database at one window may now be materialised over DIFFERENT columns, and a - # narrower later build would otherwise replace a wider earlier one under a spec the memo is - # still holding, so a query would address a column its own table no longer has. slot = hashlib.sha1( json.dumps([metric_window, None if needed is None else sorted(needed)], sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12] - at_window = f'{spec["table"]}__at_{slot}' - cur.execute(f'CREATE OR REPLACE TEMP TABLE "{at_window}" AS ' - f'SELECT * FROM "{spec["table"]}"') - spec = {**spec, "table": at_window} + spec = rel.registry_grid_spec(cur, f"{table_key}__at_{slot}", spec_fields, rows, bulk=True) + if join_agent: + cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id = partner.agent_id ' + 'FROM res_partner AS partner WHERE grid.pid = partner.id') + # Window and column identities name the table before insertion, preserving every memo slot. # ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal` # has to name WHY a column cannot be totalled, and the normalised copy has already thrown that # away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds @@ -1123,6 +1165,22 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede return cur, spec, rot._filter_columns(spec, spec_fields) +class _PeriodDerived: + """Read borrowed metric maps per row without copying their full population.""" + + def __init__(self, base, cleared): + self.base, self.cleared, self.columns = base, cleared, [] + + def get(self, pid, default=None): + cells = {key: value for key, value in (self.base.get(pid) or {}).items() + if key not in self.cleared} + for key, column in self.columns: + values = column.get(pid) or {} + if key in values: + cells[key] = values[key] + return cells or default + + def _metric_derived(session: Session, table_key: str, g, metric_window): """`(derived, bounded_keys, unbounded)` for one assembly at the cell's period. @@ -1156,10 +1214,7 @@ def _metric_derived(session: Session, table_key: str, g, metric_window): seen.update(cells or ()) live = [f for f in mfields if f.get("key") in seen] - out = {pid: dict(cells or {}) for pid, cells in derived.items()} - for f in mfields: # every Metric column starts blank at a new window - for cells in out.values(): - cells.pop(f.get("key"), None) + out = _PeriodDerived(derived, {f.get("key") for f in mfields}) bounded, unbounded = [], {} for f in live: @@ -1170,12 +1225,9 @@ def _metric_derived(session: Session, table_key: str, g, metric_window): cells = _measure_cells_at(session, table_key, g, rewritten) except Exception: # noqa: BLE001 cells = {} - got = False - for pid, values in (cells or {}).items(): - if key in values: - out.setdefault(pid, {})[key] = values[key] - got = True + got = any(key in values for values in (cells or {}).values()) if got: + out.columns.append((key, cells)) bounded.append(key) else: unbounded[key] = sheet_mod.metric_window_limit( @@ -1209,10 +1261,12 @@ def _measure_cells_at(session: Session, table_key: str, g, field): import routes_products as rp return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"), g.get("today"), "sheet-period", {}, - rp.product_measures(session)) + rp.product_measures(session), + cur=getattr(session.runtime, '_canonical_source_cursor', None)) from core import measure_resolve return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(), - g.get("today"), "sheet-period", {}) + g.get("today"), "sheet-period", {}, + cur=getattr(session.runtime, '_canonical_source_cursor', None)) def _offered_view(session: Session, database: str, view_id: str): @@ -1570,8 +1624,21 @@ def activity_read_policy(session: Session, object): source_identity = identity[:3] + (databases,) sources = memo.setdefault("activity_sources", {}) if isinstance(memo, dict) else {} admitted = sources.setdefault(source_identity, {}) + policies = memo.setdefault("activity_bindings", {}) if isinstance(memo, dict) else {} + bindings = policies.setdefault(source_identity, {}) def cell_allowed(cell): + if not isinstance(cell, dict) or cell.get("kind") != "bound": + return resolve_cell(cell) + signature = json.dumps(cell, sort_keys=True, separators=(",", ":"), allow_nan=False) + if signature not in bindings: + result = resolve_cell(cell) + if len(bindings) >= 256: + bindings.clear() + bindings[signature] = result + return bindings[signature] + + def resolve_cell(cell): if cell is None: return True if not isinstance(cell, dict): @@ -4785,7 +4852,7 @@ def _metric_column_aggregate(session, database, aggregate, window): # assembly for another database: that would create a field on the wrong database. if database not in ("customer_data", "product_data"): refuse("measure_not_offered") - g = _registry_assembly(session, database) + g = _registry_assembly(session, database, with_rows=False) # Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue # with the served offer too: product has a narrower entity catalogue, and revoked Metric # capability makes the account's offer empty. A browser key can never widen either offer. @@ -4845,7 +4912,7 @@ def _metric_column_aggregate(session, database, aggregate, window): if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys or raw["key"] in {f.get("key") for f in g["fields"]}): _forget_sheet_database(session, database) - found = existing(_registry_assembly(session, database).get("fields")) + found = existing(_registry_assembly(session, database, with_rows=False).get("fields")) if found is not None: return column(found) refuse("metric_column_conflict", label) @@ -4897,6 +4964,7 @@ def _sheet_request_scope(session): previous = getattr(session, "_sheet_read_memo", None) previous_assemblies = getattr(session, "_sheet_assemblies", None) previous_context = getattr(session, "_sheet_scope_context", None) + previous_cursor = getattr(session, '_sheet_drill_cursor', None) sources = contextlib.ExitStack() memo = {"today": _sheet_today(session), "sources": sources, "stats": {"engines": 0, "resolves": 0, "input_hits": 0}} @@ -4916,6 +4984,7 @@ def _sheet_request_scope(session): session._sheet_read_memo = previous session._sheet_assemblies = previous_assemblies session._sheet_scope_context = previous_context + session._sheet_drill_cursor = previous_cursor def _request_scoped(route): @@ -5681,6 +5750,16 @@ def _binding_related_entry(session, database, name): return entry +def _remaining_row_wall(spec, nodes, conj): + """Trust a scoped source only when its declared wall vocabulary covers every operand.""" + if not spec.get("scope_applied"): + return nodes, conj + from modules import sheet as sheet_mod + if "scope_fields" in spec and sheet_mod.filter_columns_named(nodes) - set(spec["scope_fields"]): + raise err(409, "scope_not_applicable", "Your row restriction cannot be applied to this database. Ask an administrator to review it.") + return [], "and" + + def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None): """Materialise the far side on the near cursor, then hand metadata to the compiler.""" from core import perm_scope, relations @@ -5727,8 +5806,11 @@ def _binding_related_query(session, database, entry, cur, spec, columns, *, fiel raise err(409, "relation_cursor_mismatch", f"Relationship '{name}' cannot be queried on the same live connection as " f"'{database}'. Retry after both databases share a connection.") + wall_nodes, wall_conj = _remaining_row_wall(rspec, wall_nodes, wall_conj) hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [], st=session.runtime) + # The far-side wall compiles against the full map: a wall column is often hidden from its reader. + wall_cols = dict(rcols) rcols = {key: value for key, value in rcols.items() if key not in hidden} declared = {field["key"]: field for field in rspec.get("fields") or []} @@ -5750,7 +5832,7 @@ def _binding_related_query(session, database, entry, cur, spec, columns, *, fiel raise unknown_field(key) return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key, "spec": rspec, "columns": rcols, "where": entry.get("where"), - "wall_nodes": wall_nodes, "wall_conj": wall_conj, + "wall_nodes": wall_nodes, "wall_conj": wall_conj, "wall_columns": wall_cols, "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]} @@ -5869,11 +5951,12 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed) cols = dict(cols) if spec.get("scope_applied"): - wall_nodes, wall_conj = [], "and" + wall_nodes, wall_conj = _remaining_row_wall(spec, wall_nodes, wall_conj) unreadable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols) - sheet_mod.related_filter_ids(scope_nodes)) field = binding["aggregate"].get("field") - if field and field not in cols: + known_fields = {f.get("key") for f in spec.get("fields", [])} + if field and field not in cols and ("declared_fields" in spec or field in known_fields): unreadable.add(field) if unreadable: declared = {field.get("key") for field in spec.get("declared_fields", [])} @@ -6307,7 +6390,7 @@ def view_rows(workbook_id: str, body: object = Body(default=None), raise _refuse(exc) _require_reader(session, book) database = _require_database(session, binding["database"]) - fields = list(_registry_assembly(session, database).get("fields") or []) + fields = list(_registry_assembly(session, database, with_rows=False).get("fields") or []) hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime) visible = {field["key"]: field for field in fields if field.get("key") not in hidden} requested = binding["fields"] @@ -6399,15 +6482,36 @@ _DRILL_MIRROR_LOCK = threading.RLock() _DRILL_MIRROR_MAX = 6 +@contextlib.contextmanager +def _drill_mirror_lock(): + """Include shared-cursor contention in the caller's absolute request budget.""" + from core import request_timing + timeout = request_timing.remaining_budget() + acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None + else _DRILL_MIRROR_LOCK.acquire(timeout=timeout)) + if not acquired: + request_timing.remaining_budget() + budget = request_timing.current_budget() + error = budget[1] if budget is not None else request_timing.RequestDeadlineExceeded + raise error('drill mirror lock deadline exceeded') + try: + request_timing.remaining_budget() + yield + finally: + _DRILL_MIRROR_LOCK.release() + + def _drill_tokens(session, databases): """Unknown freshness disables reuse. No workspace documents are retained here.""" from core import sheet_binding + from core.request_timing import raise_if_resource_error tokens = {} for db in sorted(databases): try: import harness.mirror_tokens # noqa: F401 tokens[db] = sheet_binding.database_change_token(_store(session), db) except Exception as exc: + raise_if_resource_error(exc) _token_unread(session, db, exc) tokens[db] = None return tokens @@ -6417,6 +6521,7 @@ def _drill_tokens(session, databases): def _drill_mirror(session, database, window, needed, databases): """A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction.""" from core import sheet_binding, perm_scope + from core.request_timing import raise_if_resource_error import datetime if _native_table_definition(session, database) is not None: @@ -6433,7 +6538,8 @@ def _drill_mirror(session, database, window, needed, databases): if callable(stamp): try: wall = stamp(session.tenant, session.uname, database, user=session.user) - except Exception: + except Exception as exc: + raise_if_resource_error(exc) pass if not wall: token = None @@ -6446,7 +6552,7 @@ def _drill_mirror(session, database, window, needed, databases): "_sheet_drill_entries") absent = object() previous = {name: getattr(session, name, absent) for name in attrs} - with _DRILL_MIRROR_LOCK: + with _drill_mirror_lock(): hit = _DRILL_MIRRORS.get(key) if token else None session._sheet_drill_isolated = True session._sheet_drill_cursor = hit["owned"] if hit else None @@ -6492,16 +6598,23 @@ def _drill_mirror(session, database, window, needed, databases): setattr(session, name, value) -def _drill_page(session, database, record_ids, offset, limit): +def _drill_page(session, database, record_ids, offset, limit, total): """Use fetchTableWindow's server door, including its row and field walls and wire shape.""" import routes_odoo_tables as rot from harness import datastore size = min(limit, datastore.WINDOW_MAX) - ids = record_ids[offset:offset + size] - payload = rot.odoo_table_rows(database, offset=0, limit=size, filters=None, - filterConj="and", sorts=None, search=None, related=None, groupBy=None, groupKey=None, - session=session, body={"ids": ids}) if ids else {"rows": [], "fields": []} + ids = record_ids + import odoo_relational as rel + stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY} + and not database.startswith('ut_amzn_') and rot._materialised_table(database)) + if ids and stored: + import registry_source + payload = registry_source.stored_page(session, database, ids) + else: + payload = rot.odoo_table_rows(database, offset=0, limit=size, filters=None, + filterConj="and", sorts=None, search=None, related=None, groupBy=None, groupKey=None, + session=session, body={"ids": ids}) if ids else {"rows": [], "fields": []} by_id = {int(row["pid"]): row for row in payload["rows"]} limits = list(payload.get("limits") or []) if size != limit: @@ -6509,10 +6622,117 @@ def _drill_page(session, database, record_ids, offset, limit): "cause": f"One response carries at most {datastore.WINDOW_MAX} rows.", "recommendation": "Continue paging with offset. Every matching row remains available."}) return {**payload, "offset": offset, "limit": size, "limits": limits, - "total": len(record_ids), "totalUnfiltered": len(record_ids), + "total": total, "totalUnfiltered": total, "rows": [by_id[pid] for pid in ids if pid in by_id]} +def _drill_query(raw): + """Accept structured narrowing and presentation options, never caller SQL.""" + import routes_odoo_tables as rot + import aios_grid + try: + value = json.loads(raw) if isinstance(raw, str) else {} + except ValueError: + raise err(400, "bad_drill_query", "The record query could not be read. Reopen View data.") + allowed = {"filters", "filterConj", "sorts", "search", "ids", "group", "aggs", + "groupBy", "groupOffset", "groupLimit"} + if not isinstance(value, dict) or set(value) - allowed: + raise err(400, "bad_drill_query", "Use record filters and paging options for this query.") + query = {**value, **rot._selection_query(value)} + budget = [0] + + def check_nodes(nodes, depth=1): + # Calendar range queries add one group and one date rule to an authored filter tree. + if (depth > aios_grid.MAX_FILTER_DEPTH + 1 or not isinstance(nodes, list) + or len(nodes) > aios_grid.MAX_FILTER_SIBLINGS): + raise err(400, "bad_drill_query", "Use a valid record filter tree.") + for node in nodes: + budget[0] += 1 + if budget[0] > aios_grid.MAX_FILTER_NODES + 2 or not isinstance(node, dict): + raise err(400, "bad_drill_query", "Use a valid record filter tree.") + if "children" in node: + if node.get("conj", "and") not in ("and", "or"): + raise err(400, "bad_drill_query", "Choose all or any for this filter group.") + check_nodes(node["children"], depth + 1) + elif (not isinstance(node.get("colId"), str) + or node.get("op") not in aios_grid.FILTER_OPS | aios_grid.COHORT_OPS | aios_grid.VIEW_OPS): + raise err(400, "bad_drill_query", "Choose a field and condition for this filter.") + + check_nodes(query["filters"]) + if "ids" in value: + query["ids"] = rot._record_ids(value) + if "aggs" in value and (not isinstance(value["aggs"], list) + or len(value["aggs"]) > rot.MAX_AGGREGATES + or any(not isinstance(agg, dict) or not isinstance(agg.get("colId"), str) + or not isinstance(agg.get("agg"), str) for agg in value["aggs"])): + raise err(400, "bad_drill_query", "Choose a list of field summaries.") + if query.get("groupBy") is not None and not isinstance(query["groupBy"], str): + raise err(400, "bad_drill_query", "Choose a grouping field.") + for key, low, high in (("groupOffset", 0, None), ("groupLimit", 1, rot.MAX_GROUPS)): + if key in query and (type(query[key]) is not int or query[key] < low + or (high is not None and query[key] > high)): + raise err(400, "bad_drill_query", "Choose a valid group offset and page size.") + if query.get("group") is not None and (not isinstance(query["group"], dict) + or set(query["group"]) != {"by", "key"} + or not isinstance(query["group"]["by"], str)): + raise err(400, "bad_drill_query", "Choose a grouping field and value.") + if any(not isinstance(sort, dict) or not isinstance(sort.get("colId"), str) + or sort.get("dir") not in ("asc", "desc") for sort in query["sorts"]): + raise err(400, "bad_drill_query", "Choose a field and direction for each sort.") + return query + + +def _drill_read_sql(session, database, cur, spec, cols, count_sql, params, query, limits): + """Intersect grid options with the cell population before counting or paging.""" + import routes_odoo_tables as rot + source = spec.get("from_sql") or spec["table"] + prefix = f"SELECT COUNT(*) FROM {source}" + if not count_sql.startswith(prefix): + raise err(409, "scope_not_applicable", "This cell's record scope cannot be paged.") + where = count_sql[len(prefix):].strip() + where = where[len("WHERE "):] if where else "TRUE" + where = f"({where})" + params = list(params) + tree = query["filters"] + named = rot._leaf_cols(tree) - rot._PSEUDO_COLS + named.update(sort["colId"] for sort in query["sorts"]) + if query.get("group"): + named.add(query["group"]["by"]) + if named - set(cols): + raise err(403, "field_unreadable", "This record query names an unavailable field.") + try: + had_cohort = bool(rot._active_cohort_leaves(tree)) + tree, sets = rot._views_as_cohorts(tree, database, session, spec, cols, + where, params, cur, limits) + tree, sets = rot._ranks_as_cohorts(tree, query["filterConj"], spec, cols, + where, params, cur, sets, had_cohort, rot._today(), limits) + pred = rot._fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols, + id_sql=spec["id"], today=rot._today(), cohort_sets=sets) + if pred is not None: + if pred.uses_aggregate: + raise ValueError("A record filter cannot use an aggregate.") + where += f" AND ({pred.sql})" + params.extend(pred.params) + if query["search"].strip(): + search = rot._fs().compile_search(query["search"].strip(), cols) + if search is not None: + where += f" AND ({search.sql})" + params.extend(search.params) + if query.get("group"): + group = query["group"] + sql, values = rot._group_pin(cols, group["by"], json.dumps(group["key"])) + where += f" AND ({sql})" + params.extend(values) + if query.get("ids") is not None: + ids = query["ids"] + where += f" AND {spec['id']} IN (SELECT unnest(?::BIGINT[]))" + params.append(ids) + order = rot._fs().compile_order_by(query["sorts"], cols, tiebreak_sql=spec["id"]) + except ValueError as exc: + raise err(400, "filter_unsupported", str(exc)) + return source, f"({where})", params, order or spec["id"] + + def _drill_draft_binding(cell, binding, draft_filter): """The stored binding with View data's UNSAVED filter in place of its own (S68-T23 finding). @@ -6546,35 +6766,20 @@ def _drill_draft_binding(cell, binding, draft_filter): def drill_cell(workbook_id: str, addr: str, tab: str | None = None, session: Session = Depends(require_session), offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1), - draft_filter: str | None = Query(default=None, alias="filter")): - """`{database, view_id?, filter?, record_ids, count, id_limit}` for `LinkGridModal` (R6, R13, R17). - - ⭐⭐ S66-T48 — `tab` (an id or a name, as `GET /sheets/{id}/tabs/{tab}` accepts) NAMES THE TAB - THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a - Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old - first-tab rule still answers, so an older client keeps working. - - ⭐⭐ S66-T48 — NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching - id; the popup pages them and sends the set in a POST body. Only a database whose records a - person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so - the popup says "showing X of N" rather than presenting X as the whole. - - ⛔ A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input - would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person - clicked on. - - ⭐ THE IDS ARE READ LIVE, NOT TAKEN FROM THE CELL (R13). A stale cell opens the same popup with - no comparison text, and the count beside those ids is the LIVE count, because `LinkGridModal` - prints the length of what it is handed and runs no query of its own. - """ - from core import sheets + draft_filter: str | None = Query(default=None, alias="filter"), + query: str | None = Query(default=None)): + """Return exact counts and one SQL page, with page IDs explicitly marked as incomplete.""" + from core import sheets, sheet_binding from modules import sheet as sheet_mod import core.perm_scope as perm_scope + import routes_odoo_tables as rot + from harness import datastore # Python callers omit FastAPI's Query sentinels. Offset has no maximum; only the page does. offset = offset if isinstance(offset, int) else 0 limit = limit if isinstance(limit, int) else 100 draft_filter = draft_filter if isinstance(draft_filter, str) else None + query = _drill_query(query) if offset < 0 or limit < 1: raise err(400, "bad_drill_page", "Choose a nonnegative record offset and a positive page size.") @@ -6645,15 +6850,31 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None, raise err(400, exc.code, exc.message) needed = (sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes)) + needed.update(rot._leaf_cols(query["filters"]) - rot._PSEUDO_COLS) + needed.update(sort["colId"] for sort in query["sorts"]) + needed.update(agg.get("colId") for agg in query.get("aggs", []) if agg.get("colId")) + if query.get("groupBy"): + needed.add(query["groupBy"]) + if query.get("group"): + needed.add(query["group"]["by"]) + for leaf in rot._fe()._rank_leaves(query["filters"], {key: {} for key in needed}): + _mode, group_field = rot._fe()._rank_scope(leaf, None) + if group_field: + needed.add(group_field) + if query["search"].strip(): + from routes_grid import _source_fields + needed.update(field["key"] for field in _source_fields(session, database)) related_bags = _related_leaf_bags(session, database, scope_nodes, needed) databases = {database} databases.update(_binding_related_entry(session, database, via)["target"] for via in sheet_mod.related_vias(scope_nodes)) + sources = _drill_tokens(session, databases) with _drill_mirror(session, database, window, needed, databases) as (made, token): cur, spec, cols = made cols = dict(cols) # Apply the aggregate's field wall before compiling either the count or the IDs. hidden = perm_scope.hidden_keys(session.user, database, spec.get("fields") or [], st=session.runtime) + _require_aggregate_field(session, binding, hidden=hidden) if spec.get("scope_applied"): wall_nodes, wall_conj = [], "and" unavailable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols) @@ -6672,9 +6893,6 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None, count_sql, count_params, limits = sheet_mod.build_aggregate_query( spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related) - ids_sql, ids_params, _ = sheet_mod.build_ids_query( - spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj, - wall_nodes=wall_nodes, wall_conj=wall_conj, related=related) except sheet_mod.SheetRefusal as exc: raise err(400, exc.code, exc.message) except ValueError: @@ -6683,25 +6901,52 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None, "connection. Edit the scope or ask an administrator to update the restriction.") if spec.get("scope_applied") and limits: raise err(409, "scope_not_applicable", "This cell's filter cannot be applied to this database.") - total = (cur.execute(count_sql, count_params).fetchone() or [0])[0] - - # S66-T48: preserve the existing mutable-table cap and every legacy answer key. - from core import user_tables - id_limit = None - if (int(total or 0) > user_tables.MAX_ROWS - and user_tables.records_mutable(database, st=session.runtime)): - id_limit = user_tables.MAX_ROWS - ids_sql = f"{ids_sql} LIMIT {int(id_limit)}" - record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]] + cols = {key: value for key, value in cols.items() if key not in hidden} + population_total = int(cur.execute(count_sql, count_params).fetchone()[0]) + source, where, params, order = _drill_read_sql( + session, database, cur, spec, cols, count_sql, count_params, query, limits) + narrowed = bool(query["filters"] or query["search"].strip() or query.get("group") + or query.get("ids") is not None) + total = int(cur.execute(f"SELECT count(*) FROM {source} WHERE {where}", params).fetchone()[0]) \ + if narrowed else population_total + size = min(limit, datastore.WINDOW_MAX) + summary = None + if "aggs" in query: + summary = rot._aggregate_over(cur, spec, cols, where, params, {"from_sql": source}, + query["aggs"], group_by=query.get("groupBy"), limits=limits, + group_offset=query.get("groupOffset", 0), group_limit=query.get("groupLimit")) + record_ids = [] + else: + ids_sql = (f"SELECT {spec['id']} FROM {source} WHERE {where} ORDER BY {order} " + "LIMIT ? OFFSET ?") + record_ids = [int(row[0]) for row in cur.execute(ids_sql, [*params, size, offset]).fetchall()] + try: + wall_stamp = perm_scope.wall_stamp(session.tenant, session.uname, database, user=session.user) + except Exception as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + wall_stamp = None + population_query = {key: value for key, value in query.items() + if key not in {"groupOffset", "groupLimit"}} + population = sheet_binding.fingerprint([session.tenant, session.uname, session.user, + binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \ + if all(sources.values()) and wall_stamp else None limits = list(limits) + list(spec.get("limits") or []) out = {"database": database, "record_ids": record_ids, "count": int(total or 0), - "id_limit": id_limit, "limits": limits} + "record_ids_scope": "page", "id_limit": None, "limits": limits, + "population": population, "query": query} if scope.get("view_id"): out["view_id"] = scope["view_id"] else: out["filter"] = scope.get("filter") - out["page"] = _drill_page(session, database, record_ids, offset, limit) + if summary is not None: + out["summary"] = {"ok": True, **summary} + else: + out["page"] = _drill_page(session, database, record_ids, offset, limit, total) + out["page"]["totalUnfiltered"] = population_total + if _drill_tokens(session, databases) != sources: + raise err(409, "drill_population_changed", "These records changed while loading. Try again.") out["token"] = token return out diff --git a/api/routes_tables.py b/api/routes_tables.py index 74248dfc69f2adb4489306911f44ae60088f3951..7847b6f542fd8b2b48c538e0ca60220fa0e06fd3 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -252,7 +252,12 @@ def _table_lend(runtime, table_key, defs_only=False): ut = _ut() if defs_only: # Metadata must enter through the store projection, even on entry-capable stores. - return ut.lend_defs(runtime) + try: + return ut.lend_defs(runtime, schema_only=True) + except Exception as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc if _ut()._entry_reader(runtime) is None: return ut.lend(runtime) entry = _table_entry(runtime, table_key) @@ -578,6 +583,9 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, if isinstance(documents, dict): context_store = _ut().lend(context_store, **{ name: documents[name] for name in metadata if isinstance(documents.get(name), dict)}) + from routes_customers import _AssemblyStore + # Lend the projected catalogue outside the assembly reader's per-read copy boundary. + context_store = _ut().lend_defs(_AssemblyStore(context_store, schema_only=True), schema_only=True) from copy import copy field_session = copy(session) field_session.runtime = context_store @@ -621,9 +629,10 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, if windowed: connected.add(database) elif database in ('customer_data', 'product_data'): - from routes_grid import _builtin_history_context - _, g, _ = _builtin_history_context( - session, 'product' if database == 'product_data' else 'customer') + from routes_grid import _builtin_activity_metadata + g = _builtin_activity_metadata(field_session, + 'product' if database == 'product_data' else 'customer', st=context_store) + page_sources.add(database) else: g = None contexts[database] = g @@ -636,8 +645,26 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, contexts[database] = None except perm_scope.Unresolvable: contexts[database] = None + if database in ('customer_data', 'product_data'): + from copy import deepcopy + assemblies = getattr(session, '_sheet_assemblies', None) + if isinstance(assemblies, dict): + assemblies[(database, 'fields')] = deepcopy(contexts[database]) return contexts[database] + assemblies = getattr(session, '_sheet_assemblies', None) + if isinstance(assemblies, dict): + class ActivityBuiltinFields(dict): + def __contains__(self, slot): + # Sheet asks for a fields slot before reading any builtin binding values. + if (isinstance(slot, tuple) and len(slot) == 2 and slot[1] == 'fields' + and slot[0] in ('customer_data', 'product_data') + and not super().__contains__(slot)): + source(slot[0]) + return super().__contains__(slot) + + session._sheet_assemblies = ActivityBuiltinFields(assemblies) + def partner(obj): nonlocal applications identity = json.dumps(obj, sort_keys=True) @@ -688,9 +715,14 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, if wanted: from routes_odoo_tables import _relation_rows try: - contexts[database]['pids'] = frozenset( - row['pid'] for row in _relation_rows( - session, database, {'pid'}, ids=wanted, st=context_store)) + if database in ('customer_data', 'product_data'): + from routes_grid import _builtin_activity_pids + contexts[database]['pids'] = _builtin_activity_pids( + session, database, wanted, contexts[database], st=context_store) + else: + contexts[database]['pids'] = frozenset( + row['pid'] for row in _relation_rows( + session, database, {'pid'}, ids=wanted, st=context_store)) except HTTPException as exc: if exc.status_code not in (403, 404, 409): raise @@ -704,7 +736,7 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, continue source_pids[database] = frozenset(str(pid) for pid in contexts[database]['pids']) scoped = perm_scope.apply_row_scope(raw['records'], session.user, key, own['fields'], - st=session.runtime, defn=own) + st=context_store, defn=own) for event in scoped: obj = event['object'] clean = None @@ -725,8 +757,8 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, previous = pair.get('before') declared = {f['key'] for f in (g.get('defn') or {}).get('fields', g['fields'])} if field_key not in declared and isinstance(previous, dict): - corpus = list(g['fields']) + [previous] - hidden = _ut_hidden(session, database, corpus, st=session.runtime) + corpus = list((g.get('defn') or {}).get('fields', g['fields'])) + [previous] + hidden = _ut_hidden(session, database, corpus, st=context_store) if field_key not in hidden and field_key not in g.get('hidden', ()): visible = visible | {field_key} admitted = field_key in visible @@ -737,7 +769,7 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None, and database not in connected and changes and all(pair.get('after') is None for pair in changes.values()) and not perm_scope.row_scope_applies( - session.user, database, st=session.runtime, defn=g['defn'])): + session.user, database, st=context_store, defn=g['defn'])): admitted = True if admitted: clean = (changes if changes.keys() <= visible else @@ -1283,7 +1315,7 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None): def ut_write_ctx(session: Session, table_key: str): """The events transport borrows the same rows-free assembly as workspace and Sheet.""" return _compute_ut_assembly(session, table_key, consume_corrections=False, - with_rows=False, include_usage=False) + with_rows=False, include_usage=False, metadata_only=False) def _ut_hidden(session, table_key, fields, st=None): @@ -1367,7 +1399,9 @@ def _ut_shared_cells(session, table_key, pids): from core import shared_overlay try: return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) return {} @@ -1718,6 +1752,15 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", consume_corrections: bool = False, with_rows: bool = True, st=None, *, fields_only: bool = False, needed=None): if not with_rows and st is None: + if not fields_only: + import routes_odoo_tables as connected + # Reuse revision-validated schema bytes; classification must not reread native rows. + definitions, _token = connected._page_defs(session) + defn = _ut().get(table_key, st=definitions) + if not _ut().materialises(table_key, st=definitions, defn=defn): + # Source membership changes independently of stored and workspace revisions. + return _compute_ut_assembly(session, table_key, storage_key, False, False, + st=definitions, fields_only=False, needed=needed) from routes_customers import grid_assembly return grid_assembly(session, table_key, storage_key, build=lambda borrowed: _compute_ut_assembly( @@ -1727,10 +1770,54 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", fields_only=fields_only, needed=needed) +def _metadata_pids(session, table_key, ctx, st): + """Validate only saved membership IDs through the connected table's ordinary row wall.""" + from core import grid_events + from modules import cohort + import routes_odoo_tables as connected + + document = ctx.table.st.get(ctx.table.table_key) or {} + if isinstance(st, _ut()._Lent): + st._docs[ctx.table.table_key] = document + candidates = set() + + def collect(value): + if isinstance(value, dict): + for key, item in value.items(): + if key in {"memberPids", "members"} and isinstance(item, (list, tuple)): + candidates.update(pid for pid in item if type(pid) is int and pid > 0) + elif isinstance(item, (dict, list, tuple)): + collect(item) + elif isinstance(value, (list, tuple)): + for item in value: + collect(item) + + collect(document.get(session.uname) or {}) + from core.table_store import ASSIGNED_KEY + assigned = ((document.get(ASSIGNED_KEY) or {}).get("views") or {}).get(session.uname) or {} + collect(assigned) + collect(grid_events._granted_views(ctx, session.uname)) + cohorts = cohort.scoped(table_key, st=st) + if isinstance(st, _ut()._Lent): + st._docs[cohorts.key] = st.get(cohorts.key) or {} + collect(cohorts.all_for(session.uname)) + connected.require_amazon_mirror(table_key, session, st=st) + if not candidates: + return frozenset() + read = connected._compiled_read(table_key, session, st=st) + try: + rows = connected._relation_rows(session, table_key, [], ids=sorted(candidates), st=st, + compiled=read) + return frozenset(int(row["pid"]) for row in rows) + finally: + read["cur"].close() + + def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "", consume_corrections: bool = False, with_rows: bool = True, st=None, *, fields_only: bool = False, needed=None, include_usage: bool = True, - include_measures: bool = True, include_views: bool = True): + include_measures: bool = True, include_views: bool = True, + metadata_only: bool = True): """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so `/workspace` and the events route consume any of the three interchangeably. @@ -1750,9 +1837,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" * every materialised `ut_*` database stops allocating a dict per row and sorting them on a route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per database switch (owner item 7). - ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers - the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved - on the write door. + Connected metadata validates only explicitly saved membership IDs through the SQL row wall. + Write contexts pass `metadata_only=False` and retain complete membership admission. """ import aios_grid from core import grid_events @@ -1765,8 +1851,13 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" # reading that bucket once per question asked of it. if st is None: st = _table_lend(session.runtime, table_key, defs_only=not with_rows) + metadata_membership = False if fields_only: defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + if getattr(st, 'schema_only', False): + from copy import deepcopy + # Detach selected definitions while the request lends its projected catalogue. + defn = deepcopy(defn) fields_base = list(defn.get("fields") or []) pids, rows_src = frozenset(), [] elif with_rows: @@ -1774,7 +1865,14 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" # a READ route passes one — see `scoped_pool`'s own note and contract C5. pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) else: - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + metadata_membership = (metadata_only and table_key != "ut_activity" and + not _ut().materialises(table_key, st=session.runtime, defn=defn)) + if metadata_membership: + fields_base = list(defn.get("fields") or []) + pids = frozenset() + else: + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) rows_src = [] # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it @@ -1791,10 +1889,18 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" # that happen to carry a scoped `table`. st=session.runtime, scope_key=table_key, table=ops) + if metadata_membership: + pids = _metadata_pids(session, table_key, ctx, st) + ctx.allowed_pids = pids ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=consume_corrections, capture_document=True, **({"include_views": False} if not include_views else {})) + if metadata_membership or fields_only: + ws["views"] = {vid: dict(view, config={**(view.get("config") or {}), + "memberPids": [pid for pid in view["config"]["memberPids"] if pid in pids]}) + if isinstance(view, dict) and isinstance((view.get("config") or {}).get("memberPids"), list) + else view for vid, view in (ws.get("views") or {}).items()} workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key=storage_key, fields_base=fields_base, st=st) @@ -1932,7 +2038,9 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "" derived = aios_grid.cohort_cells(lists) if not fields_only else {} materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed] for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures, - getattr(session.runtime, 'measure_memo', {})) + getattr(session.runtime, 'measure_memo', {}), + **({'cur': session.runtime._canonical_source_cursor} + if getattr(session.runtime, '_canonical_source_cursor', None) is not None else {})) if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items(): derived.setdefault(_pid, {}).update(_cells) @@ -2026,7 +2134,7 @@ def ut_measures(table_key, session, *, include_offer=True): return [] -def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): +def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None, *, cur=None): """`{pid: {field key: value}}` for this database's Metric columns. ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need @@ -2059,11 +2167,12 @@ def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None) out = {} for (dfrom, dto), group in by_window.items(): keys = tuple(sorted({f["measure"]["key"] for f in group})) - mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) + mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys, tuple(sorted(pids))) if mkey not in memo: try: memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, - date_to=dto, team_id=team_id, offer=offer) + date_to=dto, team_id=team_id, offer=offer, group_keys=pids, + **({'cur': cur} if cur is not None else {})) except Exception as e: # noqa: BLE001 _ut_measure_err("ut-measure-column", e) transient = False @@ -2246,38 +2355,18 @@ def _module_link_targets(session, meta=None, st=None): @router.get("/tables") def list_tables(brief: bool = Query(False), session: Session = Depends(require_session)): - """This session's user tables — the list the '+ New database' surface renders. - - ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This - route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: - `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on - tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant - with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked - about every table; it is handed the document this function already holds. See - `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not - available. - - `brief=1` is for a link-field picker. It needs the tables it may target and their schemas, - not row counts or record mutability. Reading a full tenant document to decorate that one - field editor would move every stored row through Neon for no visible result, so this branch - keeps the server-side permission wall but lends its rows-free projection instead. - - ⭐⭐ W41-T15 / R9 / C5 (2026-08-24) — AND `brief=1` NOW OFFERS THE TWO REGISTRY MODULES TOO - (`_module_link_targets`), because `_clean_link` accepts them as link targets and this list is - the only producer of a link bag in the product. - - ⛔ THE BRIEF BRANCH ONLY, and the asymmetry is deliberate. The full listing is the "+ New - database" management surface: it renders row counts, `createdBy` and a delete affordance over - things a tenant OWNS, and `customer_data` is a compiled registry row with its own nav entry - and no such lifecycle. Offering it there would put a database nobody can delete into the list - of databases you delete from. Two callers, two questions, one route. - """ + """List permitted definitions with exact stored counts or explicit unknown source populations.""" from core.if_field import project_fields ut = _ut() meta = nav_meta(session) out = [] - tables = (ut.all_defs(st=session.runtime) if brief - else ut.all_tables(st=session.runtime)) + try: + tables = ut.all_defs(st=session.runtime, schema_only=True) + counts = {} if brief else ut.definition_counts(session.runtime) + except Exception as exc: + from core.request_timing import raise_if_resource_error + raise_if_resource_error(exc) + raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) for key, t in sorted(tables.items(), key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): @@ -2297,7 +2386,8 @@ def list_tables(brief: bool = Query(False), "recordsMutable": ut.records_mutable(key, st=lent), "createdBy": t.get("createdBy") or "", "created": t.get("created") or "", - "rowCount": len(t.get("rows") or {})}) + "rowCount": (counts.get(key) if ut.materialises(key, defn=t) + and type(counts.get(key)) is int and counts[key] >= 0 else None)}) out.append(row) if brief: # ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and @@ -2710,6 +2800,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session), 'recommendation': 'Use Next to read older events.'}]} _ig_forward(session, table_key) + source_needed = getattr(session.runtime, '_canonical_needed', None) # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), @@ -2721,17 +2812,17 @@ def table_rows(table_key: str, session: Session = Depends(require_session), # value it replaced (contract C5). if _ut()._entry_reader(session.runtime) is None: _lent = _table_lend(session.runtime, table_key) - g = ut_assembly(session, table_key, st=_lent, consume_corrections=False) + g = ut_assembly(session, table_key, st=_lent, consume_corrections=False, needed=source_needed) else: - from dataclasses import replace + from registry_source import with_runtime from routes_customers import _AssemblyStore # One request owns these borrowed documents. A cold memo build and its page share reads; # a warm page reads records/shared cells afresh and lends Zone documents to both readers. original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore) else session.runtime) if not isinstance(session.runtime, _AssemblyStore): - session = replace(session, runtime=_AssemblyStore(original_rt)) - g = ut_assembly(session, table_key, with_rows=False) + session = with_runtime(session, _AssemblyStore(original_rt)) + g = ut_assembly(session, table_key, with_rows=False, needed=source_needed) assembly_rt = session.runtime entry_key = (_ut().STORE_KEY, ("entry", str(table_key))) if entry_key in assembly_rt.docs: @@ -2743,7 +2834,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session), _lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {}) # The page borrows the same envelope as workspace; only records and shared cells are fresh. pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent) - g['pids'] &= pids + g['pids'] = pids from core import perm_scope g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src if row['pid'] in g['pids']] @@ -2805,8 +2896,11 @@ def table_rows(table_key: str, session: Session = Depends(require_session), # honestly instead of showing half a truncated brace. # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, # so a tenant using `json` for a short config sees no change at all. - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + projection_fields = (g['fields'] if source_needed is None + else [field for field in g['fields'] if field['key'] in source_needed]) + rows = (aios_grid.rows_from_pool( + g["rows_src"], projection_fields, _thin_json(projection_fields, merged, table_key), derived=g["derived"]) + if source_needed is None or projection_fields else [{'pid': row['pid']} for row in g['rows_src']]) rows = _ut().zone_cells(table_key, rows, allowed_row_ids=g["pids"], st=_lent) # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. diff --git a/api/sheet_warm.py b/api/sheet_warm.py index 545aa61f27f7033b865ea221a6b794ea4310c7be..d653e6fcda7a90debf784f7380167277aa778ddb 100644 --- a/api/sheet_warm.py +++ b/api/sheet_warm.py @@ -11,13 +11,15 @@ from __future__ import annotations import atexit import os +import re import threading import time +import uuid from datetime import datetime, time as dtime, timedelta, timezone from time import perf_counter import routes_sheet -from core import external_access, perms, shares, sheets, users +from core import external_access, perms, refresh_proof, shares, sheets, users from deps import Session #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader. @@ -357,6 +359,146 @@ def _desk_skip(rt, tenant): return str(text or "").startswith("pg:") and token == seen +DESK_RESULT_PREFIX = "[sheet-desk-result] " +_DESK_PROOF_REASONS = frozenset(( + "workbook_incomplete", "sheet_incomplete", "sheet_partial", "cell_malformed", "refusal_malformed", + "identity_mismatch", "original_workbook_missing", "workbook_mismatch", "result_identity_mismatch", + "inventory_changed", "execution_build_unknown", "attempt_malformed", "attempt_not_committed", + "viewer_only", "partial_scope", "held_or_unresolved", "editor_commit_unproven", + "workbook_empty", "authored_definitions_changed", "proof_capacity_exceeded", + "generated_manifest_invalid", "generated_output_missing", "generated_output_invalid", +)) + + +def desk_result_census(book): + """Validate a complete workbook and count its cells without retaining their values.""" + identity = refresh_proof.snapshot(book) + return identity["census"], identity["inventory"], identity["unresolved"] + + +def _desk_input_identity(book): + try: + return refresh_proof.snapshot(book) + except refresh_proof.ProofCapacityError as exc: + return {"error": "proof_capacity_exceeded", "capacityCause": str(exc)} + except ValueError as exc: + return {"error": str(exc) if str(exc) in _DESK_PROOF_REASONS else "cell_malformed"} + except Exception: # noqa: BLE001 + return {"error": "cell_malformed"} + + +def desk_result_capture(row, tenant, session, before, result, *, admitted=None): + """Describe only the original resolver return and the worker's admitted pre-run inventory.""" + out = {"schema": "sheet-desk-result/v1", "source": "_refresh_desk_run.return", + "executionId": uuid.uuid4().hex, + "requestId": row.get("id") if isinstance(row.get("id"), str) else None, + "tenant": tenant if isinstance(tenant, str) else None, + "reader": row.get("reader") if isinstance(row.get("reader"), str) else None, + "workbook": row.get("workbook") if isinstance(row.get("workbook"), str) else None, + "version": os.environ.get("AIOS_VERSION", ""), + "complete": False, "persisted": False, "reason": None, "revision": None, + "priorRevision": None, "attempt": None, "census": None, "inventory": None, "sheets": None, + "authored": None, "priorAuthored": None, "recommendation": None, "capacityCause": None} + try: + if (session is None or row.get("tenant") != tenant or session.tenant != tenant + or session.uname != row.get("reader") or before.get("id") != row.get("workbook") + or session.user.get("username") != row.get("reader") + or session.user.get("tenant", tenant) != tenant + or any(key and key != tenant for key in + (getattr(session.runtime, "key", None), + getattr(getattr(session.runtime, "tenant", None), "key", None)))): + raise ValueError("identity_mismatch") + if not isinstance(result, dict) or not isinstance(result.get("workbook"), dict): + raise ValueError("original_workbook_missing") + book = result["workbook"] + if book.get("id") != row.get("workbook"): + raise ValueError("workbook_mismatch") + if (any(item.get("tenant", tenant) != tenant for item in (result, book)) + or result.get("reader", session.uname) != session.uname): + raise ValueError("result_identity_mismatch") + prior = admitted if admitted is not None else _desk_input_identity(before) + if prior.get("error"): + out["capacityCause"] = prior.get("capacityCause") + raise ValueError(prior["error"]) + returned = refresh_proof.snapshot(book) + out["census"], unresolved = returned["census"], returned["unresolved"] + out["inventory"] = refresh_proof.digest(returned["inventory"]) + out["sheets"] = returned["sheets"] + out["authored"], out["priorAuthored"] = returned["authored"], prior["authored"] + out["revision"] = book.get("rev") if type(book.get("rev")) is int else None + out["priorRevision"] = prior["revision"] if type(prior["revision"]) is int else None + if (returned["inventory"] != prior["inventory"] or returned["sheets"] != prior["sheets"]): + raise ValueError("inventory_changed") + if returned["authored"] != prior["authored"]: + raise ValueError("authored_definitions_changed") + if not re.search(r"\b[0-9a-f]{7,40}\b", out["version"]): + raise ValueError("execution_build_unknown") + if result.get("attempt") is not None and not isinstance(result["attempt"], dict): + raise ValueError("attempt_malformed") + if isinstance(result.get("attempt"), dict): + attempt = result["attempt"] + if any(attempt.get(key) is not None and not isinstance(attempt[key], str) for key in ("id", "state")): + raise ValueError("attempt_malformed") + out["attempt"] = {key: attempt.get(key) for key in ("id", "state")} + if attempt.get("state") != "committed": + raise ValueError("attempt_not_committed") + role = routes_sheet._role_on_workbook(session, before) + if role not in ("owner", "edit") or "refreshedFor" in book: + raise ValueError("viewer_only") + args = row.get("args") or {} + if (args != {"kind": "workbook"} and args != {"kind": "scope", "scope": {"workbook": True}}): + raise ValueError("partial_scope") + report = result.get("binding_refresh") + statuses = report.get("databases") if isinstance(report, dict) else None + if (not isinstance(statuses, dict) or any(state not in ("resolved", "skipped") + for state in statuses.values()) or unresolved or result.get("reused") + or result.get("held") or book.get("held") or result.get("partial") or book.get("partial") + or result.get("viewer") or result.get("viewer_only") + or result.get("state") not in (None, "committed") + or result.get("outcome") not in (None, "committed") + or result.get("persisted") is False or book.get("persisted") is False + or result.get("ok") is False): + raise ValueError("held_or_unresolved") + if (type(book.get("rev")) is not int or type(prior["revision"]) is not int + or prior["revision"] < 0 or book["rev"] != prior["revision"] + 1 + or not isinstance(book.get("refreshed_at"), str) or not book["refreshed_at"] + or (any(_is_bound(cell) for tab in book["sheets"] for cell in tab["cells"].values()) + and not statuses)): + raise ValueError("editor_commit_unproven") + empty_noop = prior["census"]["cells"] == returned["census"]["cells"] == 0 + out.update(complete=True, persisted=True, + reason="empty_noop" if empty_noop else "editor_revision_advanced") + refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES) + except refresh_proof.ProofCapacityError as exc: + out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None, + recommendation=refresh_proof.RECOMMENDATION, capacityCause=str(exc)) + except (ValueError, TypeError, AttributeError) as exc: + reason = str(exc) if isinstance(exc, ValueError) else "capture_malformed" + out.update(complete=False, persisted=False) + out["reason"] = reason if reason in _DESK_PROOF_REASONS else "capture_malformed" + if out["reason"] == "proof_capacity_exceeded": + out["recommendation"] = refresh_proof.RECOMMENDATION + try: + refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES) + except (ValueError, TypeError): + out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None, + sheets=None, attempt=None, recommendation=refresh_proof.RECOMMENDATION, + capacityCause="log_line_capacity") + for key in ("requestId", "tenant", "reader", "workbook", "version"): + if isinstance(out[key], str) and len(out[key]) > 1024: + out[key] = None + return out + + +def _desk_result_log(row, tenant, session, before, result, *, admitted=None): + """A logging failure supplies no proof and never changes the desk's business outcome.""" + try: + capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted) + print(DESK_RESULT_PREFIX + refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES), flush=True) + except Exception: # noqa: BLE001 + pass + + def desk_run_one(rt, tenant): """Claim and run the oldest queued refresh of one tenant. Returns True when one ran.""" row = routes_sheet._refresh_desk_claim(rt) @@ -364,6 +506,7 @@ def desk_run_one(rt, tenant): return False started = perf_counter() error = None + book, session, result, admitted = None, None, None, None try: book = sheets.get_workbook(rt, row["workbook"]) rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt) @@ -371,9 +514,11 @@ def desk_run_one(rt, tenant): if session is None: error = {"code": "workbook_missing", "message": "That workbook no longer exists."} else: - routes_sheet._refresh_desk_run(session, row) + admitted = _desk_input_identity(book) + result = routes_sheet._refresh_desk_run(session, row) except Exception as exc: # noqa: BLE001 error = routes_sheet._refresh_desk_error(exc) + _desk_result_log(row, tenant, session, book, result, admitted=admitted) routes_sheet._refresh_desk_finish(rt, row["id"], error) print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} " f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}" diff --git a/platform/core/activity_log.py b/platform/core/activity_log.py index 7b91a45ca62d7b59e8c036ef919b170f44848ec6..e704380f2387500cf850adcb516b61e97da42f83 100644 --- a/platform/core/activity_log.py +++ b/platform/core/activity_log.py @@ -5,6 +5,9 @@ from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone import json +import hashlib +import heapq +from itertools import islice import re DATABASE_KEY = 'ut_activity' @@ -27,8 +30,8 @@ _READ_SCOPE = ContextVar('activity_read_scope', default=None) @contextmanager def read_scope(rt): - """Share one outbox drain and bounded history pages within one read-only request.""" - state = {'runtime': rt, 'backend': None, 'drained': False, 'now': None, + """Share committed outbox input and bounded history pages within one read-only request.""" + state = {'runtime': rt, 'backend': None, 'outbox': None, 'now': None, 'pending': [], 'histories': {}} token = _READ_SCOPE.set(state) try: @@ -228,9 +231,10 @@ def drain(rt, *, keys=None): pending = table.get(OUTBOX_KEY) or {} delivered = {} try: - for key, entry in pending.items(): - if keys is not None and key not in keys: - continue + # A successful writer also retries older committed deliveries left by failed writers. + ordered = sorted(pending, key=lambda key: key not in keys) if keys is not None else pending + for key in ordered: + entry = pending[key] append(rt, **entry) delivered[key] = entry['_delivery']['version'] finally: @@ -249,6 +253,68 @@ def drain(rt, *, keys=None): return len(delivered) +def _delivery_id(key): + """Reserve an exact JSON integer for a delivery before its first persistent insert.""" + digest = hashlib.sha256(_text(key, 'delivery key').encode()).digest() + return (1 << 52) | (int.from_bytes(digest[:8], 'big') & ((1 << 52) - 1)) + + +def _outbox(rt): + """Read only this tenant's private committed Activity entry.""" + from core import user_tables + while isinstance(rt, user_tables._Lent): + rt = rt._runtime + target = rt._store() if callable(getattr(rt, '_store', None)) else rt + if callable(getattr(rt, 'get_entry', None)): + target = rt + reader = getattr(target, 'get_entry', None) + if callable(reader): + table = reader(user_tables.STORE_KEY, DATABASE_KEY) or {} + elif isinstance(target, FixtureAdapter) and not callable(getattr(target, 'get', None)): + return {} + else: + table = (target.get(user_tables.STORE_KEY) or {}).get(DATABASE_KEY) or {} + return table.get(OUTBOX_KEY) or {} + + +def _pending_page(backend, pending, *, target, actor, since, until, before, limit): + """Project a bounded pending page with persistent delivery identities and version deduplication.""" + def events(): + entries = iter(pending.items()) + while chunk := list(islice(entries, 256)): + delivered = backend.deliveries([key for key, _ in chunk]) + occupied = {receipt['id']: key for key, (_, receipt) in delivered.items()} + for key, entry in chunk: + delivery = entry['_delivery'] + if delivery['key'] != key: + raise ValueError('Activity delivery key does not match its committed intent.') + prior = delivered.get(key) + stamp = _time(delivery['occurred_at']).isoformat() + kind, name, identity = _object(entry['object']) + event_id = prior[1]['id'] if prior is not None else _delivery_id(key) + if event_id in occupied and occupied[event_id] != key: + raise ValueError('Activity delivery identity conflicts with another event.') + if (not since <= stamp <= until or (actor is not None and entry['by'] != actor) + or (target is not None and (kind, name) != target[:2]) + or (target is not None and target[2] is not None and identity != target[2])): + continue + if before is not None and (stamp, event_id) >= before: + continue + if prior is not None and prior[0] >= delivery['version']: + yield prior[1] + continue + changes, truncated, original_bytes = _bounded_changes(entry.get('changes')) + yield _event((event_id, stamp, entry['kind'], kind, name, identity, + entry['by'], entry.get('via'), entry.get('on_behalf_of'), + changes, truncated, original_bytes), + decoded_json=(_decode_json(identity), changes)) + + page = heapq.nlargest(limit + 1, events(), key=lambda event: (event['occurred_at'], event['id'])) + if len({event['id'] for event in page}) != len(page): + raise ValueError('Activity pending deliveries have conflicting identities.') + return EventPage(page[:limit], has_more=len(page) > limit) + + def prune(rt, *, limit=PRUNE_BATCH, now=None): """Remove a bounded oldest-first batch strictly older than thirteen calendar months.""" if type(limit) is not int or not 1 <= limit <= MAX_SCAN: @@ -287,10 +353,12 @@ def read(rt, *, can_read_object, object=None, by=None, since=None, until=None, or type(cursor[1]) is not int or cursor[1] < 1): raise ValueError('Use the next_cursor returned by the activity reader.') before = (_time(cursor[0]).isoformat(), cursor[1]) - if scope is None or not scope['drained']: - drain(rt) - if scope is not None: - scope['drained'], scope['now'] = True, now + if scope is not None: + if scope['outbox'] is None: + scope['outbox'], scope['now'] = _outbox(rt), now + pending = scope['outbox'] + else: + pending = _outbox(rt) prefetched = None if scope is not None and whole_history: workbook = target[1] @@ -310,6 +378,17 @@ def read(rt, *, can_read_object, object=None, by=None, since=None, until=None, batch = backend.page(target=target, actor=actor, since=since.isoformat(), until=until.isoformat(), before=before, limit=batch_limit) + if pending: + overlay = _pending_page(backend, pending, target=target, actor=actor, + since=since.isoformat(), until=until.isoformat(), + before=before, limit=batch_limit) + merged = {event['id']: event for event in batch} + merged.update((event['id'], event) for event in overlay) + ordered = heapq.nlargest(batch_limit + 1, merged.values(), + key=lambda event: (event['occurred_at'], event['id'])) + batch = EventPage(ordered[:batch_limit], has_more=(len(ordered) > batch_limit + or getattr(batch, 'has_more', len(batch) == batch_limit) + or overlay.has_more)) if not batch: exhausted = True break @@ -422,10 +501,12 @@ class PgActivityStore: columns = _COLUMNS[1:] conflict = '' if delivery is not None: + columns = ('id', *columns) + values = (_delivery_id(delivery[0]), *values) columns += ('delivery_key', 'delivery_version') values = (*values, *delivery) assignments = ', '.join(f'{column} = EXCLUDED.{column}' for column in columns - if column != 'delivery_key') + if column not in ('id', 'delivery_key')) conflict = (f' ON CONFLICT (delivery_key) DO UPDATE SET {assignments} ' f'WHERE {self.table}.delivery_version < EXCLUDED.delivery_version') stored = con.execute( @@ -501,6 +582,26 @@ class PgActivityStore: events.append(_event(row[:-1], decoded_json=(obj_id, changes))) return events + def deliveries(self, keys): + """Resolve at most 256 committed delivery receipts without changing the store.""" + if not keys or len(keys) > 256: + raise ValueError('Read between 1 and 256 delivery receipts.') + columns = ', '.join('CAST(changes AS TEXT) AS changes' if col == 'changes' else col + for col in _COLUMNS) + try: + with self._connection() as con: + reserved = [_delivery_id(key) for key in keys] + placeholders = ', '.join(['%s'] * len(keys)) + values = con.execute(f'SELECT {columns}, delivery_key, delivery_version ' + f'FROM {self.table} WHERE delivery_key IN ' + f'({placeholders}) OR id IN ({placeholders})', + [*keys, *reserved]).fetchall() + except Exception as exc: + if getattr(exc, 'sqlstate', None) != '42P01': + raise + return {} + return {row[-2]: (row[-1], _event(row[:-2])) for row in values} + def workbook_pages(self, workbooks, *, since, until, limit): """Read bounded first history pages through the existing tenant and object-time index.""" if not 1 <= len(workbooks) <= 16 or not 1 <= limit <= 256: diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 808023949f5f84d1080a3c5686b1cc5937de4138..944618258378e0474f2e7acf09bcc5944fff19a6 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -2083,6 +2083,10 @@ def handle_one(event, ctx): shared_prior = {**base_field, **stored_choice, 'key': key, 'source': 'overlay', 'shared': True, 'custom': True, 'type': base_field['type']} + # Registry admission comes from the server context, even when the native + # definition stores granted=False. The stored marker is persistence only. + if base_field.get('granted') is True: + shared_prior['granted'] = True shared_field = True if shared_field: if not _may_edit_field_definition(ctx, key, shared_prior): @@ -2095,6 +2099,13 @@ def handle_one(event, ctx): # cannot turn an image field into another schema type while saving # its permissions. field = dict(shared_prior) + # The Product choice projection may derive granted=True from the registry. + # Keep it for admission above, but persist only the native marker state. + if spreadsheet_choice: + if 'granted' in stored_choice: + field['granted'] = stored_choice['granted'] + else: + field.pop('granted', None) field['label'] = str(raw.get('label') or field.get('label') or 'Untitled')[:120] field['note'] = str(raw.get('note') or '')[:2000] fmt = _ag2._clean_format(raw.get('format'), field.get('type')) diff --git a/platform/core/measure_resolve.py b/platform/core/measure_resolve.py index bb65aa676899b4f42a373e866b4a8c28f4ab3597..6b729f19684b3e5e25bf46b50c10cf3d454e1a1c 100644 --- a/platform/core/measure_resolve.py +++ b/platform/core/measure_resolve.py @@ -142,7 +142,8 @@ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp, return out -def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None): +def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None, + *, cur=None): """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`, handed to `rows_from_pool(derived=…)` beside the cohort cells. @@ -171,7 +172,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No if key not in memo: try: memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'), - today, team_id, allowed_pids, related=related) + today, team_id, allowed_pids, related=related, + **({'cur': cur} if cur is not None else {})) except Exception as e: if on_error: on_error('measure-column', e) diff --git a/platform/core/odoo.py b/platform/core/odoo.py index 5e1506a7fa513a5818576107b578885d649cbbc9..3847c97cf8c589f55d0c1175808f608b40a74f88 100644 --- a/platform/core/odoo.py +++ b/platform/core/odoo.py @@ -10,9 +10,16 @@ a single source of truth for the connection. Adds: NOTHING here writes to Odoo — the underlying client hard-blocks writes. """ import os +import math import sys import queue as _queue import threading +import contextvars +import http.client +import io +import socket +import time +import xmlrpc.client from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from functools import lru_cache @@ -23,7 +30,176 @@ from pathlib import Path _ROOT = Path(__file__).resolve().parents[1] # platform/ if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) -from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded) +from odoo_client import OdooClient as _BaseOdooClient, _RPC_TIMEOUT +from core import request_timing as _timing + + +class OdooRequestTimeout(TimeoutError): + """An Odoo request transport exhausted its absolute RPC budget.""" + + +def _rpc_remaining(): + return _timing.remaining_budget(_RPC_TIMEOUT) + + +def _create_rpc_connection(address, timeout=None, source_address=None): + """Bound each address attempt while checking the budget around synchronous DNS resolution.""" + _timing.remaining_budget() + addresses = socket.getaddrinfo(address[0], address[1], 0, socket.SOCK_STREAM) + _timing.remaining_budget() + error = None + for family, kind, protocol, _, target in addresses: + _timing.remaining_budget() + con = socket.socket(family, kind, protocol) + try: + con.settimeout(_rpc_remaining()) + if source_address: + con.bind(source_address) + con.connect(target) + con.settimeout(_rpc_remaining()) + return con + except BaseException as exc: + con.close() + if not isinstance(exc, OSError): + raise + error = exc + if error is not None: + raise error + raise OSError("Odoo DNS resolution returned no addresses") + + +class _SocketReader(io.RawIOBase): + def __init__(self, sock): + super().__init__() + self.sock = sock + + def readable(self): + return True + + def readinto(self, buffer): + return self.sock.recv_into(buffer) + + +class _BudgetSocket: + def __init__(self, sock): + self.sock = sock + + def __getattr__(self, name): + return getattr(self.sock, name) + + def _call(self, method, *args): + self.sock.settimeout(_rpc_remaining()) + try: + result = getattr(self.sock, method)(*args) + except socket.timeout as exc: + if _timing.current_budget() is not None: + _timing.remaining_budget() + raise OdooRequestTimeout("Odoo RPC socket timed out") from exc + raise + _timing.remaining_budget() + return result + + def recv_into(self, buffer): + return self._call("recv_into", buffer) + + def sendall(self, data): + return self._call("sendall", data) + + def makefile(self, mode, buffering=None): + if mode != "rb": + raise ValueError("Odoo transport only reads binary responses") + return io.BufferedReader(_SocketReader(self)) + + +class _BudgetHTTPConnection(http.client.HTTPConnection): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._create_connection = _create_rpc_connection + + def connect(self): + self.timeout = _rpc_remaining() + super().connect() + self.sock = _BudgetSocket(self.sock) + _timing.remaining_budget() + + +class _BudgetHTTPSConnection(http.client.HTTPSConnection): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._create_connection = _create_rpc_connection + + def connect(self): + self.timeout = _rpc_remaining() + super().connect() + self.sock = _BudgetSocket(self.sock) + _timing.remaining_budget() + + +class _BudgetTransport(xmlrpc.client.Transport): + connection_type = _BudgetHTTPConnection + + def make_connection(self, host): + _timing.remaining_budget() + if self._connection and host == self._connection[0]: + return self._connection[1] + chost, self._extra_headers, _ = self.get_host_info(host) + con = self.connection_type(chost, timeout=_rpc_remaining()) + self._connection = host, con + return con + + def single_request(self, *args, **kwargs): + # A single deadline covers headers and every body read, including trickling responses. + deadline = time.monotonic() + _RPC_TIMEOUT + try: + with _timing.budget_scope(deadline, OdooRequestTimeout): + return super().single_request(*args, **kwargs) + except _timing.RequestDeadlineExceeded: + self.close() + raise + except socket.timeout as exc: + self.close() + _timing.remaining_budget() + raise OdooRequestTimeout("Odoo RPC socket timed out") from exc + except BaseException: + self.close() + raise + + +class _BudgetSafeTransport(_BudgetTransport, xmlrpc.client.SafeTransport): + connection_type = _BudgetHTTPSConnection + + +def _bounded_proxy(uri): + transport = _BudgetSafeTransport() if uri.startswith("https") else _BudgetTransport() + return xmlrpc.client.ServerProxy(uri, transport=transport) + + +class OdooClient(_BaseOdooClient): + def _connect(self, _retried=False): + try: + with _bounded_proxy(f'{self.url}/xmlrpc/2/common') as common: + self.uid = common.authenticate(self.db, self.user, self.api_key, {}) + except (http.client.HTTPException, ConnectionError, OSError): + if _retried or _timing.current_budget() is not None: + raise + time.sleep(3) + return self._connect(_retried=True) + if not self.uid: + raise RuntimeError("Odoo authentication failed; check credentials.") + self.models = _bounded_proxy(f'{self.url}/xmlrpc/2/object') + + def execute_kw(self, model, method, args, kwargs=None): + if _timing.current_budget() is None: + return super().execute_kw(model, method, args, kwargs) + self._guard(method) + if not self._xlock.acquire(timeout=_rpc_remaining()): + _timing.remaining_budget() + raise OdooRequestTimeout("Odoo client lock timed out") + try: + _timing.remaining_budget() + return self.models.execute_kw(self.db, self.uid, self.api_key, model, method, args, kwargs or {}) + finally: + self._xlock.release() # ---- business constants (TENANT #0's scope, kept as module attributes for its modules) ---- # These are Royal Imports' config — the default a request gets when nothing binds a tenant. @@ -151,16 +327,47 @@ _pool_n = 0 _pool_lock = threading.Lock() +class OdooPoolTimeout(TimeoutError): + """All read-only Odoo clients remained checked out past the acquisition budget.""" + + +def _checkout_timeout_s(): + try: + value = float(os.environ.get("AIOS_ODOO_POOL_TIMEOUT_S", "10")) + return max(.01, value) if math.isfinite(value) else 10.0 + except ValueError: + return 10.0 + + def _checkout(): global _pool_n + _timing.remaining_budget() try: - return _pool_q.get_nowait() + client = _pool_q.get_nowait() except _queue.Empty: with _pool_lock: grow = _pool_n < _POOL_MAX if grow: _pool_n += 1 - return OdooClient() if grow else _pool_q.get() # else block until one frees + if grow: + try: + client = OdooClient() + _timing.remaining_budget() + return client + except BaseException: + with _pool_lock: + _pool_n -= 1 + raise + try: + client = _pool_q.get(timeout=_timing.remaining_budget(_checkout_timeout_s())) + except _queue.Empty: + raise OdooPoolTimeout("Odoo read pool acquisition timed out") from None + try: + _timing.remaining_budget() + except BaseException: + _checkin(client) + raise + return client def _checkin(c): @@ -205,7 +412,10 @@ def parallel(tasks): if not tasks: return [] if len(tasks) == 1: - return [tasks[0]()] + _timing.remaining_budget() + result = tasks[0]() + _timing.remaining_budget() + return [result] caller_mode = doc_mode() # propagate the submitter's basis to each worker caller_ctx = _ctx() # and the tenant binding — a worker thread must @@ -216,14 +426,17 @@ def parallel(tasks): _tlocal.doc_mode = caller_mode _tlocal.tenant_ctx = caller_ctx try: - return fn() + result = fn() + _timing.remaining_budget() + return result finally: _tlocal.client = None _tlocal.doc_mode = 'order' _tlocal.tenant_ctx = None _checkin(c) with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex: - return list(ex.map(_run, tasks)) + contexts = [contextvars.copy_context() for _ in tasks] + return list(ex.map(lambda pair: pair[0].run(_run, pair[1]), zip(contexts, tasks))) def parallel_map(task_dict): diff --git a/platform/core/refresh_proof.py b/platform/core/refresh_proof.py new file mode 100644 index 0000000000000000000000000000000000000000..341d004fcf0e6ab0f7ddce44ff360dd5e39fd37f --- /dev/null +++ b/platform/core/refresh_proof.py @@ -0,0 +1,295 @@ +"""Pure, bounded identities for authored Sheet inputs and complete returned outputs.""" +from __future__ import annotations + +import hashlib +import json +import math +import re + +MAX_CELLS = 100_000 +MAX_TABS = 128 +MAX_DOCUMENT_BYTES = 32 * 1024 * 1024 +MAX_LINE_BYTES = 256 * 1024 +MAX_LOG_BYTES = 8 * 1024 * 1024 +MAX_LOG_ROWS = 4096 +MAX_STATE_BYTES = 16 * 1024 * 1024 +MAX_STATE_ROWS = 16_384 +RECOMMENDATION = "Increase the named proof budget and collect a fresh complete proof." +_UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00" + + +class ProofCapacityError(ValueError): + """A complete proof cannot fit the named finite budget.""" + + +def bounded_json(value, limit=MAX_DOCUMENT_BYTES): + """Check depth, nodes and scalar sizes before allocating encoded JSON.""" + remaining, nodes = [limit], [0] + + def check(item, depth=0): + nodes[0] += 1 + if depth > 40 or nodes[0] > 1_000_000: + raise ProofCapacityError("json_structure_capacity") + if isinstance(item, str): + if len(item) > remaining[0]: + raise ProofCapacityError("json_byte_capacity") + cost = len(item.encode("utf-8")) + 2 + elif isinstance(item, (dict, list, tuple)): + if len(item) > limit: + raise ProofCapacityError("json_structure_capacity") + cost = 2 + 2 * len(item) + elif item is None or type(item) in (bool, int, float): + if type(item) is int and item.bit_length() > 256: + raise ProofCapacityError("json_scalar_capacity") + if type(item) is float and not math.isfinite(item): + raise ValueError("json_nonfinite") + cost = 32 + else: + raise ValueError("json_malformed") + remaining[0] -= cost + if remaining[0] < 0: + raise ProofCapacityError("json_byte_capacity") + if isinstance(item, dict): + for key, child in item.items(): + if not isinstance(key, str): + raise ValueError("json_key_malformed") + check(key, depth + 1) + check(child, depth + 1) + elif isinstance(item, (list, tuple)): + for child in item: + check(child, depth + 1) + + check(value) + chunks, size = [], 0 + for chunk in json.JSONEncoder(sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False).iterencode(value): + size += len(chunk.encode("utf-8")) + if size > limit: + raise ProofCapacityError("json_byte_capacity") + chunks.append(chunk) + return "".join(chunks) + + +def digest(value): + return hashlib.sha256(bounded_json(value).encode("utf-8")).hexdigest() + + +def _address(addr): + if not isinstance(addr, str) or not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]{0,6}", addr): + return False + col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups() + n = 0 + for char in col: + n = n * 26 + ord(char) - 64 + return n <= 16384 and int(row) <= 1048576 + + +def _definition(cell): + kind = cell["kind"] + if kind == "literal": + return {key: cell[key] for key in ("kind", "text", "value") if key in cell} + if kind == "formula": + meta = cell.get("formulaMeta") or {} + return {"kind": kind, "formula": cell["formula"], + "formulaMeta": {k: v for k, v in meta.items() + if k not in ("spill", "inferredFormat")}} + if kind == "bound": + return {"kind": kind, "binding": cell["binding"]} + return {"kind": kind} + + +def _position(addr): + if not _address(addr): + raise ValueError("generated_manifest_invalid") + col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups() + number = 0 + for char in col: + number = number * 26 + ord(char) - 64 + return number, int(row) + + +def _cell_address(col, row): + letters = "" + while col: + col, digit = divmod(col - 1, 26) + letters = chr(65 + digit) + letters + return letters + str(row) + + +def _generated_manifest(tab, scope, remaining): + """Expand production spill extents and placed Grid blocks before accepting their children.""" + expected = {} + for addr, root in tab["cells"].items(): + if not isinstance(root, dict): + raise ValueError("cell_malformed") + meta = root.get("formulaMeta") or {} + if not isinstance(meta, dict): + raise ValueError("cell_malformed") + has_spill = "spill" in root or "spill" in meta + spill = root.get("spill", meta.get("spill")) + binding = root.get("binding") or {} + view = (root.get("kind") == "bound" and isinstance(binding, dict) + and binding.get("mode", binding.get("kind")) == "view") + block = binding.get("block") if view and binding.get("viewType") == "grid" else None + if not has_spill and block is None: + continue + col, row = _position(addr) + extent = spill if has_spill else block + if (not isinstance(extent, dict) or any(type(extent.get(k)) is not int or extent[k] < 1 + for k in ("rows", "cols"))): + raise ValueError("generated_manifest_invalid") + rows, cols = extent["rows"], extent["cols"] + if col + cols - 1 > 16384 or row + rows - 1 > 1048576: + raise ValueError("generated_manifest_invalid") + if rows * cols > remaining - len(expected): + raise ProofCapacityError("generated_cell_capacity") + if has_spill: + if (root.get("kind") != "formula" or set(spill) != {"version", "origin", "rows", "cols", "ref"} + or type(spill.get("version")) is not int or spill["version"] != 1 + or spill.get("origin") != scope + "!" + addr + or spill.get("ref") != addr + ":" + _cell_address(col + cols - 1, row + rows - 1) + or any(key in holder and holder[key] != spill for holder, key in + ((root, "spill"), (meta, "spill")))): + raise ValueError("generated_manifest_invalid") + else: + fields = binding.get("fields") + if (rows < 2 or not isinstance(fields, list) or len(fields) != cols + or any(not isinstance(field, str) or not field for field in fields) + or len(set(fields)) != len(fields)): + raise ValueError("generated_manifest_invalid") + for dy in range(rows): + for dx in range(cols): + child_addr = _cell_address(col + dx, row + dy) + if child_addr in expected: + raise ValueError("generated_manifest_invalid") + expected[child_addr] = (("spill", scope + "!" + addr) if has_spill else + ("view", addr, fields[dx], -1 if dy == 0 else -2 if dy == rows - 1 else dy - 1)) + if child_addr not in tab["cells"]: + raise ValueError("generated_output_missing") + return expected + + +def snapshot(book): + """Bind authored definitions and revision while censusing every validated output cell.""" + from core import sheets + + if not isinstance(book, dict) or not isinstance(book.get("sheets"), list) or not book["sheets"]: + raise ValueError("workbook_incomplete") + if len(book["sheets"]) > MAX_TABS: + raise ProofCapacityError("sheet_capacity") + total = 0 + for tab in book["sheets"]: + if isinstance(tab, dict) and isinstance(tab.get("cells"), dict): + total += len(tab["cells"]) + if total > MAX_CELLS: + raise ProofCapacityError("cell_capacity") + # Bound the existing document before canonical cleaners copy any nested cell object. + bounded_json(book) + census = {"cells": 0, "bound": 0, "formulas": 0, "refusals": []} + rows, definitions, tabs, names, unresolved = [], [], set(), set(), False + generated_cells = 0 + for index, tab in enumerate(book["sheets"]): + if (not isinstance(tab, dict) or not isinstance(tab.get("id"), str) or not tab["id"] + or tab["id"] in tabs or not isinstance(tab.get("cells"), dict) + or (tab.get("name") is not None and not isinstance(tab["name"], str))): + raise ValueError("sheet_incomplete") + tabs.add(tab["id"]) + if "cellCount" in tab and (type(tab["cellCount"]) is not int or tab["cellCount"] != len(tab["cells"])): + raise ValueError("sheet_partial") + if census["cells"] + len(tab["cells"]) > MAX_CELLS: + raise ProofCapacityError("cell_capacity") + name = tab.get("name") or "" + scope = name.upper() if name and name.upper() not in names else (_UNADDRESSABLE_SHEET + str(index)).upper() + names.add(scope) + generated = _generated_manifest(tab, scope, MAX_CELLS - generated_cells) + generated_cells += len(generated) + for addr, cell in tab["cells"].items(): + if (not _address(addr) or not isinstance(cell, dict) + or cell.get("kind") not in sheets.CELL_KINDS + or cell.get("state") not in sheets.CELL_STATES): + raise ValueError("cell_malformed") + try: + sheets.clean_cell(cell, addr) + except sheets.SheetRefusal as exc: + raise ValueError("cell_malformed") from exc + kind = cell["kind"] + if ((kind == "formula" and (not isinstance(cell.get("formula"), str) + or not cell["formula"].startswith("=") or len(cell["formula"]) < 2)) + or (kind == "bound" and (not isinstance(cell.get("binding"), dict) or not cell["binding"])) + or (kind == "literal" and "value" not in cell and "text" not in cell)): + raise ValueError("cell_malformed") + census["cells"] += 1 + census["bound"] += kind == "bound" + census["formulas"] += kind == "formula" + unresolved |= kind in ("bound", "formula") and cell["state"] not in ("fresh", "refused") + unresolved |= cell.get("staleReason") == "refresh_held" + if cell["state"] == "refused": + refusal, error = cell.get("refusal") or {}, cell.get("error") or {} + if not isinstance(refusal, dict) or not isinstance(error, dict): + raise ValueError("refusal_malformed") + code = refusal.get("code") or error.get("code") or "refused" + if not isinstance(code, str) or not code: + raise ValueError("refusal_malformed") + census["refusals"].append({"cell": "%s!%s" % (name or tab["id"], addr), + "kind": kind, "code": code}) + authored = cell + meta = cell.get("formulaMeta") or {} + marker = meta.get("spillChild") + declared = generated.get(addr) + if marker is not None: + key = scope + "!" + addr + origin = marker.get("origin") if isinstance(marker, dict) else None + root_addr = origin.rpartition("!")[2] if isinstance(origin, str) else "" + root = tab["cells"].get(root_addr, {}) + if (not isinstance(marker, dict) or type(marker.get("version")) is not int + or marker["version"] != 1 or marker.get("address") != key + or origin != scope + "!" + root_addr or origin == key + or cell.get("spillParent") != origin or root.get("kind") != "formula" + or set(marker) - {"version", "origin", "address", "base"} + or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell): + raise ValueError("cell_malformed") + base = marker.get("base") + if base and (not isinstance(base, dict) or base.get("kind") != "empty" + or any(base.get(k) is not None for k in ("value", "text", "formula", "binding"))): + raise ValueError("cell_malformed") + if meta.get("spillClear") is True: + if (declared is not None or cell.get("value") is not None or cell.get("display") != "" + or cell["state"] != "typed" or cell.get("refusal") or cell.get("error")): + raise ValueError("generated_output_invalid") + elif ("spillClear" in meta or declared != ("spill", origin) + or cell["state"] not in ("fresh", "refused")): + raise ValueError("generated_output_invalid") + authored = base if base else None + elif "spillParent" in cell or "spillClear" in meta: + raise ValueError("cell_malformed") + view = cell.get("viewChild") + if view is not None: + root = tab["cells"].get(view.get("origin"), {}) if isinstance(view, dict) else {} + binding = root.get("binding") or {} + if (marker is not None or not isinstance(view, dict) or not _address(view.get("origin")) + or view["origin"] == addr or not isinstance(view.get("field"), str) or not view["field"] + or type(view.get("line")) is not int or view["line"] < -2 + or root.get("kind") != "bound" or not isinstance(binding, dict) + or (binding.get("mode") != "view" and binding.get("kind") != "view") + or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell): + raise ValueError("cell_malformed") + if declared != ("view", view["origin"], view["field"], view["line"]): + raise ValueError("generated_output_invalid") + authored = None + if declared is not None: + is_root = (declared[1] == (scope + "!" + addr if declared[0] == "spill" else addr)) + if not is_root and ((declared[0] == "spill" and marker is None) + or (declared[0] == "view" and view is None)): + raise ValueError("generated_output_invalid") + if authored is not None: + rows.append({"sheet": tab["id"], "address": addr, "kind": authored["kind"]}) + definitions.append({"sheet": tab["id"], "address": addr, "definition": _definition(authored)}) + if census["cells"] and not rows: + raise ValueError("workbook_empty") + rows.sort(key=lambda row: (row["sheet"], row["address"])) + definitions.sort(key=lambda row: (row["sheet"], row["address"])) + authored = digest({"sheets": [(tab["id"], tab.get("name")) for tab in book["sheets"]], + "cells": definitions, "namedRanges": book.get("namedRanges"), + "dateSystem": book.get("dateSystem")}) + return {"revision": book.get("rev"), "inventory": rows, "authored": authored, + "sheets": sorted(tabs), "census": census, "unresolved": bool(unresolved)} diff --git a/platform/core/request_timing.py b/platform/core/request_timing.py new file mode 100644 index 0000000000000000000000000000000000000000..9f94934149785c6102f3788d211b6e6ad39cad07 --- /dev/null +++ b/platform/core/request_timing.py @@ -0,0 +1,412 @@ +"""Request-local measurements shared by async tasks and copied threadpool contexts.""" +from __future__ import annotations + +import re +import hashlib +import sys +import threading +import time +from collections import Counter +from contextlib import contextmanager +from contextvars import ContextVar +from functools import wraps + +CURRENT = ContextVar("aios_request_timing", default=None) +READ_MEMO_SECONDS = 2.0 +READ_MEMO_ENTRIES = 64 +READ_MEMO_BYTES = 8 * 1024 * 1024 +_STORE_SPAN = ContextVar("aios_store_timing_span", default=None) +_MEMO_READ = ContextVar("aios_request_memo_read", default=None) +_THREAD_LOCK = threading.Lock() +_THREADS = {} +_DEADLINE_ARM = None +_LOCK_PROBE_ARM = None +_BUDGET = ContextVar("aios_operation_budget", default=None) +_SQL_VALUES = re.compile( + r"--[^\n]*|/\*.*?\*/|\$([A-Za-z_][A-Za-z_0-9]*|)\$.*?\$\1\$|" + r"(?:[eEbBxX])?'(?:''|\\.|[^'])*'|\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b", re.S) + + +class Accumulator: + """Mutable request state, including measurements from anyio's copied contexts.""" + + def __init__(self): + self.lock = threading.RLock() + self.started = time.perf_counter() + self.closed = False + self.queue_ms = 0.0 + self.store_ms = 0.0 + self.store_reads = 0 + self.store_bytes = 0 + self.store_write_bytes = 0 + self.store_operations = Counter() + self.store_keys = {} + self.duckdb_ms = 0.0 + self.duckdb_count = 0 + self.slowest_sql = "" + self.slowest_sql_ms = 0.0 + self.response_bytes = 0 + self.samples = Counter() + self.deadline = None + self.cancelled = False + self.read_memo = {} + self.read_memo_bytes = 0 + self.read_memo_epoch = 0 + self.read_memo_invalidated_at = 0.0 + self.read_memo_ended = False + + def snapshot(self): + with self.lock: + # The middleware takes its first snapshot before sending response headers. + clear_read_memo(self, end=True) + return {"queue_ms": self.queue_ms, "store_ms": self.store_ms, + "store_reads": self.store_reads, "store_bytes": self.store_bytes, + "store_write_bytes": self.store_write_bytes, + "store_operations": dict(self.store_operations), + "store_keys": {key: dict(value) for key, value in self.store_keys.items()}, + "duckdb_ms": self.duckdb_ms, "duckdb_count": self.duckdb_count, + "slowest_sql": self.slowest_sql, "slowest_sql_ms": self.slowest_sql_ms, + "response_bytes": self.response_bytes} + + +@contextmanager +def request_scope(accumulator=None): + """Bind a mutable accumulator and restore the caller's context on every exit.""" + acc = accumulator if accumulator is not None else Accumulator() + token = CURRENT.set(acc) + try: + yield acc + finally: + clear_read_memo(acc, end=True) + CURRENT.reset(token) + + +def clear_read_memo(acc=None, *, end=False): + """Invalidate in-flight memo fills as well as completed values in copied contexts.""" + acc = CURRENT.get() if acc is None else acc + if acc is not None: + with acc.lock: + acc.read_memo.clear() + acc.read_memo_bytes = 0 + acc.read_memo_epoch += 1 + acc.read_memo_invalidated_at = time.monotonic() + acc.read_memo_ended |= end + + +def memo_checkpoint(): + """Capture the active request epoch before a backing read or dependent fill.""" + acc = CURRENT.get() + if acc is None: + return None + with acc.lock: + parent = _MEMO_READ.get() + if parent is not None and parent[0] is acc and parent[1] != acc.read_memo_epoch: + return None + if (acc.closed or acc.cancelled or acc.read_memo_ended + or time.perf_counter() - acc.started >= READ_MEMO_SECONDS): + clear_read_memo(acc, end=True) + return None + return acc, acc.read_memo_epoch + + +def memo_read(fn): + """Revoke request snapshots on failure and keep nested fills in their original epoch.""" + @wraps(fn) + def guarded(*args, **kwargs): + acc = CURRENT.get() + if acc is None: + return fn(*args, **kwargs) + parent = _MEMO_READ.get() + checkpoint = parent if parent is not None and parent[0] is acc else memo_checkpoint() + token = _MEMO_READ.set(checkpoint) + try: + return fn(*args, **kwargs) + except BaseException: + clear_read_memo(acc) + raise + finally: + _MEMO_READ.reset(token) + return guarded + + +def memo_peek(key): + """Find immutable text only while this request's bounded memo remains active.""" + checkpoint = memo_checkpoint() + if checkpoint is not None: + acc, epoch = checkpoint + with acc.lock: + if epoch == acc.read_memo_epoch and key in acc.read_memo: + return True, acc.read_memo[key] + return False, None + + +def memo_seed(key, value, checkpoint): + """Publish successful backing-read text without reviving an expired or invalidated epoch.""" + if checkpoint is not None and (value is None or isinstance(value, str)): + acc, epoch = checkpoint + def key_size(part): + return sys.getsizeof(part) + (sum(key_size(item) for item in part) + if isinstance(part, tuple) else 0) + size = key_size(key) + sys.getsizeof(value) + with acc.lock: + if (CURRENT.get() is acc and epoch == acc.read_memo_epoch + and not acc.closed and not acc.cancelled + and not acc.read_memo_ended + and time.perf_counter() - acc.started < READ_MEMO_SECONDS + and key not in acc.read_memo + and len(acc.read_memo) < READ_MEMO_ENTRIES + and acc.read_memo_bytes + size <= READ_MEMO_BYTES): + acc.read_memo[key] = value + acc.read_memo_bytes += size + + +@memo_read +def memo_text(key, read, *, fresh=False): + """Reuse bounded immutable backing-read text during the short pre-header request phase.""" + if fresh: + clear_read_memo() + return read() + checkpoint = memo_checkpoint() + found, value = memo_peek(key) + if found: + return value + value = read() + memo_seed(key, value, checkpoint) + return value + + +@contextmanager +def thread_scope(): + """Associate only this operation's thread with the current request for sampling.""" + acc = CURRENT.get() + if acc is None or acc.closed: + yield + return + tid = threading.get_ident() + with _THREAD_LOCK: + previous = _THREADS.get(tid) + _THREADS[tid] = acc + try: + yield + finally: + with _THREAD_LOCK: + if previous is None: + _THREADS.pop(tid, None) + else: + _THREADS[tid] = previous + + +def associated_threads(): + with _THREAD_LOCK: + return dict(_THREADS) + + +class RequestDeadlineExceeded(TimeoutError): + """A request exhausted its analytical query budget.""" + + +def current_budget(): + """Return the earliest absolute deadline and its named failure.""" + budget = _BUDGET.get() + acc = CURRENT.get() + if acc is not None: + if acc.cancelled: + raise RequestDeadlineExceeded("request cancelled") + if acc.deadline is not None and (budget is None or acc.deadline <= budget[0]): + budget = (acc.deadline, RequestDeadlineExceeded) + return budget + + +def remaining_budget(limit=None): + """Check the absolute budget before lending its remaining seconds to a blocking operation.""" + budget = current_budget() + if budget is None: + return limit + remaining = budget[0] - time.monotonic() + if remaining <= 0: + raise budget[1]("absolute operation deadline exceeded") + return remaining if limit is None else min(limit, remaining) + + +@contextmanager +def budget_scope(deadline, error=RequestDeadlineExceeded): + """Narrow a controlled operation's budget without extending its caller's deadline.""" + parent = current_budget() + token = _BUDGET.set(parent if parent is not None and parent[0] <= deadline else (deadline, error)) + try: + remaining_budget() + yield + remaining_budget() + finally: + _BUDGET.reset(token) + + +def raise_if_resource_error(error): + """Keep request resource refusals out of readiness fallbacks.""" + if CURRENT.get() is None: + return + known = { + "RequestDeadlineExceeded", "DeadlineCapacityExceeded", "PoolTimeout", "TooManyRequests", + "QueryCanceled", "LockNotAvailable", "SharedReadTimeout", "OdooPoolTimeout", + "OperationalError", "InterfaceError", "IdleInTransactionSessionTimeout", + "OutOfMemoryException", "ConnectionTimeout", "PostgresOperationTimeout", "OdooRequestTimeout"} + seen, probe = set(), error + while probe is not None and id(probe) not in seen: + seen.add(id(probe)) + if type(probe).__name__ in known: + raise error + probe = probe.__cause__ or probe.__context__ + + +def configure_deadlines(arm): + """Install the runtime's registered statement watchdog without an upward import.""" + global _DEADLINE_ARM + _DEADLINE_ARM = arm + + +def configure_lock_probes(arm): + """Install bounded lock diagnostics supplied by the runtime.""" + global _LOCK_PROBE_ARM + _LOCK_PROBE_ARM = arm + + +@contextmanager +def lock_wait_probe(waiter_pid, wait_s, diagnose): + """Sample blockers while the row transaction waits, using a separate connection.""" + handle = _LOCK_PROBE_ARM(waiter_pid, wait_s, diagnose) if ( + waiter_pid is not None and _LOCK_PROBE_ARM is not None) else None + try: + yield handle + finally: + if handle is not None: + handle.close() + + +@contextmanager +def duckdb_operation(connection): + """Arm only the active cursor call and disarm before its connection can be reused.""" + acc = CURRENT.get() + arm = _DEADLINE_ARM + if acc is None or acc.deadline is None or arm is None: + yield + return + if acc.cancelled or time.monotonic() >= acc.deadline: + raise RequestDeadlineExceeded("request query deadline exceeded") + handle = arm(connection, acc) + error = None + try: + yield + except BaseException as exc: + error = exc + raise + finally: + expired = handle.close() + if expired and (error is None or type(error).__name__ == "InterruptException"): + raise RequestDeadlineExceeded("request query deadline exceeded") from error + + +def text_bytes(value): + """Measure document payload bytes without serializing decoded caller objects.""" + if isinstance(value, str): + return len(value.encode("utf-8")) + if isinstance(value, (bytes, bytearray, memoryview)): + return len(value) + return 0 + + +def record_store(operation, keys, milliseconds=0.0, *, bytes_read=0, bytes_written=0, count=1): + """Record exclusive operation time and measured payload bytes, without document values.""" + acc = CURRENT.get() + if acc is None or acc.closed: + return + keys = [keys] if isinstance(keys, str) else list(keys) + keys = keys or [""] + with acc.lock: + if acc.closed: + return + acc.store_ms += milliseconds + if count: + acc.store_operations[operation] += count + acc.store_reads += count if operation != "update" else 0 + acc.store_bytes += bytes_read + acc.store_write_bytes += bytes_written + for key in keys: + key = str(key) + if not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", key): + prefix = re.split(r"[/.:]", key, maxsplit=1)[0] + prefix = prefix if re.fullmatch(r"[a-z][a-z_]{0,31}", prefix) else "document" + key = prefix + "#" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] + if key not in acc.store_keys and len(acc.store_keys) >= 64: + key = "" + row = acc.store_keys.setdefault(key, {"count": 0, "ms": 0.0, "bytes": 0, "write_bytes": 0}) + row["count"] += count + row["ms"] += milliseconds / len(keys) + row["bytes"] += bytes_read / len(keys) + row["write_bytes"] += bytes_written / len(keys) + + +def store_operation(operation, *, many=False): + """Measure caller wait, acquisition and parse once, excluding nested store operation time.""" + def decorate(fn): + @wraps(fn) + def measured(*args, **kwargs): + if CURRENT.get() is None: + return fn(*args, **kwargs) + keys = args[0] if args else kwargs.get("names" if many else "name", "") + if many: + keys = tuple(dict.fromkeys(str(key) for key in (keys or ()) if str(key))) + if args: + args = (keys,) + args[1:] + else: + kwargs["names"] = keys + else: + keys = [str(keys)] + parent = _STORE_SPAN.get() + span = [0.0] + token = _STORE_SPAN.set(span) + started = time.perf_counter() + try: + with thread_scope(): + return fn(*args, **kwargs) + except BaseException: + clear_read_memo() + raise + finally: + elapsed = (time.perf_counter() - started) * 1000 + _STORE_SPAN.reset(token) + if parent is not None: + parent[0] += elapsed + record_store(operation, keys, max(0.0, elapsed - span[0])) + return measured + return decorate + + +def sql_text(statement): + """Keep bounded SQL structure and remove inline literals and comments.""" + if not isinstance(statement, str): + return "" + return " ".join(_SQL_VALUES.sub("?", statement).split())[:300] + + +def record_duckdb(statement, milliseconds, *, count=1, statement_ms=None): + """Record execution or fetch time without accepting a parameter argument.""" + acc = CURRENT.get() + if acc is None or acc.closed: + return + with acc.lock: + if acc.closed: + return + acc.duckdb_ms += milliseconds + acc.duckdb_count += count + duration = milliseconds if statement_ms is None else statement_ms + if duration >= acc.slowest_sql_ms: + acc.slowest_sql_ms = duration + acc.slowest_sql = sql_text(statement) + + +def record_queue(milliseconds): + acc = CURRENT.get() + if acc is not None and not acc.closed: + with acc.lock: + if not acc.closed: + acc.queue_ms += milliseconds diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index f58d002edc45a67921ae89b42e86aecd232227b5..d67d3871bb429c8f63d357634ccdecce27ab9f4a 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -48,12 +48,14 @@ import tempfile import threading import time from collections import OrderedDict +from contextlib import contextmanager from functools import lru_cache from pathlib import Path from urllib.parse import urlparse import core.data_binding as data_binding from core import freeze_notice +from core import request_timing as _request_timing from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER _URL_ENV = 'DATABASE_URL' @@ -341,11 +343,18 @@ def _holder_pool_class(base, timeout_error): super().__init__(*a, **kw) def getconn(self, timeout=None): + timeout = _request_timing.remaining_budget( + timeout if timeout is not None else getattr(self, 'timeout', _pg_limits()['acquire'])) try: conn = super().getconn(timeout=timeout) except timeout_error: self._log_exhausted() raise + try: + _request_timing.remaining_budget() + except BaseException: + super().putconn(conn) + raise label = _holder_label() with self._holders_lock: self._holders[id(conn)] = (time.monotonic(), label) @@ -356,6 +365,19 @@ def _holder_pool_class(base, timeout_error): self._holders.pop(id(conn), None) super().putconn(conn) + @contextmanager + def connection(self, timeout=None): + # SET LOCAL survives transaction pooling and rolls back with the caller's writes. + with super().connection(timeout=timeout) as con: + _request_timing.remaining_budget() + with con.transaction(): + limits = _pg_limits() + con.execute("SELECT set_config('statement_timeout', %s, true), " + "set_config('idle_in_transaction_session_timeout', %s, true)", + (f"{int(limits['statement'] * 1000)}ms", + f"{int(limits['idle'] * 1000)}ms")) + yield con + def _log_exhausted(self): now = time.monotonic() with self._holders_lock: @@ -381,6 +403,67 @@ def _lock_timeout_s(): return max(1.0, value) +def _bound_env(name, default, minimum=.01): + try: + value = float(os.environ.get(name, default)) + return max(minimum, value) if math.isfinite(value) else float(default) + except ValueError: + return float(default) + + +def _pg_limits(): + worker = os.environ.get('AIOS_PROCESS_ROLE') == 'worker' + return { + 'connect': int(_bound_env('AIOS_PG_CONNECT_TIMEOUT_S', 5, 2)), + 'acquire': _bound_env('AIOS_PG_POOL_TIMEOUT_S', 10), + 'waiting': int(_bound_env('AIOS_PG_MAX_WAITING', 16, 1)), + 'statement': _bound_env('AIOS_PG_STATEMENT_TIMEOUT_S', 300 if worker else 30), + 'idle': _bound_env('AIOS_PG_IDLE_TRANSACTION_TIMEOUT_S', 300 if worker else 60), + 'network_ms': int(_bound_env('AIOS_PG_TCP_USER_TIMEOUT_MS', 15000, 1)), + } + + +def _read_timeout_s(): + limits = _pg_limits() + return limits['acquire'] + limits['statement'] + limits['network_ms'] / 1000 + + +class PostgresOperationTimeout(TimeoutError): + """A PostgreSQL socket operation exhausted its client-side limit.""" + + +def _checked_pg_wait(connection, gen): + """Check each libpq poll and discard expired connections without asynchronous cancellation.""" + deadline = time.monotonic() + _pg_limits()['statement'] + try: + with _request_timing.budget_scope(deadline, PostgresOperationTimeout): + ready = None + while True: + _request_timing.remaining_budget() + try: + state = gen.send(ready) + except StopIteration as result: + _request_timing.remaining_budget() + return result.value + ready = yield state + except (PostgresOperationTimeout, SharedReadTimeout, _request_timing.RequestDeadlineExceeded): + # No cancel packet can arrive after this socket is returned to the pool. + connection.close() + raise + finally: + gen.close() + + +@lru_cache(maxsize=None) +def _budget_connection_class(base): + class BudgetConnection(base): + def wait(self, gen, interval=.01): + # libpq yields on socket waits, including BEGIN, settings, fetch and COMMIT. + return super().wait(_checked_pg_wait(self, gen), interval=.01) + + return BudgetConnection + + def _pool(): """The psycopg connection pool, opened once per process. @@ -399,6 +482,7 @@ def _pool(): return _POOL['pool'] try: from psycopg_pool import ConnectionPool, PoolTimeout + from psycopg import Connection except ImportError as e: # pragma: no cover - depends on B-3 raise RuntimeError( 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in ' @@ -415,11 +499,17 @@ def _pool(): # real requests, not be held awake by this process's idle pool for psycopg's 10-minute # default. Thirty seconds still reuses a normal request burst without creating a resident # database client. - options = {'kwargs': {'sslrootcert': sslrootcert}} if sslrootcert is not None else {} - # The subclass only records holders and names them on PoolTimeout. Sizes are unchanged. + limits = _pg_limits() + connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1, + keepalives_idle=30, keepalives_interval=10, keepalives_count=3, + tcp_user_timeout=limits['network_ms']) + if sslrootcert is not None: + connection_kwargs['sslrootcert'] = sslrootcert + # The pool applies transaction settings and lends connections with checked socket polls. pool_cls = _holder_pool_class(ConnectionPool, PoolTimeout) - _POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=10.0, - max_idle=30.0, **options) + _POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=limits['acquire'], + max_idle=30.0, max_waiting=limits['waiting'], kwargs=connection_kwargs, + connection_class=_budget_connection_class(Connection)) _POOL['url'] = url _POOL['sslrootcert'] = sslrootcert return _POOL['pool'] @@ -434,6 +524,7 @@ def available(): """ if not _url(): return False + _request_timing.remaining_budget() if _POOL.get('ok'): return True try: @@ -441,7 +532,8 @@ def available(): con.execute('SELECT 1') _POOL['ok'] = True return True - except Exception: + except Exception as exc: + _request_timing.raise_if_resource_error(exc) return False @@ -477,7 +569,7 @@ def _q(template, kind, tenant_slug=None): #: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for #: the SAME document. So a read that finds an identical read already in flight (same tenant, key and #: projection) waits for it and parses the same TEXT itself, instead of opening another backend. -#: ⚠ NOT A CACHE: nothing outlives the SELECT that produced it, so `fresh` still means nothing here. +#: Completed reads may retain immutable text, validated by a new SELECT before reuse. #: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it #: finishes, and a read never joins a flight started under an older generation. #: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers @@ -488,34 +580,57 @@ _READ_FLIGHT_LOCK = threading.Lock() class _ReadFlight: - __slots__ = ('done', 'text', 'error', 'generation') + __slots__ = ('done', 'text', 'error', 'generation', 'deadline', 'started') def __init__(self, generation): self.done = threading.Event() self.text = None self.error = None self.generation = generation + self.started = time.monotonic() + self.deadline = self.started + _read_timeout_s() + budget = _request_timing.current_budget() + if budget is not None: + self.deadline = min(self.deadline, budget[0]) + + +class SharedReadTimeout(TimeoutError): + """A coalesced follower exhausted the original leader's read budget.""" def _bump_write_generation(name, tenant_slug): + _request_timing.clear_read_memo() key = (_effective_slug(tenant_slug), str(name)) with _READ_FLIGHT_LOCK: _WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1 + with _WHOLE_TEXT_LOCK: + for content_key in list(_WHOLE_TEXT): + if content_key[1:] == key: + _forget_whole(content_key) -def _shared_read(name, drops, tenant_slug, fetch_text): - """`fetch_text()` once for every concurrent identical read. -> the value's JSON text, or None.""" +def _shared_read(name, drops, tenant_slug, fetch_text, *, pool=None): + """Share one immutable store-controlled result across concurrent identical reads.""" doc = (_effective_slug(tenant_slug), str(name)) - key = doc + (drops,) + key = doc + (drops,) if pool is None else (pool,) + doc + (drops,) + acc = _request_timing.CURRENT.get() + invalidated_at = 0.0 + if acc is not None: + with acc.lock: + invalidated_at = acc.read_memo_invalidated_at with _READ_FLIGHT_LOCK: generation = _WRITE_GENERATION.get(doc, 0) flight = _READ_FLIGHTS.get(key) - leader = flight is None or flight.generation != generation + leader = (flight is None or flight.generation != generation + or flight.started <= invalidated_at) if leader: flight = _ReadFlight(generation) _READ_FLIGHTS[key] = flight if not leader: - flight.done.wait() + with _request_timing.budget_scope(flight.deadline, SharedReadTimeout): + if not flight.done.wait(_request_timing.remaining_budget()): + _request_timing.remaining_budget() + raise SharedReadTimeout('shared store read deadline exceeded') if flight.error is not None: # ⛔ THE LEADER'S OWN EXCEPTION, never a wrapper: routes and retries classify store # failures by TYPE (`PoolTimeout`/`OperationalError` -> 503 "try again", a transient @@ -524,7 +639,10 @@ def _shared_read(name, drops, tenant_slug, fetch_text): raise flight.error return flight.text try: - flight.text = fetch_text() + # Only store-controlled reads use this callback. Python decoding is cooperative. + with _request_timing.budget_scope(flight.deadline, SharedReadTimeout): + text = fetch_text() + flight.text = text except BaseException as exc: flight.error = exc raise @@ -538,26 +656,172 @@ def _shared_read(name, drops, tenant_slug, fetch_text): def _parsed(text): """The caller's own object from the shared JSON text (`None` = the key is absent).""" - if text is None: - return {} - value = json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text - return dict(value) if isinstance(value, dict) else value + try: + _request_timing.remaining_budget() + value = {} if text is None else (json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text) + value = dict(value) if isinstance(value, dict) else value + _request_timing.remaining_budget() + return value + except BaseException: + _request_timing.clear_read_memo() + raise + + +def _parse_read(name, operation, text): + """Keep independent parsing in H1 store time without counting another backing read.""" + if _request_timing.CURRENT.get() is None: + return _parsed(text) + started = time.perf_counter() + try: + return _parsed(text) + finally: + elapsed = (time.perf_counter() - started) * 1000 + parent = _request_timing._STORE_SPAN.get() + if parent is not None: + parent[0] += elapsed + _request_timing.record_store(operation, str(name), elapsed, count=0) + + +def _request_key(name, shape, tenant_slug, pool=None): + """Address a read shape by pool, tenant and local write generation.""" + pool = _pool() if pool is None else pool + doc = (_effective_slug(tenant_slug), str(name)) + with _READ_FLIGHT_LOCK: + generation = _WRITE_GENERATION.get(doc, 0) + return (pool,) + doc + (shape, generation) + + +@_request_timing.memo_read +def _request_text(name, shape, tenant_slug, read, *, fresh=False): + """Share only this request's validated backing read, including independently acquired handles.""" + if _request_timing.CURRENT.get() is None: + return read() + return _request_timing.memo_text(_request_key(name, shape, tenant_slug), read, fresh=fresh) + + +def _revision_value(raw_rev, stamp): + rev = int(raw_rev) if raw_rev is not None else 0 + return {'rev': rev, 'updated_at': stamp.timestamp() if hasattr(stamp, 'timestamp') else None, + 'token': f'pg:{rev}'} + + +def _seed_read_token(key, checkpoint, revision, stamp): + """A successful content read validates both revision columns for every request read shape.""" + _request_timing.memo_seed(key, json.dumps(_revision_value(revision, stamp)), checkpoint) + + +def _validated_content(name, drops, tenant_slug): + """Reuse retained content only against this request's checked revision AND timestamp.""" + checkpoint = _request_timing.memo_checkpoint() + if checkpoint is None: + return False, None + key = _request_key(name, 'revision', tenant_slug) + found, text = _request_timing.memo_peek(key) + if not found: + return False, None + token = _parsed(text) + if token.get('updated_at') is None: + return False, None + cache_key = key[:3] if not drops else (*key[:3], drops) + cache, lock = ((_WHOLE_TEXT, _WHOLE_TEXT_LOCK) if not drops else + (_PROJECTION_TEXT, _PROJECTION_TEXT_LOCK)) + with lock: + cached = cache.get(cache_key) + if cached is not None and _revision_value(*cached[:2]) == token: + shape = 'whole' if drops is None else ('projection', drops) + content_key = (*key[:3], shape, key[-1]) + _request_timing.memo_seed(content_key, cached[2], checkpoint) + return _request_timing.memo_peek(content_key) + return False, None def get(name, fresh=False, tenant_slug=None): + """Parse an independent document from this request's backing read or a new validated read.""" + def read(): + found, text = (False, None) if fresh or str(name).endswith('user_tables') else ( + _validated_content(name, None, tenant_slug)) + return text if found else _get_text(name, fresh=fresh, tenant_slug=tenant_slug) + # Whole row documents never become request metadata snapshots. + text = read() if str(name).endswith('user_tables') and not fresh else _request_text( + name, 'whole', tenant_slug, read, fresh=fresh) + return _parse_read(name, 'get', text) + + +@_request_timing.store_operation("get") +def _get_text(name, fresh=False, tenant_slug=None): """The value stored under `name`, or `{}` when the key is genuinely absent. - Ordinary reads share identical SELECTs already in flight. A fresh read starts its own SELECT - so deletion readback cannot join a query begun before another process's committed mutation. + Ordinary reads validate both live tokens before reusing immutable text and share concurrent + SELECTs. A fresh read bypasses both the content token and any existing flight. """ + pool = _pool() + doc = (_effective_slug(tenant_slug), str(name)) + content_key = (pool,) + doc + token_key = _request_key(name, 'revision', tenant_slug, pool) + checkpoint = None if fresh else _request_timing.memo_checkpoint() + def fetch(): - with _pool().connection() as con: + with _READ_FLIGHT_LOCK: + generation = _WRITE_GENERATION.get(doc, 0) + with _WHOLE_TEXT_LOCK: + cached = _WHOLE_TEXT.get(content_key) if not fresh else None + if cached is not None: + _WHOLE_TEXT.move_to_end(content_key) + with pool.connection() as con: row = con.execute( - _q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), - (str(name),)).fetchone() - return row[0] if row else None + _q('SELECT kv.rev, kv.updated_at, CASE ' + 'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ' + 'ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s', + 'store_kv', tenant_slug), + (cached[0] if cached else None, cached[1] if cached else None, + str(name))).fetchone() + text = row[2] if row else None + _request_timing.record_store("get", str(name), bytes_read=_request_timing.text_bytes(text), count=0) + if row and text is None: + if cached is None or row[:2] != cached[:2]: + raise RuntimeError('A whole read returned no content for its current revision') + text = cached[2] + _remember_whole(content_key, generation, row[0] if row else None, + row[1] if row else None, text) + return text, row[0] if row else None, row[1] if row else None + + text, rev, stamp = fetch() if fresh else _shared_read(name, None, tenant_slug, fetch, pool=pool) + _seed_read_token(token_key, checkpoint, rev, stamp) + return text + + +_WHOLE_TEXT = OrderedDict() +_WHOLE_TEXT_LOCK = threading.Lock() +_WHOLE_TEXT_BYTES = 0 +_WHOLE_TEXT_LIMIT = 128 * 1024 * 1024 +_WHOLE_TEXT_ENTRIES = 64 + - return _parsed(fetch() if fresh else _shared_read(name, None, tenant_slug, fetch)) +def _forget_whole(key): + """Remove one retained string while holding `_WHOLE_TEXT_LOCK`.""" + global _WHOLE_TEXT_BYTES + previous = _WHOLE_TEXT.pop(key, None) + if previous is not None: + _WHOLE_TEXT_BYTES -= previous[3] + + +def _remember_whole(key, generation, revision, stamp, text): + """Retain bounded immutable text only while its local write generation remains current.""" + global _WHOLE_TEXT_BYTES + _request_timing.remaining_budget() + size = sys.getsizeof(text) if isinstance(text, str) else 0 + with _READ_FLIGHT_LOCK: + if _WRITE_GENERATION.get(key[1:], 0) != generation: + return + with _WHOLE_TEXT_LOCK: + _request_timing.remaining_budget() + _forget_whole(key) + if revision is not None and stamp is not None and 0 < size <= _WHOLE_TEXT_LIMIT: + _WHOLE_TEXT[key] = (revision, stamp, text, size) + _WHOLE_TEXT_BYTES += size + while _WHOLE_TEXT and (len(_WHOLE_TEXT) > _WHOLE_TEXT_ENTRIES + or _WHOLE_TEXT_BYTES > _WHOLE_TEXT_LIMIT): + _forget_whole(next(iter(_WHOLE_TEXT))) _PROJECTION_TEXT = OrderedDict() @@ -569,8 +833,10 @@ _PROJECTION_TEXT_LIMIT = 16 * 1024 * 1024 def _remember_projection(key, revision, stamp, text): """Retain bounded immutable content whose revision must pass the next database read.""" global _PROJECTION_TEXT_BYTES + _request_timing.remaining_budget() size = len(text.encode('utf-8')) if isinstance(text, str) else 0 with _PROJECTION_TEXT_LOCK: + _request_timing.remaining_budget() previous = _PROJECTION_TEXT.pop(key, None) if previous is not None: _PROJECTION_TEXT_BYTES -= previous[3] @@ -584,6 +850,27 @@ def _remember_projection(key, revision, stamp, text): def get_projection(name, drop=(), tenant_slug=None): + """Parse independent projected definitions from the current request's backing read.""" + import core.store as _hf + drops = tuple(sorted(set(str(d) for d in drop))) + can_memo = not str(name).endswith('user_tables') or 'rows' in drops + def read(): + found, text = _validated_content(name, drops, tenant_slug) if can_memo else (False, None) + return text if found else _get_projection_text(name, drops, tenant_slug) + text = (read() if not can_memo else + _request_text(name, ('projection', drops), tenant_slug, read)) + data = _parse_read(name, 'get_projection', text) + dropped = frozenset(drops) + if not isinstance(text, (str, bytes, bytearray)): + return _hf._project(data, dropped) + if not isinstance(data, dict): + return data + return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped) + if isinstance(value, dict) else value for key, value in data.items()} + + +@_request_timing.store_operation("get_projection") +def _get_projection_text(name, drop=(), tenant_slug=None): """`core.store.get_projection` with the DROP performed inside Postgres. `user_tables` is one JSONB value, and tenant #0's stored ``rows`` have measured at 28.6 MB @@ -597,11 +884,11 @@ def get_projection(name, drop=(), tenant_slug=None): so marking those dictionaries needs no second JSON serialization and parse. The ``CASE`` retains the generic store contract: non-object JSON values are returned untouched rather than making ``jsonb_each`` error. """ - import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class - drops = tuple(str(d) for d in drop) pool = _pool() content_key = (pool, _effective_slug(tenant_slug), str(name), tuple(sorted(drops))) + token_key = _request_key(name, 'revision', tenant_slug, pool) + checkpoint = _request_timing.memo_checkpoint() def fetch(): with _PROJECTION_TEXT_LOCK: @@ -612,9 +899,13 @@ def get_projection(name, drop=(), tenant_slug=None): con.execute("SET LOCAL work_mem = '64MB'") if not drops: row = con.execute( - _q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + _q('SELECT kv.rev, kv.updated_at, kv.value::text FROM {tbl} AS kv WHERE kv.key = %s', + 'store_kv', tenant_slug), (str(name),)).fetchone() - return row[0] if row else None + text = row[2] if row else None + _request_timing.record_store("get_projection", str(name), + bytes_read=_request_timing.text_bytes(text), count=0) + return text, row[0] if row else None, row[1] if row else None row = con.execute( _q('SELECT kv.rev, kv.updated_at, CASE ' 'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ELSE (CASE ' @@ -629,23 +920,21 @@ def get_projection(name, drop=(), tenant_slug=None): list(drops), str(name))).fetchone() # The SELECT always checks the live row. Only its unchanged JSON reconstruction is reused. text = row[2] if row else None + _request_timing.record_store("get_projection", str(name), + bytes_read=_request_timing.text_bytes(text), count=0) if row and text is None: if cached is None or row[:2] != cached[:2]: raise RuntimeError('A projected read returned no content for its current revision') text = cached[2] _remember_projection(content_key, row[0] if row else None, row[1] if row else None, text) - return text + return text, row[0] if row else None, row[1] if row else None # D-2640: the projection is the read 14 concurrent backends were making; share it too. - text = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch) - data = _parsed(text) - dropped = frozenset(drops) - if not isinstance(text, (str, bytes, bytearray)): - return _hf._project(data, dropped) - if not isinstance(data, dict): - return data - return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped) - if isinstance(value, dict) else value for key, value in data.items()} + text, rev, stamp = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch, pool=pool) + _seed_read_token(token_key, checkpoint, rev, stamp) + if not drops and not str(name).endswith('user_tables'): + _request_timing.memo_seed((*token_key[:3], 'whole', token_key[-1]), text, checkpoint) + return text def _get_workbook_projection(name, *, metadata_only=False, tab_ids=None, cells=None, @@ -769,6 +1058,7 @@ def _amazon_backup_policy(*, trigger_at, checked_at, tenant_slug=None): 'cause': 'provider_backup_expiry_verification_unavailable'} +@_request_timing.store_operation("get_projections", many=True) def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): """Several projected documents in one round trip, keyed by their requested store key. @@ -782,7 +1072,10 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): if not keys: return {} drops = tuple(str(d) for d in drop) - with _pool().connection() as con: + pool = _pool() + request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys} + checkpoint = _request_timing.memo_checkpoint() + with pool.connection() as con: if timeout_s is not None: timeout_ms = max(1, int(float(timeout_s) * 1000)) # Transaction-local: a pooled connection must not lend this caller's budget to the @@ -791,16 +1084,34 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None): con.execute("SELECT set_config('statement_timeout', %s, true)", (str(timeout_ms),)) rows = con.execute( - _q('SELECT kv.key, CASE ' + _q('SELECT kv.key, (CASE ' "WHEN jsonb_typeof(kv.value) = 'object' THEN " "COALESCE((SELECT jsonb_object_agg(e.entry_key, " "CASE WHEN jsonb_typeof(e.entry_value) = 'object' " 'THEN e.entry_value - %s::text[] ELSE e.entry_value END) ' 'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), ' - "'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv " + "'{{}}'::jsonb) ELSE kv.value END)::text, kv.rev, kv.updated_at FROM {tbl} AS kv " 'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug), (list(drops), keys)).fetchall() - found = {str(key): value for key, value in rows} + found = {} + for key, value, rev, stamp in rows: + _request_timing.record_store("get_projections", str(key), + bytes_read=_request_timing.text_bytes(value), count=0) + found[str(key)] = _parsed(value) + _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp) + if not str(key).endswith('user_tables') or 'rows' in drops: + shape_key = (*request_keys[str(key)][:3], ('projection', tuple(sorted(set(drops)))), + request_keys[str(key)][-1]) + _request_timing.memo_seed(shape_key, value, checkpoint) + if not drops: + _request_timing.memo_seed((*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1]), + value, checkpoint) + for key in keys: + if key not in found: + _seed_read_token(request_keys[key], checkpoint, None, None) + if not key.endswith('user_tables') or 'rows' in drops: + _request_timing.memo_seed((*request_keys[key][:3], ('projection', tuple(sorted(set(drops)))), + request_keys[key][-1]), None, checkpoint) dropped = frozenset(drops) return { key: _hf._project( @@ -832,17 +1143,221 @@ def _get_entry(name, entry, tenant_slug=None): return dict(value) if isinstance(value, dict) else None +@_request_timing.store_operation("json_paths") +def _get_json_paths(name, paths, tenant_slug=None): + """Read explicit nested JSON values without transferring their containing document.""" + import itertools + paths = [list(map(str, path)) for path in itertools.islice(iter(paths), 4097)] + if len(paths) > 4096: + raise ValueError("A JSON path read accepts at most 4096 paths per batch") + if not paths: + return [] + with _pool().connection() as con: + row = con.execute(_q( + "SELECT ARRAY(SELECT (kv.value #> ARRAY(SELECT jsonb_array_elements_text(p.path)))::text " + "FROM jsonb_array_elements(%s::jsonb) WITH ORDINALITY AS p(path, n) ORDER BY p.n) " + "FROM {tbl} AS kv WHERE kv.key = %s", 'store_kv', tenant_slug), + (json.dumps(paths), str(name))).fetchone() + return [json.loads(value) if value is not None else None for value in row[0]] if row else [None] * len(paths) + + +def _iter_json_items(name, path=(), *, batch_size=1024, tenant_slug=None): + """Stream one JSON object through a server cursor within the existing checked connection.""" + import uuid + size = int(batch_size) + if not 1 <= size <= 4096: + raise ValueError('A JSON object batch accepts between 1 and 4096 entries') + with _pool().connection() as con: + with con.cursor(name='registry_' + uuid.uuid4().hex) as cur: + cur.execute(_q( + 'SELECT jsonb_typeof(kv.value #> %s::text[]), e.key, e.value::text FROM {tbl} AS kv ' + 'LEFT JOIN LATERAL jsonb_each(CASE ' + "WHEN jsonb_typeof(kv.value #> %s::text[]) = 'object' " + "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) AS e ON TRUE WHERE kv.key = %s", + 'store_kv', tenant_slug), (list(map(str, path)), list(map(str, path)), + list(map(str, path)), str(name))) + found = False + while True: + _request_timing.remaining_budget() + rows = cur.fetchmany(size) + if not rows: + if not found: + raise RuntimeError('canonical_source_unavailable: the stored document disappeared') + return + found = True + if any(kind not in (None, 'null', 'object') for kind, _key, _value in rows): + raise RuntimeError('canonical_source_invalid: the stored rows are not an object') + _request_timing.record_store('json_items', str(name), + bytes_read=sum(len(value.encode('utf-8')) for _kind, _key, value in rows if value is not None)) + yield [(key, json.loads(value)) for _kind, key, value in rows if key is not None] + rows.clear() + + +def _iter_json_values(name, path=(), *, drop=(), batch_size=1024, tenant_slug=None): + """Stream an array with selected object operands removed before transport.""" + import uuid + size = int(batch_size) + if not 1 <= size <= 4096: + raise ValueError('A JSON array batch accepts between 1 and 4096 values') + with _pool().connection() as con: + with con.cursor(name='canonical_' + uuid.uuid4().hex) as cur: + cur.execute(_q("SELECT e.n-1, (CASE WHEN jsonb_typeof(e.v)='object' " + "THEN e.v - %s::text[] ELSE e.v END)::text FROM {tbl} kv " + "CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' " + "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) " + 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug), + (list(drop), list(map(str, path)), list(map(str, path)), str(name))) + while True: + _request_timing.remaining_budget() + rows = cur.fetchmany(size) + if not rows: + return + _request_timing.record_store('json_values', str(name), + bytes_read=sum(len(value.encode('utf-8')) for _, value in rows)) + yield [(str(index), json.loads(value)) for index, value in rows] + rows.clear() + + +class _ViewWorkspace(dict): + pass + + +def _get_view_workspace(name, tenant_slug=None): + """Saved-view metadata without authored memberships crossing the JSON decoder.""" + with _pool().connection() as con: + row = con.execute(_q("SELECT kv.rev, kv.updated_at, CASE WHEN jsonb_typeof(kv.value)='object' THEN " + "coalesce((SELECT jsonb_object_agg(u.k, CASE WHEN jsonb_typeof(u.v)='object' THEN " + "(u.v - 'overlays' - 'trash') || CASE WHEN jsonb_typeof(u.v->'views')='object' THEN " + "jsonb_build_object('views', coalesce((SELECT jsonb_object_agg(v.k, CASE " + "WHEN jsonb_typeof(v.v->'config')='object' THEN jsonb_set(v.v, '{{config}}', " + "CASE WHEN jsonb_typeof(v.v->'config'->'memberPids')='array' THEN " + "(v.v->'config') - 'memberPids' ELSE v.v->'config' END) ELSE v.v END) FROM jsonb_each(u.v->'views') v(k,v)), " + "'{{}}'::jsonb)) ELSE '{{}}'::jsonb END ELSE u.v END) FROM jsonb_each(kv.value) u(k,v)), " + "'{{}}'::jsonb) ELSE kv.value END::text FROM {tbl} kv WHERE kv.key=%s", + 'store_kv', tenant_slug), (str(name),)).fetchone() + text = row[2] if row else None + _request_timing.record_store('view_workspace', str(name), bytes_read=_request_timing.text_bytes(text), count=1) + parsed = _parse_read(name, 'view_workspace', text) + if not isinstance(parsed, dict): + return parsed + result = _ViewWorkspace(parsed) + result.source_revision = row[:2] if row else None + return result + + +def _iter_view_members(name, path, revision, *, batch_size=2048, tenant_slug=None): + """Stream the exact metadata generation, refusing disappearance or concurrent edits.""" + import uuid + size = int(batch_size) + if not 1 <= size <= 4096: + raise ValueError('A view membership batch accepts between 1 and 4096 values') + with _pool().connection() as con: + with con.cursor(name='view_members_' + uuid.uuid4().hex) as cur: + cur.execute(_q("SELECT kv.rev, kv.updated_at, e.n, e.v::text FROM {tbl} kv " + "LEFT JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' " + "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) ON TRUE " + 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug), + (list(map(str, path)), list(map(str, path)), str(name))) + found = False + while True: + _request_timing.remaining_budget() + rows = cur.fetchmany(size) + if not rows: + if not found: + raise RuntimeError('canonical_source_unavailable: the saved-view document disappeared') + return + found = True + if any(tuple(row[:2]) != tuple(revision or ()) for row in rows): + raise RuntimeError('canonical_source_unavailable: the saved view changed during this read') + _request_timing.record_store('view_members', str(name), + bytes_read=sum(len(value.encode('utf-8')) for _, _, _, value in rows if value is not None)) + yield [(str(index - 1), json.loads(value)) for _, _, index, value in rows if index is not None] + rows.clear() + + +def _get_json_object(name, path=(), *, drop=(), omit=(), tenant_slug=None): + """Read nested object definitions while removing their population operands in SQL.""" + with _pool().connection() as con: + row = con.execute(_q("SELECT coalesce(jsonb_object_agg(e.k, CASE WHEN jsonb_typeof(e.v)='object' " + "THEN e.v - %s::text[] ELSE e.v END), '{{}}'::jsonb)::text FROM {tbl} kv " + "CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='object' " + "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) e(k,v) WHERE kv.key=%s AND NOT e.k=ANY(%s::text[])", + 'store_kv', tenant_slug), (list(drop), list(map(str, path)), list(map(str, path)), str(name), list(omit))).fetchone() + _request_timing.record_store('json_object', str(name), bytes_read=_request_timing.text_bytes(row[0]), count=1) + return json.loads(row[0]) + + +def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant_slug=None): + """Read complete selected membership without transferring an entire stored array.""" + wanted = list(dict.fromkeys(str(int(pid)) for pid in ids)) + if len(wanted) > 4096: + raise ValueError('A membership read accepts at most 4096 identities') + if not wanted: + return [] + with _pool().connection() as con: + rows = con.execute(_q("WITH members AS (SELECT e.v, e.n, CASE WHEN %s::text IS NULL " + "THEN e.v ELSE e.v -> %s END ident FROM {tbl} kv " + "CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' " + "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) " + 'WHERE kv.key=%s AND (%s::bigint IS NULL OR e.n <= %s)), ' + "normalized AS (SELECT v,n,CASE WHEN jsonb_typeof(ident)='number' THEN trunc((ident #>> '{{}}')::numeric)::text " + "WHEN ident='true'::jsonb THEN '1' WHEN ident='false'::jsonb THEN '0' ELSE trim(ident #>> '{{}}') END id FROM members), " + "selected AS (SELECT v,n,CASE WHEN id ~ '^[+-]?[0-9]+$' THEN id::numeric::text END id FROM normalized), " + 'ranked AS (SELECT v,n,row_number() OVER (PARTITION BY id ORDER BY n) r FROM selected WHERE id=ANY(%s::text[])) ' + 'SELECT v::text FROM ranked WHERE r=1 ORDER BY n', 'store_kv', tenant_slug), + (id_key, id_key, list(map(str, path)), list(map(str, path)), str(name), + source_limit, source_limit, wanted)).fetchall() + _request_timing.record_store('json_members', str(name), bytes_read=sum(len(row[0]) for row in rows), count=1) + return [json.loads(row[0]) for row in rows] + + +@_request_timing.store_operation("definition_counts") +def _definition_counts_text(name, tenant_slug=None): + """Count stored row keys in Postgres without returning row values.""" + with _pool().connection() as con: + row = con.execute(_q( + "SELECT COALESCE(json_object_agg(e.key, CASE " + "WHEN jsonb_typeof(e.value->'rows') = 'object' THEN " + "(SELECT count(*) FROM jsonb_object_keys(e.value->'rows')) " + "WHEN NOT (e.value ? 'rows') OR e.value->'rows' = 'null'::jsonb THEN 0 " + "ELSE NULL END), '{{}}'::json)::text FROM {tbl} AS kv " + "CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value) = 'object' " + "THEN kv.value ELSE '{{}}'::jsonb END) AS e WHERE kv.key = %s", + 'store_kv', tenant_slug), (str(name),)).fetchone() + text = row[0] if row else None + _request_timing.record_store('definition_counts', str(name), + bytes_read=_request_timing.text_bytes(text), count=0) + return text + + +@_request_timing.store_operation('get_many', many=True) def _get_many(names, tenant_slug=None): """Several ordinary documents in one round trip, keyed by requested store key.""" keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name))) if not keys: return {} - with _pool().connection() as con: + pool = _pool() + request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys} + checkpoint = _request_timing.memo_checkpoint() + with pool.connection() as con: rows = con.execute( - _q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])', + _q('SELECT key, value::text, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])', 'store_kv', tenant_slug), (keys,)).fetchall() - found = {str(key): value for key, value in rows} + found = {} + for key, text, rev, stamp in rows: + found[str(key)] = _parsed(text) + _request_timing.record_store('get_many', str(key), bytes_read=_request_timing.text_bytes(text), count=0) + _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp) + if not str(key).endswith('user_tables'): + shape_key = (*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1]) + _request_timing.memo_seed(shape_key, text, checkpoint) + for key in keys: + if key not in found: + _seed_read_token(request_keys[key], checkpoint, None, None) + if not key.endswith('user_tables'): + _request_timing.memo_seed((*request_keys[key][:3], 'whole', request_keys[key][-1]), + None, checkpoint) return { key: (dict(found[key]) if isinstance(found.get(key), dict) else (found[key] if key in found else {})) @@ -957,6 +1472,7 @@ def put(name, data, tenant_slug=None): return data +@_request_timing.store_operation("update") def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=None): """Read-modify-write inside ONE transaction, with the row LOCKED for the duration. @@ -1006,7 +1522,36 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row= def _update_locked(name, fn, tenant_slug, expect_rev, audit_row): - """`update`'s transaction body (split out so the D-2640 generation bump wraps all of it).""" + """Diagnose lock refusal only after the aborted row transaction has rolled back.""" + from psycopg.errors import LockNotAvailable + try: + return _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row) + except LockNotAvailable as exc: + if not getattr(exc, '_aios_lock_diagnosed', False): + _log_lock_blockers(getattr(exc, '_aios_waiter_pid', None)) + raise + + +def _log_lock_blockers(waiter_pid): + """Read blocker identities on a separate transaction without logging statement values.""" + if waiter_pid is None: + return + try: + with _pool().connection(timeout=1.0) as diagnostic: + with diagnostic.transaction(): + diagnostic.execute("SELECT set_config('statement_timeout', '1000ms', true)") + rows = diagnostic.execute( + 'SELECT a.pid, a.query FROM pg_stat_activity AS a ' + 'WHERE a.pid = ANY(pg_blocking_pids(%s))', (int(waiter_pid),)).fetchall() + blockers = [{'pid': int(pid), 'query': _request_timing.sql_text(query)} for pid, query in rows] + print(f'[store-pg] lock blockers for pid {int(waiter_pid)}: ' + f'{json.dumps(blockers, ensure_ascii=True)}', flush=True) + except Exception as exc: + print(f'[store-pg] lock blocker diagnosis unavailable ({type(exc).__name__})', flush=True) + + +def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row): + """Run the callback and write under the existing row lock and transaction.""" from psycopg.errors import LockNotAvailable wait_s = _lock_timeout_s() with _pool().connection() as con: @@ -1015,21 +1560,28 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row): # lock gives its pooled connection back after `wait_s` instead of holding it forever. con.execute("SELECT set_config('lock_timeout', %s, true)", (f'{int(wait_s * 1000)}ms',)) - try: - row = con.execute( - _q('SELECT value, rev FROM {tbl} WHERE key = %s FOR UPDATE', - 'store_kv', tenant_slug), - (str(name),)).fetchone() - except LockNotAvailable: - print(f'[store-pg] lock wait over {wait_s:g}s on ' - f'{_effective_slug(tenant_slug)}/{name}', flush=True) - raise + pid_row = con.execute('SELECT pg_backend_pid()').fetchone() + waiter_pid = pid_row[0] if pid_row else None + with _request_timing.lock_wait_probe(waiter_pid, wait_s, _log_lock_blockers) as probe: + try: + row = con.execute( + _q('SELECT value::text, rev FROM {tbl} WHERE key = %s FOR UPDATE', + 'store_kv', tenant_slug), + (str(name),)).fetchone() + except LockNotAvailable as exc: + exc._aios_waiter_pid = waiter_pid + exc._aios_lock_diagnosed = probe is not None and probe.fired + print(f'[store-pg] lock wait over {wait_s:g}s on ' + f'{_effective_slug(tenant_slug)}/{name}', flush=True) + raise + _request_timing.record_store("update", str(name), + bytes_read=_request_timing.text_bytes(row[0] if row else None), count=0) if expect_rev is not None: current = int(row[1]) if row and row[1] is not None else 0 if current != int(expect_rev): from core.view_audit import ViewRevisionStale raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev)) - data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {}) + data = _parsed(row[0] if row else None) result = fn(data) data = result if result is not None else data payload = _payload(name, data, con, tenant_slug) @@ -1038,6 +1590,8 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row): 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug), (str(name), payload)) + _request_timing.record_store("update", str(name), + bytes_written=_request_timing.text_bytes(payload), count=0) # ⚠ A LIST, OR ONE DICT, AND THE LIST IS NOT A CONVENIENCE. Moving views between # folders is ONE workspace write that changes several views' placements, so the # trail for it is several rows or it is wrong; a single-row keyword would have @@ -1096,6 +1650,46 @@ def delete_path(path_in_repo, tenant_slug=None): def revision(name, tenant_slug=None): + """Return an independent revision object, reusing only this request's backing validation.""" + return _parsed(_request_text(name, 'revision', tenant_slug, + lambda: json.dumps(_revision(name, tenant_slug)))) + + +@_request_timing.memo_read +def _revision_many(names, tenant_slug=None): + """Batch only unchecked keys and lend complete tokens to every request read shape.""" + keys = tuple(sorted(set(map(str, names)))) + checkpoint = _request_timing.memo_checkpoint() + request_keys = {name: _request_key(name, 'revision', tenant_slug) for name in keys} + found, missing = {}, [] + for name, key in request_keys.items(): + hit, text = _request_timing.memo_peek(key) + if hit: + found[name] = _parsed(text) + else: + missing.append(name) + if missing: + read = _read_revisions(missing, tenant_slug) + for name in missing: + token = read.get(name, _revision_value(None, None)) + _request_timing.memo_seed(request_keys[name], json.dumps(token), checkpoint) + found[name] = token + return {name: found[name] for name in keys} + + +@_request_timing.store_operation('revisions', many=True) +def _read_revisions(names, tenant_slug=None): + with _pool().connection() as con: + rows = con.execute(_q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])', + 'store_kv', tenant_slug), (list(names),)).fetchall() + found = {} + for name, raw_rev, stamp in rows: + found[name] = _revision_value(raw_rev, stamp) + return found + + +@_request_timing.store_operation("revision") +def _revision(name, tenant_slug=None): """`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6. ⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and @@ -1115,8 +1709,12 @@ def revision(name, tenant_slug=None): """ with _pool().connection() as con: row = con.execute( - _q('SELECT rev, updated_at FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + _q('SELECT revision.rev, revision.updated_at, octet_length(revision.rev::text) + ' + 'octet_length(revision.updated_at::text) ' + 'FROM (SELECT rev, updated_at FROM {tbl} WHERE key = %s) AS revision', 'store_kv', tenant_slug), (str(name),)).fetchone() + _request_timing.record_store("revision", str(name), bytes_read=int(row[2] or 0) if row and len(row) > 2 else 0, + count=0) rev = int(row[0]) if row and row[0] is not None else 0 stamp = row[1] if row else None # ⚠ EPOCH SECONDS ON BOTH BACKENDS. `core.store` records `time.time()`; this column is a @@ -1163,8 +1761,8 @@ class PgStore: """`core.store.Store`'s interface for ONE tenant's schema. Deliberately a thin binder over the module functions rather than a reimplementation: every - behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh` - ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift + behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, fresh reads + bypassing text reuse) is stated once, up there, and cannot drift between the two entry points. ⭐⭐ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and @@ -1214,6 +1812,27 @@ class PgStore: return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug, timeout_s=timeout_s) + def get_json_paths(self, name, paths): + return _get_json_paths(name, paths, tenant_slug=self.tenant_slug) + + def iter_json_items(self, name, path=(), *, batch_size=1024): + return _iter_json_items(name, path, batch_size=batch_size, tenant_slug=self.tenant_slug) + + def iter_json_values(self, name, path=(), *, drop=(), batch_size=1024): + return _iter_json_values(name, path, drop=drop, batch_size=batch_size, tenant_slug=self.tenant_slug) + + def get_json_object(self, name, path=(), *, drop=(), omit=()): + return _get_json_object(name, path, drop=drop, omit=omit, tenant_slug=self.tenant_slug) + + def get_view_workspace(self, name): + return _get_view_workspace(name, tenant_slug=self.tenant_slug) + + def iter_view_members(self, name, path, revision, *, batch_size=2048): + return _iter_view_members(name, path, revision, batch_size=batch_size, tenant_slug=self.tenant_slug) + + def get_json_members(self, name, path, ids, *, id_key=None, source_limit=None): + return _get_json_members(name, path, ids, id_key=id_key, source_limit=source_limit, tenant_slug=self.tenant_slug) + def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None, expected_rev=None): return _get_workbook_projection(name, metadata_only=metadata_only, tab_ids=tab_ids, @@ -1244,6 +1863,11 @@ class PgStore: """Bound multi-key read used by request-scoped read snapshots.""" return _get_many(names, tenant_slug=self.tenant_slug) + def get_definition_counts(self, name): + """Exact stored counts only; a read-through source's population is not measured here.""" + return _parsed(_request_text(name, 'definition_counts', self.tenant_slug, + lambda: _definition_counts_text(name, self.tenant_slug))) + def get_entry(self, name, entry): """Bound single-entry read; None means absent, and the caller falls back to `get`.""" return _get_entry(name, entry, tenant_slug=self.tenant_slug) diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 5819015df17953ed6e1c0e17159c73c4faefe70c..a7b50713b10cee34a9dec1fea13241343994f5b9 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -1916,8 +1916,11 @@ def all_tables(st=None): ⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below. """ + reader = _st(st) + if getattr(reader, 'schema_only', False): + return all_defs(reader, schema_only=True) try: - return _with_activity(_st(st).get(STORE_KEY)) + return _with_activity(reader.get(STORE_KEY)) except Exception: return {} @@ -1927,7 +1930,27 @@ def all_tables(st=None): DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit') -def all_defs(st=None): +def definition_counts(st=None): + """Exact stored counts from a bound capability, or unknown without reading rows.""" + reader = _st(st) + # Snapshot wrappers cannot borrow live counts from a different document owner. + if isinstance(reader, _Lent) or type(reader).__name__ == '_AssemblyStore': + return {} + backend = reader._store() if callable(getattr(reader, '_store', None)) else reader + if callable(getattr(backend, '_d', None)): + backend = backend._d() + count = getattr(backend, 'get_definition_counts', None) + if not callable(count): + return {} + name = reader.store_key(STORE_KEY) if callable(getattr(reader, 'store_key', None)) else STORE_KEY + counts = count(name) + if not isinstance(counts, dict): + from core.shared_overlay import SchemaUnavailable + raise SchemaUnavailable('stored row counts are unavailable') + return counts + + +def all_defs(st=None, *, schema_only=False): """{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row. ⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for @@ -1940,8 +1963,9 @@ def all_defs(st=None): from a database that genuinely has no rows, and the caller least able to notice is the one that wanted rows. If you get that error you wanted `all_tables`. - ⚠ Falls back to `all_tables` on ANY failure, including a store handle too old to project (a - fake store installed by a gate, a `_Lent` handle serving a document somebody else read). The + Schema-only requests propagate read failures and never fall back to a whole read. + Other readers fall back to `all_tables` on ANY failure, including a store handle too old to + project (a fake store installed by a gate, a `_Lent` handle serving an already-read document). The fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for — which is the one direction a fallback here is allowed to be wrong in. @@ -1952,7 +1976,7 @@ def all_defs(st=None): databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]). Nothing had noticed because the projection does not fail on a healthy store — a fallback is dead code until the day it is the only code running. **Every failure now falls back to the - whole read**, which keeps `all_tables`' own `except: return {}` as the single backstop instead + whole read for ordinary readers**, which keeps `all_tables`' own `except: return {}` as the single backstop instead of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on this function, so an empty answer here stops being a blank nav and becomes a 404 on every database in the tenant. @@ -1960,13 +1984,22 @@ def all_defs(st=None): # Schema-only handles cannot use the historical whole-reader fallback described above: # their caller must distinguish an unavailable definition snapshot from an empty one. reader = _st(st) + from core import request_timing + schema_only = schema_only or getattr(reader, 'schema_only', False) or request_timing.CURRENT.get() is not None try: - return _with_activity(reader.get_projection(STORE_KEY, drop=DEF_ONLY_DROPS)) - except Exception: + project = getattr(reader, 'get_projection', None) + if schema_only and not callable(project): + from core.shared_overlay import SchemaUnavailable + raise SchemaUnavailable('table definitions require a projected store reader') + return _with_activity(project(STORE_KEY, drop=DEF_ONLY_DROPS)) + except Exception as exc: # A schema-only request promises not to materialize rows, including on an outage. - # Its existing route maps this failure to a named unavailable response. Ordinary - # full readers retain the correct-but-slow fallback below. - if getattr(reader, 'schema_only', False): + # Preserve the original failure type for request callers to classify. + if schema_only: + request_timing.raise_if_resource_error(exc) + if request_timing.CURRENT.get() is not None: + from core.shared_overlay import SchemaUnavailable + raise SchemaUnavailable('table definitions are unavailable') from exc raise # ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was # only ever ONE of the ways this can fail, and singling it out is what made the other ways @@ -1982,6 +2015,8 @@ def _entry_reader(st): """Use an entry door only when it belongs to the same reader as the bound snapshot.""" # The API's request reader defines get_entry even when its wrapped snapshot does not. # Judge that runtime's own static capability and owner, then retain the request read door. + if getattr(st, 'schema_only', False): + return None bound = st if type(st).__name__ == '_AssemblyStore': st = st._runtime @@ -5430,6 +5465,15 @@ class _Lent: self._runtime, self._docs = runtime, dict(docs or {}) def get(self, name, fresh=False): + if fresh: + from core.request_timing import clear_read_memo + clear_read_memo() + if name == STORE_KEY and getattr(self, 'schema_only', False): + if fresh: + self._docs.pop(name, None) + if fresh or name not in self._docs: + self._docs[name] = all_defs(self._runtime, schema_only=True) + return self._docs[name] if fresh or (name not in self._docs and name not in _LENDABLE and not str(name).endswith('__shared')): return self._runtime.get(name, fresh) if fresh else self._runtime.get(name) @@ -5470,7 +5514,7 @@ def lend(st=None, **docs): return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None}) -def lend_defs(st=None): +def lend_defs(st=None, *, schema_only=False): """`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows. ⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02 @@ -5489,7 +5533,12 @@ def lend_defs(st=None): ⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY should take the whole read once rather than this plus that. """ - return lend(st, **{STORE_KEY: all_defs(st)}) + reader = _st(st) + strict = schema_only or getattr(reader, 'schema_only', False) + lent = lend(reader, **{STORE_KEY: all_defs(reader, schema_only=strict)}) + if strict: + lent.schema_only = True + return lent def _utc_now(): diff --git a/platform/harness/canonical_source.py b/platform/harness/canonical_source.py new file mode 100644 index 0000000000000000000000000000000000000000..742552bc096a0631090aff41d58591ff504589c0 --- /dev/null +++ b/platform/harness/canonical_source.py @@ -0,0 +1,383 @@ +"""Bounded, read-only operands for the existing compiled customer and product projections.""" +import contextvars +import itertools +import json +import uuid +from contextlib import contextmanager + +from core import odoo as _odoo + +BATCH = 1024 +_ACTIVE = contextvars.ContextVar('canonical_source', default=None) + + +def code_sql(alias): + raw = f"json_extract_string({alias}.body, '/default_code')" + return (f"CASE WHEN {raw} IS NULL OR {raw} = '' OR json_type({alias}.body, '/default_code') = 'BOOLEAN' " + f"THEN 'pid:' || json_extract_string({alias}.body, '/id') ELSE trim({raw}) END") + + +class TierProjection(list): + """Retain displayed tiers and the complete overflow receipt from SQL.""" + def __init__(self): + super().__init__() + self.overflow = {} + + +class Rows: + """Reiterate a SQL snapshot without retaining its Python records.""" + + def __init__(self, owner, table): + self.owner, self.table = owner, table + + def __iter__(self): + last = -1 + while True: + batch = self.owner.cur.execute( + f'SELECT ordinal, body FROM "{self.table}" WHERE ordinal > ? ORDER BY ordinal LIMIT ?', + [last, BATCH]).fetchall() + if not batch: + return + for _, body in batch: + yield json.loads(body) + last = batch[-1][0] + batch.clear() + + def __len__(self): + return self.owner.cur.execute(f'SELECT count(*) FROM "{self.table}"').fetchone()[0] + + def __bool__(self): + return bool(len(self)) + + +class Source: + """Own bounded source operands and release every temporary snapshot on exit.""" + + def __init__(self, cur, kind, identities, store=None): + self.cur, self.kind, self.identities, self.store = cur, kind, list(identities), store + self.tables = [] + self.cache = {} + self.failure = None + + def snapshot(self, records): + name = 'canonical_' + uuid.uuid4().hex + self.cur.execute(f'CREATE TEMP TABLE "{name}" (ordinal BIGINT, body JSON)') + self.tables.append(name) + records, offset = iter(records), 0 + try: + while True: + batch = list(itertools.islice(records, BATCH)) + if not batch: + break + self.cur.execute(f'INSERT INTO "{name}" SELECT unnest(?::BIGINT[]), unnest(?::JSON[])', + [list(range(offset, offset + len(batch))), [json.dumps(r, default=str) for r in batch]]) + offset += len(batch) + batch.clear() + except BaseException as exc: + self.failure = exc + raise + finally: + close = getattr(records, 'close', None) + if close: + close() + return Rows(self, name) + + def domain(self, model, domain): + domain = list(domain or []) + if self.kind == 'customer': + field = {'res.partner': 'id', 'sale.order': 'partner_id', + 'sale.order.line': 'order_partner_id', 'account.move': 'partner_id'}.get(model) + # Explicit partner lookup domains also resolve agents and must retain their ids. + if model == 'res.partner' and any(isinstance(d, (tuple, list)) and d[0] in {'id', 'name'} + for d in domain): + field = None + if field: + domain.append((field, 'in', self.identities)) + elif self.kind == 'product' and model == 'product.product': + # Match the canonical stripped code, including every duplicate and archived variant. + codes = list(self.identities) + ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()] + terms = [('default_code', '=like', '%' + code + '%') for code in codes] + terms.append(('id', 'in', ids)) + domain += ['|'] * (len(terms) - 1) + terms + elif self.kind == 'product' and model in {'sale.order.line', 'stock.quant'}: + codes = list(self.identities) + ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()] + terms = [('product_id.default_code', '=like', '%' + code + '%') for code in codes] + terms.append(('product_id', 'in', ids)) + domain += ['|'] * (len(terms) - 1) + terms + return domain + + def products(self): + key = ('products',) + if key not in self.cache: + self.cache[key] = self.search_read('product.product', [('active', 'in', [True, False])], + ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active']) + return self.cache[key] + + def sku_revenue(self, groups): + products = self.products() + code = code_sql('p') + rows = self.cur.execute(f"""SELECT {code} code, + first(json_extract_string(g.body, '/product_id/1') ORDER BY g.ordinal), + sum(coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0)), + sum(coalesce(try_cast(json_extract_string(g.body, '/product_uom_qty') AS DOUBLE), 0)), + count(DISTINCT nullif(json_extract_string(g.body, '/order_id/0'), '0')) + FROM "{groups.table}" g JOIN "{products.table}" p + ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id') + WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL + GROUP BY code""").fetchall() + return {code: {'name': name, 'rev': rev, 'qty': qty, 'orders': orders} + for code, name, rev, qty, orders in rows} + + def customer_mix(self, groups): + def products(): + last = 0 + while True: + ids = [r[0] for r in self.cur.execute(f"""SELECT DISTINCT + try_cast(json_extract_string(body, '/product_id/0') AS BIGINT) id + FROM "{groups.table}" WHERE id > ? ORDER BY id LIMIT ?""", [last, BATCH]).fetchall()] + if not ids: + return + yield from _odoo.search_read('product.product', [('id', 'in', ids)], + ['id', 'name', 'categ_id'], limit=BATCH, order='id asc') + last = ids[-1] + prods = self.snapshot(products()) + query = f"""WITH lines AS ( + SELECT try_cast(json_extract_string(g.body, '/order_partner_id/0') AS BIGINT) pid, + json_extract_string(g.body, '/product_id/0') product, + coalesce(nullif(trim(list_last(string_split(json_extract_string(p.body, '/categ_id/1'), ' / '))), ''), '(none)') cat, + coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0) val, + g.ordinal, json_extract_string(p.body, '/name') AS product_name + FROM "{groups.table}" g LEFT JOIN "{prods.table}" p + ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id') + WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL), + cats AS (SELECT pid, cat, sum(val) val, min(ordinal) ordinal FROM lines GROUP BY pid, cat), + topcat AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM cats), + skus AS (SELECT pid, product, first(product_name ORDER BY ordinal) AS product_name, sum(val) val, + min(ordinal) ordinal FROM lines GROUP BY pid, product), + topsku AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM skus), + totals AS (SELECT pid, count(DISTINCT product) n, sum(val) total FROM lines GROUP BY pid) + SELECT t.pid, c.cat, CASE WHEN t.total != 0 THEN c.val / t.total END, t.n, + coalesce(s.product_name, '(none)') FROM totals t JOIN topcat c ON t.pid=c.pid AND c.rn=1 + JOIN topsku s ON t.pid=s.pid AND s.rn=1""" + return {pid: {'top_category': cat, 'top_category_pct': pct, 'sku_count': n, 'top_sku': name} + for pid, cat, pct, n, name in self.cur.execute(query).fetchall()} + + def categories(self): + rows = self.products() + cats = self.search_read('product.category', [], ['id', 'complete_name']) + return {code: category for code, category in self.cur.execute(f"""SELECT + {code_sql('p')} code, + first(coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''), + '(uncategorized)') ORDER BY p.ordinal) + FROM "{rows.table}" p LEFT JOIN "{cats.table}" c + ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id') + WHERE json_extract_string(p.body, '/active') = 'true' + AND json_type(p.body, '/default_code') = 'VARCHAR' AND json_extract_string(p.body, '/default_code') != '' + GROUP BY code""").fetchall()} + + def backlog(self, groups): + products = self.products() + return {code: qty for code, qty in self.cur.execute(f"""WITH per AS ( + SELECT json_extract_string(body, '/product_id/0') id, + greatest(0, coalesce(try_cast(json_extract_string(body, '/product_uom_qty') AS DOUBLE), 0) + - coalesce(try_cast(json_extract_string(body, '/qty_delivered') AS DOUBLE), 0)) qty + FROM "{groups.table}") + SELECT {code_sql('p')} code, + sum(per.qty) FROM per JOIN "{products.table}" p + ON per.id=json_extract_string(p.body, '/id') + WHERE per.id IS NOT NULL AND per.qty > 1e-9 GROUP BY code""").fetchall()} + + def inventory_frame(self, t): + from core import periods + from modules import inventory + import pandas as pd + lf, lt = periods.ltm(t) + quant = self.read_group('stock.quant', [('location_id.usage', '=', 'internal')], + ['product_id', 'quantity:sum'], ['product_id'], lazy=False) + sales = self.read_group('sale.order.line', _odoo.sale_line_domain(lf, lt, all_channels=True), + ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) + products = self.search_read('product.product', [('default_code', '!=', False)], + ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type']) + cats = self.search_read('product.category', [], ['id', 'complete_name']) + query = f"""WITH raw AS (SELECT p.ordinal, + trim(json_extract_string(p.body, '/default_code')) sku, + json_extract_string(p.body, '/name') AS product_name, + coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''), '(uncategorized)') category, + json_extract_string(p.body, '/type') = 'product' storable, + coalesce(try_cast(json_extract_string(p.body, '/standard_price') AS DOUBLE), 0) AS cost, + coalesce(try_cast(json_extract_string(q.body, '/quantity') AS DOUBLE), 0) on_hand, + coalesce(try_cast(json_extract_string(s.body, '/product_uom_qty') AS DOUBLE), 0) qty_ltm, + coalesce(try_cast(json_extract_string(s.body, '/price_subtotal') AS DOUBLE), 0) rev_ltm + FROM "{products.table}" p + LEFT JOIN "{quant.table}" q ON json_extract_string(p.body, '/id')=json_extract_string(q.body, '/product_id/0') + LEFT JOIN "{sales.table}" s ON json_extract_string(p.body, '/id')=json_extract_string(s.body, '/product_id/0') + LEFT JOIN "{cats.table}" c ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id')) + SELECT sku, first(product_name ORDER BY ordinal) AS name, first(category ORDER BY ordinal) category, + bool_or(storable) storable, sum(on_hand) on_hand, sum(on_hand*cost) inv_value, + sum(qty_ltm*cost) cogs_ltm, sum(qty_ltm) qty_ltm, sum(rev_ltm) rev_ltm + FROM raw GROUP BY sku""" + result = self.cur.execute(query) + names = [d[0] for d in result.description] + return pd.DataFrame(result.fetchall(), columns=names), (lf, lt) + + def tiers(self, prods): + from core import periods + from modules import products + lists = self.search_read('product.pricelist', [], ['id', 'name']) + today = periods.today().isoformat() + rules = self.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product']), + '|', ('date_start', '=', False), ('date_start', '<=', today), + '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)], + ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', + 'fixed_price', 'packaging_id', 'packaging_price']) + normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2), + canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules) + prods = products.active_products() if prods is None else prods + declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS}) + query = f"""WITH eligible AS ( + SELECT {code_sql('p')} code, p.ordinal product_order, r.ordinal rule_order, + trim(json_extract_string(l.body, '/name')) pricelist, + try_cast(json_extract_string(r.body, '/canonical_price') AS DOUBLE) price, + json_extract_string(r.body, '/canonical_unit') unit + FROM "{prods.table}" p JOIN "{normalized.table}" r ON ( + (json_extract_string(r.body, '/applied_on')='0_product_variant' + AND json_extract_string(r.body, '/product_id/0')=json_extract_string(p.body, '/id')) OR + (json_extract_string(r.body, '/applied_on')='1_product' + AND json_extract_string(r.body, '/product_tmpl_id/0')=json_extract_string(p.body, '/product_tmpl_id/0') + AND NOT EXISTS (SELECT 1 FROM "{normalized.table}" v + WHERE json_extract_string(v.body, '/applied_on')='0_product_variant' + AND json_extract_string(v.body, '/pricelist_id/0')=json_extract_string(r.body, '/pricelist_id/0') + AND json_extract_string(v.body, '/product_id/0')=json_extract_string(p.body, '/id')))) + JOIN "{lists.table}" l ON json_extract_string(l.body, '/id')=json_extract_string(r.body, '/pricelist_id/0')), + identified AS (SELECT *, min(product_order) OVER (PARTITION BY code) first_product, + row_number() OVER (PARTITION BY code, pricelist, price, unit ORDER BY product_order, rule_order) ident FROM eligible), + merged AS (SELECT * FROM identified WHERE product_order=first_product OR ident=1), + ranked AS (SELECT *, row_number() OVER (PARTITION BY code ORDER BY price, lower(unit), lower(pricelist), product_order, rule_order) pooled, + row_number() OVER (PARTITION BY code, pricelist ORDER BY price, lower(unit), product_order, rule_order) tier, + count(*) OVER (PARTITION BY code, pricelist) n FROM merged) + SELECT code, pricelist, price, unit, n FROM ranked + WHERE pooled <= 3 OR (pricelist IN (SELECT unnest(?::VARCHAR[])) AND tier <= ?) + ORDER BY code, price, lower(unit), lower(pricelist), product_order, rule_order""" + result = self.cur.execute(query, [declared, products.PRICELIST_TIER_DEPTH]) + out = {} + while True: + batch = result.fetchmany(BATCH) + if not batch: + break + for code, name, price, unit, n in batch: + entry = out.setdefault(code, TierProjection()) + entry.append({'pricelist': name, 'unit_price': price, 'unit': unit}) + if name in declared and n > products.PRICELIST_TIER_DEPTH: + entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH + return out, {'bounded': True} + + def search_read(self, model, domain=None, fields=None, **kw): + domain = self.domain(model, domain) + # Keyset traversal avoids a connector's default limit and never silently caps connected rows. + kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset', 'order'}} + def records(): + last = 0 + while True: + page = _odoo.search_read(model, domain + [('id', '>', last)], + list(dict.fromkeys(['id', *(['default_code'] if self.kind == 'product' and model == 'product.product' else []), + *(fields or [])])), limit=BATCH, order='id asc', **kw) + if not page: + return + nxt = max(int(r['id']) for r in page) + if nxt <= last: + raise RuntimeError('canonical_source_invalid: source keyset did not advance') + if self.kind == 'product' and model == 'product.product': + selected = set(self.identities) + for row in page: + code = str(row['default_code']).strip() if row.get('default_code') else f"pid:{row['id']}" + if code in selected: + yield row + else: + yield from page + last = nxt + page.clear() + return self.snapshot(records()) + + def read_group(self, model, domain, fields, groupby, **kw): + domain = self.domain(model, domain) + kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset'}} + def records(): + offset = 0 + while True: + options = dict(kw, offset=offset, limit=BATCH) + if groupby: + options.setdefault('orderby', ','.join(groupby)) + page = _odoo.get_odoo().execute_kw(model, 'read_group', [domain, fields, groupby], options) + if not page: + return + for row in page: + yield {key: value for key, value in row.items() + if not key.startswith('__') or key == '__count'} + offset += len(page) + page.clear() + return self.snapshot(records()) + + def search_count(self, model, domain=None, **kw): + if self.kind == 'product' and model == 'product.product': + return len(self.search_read(model, domain, ['id', 'default_code'])) + return _odoo.get_odoo().search_count(model, self.domain(model, domain), **kw) + + def agent_members(self, name): + if name == '(none)': + return self.search_read('res.partner', [('customer_rank', '>', 0), + ('agent_ids', '=', False), ('active', 'in', [True, False])], ['id']) + agents = self.search_read('res.partner', [('name', '=', name), + ('active', 'in', [True, False])], ['id']) + def records(): + source = iter(agents) + while True: + batch = list(itertools.islice(source, BATCH)) + if not batch: + return + rows = self.search_read('res.partner', [('agent_ids', 'in', [r['id'] for r in batch]), + ('active', 'in', [True, False])], ['id']) + try: + yield from rows + finally: + self.cur.execute(f'DROP TABLE IF EXISTS "{rows.table}"') + self.tables.remove(rows.table) + return self.snapshot(records()) + + def close(self): + for table in reversed(self.tables): + self.cur.execute(f'DROP TABLE IF EXISTS "{table}"') + self.tables.clear() + self.cache.clear() + + +def active(): + return _ACTIVE.get() + + +@contextmanager +def bounded(cur, kind, identities, store=None): + source = Source(cur, kind, identities, store) + token = _ACTIVE.set(source) + try: + yield source + if source.failure is not None: + raise RuntimeError('canonical_source_unavailable: ' + str(source.failure)) from source.failure + finally: + _ACTIVE.reset(token) + source.close() + + +class _Odoo: + def __getattr__(self, name): + source = active() + if source is not None and name in {'search_read', 'read_group', 'search_count'}: + return getattr(source, name) + if source is not None and name == 'get_odoo': + return lambda: self + return getattr(_odoo, name) + + +odoo = _Odoo() diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index a79a0c3a6f8d4fb0aa99bd103d6c42bc17e07e8a..64654ea3a670e1e52662e393df24989d4c9280ee 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -17,18 +17,23 @@ Design rules: - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo. """ import datetime as dt +import hashlib import json import math import os +import pickle import re import sys import time +import tempfile +from collections import OrderedDict from contextlib import nullcontext from pathlib import Path import duckdb import core.odoo as O +from core import request_timing as _request_timing _STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store" @@ -136,16 +141,46 @@ def path_for(tenant_key): def apply_limits(con): - """Cap one DuckDB instance from AIOS_DUCKDB_MEMORY and AIOS_DUCKDB_THREADS. Unset keeps the default.""" - memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or "").strip() - threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or "").strip() + """Bound each instance's memory, execution threads and local spill space.""" + web = os.environ.get("AIOS_PROCESS_ROLE") == "web" + memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or ( + os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB" if web else "4GB")).strip() + threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or ( + os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2" if web else "1")).strip() + spill = str(os.environ.get("AIOS_DUCKDB_MAX_TEMP_SIZE") or "4GB").strip() + temp = Path(os.environ.get("AIOS_DUCKDB_TEMP_DIR") or ( + Path(tempfile.gettempdir()) / f"aios-duckdb-{os.getpid()}")) + rows = con.execute("PRAGMA database_list").fetchall() + identity = next((str(row[2]) for row in rows if row[2]), str(id(con))) + temp = temp / hashlib.sha256(identity.encode()).hexdigest()[:16] + temp.mkdir(parents=True, exist_ok=True, mode=0o700) + if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE): + raise ValueError("Invalid AIOS_DUCKDB_MEMORY") + if not threads.isdigit() or int(threads) < 1: + raise ValueError("Invalid AIOS_DUCKDB_THREADS") + if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", spill, re.IGNORECASE): + raise ValueError("Invalid AIOS_DUCKDB_MAX_TEMP_SIZE") if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE): con.execute(f"SET memory_limit='{memory}'") if threads.isdigit() and int(threads) > 0: con.execute(f"SET threads={int(threads)}") + con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'") + con.execute(f"SET max_temp_directory_size='{spill}'") return con +def bounded_connection(con): + """Apply instance budgets and request deadlines to a temporary analytical connection.""" + try: + return _TimedCursor(apply_limits(con)) + except BaseException: + try: + con.close() + except Exception: + pass + raise + + def use_path(path): """Repoint the store at `path`, CLOSING the process singleton first. @@ -180,6 +215,7 @@ def use_path(path): _READY["ok"] = False _GENERATION[0] += 1 # every other thread's cached cursor is now stale DB_PATH = Path(path) + _invalidate_counts() return DB_PATH # Entity specs: Odoo model → store table. m2o fields land as _id + _name. @@ -505,7 +541,7 @@ def snapshot_status(tenant_key, source, *, rt=None): result.update(saved) if not path.exists(): return result - con = ro_cursor() if path.resolve() == DB_PATH.resolve() else duckdb.connect(str(path)) + con = ro_cursor() if path.resolve() == DB_PATH.resolve() else _TimedCursor(apply_limits(duckdb.connect(str(path)))) try: manifest = _publication(con, source) if manifest: @@ -826,11 +862,23 @@ _GENERATION = [0] #: Bumped after every mirror snapshot install (`MirrorStore._install_snapshot`). D-2670: the #: Sheet change token caches its table scan per generation, so an install must move this. _MIRROR_INSTALLS = [0] +_MIRROR_WRITES = [0] +_ACTIVE_WRITES = [0] +_COUNT_CACHE = OrderedDict() +_COUNT_CACHE_MAX = 256 +_COUNT_CACHE_LOCK = _thr.RLock() + + +def _invalidate_counts(): + with _COUNT_CACHE_LOCK: + _MIRROR_WRITES[0] += 1 + _COUNT_CACHE.clear() def note_mirror_install(): """Mark that a mirror install committed or rolled back, so cached change tokens recompute.""" _MIRROR_INSTALLS[0] += 1 + _invalidate_counts() def mirror_generation(): @@ -868,9 +916,98 @@ def _instance(): return _INSTANCE["con"] +class _TimedCursor: + """Keep cursor chaining and thread-local identity while timing execute and result conversion.""" + + _FETCH = {"fetchone", "fetchall", "fetchmany", "fetchdf", "fetch_df", "df", "fetchnumpy", + "fetch_arrow_table", "fetch_record_batch", "arrow", "pl", "fetch_df_chunk"} + + def __init__(self, cursor): + self._cursor = cursor + self._statement = "" + self._statement_ms = 0.0 + self._request = None + self._in_transaction = False + + def _call(self, name, *args, **kwargs): + acc = _request_timing.CURRENT.get() + execute = name in {"execute", "executemany"} + writes = False + transaction = None + if execute: + statement = args[0] if args else kwargs.get("query", "") + self._statement = _request_timing.sql_text(statement) if acc is not None else "" + self._statement_ms = 0.0 + self._request = acc + try: + statements = self._cursor.extract_statements(statement) + writes = any(s.type.name not in {"SELECT", "TRANSACTION"} for s in statements) + for parsed in statements: + if parsed.type.name == "TRANSACTION": + transaction = not re.match(r"\s*(COMMIT|ROLLBACK|END)\b", parsed.query, re.I) + except Exception as exc: + _request_timing.raise_if_resource_error(exc) + # Unclassified statements cannot establish a reusable read snapshot. + writes, self._in_transaction = True, True + if transaction is not None: + # A failed statement batch may leave its BEGIN active. + self._in_transaction = True + if writes: + with _COUNT_CACHE_LOCK: + _ACTIVE_WRITES[0] += 1 + _invalidate_counts() + started = time.perf_counter() + try: + with _request_timing.thread_scope(), _request_timing.duckdb_operation(self._cursor): + result = getattr(self._cursor, name)(*args, **kwargs) + if transaction is not None: + self._in_transaction = transaction + return self if result is self._cursor else result + finally: + if writes: + with _COUNT_CACHE_LOCK: + _ACTIVE_WRITES[0] -= 1 + _invalidate_counts() + if acc is not None and self._request is acc: + elapsed = (time.perf_counter() - started) * 1000 + self._statement_ms += elapsed + _request_timing.record_duckdb(self._statement, elapsed, count=int(execute), + statement_ms=self._statement_ms) + + def execute(self, *args, **kwargs): + return self._call("execute", *args, **kwargs) + + def executemany(self, *args, **kwargs): + return self._call("executemany", *args, **kwargs) + + def cursor(self, *args, **kwargs): + return _TimedCursor(self._cursor.cursor(*args, **kwargs)) + + def __getattr__(self, name): + if name in self._FETCH: + return lambda *args, **kwargs: self._call(name, *args, **kwargs) + return getattr(self._cursor, name) + + def __enter__(self): + self._cursor.__enter__() + return self + + def __exit__(self, *args): + return self._cursor.__exit__(*args) + + def __iter__(self): + return self + + def __next__(self): + row = self.fetchone() + if row is None: + raise StopIteration + return row + + def ro_cursor(): """A read cursor on the shared instance — the contention-free way to read the store.""" - return _instance().cursor() + return _TimedCursor(_instance().cursor()) def ready(): @@ -891,7 +1028,8 @@ def ready(): cur.execute("SELECT entity, phase FROM _sync_state").fetchall()} finally: cur.close() - except Exception: + except Exception as exc: + _request_timing.raise_if_resource_error(exc) return False # ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN # OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every @@ -943,7 +1081,8 @@ def entity_live(key): columns_ready = _required_columns_ready(cur, key, ENTITIES.get(key) or {}) finally: cur.close() - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 + _request_timing.raise_if_resource_error(exc) return False return bool(row) and row[0] == "live" and columns_ready @@ -975,7 +1114,8 @@ def ro_con(): try: cur.execute("SELECT 1") # cheap liveness probe return cur - except Exception: + except Exception as exc: + _request_timing.raise_if_resource_error(exc) _RO_TLS.cur = None # stale → reopen below if not ready(): raise RuntimeError("the tenant data store is completing its FIRST sync on this " @@ -1034,11 +1174,335 @@ _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen. _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE) +_ORDINAL_ORDER = re.compile( + r"(?:\(\s*)*[+-]?\d[\d_]*(?:\s*\))*\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I) +_ALL_ORDER = re.compile(r"ALL\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I) + + +def _sql_terms(sql): + """Split trusted SQL at unquoted, top-level commas, or decline unfamiliar syntax.""" + depth, quote, start, index = 0, None, 0, 0 + terms = [] + while index < len(sql): + char = sql[index] + if quote: + if char == "\\": + return None + if char == quote: + if index + 1 < len(sql) and sql[index + 1] == quote: + index += 2 + continue + quote = None + elif char in "'\"": + quote = char + elif (not (char.isalnum() or char.isspace() or char in "_.,()+-*/%=<>!|:&~^") + or sql[index:index + 2] in ("--", "/*", "*/", "->", "=>", ":=")): + return None + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth < 0: + return None + elif char == "," and depth == 0: + terms.append(sql[start:index].strip()) + start = index + 1 + index += 1 + if quote or depth: + return None + terms.append(sql[start:].strip()) + return terms if all(terms) else None + + +def _sql_tokens(sql): + """Use DuckDB's lexer for identifier quoting and expression comparisons.""" + tokens = duckdb.tokenize(sql) + encoded = sql.encode("utf-8") + result = [(encoded[start:tokens[i + 1][0] if i + 1 < len(tokens) else len(encoded)].decode("utf-8").strip(), kind) + for i, (start, kind) in enumerate(tokens)] + if any(text.upper() in ("COLUMNS", "EXCLUDE", "REPLACE", "USING") for text, _kind in result): + raise ValueError("Unrecognized page ordering syntax") + return result + + +def _order_needs_projection(order, projection=""): + """Keep positional, ALL, alias-dependent and unfamiliar orders after projection.""" + terms = _sql_terms(order) + if terms is None or any(_ORDINAL_ORDER.fullmatch(t) or _ALL_ORDER.fullmatch(t) for t in terms): + return True + selected = _sql_terms(projection) if projection else [] + if selected is None: + return True + try: + fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + def normalized(tokens): + return [(text[1:-1].replace('""', '"').translate(fold) if text.startswith('"') + else text if text.startswith("'") else text.translate(fold), kind) for text, kind in tokens] + ordered = normalized(_sql_tokens(order)) + aliases = {} + for term in selected: + tokens = _sql_tokens(term) + depth, top = 0, [] + for i, (text, _kind) in enumerate(tokens): + if text == ")": + depth -= 1 + if depth == 0: + top.append(i) + if text == "(": + depth += 1 + if not top or any(tokens[i][0] in ("*", ":", ":=") for i in top): + return True + last = top[-1] + if len(top) >= 2 and tokens[top[-2]][0].upper() == "AS": + expression = tokens[:top[-2]] + elif (last > 0 and tokens[last][1] == duckdb.token_type.identifier + and tokens[last - 1][0] != "."): + expression = tokens[:last] + else: + continue + alias = normalized([tokens[last]])[0][0] + if tokens[last][0].startswith("'"): + alias = tokens[last][0][1:-1].replace("''", "'").translate(fold) + aliases.setdefault(alias, []).append(normalized(expression)) + for alias, expressions in aliases.items(): + covered = set() + # A repeated source expression is safe, including generated casts and passthrough aliases. + if all(expr == expressions[0] for expr in expressions): + expr = expressions[0] + for i in range(len(ordered) - len(expr) + 1): + if expr and ordered[i:i + len(expr)] == expr: + covered.update(range(i, i + len(expr))) + for i, token in enumerate(ordered): + if (token[0] == alias and token[1] in (duckdb.token_type.identifier, duckdb.token_type.keyword) + and i not in covered + and (i == 0 or ordered[i - 1][0] != ".") + and (i + 1 == len(ordered) or ordered[i + 1][0] not in (".", "("))): + return True + except (duckdb.Error, AttributeError, ValueError, UnicodeError): + return True + return False + + +def _physical_page_order(order, page_key): + """Make physical pages total without changing ALL or unfamiliar SQL ordering semantics.""" + terms = _sql_terms(order) + if terms is None or any(_ALL_ORDER.fullmatch(term) for term in terms): + return order + try: + _sql_tokens(order) + except (duckdb.Error, AttributeError, ValueError): + return order + return f"{order}, {page_key} ASC" + + +def _page_is_row_local(projection, order, cur): + """Prove outer expressions scalar with DuckDB's parser and bounded catalogue lookups.""" + metadata, verified, expanding, bound = {}, set(), set(), set() + + def parse(expressions, ordering=""): + sql = "SELECT " + expressions + (" ORDER BY " + ordering if ordering else "") + parsed = json.loads(cur.execute("SELECT json_serialize_sql(?)", [sql]).fetchone()[0]) + if parsed.get("error") or len(parsed["statements"]) != 1: + raise ValueError("Unrecognized page expression") + node = parsed["statements"][0]["node"] + if (node["type"] != "SELECT_NODE" or node["from_table"]["type"] != "EMPTY" + or node["cte_map"]["map"] or node["group_expressions"] or node["group_sets"] + or node["aggregate_handling"] != "STANDARD_HANDLING" + or any(node[key] is not None for key in ("where_clause", "having", "sample", "qualify")) + or any(m["type"] != "ORDER_MODIFIER" for m in node["modifiers"])): + raise ValueError("Projection changes the outer population") + return [node["select_list"], node["modifiers"]] + + def aggregate_is_local(value, scope): + columns = set() + + def collect(node): + if isinstance(node, list): + return all(collect(v) for v in node) + if not isinstance(node, dict): + return True + kind = node.get("class") + if kind == "COLUMN_REF": + columns.add(tuple(node["column_names"])) + elif kind is not None and kind not in { + "CONSTANT", "CAST", "CASE", "COMPARISON", "CONJUNCTION", + "OPERATOR", "FUNCTION", "PARAMETER"}: + return False + return all(collect(v) for v in node.values()) + + if not collect(value): + return False + if not columns: + return True + if scope is None or scope[1] or scope[0]["type"] != "SELECT_NODE": + return False + source = scope[0]["from_table"] + if (source["type"] != "BASE_TABLE" or source.get("column_name_alias") + or source.get("at_clause") or source.get("sample")): + return False + alias = source.get("alias") or source["table_name"] + fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + if any(len(c) != 2 or c[0].translate(fold) != alias.translate(fold) for c in columns): + return False + quote = lambda name: '"' + name.replace('"', '""') + '"' + table = '.'.join(quote(source[key]) for key in ("catalog_name", "schema_name", "table_name") + if source.get(key)) + selected = ','.join('.'.join(map(quote, c)) for c in sorted(columns)) + sql = f"DESCRIBE SELECT {selected} FROM {table} AS {quote(alias)}" + if sql not in bound: + # A matching alias alone cannot prove that a missing column will not bind outward. + cur.execute(sql).fetchall() + bound.add(sql) + return True + + def references(value, required, nested, scalar=True, scope=None): + if isinstance(value, list): + return all(references(v, required, nested, scalar, scope) for v in value) + if not isinstance(value, dict): + return True + if "cte_map" in value: + scope = (value, bool(value["cte_map"]["map"]) or bool(scope and scope[1])) + kind = value.get("class") + if scalar and kind == "SUBQUERY": + # Nested aggregates must bind locally even inside a syntactic scalar subquery. + return (value["subquery_type"] == "SCALAR" and value.get("child") is None + and references(value["subquery"], required, nested, False, scope)) + if scalar and kind is not None: + if kind not in {"COLUMN_REF", "CONSTANT", "CAST", "CASE", "COMPARISON", + "CONJUNCTION", "OPERATOR", "FUNCTION", "PARAMETER"}: + return False + if kind == "OPERATOR" and value["type"] not in { + "OPERATOR_COALESCE", "OPERATOR_NOT", "OPERATOR_IS_NULL", + "OPERATOR_IS_NOT_NULL", "COMPARE_IN", "COMPARE_NOT_IN"}: + return False + if "function_name" in value: + if value.get("catalog") or value.get("schema", "") not in ("", "main"): + return False + if scalar and (value.get("distinct") or value.get("filter") is not None + or value.get("export_state") or value.get("order_bys", {}).get("orders")): + return False + name = value["function_name"].lower() + required[name] = required.get(name, False) or scalar + if not scalar: + nested.append((name, value, scope)) + return all(references(v, required, nested, scalar, scope) for v in value.values()) + + def prove(expressions, ordering=""): + required, nested = {}, [] + if not references(parse(expressions, ordering), required, nested): + return False + missing = sorted(set(required) - metadata.keys()) + if missing: + rows = cur.execute( + "SELECT lower(function_name), function_type, internal, macro_definition " + "FROM duckdb_functions() WHERE lower(function_name) = ANY(?)", [missing]).fetchall() + metadata.update((name, []) for name in missing) + for name, function_type, internal, definition in rows: + metadata[name].append((function_type, internal, definition)) + for name, scalar in required.items(): + if (name, scalar) in verified: + continue + if not metadata[name] or name in expanding or len(expanding) >= 16: + return False + for function_type, internal, definition in metadata[name]: + if function_type == "macro": + # Only engine-owned macros with independently scalar bodies are admitted. + if not internal or not definition: + return False + expanding.add(name) + try: + if not prove(definition): + return False + finally: + expanding.remove(name) + elif function_type not in ({"scalar"} if scalar else {"scalar", "aggregate", "table"}): + return False + verified.add((name, scalar)) + for name, value, scope in nested: + if (value.get("class") != "WINDOW" + and any(kind == "aggregate" for kind, _internal, _definition in metadata[name]) + and not aggregate_is_local(value, scope)): + return False + return True + + try: + return prove(projection, order) + except Exception as exc: + # A parser/catalogue refusal never authorizes moving the original projection. + _request_timing.raise_if_resource_error(exc) + return False + + +def mirror_file_marks(cur): + """Identify the cursor's actual file and WAL, or decline an unnamed mirror.""" + row = cur.execute("SELECT path FROM duckdb_databases() " + "WHERE database_name = current_database()").fetchone() + path = row[0] if row else None + if not path or path == ":memory:": + return None + marks = [] + for name in (path, path + ".wal"): + try: + stat = os.stat(name) + marks.append((stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)) + except OSError: + marks.append(None) + return (str(Path(path).resolve()), tuple(marks)) if marks[0] is not None else None + + +def _count_revision(cur): + """Cache only managed, autocommit reads of one persistent mirror without local relations.""" + if not isinstance(cur, _TimedCursor) or cur._in_transaction or _ACTIVE_WRITES[0]: + return None + generation = mirror_generation(), _MIRROR_WRITES[0] + marks = mirror_file_marks(cur) + if marks is None: + return None + unsafe = cur.execute( + "SELECT (SELECT count(*) FROM duckdb_databases() WHERE NOT internal) <> 1 " + "OR EXISTS (SELECT 1 FROM duckdb_tables() WHERE temporary AND NOT internal) " + "OR EXISTS (SELECT 1 FROM duckdb_views() WHERE temporary AND NOT internal)").fetchone()[0] + return None if unsafe or generation != (mirror_generation(), _MIRROR_WRITES[0]) else (marks, generation) + + +def count_rows(table=None, where=None, params=(), cur=None, from_sql=None): + """Return the exact matching row population, reusing only an unchanged mirror revision.""" + # Callers supply deterministic compiled SQL, as they do for window(). + if bool(table) == bool(from_sql): + raise ValueError("datastore.count_rows: pass exactly one of `table` or `from_sql`") + name = str(from_sql or table).strip() + if table and not _IDENT_OK.fullmatch(name): + raise ValueError(f"datastore.count_rows: {table!r} is not a table identifier") + pred = str(where or "").strip() + sql = f"SELECT count(*) FROM {name}" + (f" WHERE {pred}" if pred else "") + params = tuple(params or ()) + cur = cur if cur is not None else ro_con() + revision = _count_revision(cur) + key = None + if revision is not None: + try: + # SQL and typed bindings remain private, including large IN predicates. + key = revision, hashlib.sha256(pickle.dumps((sql, params), protocol=5)).digest() + except (TypeError, ValueError, AttributeError, OverflowError, pickle.PickleError): + pass + if key is not None: + with _COUNT_CACHE_LOCK: + if key in _COUNT_CACHE: + _COUNT_CACHE.move_to_end(key) + return _COUNT_CACHE[key] + total = int(cur.execute(sql, params).fetchone()[0] or 0) + if key is not None and _count_revision(cur) == revision: + with _COUNT_CACHE_LOCK: + _COUNT_CACHE[key] = total + _COUNT_CACHE.move_to_end(key) + while len(_COUNT_CACHE) > _COUNT_CACHE_MAX: + _COUNT_CACHE.popitem(last=False) + return total def window(table=None, select=None, where=None, params=(), order_by=None, offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None, - page_first=False): + page_first=False, page_key=None): """One page of a mirror table PLUS the true count of everything the same predicate matches. Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int, @@ -1065,6 +1529,9 @@ def window(table=None, select=None, where=None, params=(), order_by=None, `page_first` (a row page whose `order_by` is an expression over the table's own columns) picks the page's rows before the projection runs, when only the projection holds a subquery. + `page_key` opts a physical table into a narrow, materialised key page before projection. + It must uniquely identify source rows, independently of the displayed record identity. + Proven row-local pages end with this physical key ascending. Fallback SQL retains its order. """ if bool(table) == bool(from_sql): raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`") @@ -1097,10 +1564,22 @@ def window(table=None, select=None, where=None, params=(), order_by=None, # ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate: # if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than # quietly returning a good count beside a broken page. - total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0) + total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur) grp = str(group_by or "").strip() fold = f" GROUP BY {grp}" if grp else "" - if (page_first and table and not grp and _SUBQUERY.search(proj) + needs_projection = _order_needs_projection(order, proj) if page_first else True + row_local = (page_first and table and not grp and not needs_projection + and _page_is_row_local(proj, order, cur)) + if row_local and page_key: + order = _physical_page_order(order, page_key) + if row_local and page_key: + # Only keys cross the materialisation boundary; identity order still uses the full population. + got = cur.execute( + f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" ' + f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) ' + f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page ' + f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall() + elif (row_local and not page_key and _SUBQUERY.search(proj) and not _SUBQUERY.search(f"{pred} {order}")): # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71). @@ -1336,7 +1815,7 @@ def _prepare_seed_mirrors(path, rt): if rt is None: return from harness import mirror_store - con = duckdb.connect(str(path)) + con = apply_limits(duckdb.connect(str(path))) try: _ensure_sync_state(con) for connector in tuple(_connector_spec.MIRRORS): @@ -1412,6 +1891,8 @@ def connect(): cur = _candidate_connection() if cur is None: cur = _instance().cursor() + if _request_timing.CURRENT.get() is not None and not isinstance(cur, _TimedCursor): + cur = _TimedCursor(cur) _ensure_sync_state(cur) return cur diff --git a/platform/harness/filter_sql.py b/platform/harness/filter_sql.py index 53e7ab0a2ccb1ad1872b5c558963f26603247b79..fd74c2c682a7f7328a4d0b970e39d7ee42980b8b 100644 --- a/platform/harness/filter_sql.py +++ b/platform/harness/filter_sql.py @@ -715,9 +715,13 @@ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql if member_ids: if not id_sql: raise ValueError('member_ids requires id_sql (the row-identity column)') - ids = [int(x) for x in member_ids] - sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))" - params = list(params) + ids + if isinstance(member_ids, SqlMembership): + sql = f"({sql} OR {id_sql} IN ({member_ids.sql}))" + params = list(params) + list(member_ids.params) + else: + ids = [int(x) for x in member_ids] + sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))" + params = list(params) + ids return Compiled(sql, list(params), state['aggregate'], frozenset(state['used'])) diff --git a/platform/harness/measure_filter.py b/platform/harness/measure_filter.py index 0941b4e474057e9527cc84c92c64a56949078f17..8c8000de59d4a8307451b2144ac60abf62dd7506 100644 --- a/platform/harness/measure_filter.py +++ b/platform/harness/measure_filter.py @@ -406,7 +406,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None, related=None): "which would be a different question with an authoritative-looking answer") con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id, related) + totals = _totals(con, key, rule.get('window'), today, team_id, related, pool) finally: con.close() # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision, @@ -523,7 +523,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None): # compare in Python — the same shape `resolve_pair` uses, and for the same reason. con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id, related) + totals = _totals(con, key, rule.get('window'), today, team_id, related, pool) finally: con.close() # No pool means we can only speak about customers a base has seen. WITH a pool, everyone @@ -535,6 +535,9 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None): frm, id_col, agg, where_sql, params = _query_parts( key, rule.get('window'), today, team_id, related) + if pool is not None: + where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))" + params = [*params, list(pool)] sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} " f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?") @@ -559,7 +562,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None): return _related_answer(matched, related) -def _totals(con, key, window, today, team_id, related=None): +def _totals(con, key, window, today, team_id, related=None, allowed_pids=None): """`{pid: value}` for one (measure, window), COMPOSITES included. For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived @@ -573,13 +576,16 @@ def _totals(con, key, window, today, team_id, related=None): """ if is_composite(key): bases, _ = _sem._expand_measures([key]) - parts = {b: _totals(con, b, window, today, team_id, related) for b in bases} + parts = {b: _totals(con, b, window, today, team_id, related, allowed_pids) for b in bases} pids = set() for got in parts.values(): pids |= set(got) return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key) for pid in pids} frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related) + if allowed_pids is not None: + where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))" + params = [*params, list(allowed_pids)] sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}") # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and @@ -620,8 +626,8 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None, related=None): pool = _pool_of(allowed_pids) con = _sem._store_con() try: - left = _totals(con, lkey, rule.get('window'), today, team_id, related) - right = _totals(con, rkey, rhs.get('window'), today, team_id, related) + left = _totals(con, lkey, rule.get('window'), today, team_id, related, pool) + right = _totals(con, rkey, rhs.get('window'), today, team_id, related, pool) finally: con.close() # With no pool we can only speak about customers at least one side has seen — the zero-vs- @@ -636,7 +642,7 @@ def _zero_satisfies(op, value): return _CMP[op](0.0, value) -def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None): +def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None, *, cur=None): """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD). @@ -669,16 +675,22 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related= f"unresolvable window {window!r} for the {key!r} column: refusing rather than " f"defaulting to all time") related = _related_scope(related, [key]) + if cur is None: + from harness.canonical_source import active + source = active() + if source is not None: + cur = source.cur pool = _pool_of(allowed_pids) - con = _sem._store_con() + con = cur if cur is not None else _sem._store_con() try: - totals = _totals(con, key, window, today, team_id, related) + totals = _totals(con, key, window, today, team_id, related, pool) guard = None metric = _sem.metrics().get(key, {}) if related is not None and not related.refusal and metric.get('agg') == 'ratio': - guard = _totals(con, metric['denominator'], window, today, team_id, related) + guard = _totals(con, metric['denominator'], window, today, team_id, related, pool) finally: - con.close() + if cur is None: + con.close() scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency') nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal diff --git a/platform/harness/mirror_tokens.py b/platform/harness/mirror_tokens.py index c41d022c1fb24f399379c6f5666621b96ee2c7eb..09c7547f1e2c6315e1c4524060edf5e0f9946d9b 100644 --- a/platform/harness/mirror_tokens.py +++ b/platform/harness/mirror_tokens.py @@ -13,7 +13,6 @@ does it. Until then that function RAISES, which `database_change_token` already prove unchanged, therefore resolve" -- the safe direction. """ from __future__ import annotations -import os import re import threading import time @@ -28,20 +27,9 @@ _TABLE_ROWS_LOCK = threading.Lock() def _file_marks(cur): - """Size and mtime of the mirror file and its WAL, or None when the mirror has no file.""" - row = cur.execute("SELECT path FROM duckdb_databases() " - "WHERE database_name = current_database()").fetchone() - path = row[0] if row else None - if not path or path == ":memory:": - return None - marks = [] - for name in (path, path + ".wal"): - try: - stat = os.stat(name) - marks.append([stat.st_size, stat.st_mtime_ns]) - except OSError: - marks.append(None) - return marks + """The shared actual-file and WAL revision, or None for an unnamed mirror.""" + from harness import datastore + return datastore.mirror_file_marks(cur) def mirror_change_token(st, database): diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index c3e29bf24d9db72aaf6b00116732511a935a56da..77f0ef56c1f74371139c56da1c2829a3252a3394 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -256,11 +256,17 @@ def render_scope(sql): return out +def _load_model_yaml(text): + """Parse model documents with the safe C loader when available.""" + loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader) + return yaml.load(text, Loader=loader) + + @functools.lru_cache(maxsize=1) def _model(): topics, metrics = {}, {} for f in sorted((MODEL_DIR / "topics").glob("*.yml")): - d = yaml.safe_load(f.read_text(encoding="utf-8")) + d = _load_model_yaml(f.read_text(encoding="utf-8")) if not d or "key" not in d: raise ModelError(f"topic file {f.name} lacks a 'key'") if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS: @@ -268,7 +274,7 @@ def _model(): # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises # a clear error if a live resolution is attempted on it topics[d["key"]] = d - metric_docs = [(f.name, yaml.safe_load(f.read_text(encoding="utf-8"))) + metric_docs = [(f.name, _load_model_yaml(f.read_text(encoding="utf-8"))) for f in sorted((MODEL_DIR / "metrics").glob("*.yml"))] # Metric libraries may own new fact topics without changing legacy topic definitions. for name, doc in metric_docs: @@ -1652,7 +1658,7 @@ def _check_shipped_links(date_from, date_to): def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None, - *, runtime=None, user=None): + *, runtime=None, user=None, cur=None, group_keys=None): """The semantic query over the tenant store — the engine behind the Analyst's run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" t = _model()["topics"].get(topic) @@ -1810,6 +1816,16 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})") params.extend(ids) + if group_keys is not None: + if len(gb) != 1: + raise ModelError('Selected entity keys require exactly one grouping dimension') + values = list(group_keys) + if values: + where.append(f"{dims[gb[0]]['col']} IN ({','.join('?' for _ in values)})") + params.extend(values) + else: + where.append('FALSE') + # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape; # this is the nested, 11-operator contract the table UI emits. Compiled by # harness/filter_sql, which is held in lock-step with the TS engine and the validator by @@ -1846,13 +1862,14 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap). sql += f" LIMIT {limit + 1}" - con = _store_con() + con = cur if cur is not None else _store_con() try: - cur = con.execute(sql, params) - cols = [d[0] for d in cur.description] - rows = [dict(zip(cols, r)) for r in cur.fetchall()] + result = con.execute(sql, params) + cols = [d[0] for d in result.description] + rows = [dict(zip(cols, r)) for r in result.fetchall()] finally: - con.close() + if cur is None: + con.close() for row in rows: # ratio/derived post-compute per group row.update(foreign_vals) # scalar cross-topic components for k in requested: @@ -2391,7 +2408,8 @@ def _one_binding_offer(topic, b, strict=False, ready=None): def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None, - exclude_services=False, offer=None, related=None): + exclude_services=False, offer=None, related=None, *, group_keys=None, + cur=None): """`{group key: {measure key: value}}` for an entity topic's lookback columns. ONE grouped query for all `keys` together — they share a topic, a scope and a window, so @@ -2421,6 +2439,10 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non `_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin nobody measured, under a row the user has no reason to doubt. """ + if group_keys is not None: + group_keys = tuple(group_keys) + if not group_keys: + return {} offer = offer if offer is not None else entity_measures(topic) by_key = {m["key"]: m for m in offer} want = [k for k in keys if k in by_key] @@ -2439,7 +2461,8 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non for (_src, _dim, _variant), ks in groups.items(): answer = entity_measure_values( topic, ks, date_from=date_from, date_to=date_to, team_id=team_id, - exclude_services=exclude_services, offer=offer, related=related) + exclude_services=exclude_services, offer=offer, related=related, + group_keys=group_keys, cur=cur) if related is not None: merged.refusals.extend(getattr(answer, 'refusals', ())) for gk, cell in answer.items(): @@ -2458,7 +2481,9 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non base_of = {k: (by_key[k].get("base_key") or k) for k in want} res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from, date_to=date_to, team_id=team_id, exclude_services=exclude_services, - limit=MAX_GROUPS, scope_variant=scope_variant, related=related) + limit=MAX_GROUPS, scope_variant=scope_variant, related=related, + **({'group_keys': group_keys} if group_keys is not None else {}), + **({'cur': cur} if cur is not None else {})) if res.get("refusals"): from core import relations return relations.MeasureAnswer(refusals=[dict(marker, measure=k) @@ -2532,7 +2557,7 @@ def entity_measure_leaves(nodes, admitted): def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None, - exclude_services=False): + exclude_services=False, *, cur=None): """`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row. ⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty @@ -2578,7 +2603,8 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe else: vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1], team_id=team_id, exclude_services=exclude_services, - offer=offer) + offer=offer, group_keys=keys_by_id, + **({'cur': cur} if cur is not None else {})) cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c} return cache[sig] @@ -2595,7 +2621,7 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe left = values_for(mkey, rule.get("window")) if left is None: continue - pool = list(keys_by_id or left) + pool = list(left if keys_by_id is None else keys_by_id) zero = m["empty"] == "zero" rhs = rule.get("rhs") try: diff --git a/platform/modules/ar.py b/platform/modules/ar.py index 3fd931778ff4f0738d3c48bac0de7e9750b8840f..f020dd547a7cafa05d6bf45af8d8f4e278688466 100644 --- a/platform/modules/ar.py +++ b/platform/modules/ar.py @@ -10,7 +10,7 @@ import math import datetime as dt from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import core.store as store @@ -32,7 +32,7 @@ DTP_KEY = 'ar_days_to_pay' DTP_MAX_AGE_HOURS = 20 -def days_to_pay_snapshot(): +def days_to_pay_snapshot(*, pids=None, st=None): """`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises. Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the @@ -40,8 +40,11 @@ def days_to_pay_snapshot(): derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about. """ try: - snap = store.get(DTP_KEY) or {} - days = snap.get('days') or {} + if pids is not None and st is not None: + days = st.selected_values(DTP_KEY, ['days'], pids) + else: + snap = store.get(DTP_KEY) or {} + days = snap.get('days') or {} out = {} for k, v in days.items(): try: @@ -123,15 +126,18 @@ def _open_docs(t=None): ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due', 'amount_total', 'amount_residual_signed']) today = t or P.today() - for d in docs: - due = d.get('invoice_date_due') - try: - d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0 - except Exception: - d['days_overdue'] = 0 - d['bucket'] = _bucket(d['days_overdue']) - d['open'] = d.get('amount_residual_signed') or 0.0 - return docs + def enriched(): + for d in docs: + due = d.get('invoice_date_due') + try: + d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0 + except Exception: + d['days_overdue'] = 0 + d['bucket'] = _bucket(d['days_overdue']) + d['open'] = d.get('amount_residual_signed') or 0.0 + yield d + from harness.canonical_source import active + return enriched() if active() is not None else list(enriched()) def _ltm_invoiced(t=None): diff --git a/platform/modules/backorders.py b/platform/modules/backorders.py index 2270809227ade65d273c4cab437fb786f7c1bd21..d8c27da48f917b2d1f278b9c7cd2e309b85258f9 100644 --- a/platform/modules/backorders.py +++ b/platform/modules/backorders.py @@ -11,7 +11,7 @@ to dealers. BU-filterable. """ import datetime as dt -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import modules.procurement as proc @@ -72,6 +72,9 @@ def sku_backlog(t=None, team_id=None): groups = O.read_group('sale.order.line', domain, ['product_uom_qty:sum', 'qty_delivered:sum'], ['product_id'], lazy=False) + from harness.canonical_source import active + if active() is not None: + return active().backlog(groups) out = {} for group in groups: pid = O.m2o_id(group.get('product_id')) @@ -85,7 +88,8 @@ def sku_backlog(t=None, team_id=None): open_by_pid = _by_product(list(common)) pre_by_pid = _by_product(list(common) + [('order_id.commitment_date', '>', end_of_day)]) - pids = sorted(set(open_by_pid) | set(pre_by_pid)) + from harness.canonical_source import active + pids = [] if active() is not None else sorted(set(open_by_pid) | set(pre_by_pid)) codes = {} for ch in _chunk(pids, 500): for product in O.search_read('product.product', @@ -101,7 +105,8 @@ def sku_backlog(t=None, team_id=None): out[code] = out.get(code, 0.0) + qty return out - pre_by_code, open_by_code = _by_code(pre_by_pid), _by_code(open_by_pid) + pre_by_code, open_by_code = ((pre_by_pid, open_by_pid) if active() is not None else + (_by_code(pre_by_pid), _by_code(open_by_pid))) report = { 'as_of': t.isoformat(), 'team_ids': teams, diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py index 939b445d82008c3ecfe78ef5997a269be75ad5c2..7d13fbadbb0e76c34bb0a1269a4e8453046ba9b8 100644 --- a/platform/modules/customer_data.py +++ b/platform/modules/customer_data.py @@ -16,7 +16,7 @@ no link (owner/CFO/admin) see the whole book. import re from urllib.parse import quote_plus -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import core.store as store import core.table_store as table_store @@ -117,7 +117,7 @@ AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60', '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'} -def _ar_attrs(t=None): +def _ar_attrs(t=None, *, pids=None, st=None): """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family. Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" — @@ -160,7 +160,9 @@ def _ar_attrs(t=None): } except Exception: pass - for pid, avg in ar.days_to_pay_snapshot().items(): + snapshot = (ar.days_to_pay_snapshot() if pids is None and st is None else + ar.days_to_pay_snapshot(pids=pids, st=st)) + for pid, avg in snapshot.items(): # A customer with an AR row but no settled invoice in the window keeps `None` — blank, # not 0. "Pays in 0 days" is a claim, and a false one. out.setdefault(pid, {})['days_to_pay'] = avg @@ -197,6 +199,10 @@ def _mix_attrs(pids=None, t=None): keep = set(pids) if pids else None rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'], ['order_partner_id', 'product_id'], lazy=False) + from harness.canonical_source import active + source = active() + if source is not None: + return source.customer_mix(rows) prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')} prod_ids.discard(None) cats = {} @@ -502,6 +508,28 @@ def _dba_attrs(pids, t): return {} +def _book_records(team_id=None): + """Yield the canonical rank, confirmed-order and posted-document identity union.""" + if team_id: + groups = O.read_group('sale.order', + [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)], + ['partner_id'], ['partner_id'], lazy=False) + for row in groups: + pid = O.m2o_id(row.get('partner_id')) + if pid is not None: + yield {'id': pid} + return + yield from O.search_read('res.partner', [('customer_rank', '>', 0), + ('active', 'in', [True, False])], ['id'], limit=200000) + for model, domain in [ + ('sale.order', [('state', 'in', ('sale', 'done'))]), + ('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))])]: + for row in O.read_group(model, domain, ['partner_id'], ['partner_id'], lazy=False): + pid = O.m2o_id(row.get('partner_id')) + if pid is not None: + yield {'id': pid} + + def _book_pids(team_id=None, agent_pids=None): """⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently. @@ -548,36 +576,7 @@ def _book_pids(team_id=None, agent_pids=None): the whole permissioning model rests on. """ try: - if team_id: - # Scoped: partners with confirmed orders in THIS BU, ever. `read_group` on partner_id - # rather than a search_read of orders — the group is the distinct set, and it is one - # round trip instead of paging tens of thousands of order rows. - rows = O.read_group('sale.order', - [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)], - ['partner_id'], ['partner_id'], lazy=False) - book = {O.m2o_id(r.get('partner_id')) for r in rows} - book.discard(None) - else: - rank = O.search_read('res.partner', - [('customer_rank', '>', 0), - ('active', 'in', [True, False])], - ['id'], limit=200000) - # `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or - # have had its rank reset, and an account that demonstrably bought from us belongs in - # the book whatever its flags say now. An ORDER is a fact; a rank is a setting. - rows = O.read_group('sale.order', - [('state', 'in', ('sale', 'done'))], - ['partner_id'], ['partner_id'], lazy=False) - ever = {O.m2o_id(r.get('partner_id')) for r in rows} - ever.discard(None) - docs = O.read_group( - 'account.move', - [('state', '=', 'posted'), - ('move_type', 'in', ('out_invoice', 'out_refund'))], - ['partner_id'], ['partner_id'], lazy=False) - documented = {O.m2o_id(r.get('partner_id')) for r in docs} - documented.discard(None) - book = {r['id'] for r in rank} | ever | documented + book = {r['id'] for r in _book_records(team_id)} # An agent filter NARROWS the book to that agent's partners — never widens it. The # intersection is what keeps an agent-login user inside their own book while still # gaining every no-order account assigned to them, which is the whole point of item 27. @@ -630,8 +629,85 @@ def identity_pids(agent_name=None, team_id=None, t=None): return set(_pool_identity(agent_name, team_id, t)["pids"]) -def _pool_build(agent_name, team_id, limit, fast=False): - identity = _pool_identity(agent_name, team_id) +def source_batches(cur, *, st, agent_name=None, team_id=None, ids=None, needed=None, + ledger_batches=()): + """Stream the canonical customer identity union and project only one batch at a time.""" + from harness.canonical_source import bounded, BATCH + with bounded(cur, 'identity' if ids is None else 'customer', ids or [], st) as identity: + live = identity.snapshot(_book_records(team_id)) + where = '' + if agent_name and agent_name not in {'All', 'All agents'}: + members = identity.agent_members(agent_name) + where = (f"WHERE json_extract_string(body, '/id') IN " + f"(SELECT json_extract_string(body, '/id') FROM \"{members.table}\")") + population = identity.snapshot({'id': r[0], 'live': True} for r in ()) + cur.execute(f'INSERT INTO "{population.table}" SELECT row_number() OVER (), ' + f"json_object('id', try_cast(json_extract_string(body, '/id') AS BIGINT), 'live', true) " + f'FROM "{live.table}" {where} GROUP BY json_extract_string(body, \'/id\')') + if not agent_name and team_id is None: + for batch in ledger_batches: + records = [] + for key, entry in batch: + try: + pid = int(key) + except (TypeError, ValueError): + continue + if ids is None or pid in ids: + records.append({'id': pid, 'live': False, 'entry': entry}) + if records: + cur.execute(f'INSERT INTO "{population.table}" SELECT 0, j FROM unnest(?::JSON[]) r(j) ' + f"WHERE try_cast(json_extract_string(j, '/id') AS BIGINT) NOT IN " + f"(SELECT try_cast(json_extract_string(body, '/id') AS BIGINT) FROM \"{live.table}\")", + [[__import__('json').dumps(r) for r in records]]) + records.clear() + # Sort the complete live/retained union once; pages seek a physical key, not JSON. + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS ' + f"SELECT ordinal, body, try_cast(json_extract_string(body, '/id') AS BIGINT) pid " + f'FROM "{population.table}" ORDER BY pid') + last = None + while True: + rows = cur.execute(f"SELECT pid, " + f"json_extract_string(body, '/live') = 'true', json_extract(body, '/entry') " + f'FROM "{population.table}" WHERE (?::BIGINT IS NULL OR pid > ?) ORDER BY pid LIMIT ?', + [last, last, BATCH]).fetchall() + if not rows: + return + live_ids = [pid for pid, present, _ in rows if present] + ledger = {str(pid): __import__('json').loads(entry) for pid, present, entry in rows if not present} + with bounded(cur, 'customer', live_ids, st): + if needed is not None and not needed: + projected = [dict(cust.audit_attr_defaults(), pid=pid) for pid in live_ids] + else: + projected = _pool_build(agent_name, team_id, None, selected=live_ids, + needed=needed, st=st)['rows'] if live_ids else [] + projected, _ = _reconcile_ledger(projected, len(projected), agent_name, team_id, + ledger=ledger, write=False) + yield projected + last = rows[-1][0] + projected.clear() + rows.clear() + + +def _pool_build(agent_name, team_id, limit, fast=False, *, selected=None, needed=None, st=None): + from harness.canonical_source import active + source = active() + if selected is None: + identity = _pool_identity(agent_name, team_id) + else: + t = P.today() + pids = set(selected) + wants = lambda keys: needed is None or bool(set(keys) & set(needed)) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + mf, mt = P.ltm(t) + identity = {'today': t, 'agentPids': pids, 'pids': pids, + 'this': cust._cust_rev(yf, yt, team_id, pids) if wants( + ['customer', 'status', 'revenue_ytd', 'yoy_pct', 'at_risk']) else {}, + 'last': cust._cust_rev(lf, lt, team_id, pids) if wants( + ['customer', 'status', 'revenue_ly', 'yoy_pct', 'at_risk']) else {}, + 'ltm': cust._cust_rev(mf, mt, team_id, pids) if wants(['customer', 'ltm_rev']) else {}, + 'cadence': cust._cadence_bulk(t, team_id, agent_pids=pids) if wants([ + 'orders_24m', 'aov', 'last_order', 'days_since', 'typical_gap_days', 'overdue_days']) else {}} t = identity["today"] pids = identity["agentPids"] this, last, ltm = identity["this"], identity["last"], identity["ltm"] @@ -644,8 +720,9 @@ def _pool_build(agent_name, team_id, limit, fast=False): key=lambda p: -((this.get(p) or {}).get('rev', 0.0) + (last.get(p) or {}).get('rev', 0.0))) all_pids = set(ranked[:limit]) - attrs = cust._partner_attrs(list(all_pids)) - invoice_days = {} if fast else cust._invoice_cadence_bulk(t, partner_ids=all_pids) + attrs = cust._partner_attrs(list(all_pids), **({'needed': needed} if selected is not None else {})) + invoice_days = {} if fast or (needed is not None and 'days_since_invoice' not in needed) \ + else cust._invoice_cadence_bulk(t, partner_ids=all_pids) if fast: # C1 fast phase: the slow families stay empty — their columns render blank behind the # toolbar's partial indicator until the full build swaps in. @@ -654,21 +731,25 @@ def _pool_build(agent_name, team_id, limit, fast=False): # The three families the owner asked for (2026-07-27). Each degrades to {} rather than # raising: a customer table that cannot render because AR is momentarily unreachable is # a worse failure than one with a blank column. - ar_a = _ar_attrs(t) - mix_a = _mix_attrs(all_pids, t) - sp_a = _salesperson_attrs(all_pids, t) + wants = lambda keys: needed is None or bool(set(keys) & set(needed)) + ar_a = (_ar_attrs(t, pids=all_pids, st=st) if source is not None else _ar_attrs(t)) \ + if wants(['ar_open', 'ar_overdue', 'ar_outstanding', 'ar_exposure', 'days_to_pay', + *AR_BUCKET_FIELDS.values()]) else {} + mix_a = _mix_attrs(all_pids, t) if wants(['top_category', 'top_category_pct', 'sku_count', 'top_sku']) else {} + sp_a = _salesperson_attrs(all_pids, t) if wants(['salesperson']) else {} # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache, # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the # client's map view discloses the count. - geo_a = map_mod.coords_for(list(all_pids)) + geo_a = map_mod.coords_for(list(all_pids), **({'st': st} if source is not None else {})) \ + if wants(['lat', 'lon']) else {} # C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest. - dba_a = _dba_attrs(all_pids, t) + dba_a = _dba_attrs(all_pids, t) if wants(['dba']) else {} # customers whose only activity sits in the older half of the 24-month cadence window # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly # the win-back targets this union exists for, would render as '?') unnamed = [pid for pid in all_pids if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name') - or (ltm.get(pid) or {}).get('name'))] + or (ltm.get(pid) or {}).get('name'))] if needed is None or 'customer' in needed else [] extra_names = ({r['id']: r['name'] for r in O.search_read('res.partner', [('id', 'in', unnamed), ('active', 'in', [True, False])], ['name'])} @@ -763,7 +844,7 @@ def _pool_build(agent_name, team_id, limit, fast=False): # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo. # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction, # so letting it write would mark ~1,450 living customers archived on every cold load. - if not fast: + if not fast and selected is None: rows, total = _reconcile_ledger(rows, total, agent_name, team_id) rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly'])) return {'rows': rows, 'total': total} @@ -772,7 +853,7 @@ def _pool_build(agent_name, team_id, limit, fast=False): LEDGER_KEY = 'customer_ledger' -def _reconcile_ledger(rows, total, agent_name, team_id): +def _reconcile_ledger(rows, total, agent_name, team_id, *, ledger=None, write=True): """Remember every customer we have ever seen, and keep serving the ones Odoo stopped returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately — so stored in our platform even if deleted in Odoo"). @@ -791,14 +872,14 @@ def _reconcile_ledger(rows, total, agent_name, team_id): they cannot prove they should see. """ try: - ledger = dict(store.get(LEDGER_KEY) or {}) + ledger = dict(store.get(LEDGER_KEY) or {}) if ledger is None else dict(ledger) except Exception: return rows, total # a ledger we cannot read must not break the table today = str(P.today()) live = {r['pid'] for r in rows} unscoped = agent_name is None and team_id is None - if unscoped: + if unscoped and write: for r in rows: e = ledger.get(str(r['pid'])) or {} e.update({'name': r['customer'], 'last_seen': today}) @@ -849,7 +930,7 @@ def _reconcile_ledger(rows, total, agent_name, team_id): 'days_since': None, 'days_since_invoice': None, 'typical_gap_days': None, 'overdue_days': None, }) - if store.available(): + if write and store.available(): try: store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async') except Exception: diff --git a/platform/modules/customers.py b/platform/modules/customers.py index 080a0de3654a673e586501ac7891b1b2ccb4eace..6580fb12dd5968ce858ced90d65cf7e0e98378f2 100644 --- a/platform/modules/customers.py +++ b/platform/modules/customers.py @@ -14,7 +14,7 @@ import datetime as dt import statistics from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import modules.sales as sales_mod @@ -49,7 +49,10 @@ def _cust_group_store(date_from, date_to, team_id, agent_pids, select): params += list(agent_pids) if O.doc_mode() == 'invoice': w.append("invoice_status = 'invoiced'") - return DS.ro_con().execute( + from harness.canonical_source import active + source = active() + con = source.cur if source is not None else DS.ro_con() + return con.execute( f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} " "FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id " "WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall() @@ -534,10 +537,13 @@ def _invoice_cadence_bulk(t=None, partner_ids=None): if USE_STORE: try: from harness import semantic as sem + from harness.canonical_source import active + source = active() result = sem.store_query( 'customer_invoices', ['customer_days_since_invoice'], group_by=['partner'], filters={'partner': partner_ids} if partner_ids is not None else None, - limit=sem.MAX_GROUPS, today=t.isoformat()) + limit=sem.MAX_GROUPS, today=t.isoformat(), + **({'cur': source.cur} if source is not None else {})) if result.get('truncated'): raise sem.ModelError('The invoice cadence group set was truncated.') return {int(row['partner_id']): row['customer_days_since_invoice'] @@ -650,7 +656,7 @@ def _audit_attrs(record): return out -def _partner_attrs(pids): +def _partner_attrs(pids, *, needed=None): """{pid: {street, street2, city, state, country, agent, zip, payment_terms, customer_since, tags, pricelist}} — the customer attributes the sum-level rollups slice by and the Customer table displays. @@ -667,15 +673,26 @@ def _partner_attrs(pids): pids = list(pids) if not pids: return {} + columns = ['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids', + 'zip', 'property_payment_term_id', 'create_date', 'category_id', + 'property_product_pricelist', 'active'] + [field for field, _kind in AUDIT_ATTRS.values()] + if needed is not None: + operands = {'state': ['state_id'], 'country': ['country_id'], 'agent': ['agent_ids'], + 'payment_terms': ['property_payment_term_id'], 'customer_since': ['create_date'], + '_created': ['create_date'], 'created_on': ['create_date'], 'tags': ['category_id'], + 'pricelist': ['property_product_pricelist'], 'archived': ['active'], + 'odoo_status': ['active'], 'maps_url': ['street', 'street2', 'city', 'state_id', 'country_id', 'zip']} + operands.update({key: [field] for key, (field, _kind) in AUDIT_ATTRS.items()}) + wanted = {field for key in needed for field in operands.get(key, [key])} + columns = [field for field in columns if field in wanted] + if not columns: + return {} # Odoo's active_test context otherwise hides archived partners even when their ids are named. # The explicit two-state domain makes `archived` an ordinary source attribute rather than a # conclusion inferred from a missing row. rows = O.search_read('res.partner', [('id', 'in', pids), ('active', 'in', [True, False])], - ['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids', - 'zip', 'property_payment_term_id', 'create_date', 'category_id', - 'property_product_pricelist', 'active'] - + [field for field, _kind in AUDIT_ATTRS.values()]) + columns) aids = {a for r in rows for a in (r.get('agent_ids') or [])} anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])} if aids else {}) diff --git a/platform/modules/inventory.py b/platform/modules/inventory.py index c40089da0eb316fb76e61516c28cb6adf15112f1..a53eb92fce3fce87472b57279a6a948b4f6a6679 100644 --- a/platform/modules/inventory.py +++ b/platform/modules/inventory.py @@ -9,7 +9,7 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import pandas as pd -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P @@ -273,13 +273,17 @@ def sku_inventory(target_days=180, carry_pct=_CARRY_PCT, t=None): brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand, inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value, carry_cost, turns, target_units, target_days}}.""" - df, _ = _build(t) - df = _enrich(df, target_days) - g = df.groupby('sku').agg( + from harness.canonical_source import active + if active() is not None: + g, _ = active().inventory_frame(t or P.today()) + else: + df, _ = _build(t) + df = _enrich(df, target_days) + g = df.groupby('sku').agg( name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'), on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'), cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'), - ).reset_index() + ).reset_index() out = {} for r in g.itertuples(index=False): daily = r.qty_ltm / 365.0 diff --git a/platform/modules/map.py b/platform/modules/map.py index b5079e64b81edf48401bbb85ec5ab41e968b99a0..7786b4b4b4ff73503c87130df28b2ec3fe66a294 100644 --- a/platform/modules/map.py +++ b/platform/modules/map.py @@ -18,7 +18,7 @@ import csv import io import hashlib -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import core.store as store import modules.sales as sales_mod @@ -174,7 +174,7 @@ def customer_locations(t=None, team_id=None, agent_pids=None): bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])} -def coords_for(pids): +def coords_for(pids, *, st=None): """{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map VIEW reads these off the pool rows; the standalone Map page retired the same day, and @@ -182,7 +182,10 @@ def coords_for(pids): if not pids: return {} try: - cache = store.get(_GEO_NS) if store.available() else {} + if st is None: + cache = store.get(_GEO_NS) if store.available() else {} + else: + cache = st.selected_values(_GEO_NS, [], pids) parts = O.search_read( 'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])], ['street', 'street2', 'city', 'state_id', 'zip', diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 6bb33fb0d4a181428ae3101acf546d6abc229838..b361ac5e986687a1522e91781db3799404822bd9 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -622,6 +622,45 @@ def identity_rows(): return rows +def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()): + """Stream catalogue identities, refuse global hash collisions and project bounded code batches.""" + from harness.canonical_source import bounded, BATCH + with bounded(cur, 'identity', [], st) as identity: + def codes(): + for product in identity.search_read('product.product', [('active', '=', True)], ['id', 'default_code']): + code = str(product['default_code']).strip() if product.get('default_code') else f"pid:{product['id']}" + yield {'code': code, 'pid': sku_pid(code)} + raw = identity.snapshot(codes()) + collision = cur.execute(f"SELECT json_extract_string(body, '/pid') pid " + f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone() + if collision: + raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.') + selected = '' if ids is None else " WHERE try_cast(json_extract_string(body, '/pid') AS BIGINT) IN (SELECT unnest(?::BIGINT[]))" + params = [] if ids is None else [list(ids)] + population = identity.snapshot(()) + cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"' + f'{selected} GROUP BY body', params) + # Keep the tracked snapshot name while materialising its sorted, typed seek key. + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS ' + f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid " + f'FROM "{population.table}" ORDER BY pid') + last = -1 + while True: + codes = cur.execute(f"SELECT pid, " + f"json_extract_string(body, '/code') FROM \"{population.table}\" " + 'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall() + if not codes: + return + with bounded(cur, 'product', [code for _, code in codes], st): + rows = ([{'pid': pid, 'code': code} for pid, code in codes] + if needed is not None and not needed else + pool(team_id=team_id, raw_fields=raw_fields, needed=needed)) + yield rows + last = codes[-1][0] + rows.clear() + codes.clear() + + def _fba_by_code(cat): """`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows. @@ -667,7 +706,7 @@ def _fba_by_code(cat): return out, report -def pool(team_id=None, t=None, raw_fields=()): +def pool(team_id=None, t=None, raw_fields=(), *, needed=None): """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`. CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set @@ -683,10 +722,14 @@ def pool(team_id=None, t=None, raw_fields=()): consolidated = team_id is None # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`. - inventory_snapshot = _inventory_by_code(t) + wants = lambda keys: needed is None or bool(set(keys) & set(needed)) + inventory_needed = wants(['on_hand', 'unit_cost', 'inv_value', 'qty_ltm', 'dos', 'stock_bucket', + 'inventory_value_per_quantity', 'inventory_value_per_quantity_state', 'cover_gap_units', + 'cover_gap_d', 'buy_now']) + inventory_snapshot = _inventory_by_code(t) if inventory_needed else {} inventory_source_available = isinstance(inventory_snapshot, dict) inv = inventory_snapshot if inventory_source_available else {} - bu_share = {} if consolidated else _bu_ltm_share(t, team_id) + bu_share = {} if consolidated or not inventory_needed else _bu_ltm_share(t, team_id) sup = supplier_master() # ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build: # tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the @@ -697,15 +740,19 @@ def pool(team_id=None, t=None, raw_fields=()): _prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both except Exception: # noqa: BLE001 _prods = None - tiers, _tier_report = products.tier_prices_by_code(_prods) - pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) + tiers, _tier_report = products.tier_prices_by_code(_prods) if wants( + [f'price_{n}' for n in range(1, 4)] + [f'unit_{n}' for n in range(1, 4)] + + [key for price, unit, _name, _n in products.PRICELIST_TIER_COLUMNS for key in (price, unit)]) else ({}, {}) + pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) \ + if wants(['pre_book_qty', 'open_backlog_qty']) else ({}, {}, {}) cat = _catalogue_by_code() # ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the # same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH # will sell. Degrades to `{}` on a read failure, matching every other column source here — a # blank cover gap is honest, a product grid that will not render is not. try: - fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id) + fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id) \ + if wants(['demand_fwd', 'cover_gap_units', 'cover_gap_d', 'buy_now']) else ({}, {}) except Exception: # noqa: BLE001 fwd_demand, _fwd_report = {}, {"error": "forward demand unavailable"} # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue @@ -716,8 +763,9 @@ def pool(team_id=None, t=None, raw_fields=()): # because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds # units, not brands, and pro-rating them onto a BU would print a number no Odoo query could # reproduce. Blank, never zero, when the tenant has no FBA warehouse. - fba_by_code, _fba_report = _fba_by_code(cat) - rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} + fba_by_code, _fba_report = _fba_by_code(cat) if inventory_needed or wants(['fba']) else ({}, {}) + rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} if wants( + ['rev_ytd', 'rev_ly', 'yoy_pct', 'qty_ytd', 'orders_ytd']) else {} raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p for p in (_prods or []) if p.get("default_code") or p.get("id")} diff --git a/platform/modules/products.py b/platform/modules/products.py index 7b35a3dc604a1c0f9675a295f519dfb1b9d7ed55..c11d49fe4dd935d67d6c6792e8c7c2158d2d270b 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -20,7 +20,7 @@ import datetime as dt from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from functools import lru_cache -import core.odoo as O +from harness.canonical_source import odoo as O import core.periods as P import modules.sales as sales_mod @@ -65,6 +65,10 @@ def _sku_rev(date_from, date_to, team_id=None, all_channels=False): all_channels=all_channels), ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id', 'order_id'], lazy=False) + from harness.canonical_source import active + source = active() + if source is not None: + return source.sku_revenue(g) codes = _code_map() out = {} orders_by_code = {} @@ -246,6 +250,9 @@ def velocity_leaders(t=None, limit=25, team_id=None): def _code_category(): """code -> category name. Built from _code_map (pid->code) + the sales category map (pid->category); the first record carrying a code sets that code's category.""" + from harness.canonical_source import active + if active() is not None: + return active().categories() codes = _code_map() cats = sales_mod._product_cat() out = {} @@ -630,6 +637,8 @@ def tier_cap_overflow(tiers): fourth tier is dropped — and a dropped tier that nobody counts is the silent cap the rule forbids. `validate_price_and_unit_cells` turns this into a named, reported figure. """ + if hasattr(tiers, 'overflow'): + return dict(tiers.overflow) counts = {} for tier in tiers or (): counts[tier.get("pricelist")] = counts.get(tier.get("pricelist"), 0) + 1 @@ -685,6 +694,9 @@ def tier_prices_by_code(prods=None): 2,605 price rows against **0 customers and 0 orders**, so a SKU reading several pairs is catalogue-true and commercially misleading. The entry keeps the list NAME for provenance. """ + from harness.canonical_source import active + if active() is not None: + return active().tiers(prods) report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0, "rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0, "rules_with_packaging": 0} diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index 8cb695d7b4b13327ab55338293aa5c13a20b6f90..2fae9d633fe581824a30cc62af4867610845acd8 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -1179,7 +1179,7 @@ def _related_predicate(related, columns, *, today=None, cohort_sets=None): where = [f"({part})" for part in (rspec.get("where"), related.get("where")) if part] params, limits = [], [] - def leaf_limits(nodes): + def leaf_limits(nodes, cmap): # Checking every leaf also catches a dropped condition on a column which a # different, valid condition used. The legacy column-set check cannot see it. out = [] @@ -1187,19 +1187,21 @@ def _related_predicate(related, columns, *, today=None, cohort_sets=None): if not isinstance(node, dict): raise ValueError("malformed related filter condition") if isinstance(node.get("children"), list): - out.extend(leaf_limits(node["children"])) + out.extend(leaf_limits(node["children"], cmap)) else: if "children" in node or not node.get("colId"): raise ValueError("malformed related filter condition") - part = compile_scope([node], "and", rcols, today=today, cohort_sets=cohort_sets) - out.extend(widen_limits([node], rcols, part)) + part = compile_scope([node], "and", cmap, today=today, cohort_sets=cohort_sets) + out.extend(widen_limits([node], cmap, part)) return out for nodes, conj, is_wall in ( (related.get("wall_nodes"), related.get("wall_conj", "and"), True), (related["scope_nodes"], related["scope_conj"], False)): - compiled = compile_scope(nodes, conj, rcols, today=today, cohort_sets=cohort_sets) - dropped = leaf_limits(nodes) + # A wall reads the reader's hidden columns too; authored conditions never do. + cmap = (related.get("wall_columns") or rcols) if is_wall else rcols + compiled = compile_scope(nodes, conj, cmap, today=today, cohort_sets=cohort_sets) + dropped = leaf_limits(nodes, cmap) if is_wall and dropped: raise ValueError("unapplied related row wall") limits.extend({**item, "subject": f"{related['database']}: {item['subject']}"} diff --git a/requirements.txt b/requirements.txt index 5fb1c9cb7e5d5ae888d7d73166ac770ef66b5921..a556d9b86e9f2b96224b74ac39a56ad89e13a47e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,13 +15,13 @@ # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the # import lives inside the handler, and the gate now asserts this line exists here. fastapi>=0.139 -starlette>=1.3.1 +starlette==1.3.1 uvicorn[standard]>=0.30 python-dotenv>=1.2.2 pandas>=2.0 requests>=2.28 huggingface_hub>=0.20 -duckdb>=1.5 +duckdb==1.5.4 pyyaml>=6.0 pillow>=10.0 beautifulsoup4>=4.12 diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index b2deb72ffa6e3442eb859c688c70b3d5c446bf6f..24031b784f12e122a6751a402b998a40af66f17a 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -27f1f5a90c0f4ceb5ae13bc834f4eb2e52d92512e72d6a2924151e39bad85159 +a7a3bbeebf18bb7116b669ba2c9dddd8b93014e7ecae45679ca8eb23bbc10f8d diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx index b6181021cae1b7f8f780b7b91e21497c7704b63e..e4773e3970871f5f18ef873b2c855309de13d92d 100644 --- a/web/src/automation/AutomationBuilder.tsx +++ b/web/src/automation/AutomationBuilder.tsx @@ -2360,7 +2360,8 @@ function TriggerProps({ {tables.map((t) => ( ))} diff --git a/web/src/automation/AutomationDetail.tsx b/web/src/automation/AutomationDetail.tsx index 28842c79d4dcbc4064dcbfeeb11dbbac4386543a..6cd5d0c503bf5e985cc71bc997371dfe84451f06 100644 --- a/web/src/automation/AutomationDetail.tsx +++ b/web/src/automation/AutomationDetail.tsx @@ -1597,7 +1597,8 @@ function AutomationDetail({ ) : null} {tables.map((t) => ( ))} diff --git a/web/src/automation/automationApi.ts b/web/src/automation/automationApi.ts index 862f928efe3bcd63e042ceea9ad0651cde83e6e2..590ee1dbacc7d4ecdb108fed497410c224c836f6 100644 --- a/web/src/automation/automationApi.ts +++ b/web/src/automation/automationApi.ts @@ -921,7 +921,7 @@ export interface UserTable { key: string; label: string; source: string; - rowCount: number; + rowCount: number | null; /** * `automation` is the machine-owner TAG (W23/R9's grey columns) — the id of the automation * that owns the column, or `null`. Narrowed from `unknown` in wave 24 against the route that diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx index 3bf17e0008d8f84dec9a347aab7a2af39e1952b1..7e1d451ab2118130bdca2275cfe806a3a0d1c00a 100644 --- a/web/src/customer-grid/CustomerGrid.tsx +++ b/web/src/customer-grid/CustomerGrid.tsx @@ -46,7 +46,7 @@ import { withoutAppliedLeaves } from "./drillLimits"; // ⭐⭐ W50-T03 / CONTRACT C1 — the windowed aggregate's envelope. Lane F serves the route // (W50-T31); this side is written to C1's shape and degrades to the loaded window until it // lands, which is the done-when's own observable rather than a workaround. -import type { AggregateResult } from "./apiBridge"; +import type { AggregateRequest, AggregateResult } from "./apiBridge"; import { EMPTY_WINDOW_PREDICATE, importantCountNote, limitSummary, lockedRecordsNote, pagingSummary, serverViewCount, windowedCapabilityNote, windowedFoldNote, windowPredicateKey, @@ -1361,7 +1361,9 @@ type ModeGroups = { groups: ModeGroup[]; totalGroups: number; totalRows: number; type ModeAgg = { colId: string; agg: string; p?: number }; async function mirrorModeGroups(table: string, req: RowWindowRequest, by: string, - aggs: ModeAgg[], offset: number, size: number, abort: AbortSignal): Promise { + aggs: ModeAgg[], offset: number, size: number, abort: AbortSignal, + rowSource?: EmbeddedRowSource): Promise { + if (rowSource) return rowSource.readGroups(req, by, aggs, offset, size); const response = await fetch(`${API_V1}/odoo-tables/${encodeURIComponent(table)}/aggregates`, { method: "POST", credentials: CREDENTIALS, signal: abort, headers: { "Content-Type": "application/json" }, @@ -1440,8 +1442,8 @@ function MirrorWindowPager({ offset, size, total, pending, label, onPage }: { } function useMirrorGroups(table: string, request: RowWindowRequest, field: Field, - aggs: ModeAgg[], offset: number, size: number, active = true) { - const key = JSON.stringify([table, request, field.key, aggs, offset, size, active]); + aggs: ModeAgg[], offset: number, size: number, active = true, rowSource?: EmbeddedRowSource) { + const key = JSON.stringify([table, request, field.key, aggs, offset, size, active, rowSource?.key]); const [answer, setAnswer] = useState<{ key: string; data?: ModeGroups; error?: string }>({ key: "" }); const [retry, setRetry] = useState(0); useEffect(() => { @@ -1449,7 +1451,7 @@ function useMirrorGroups(table: string, request: RowWindowRequest, field: Field, let live = true; setAnswer({ key }); if (!active) return () => { live = false; abort.abort(); }; - void mirrorModeGroups(table, request, field.key, aggs, offset, size, abort.signal).then( + void mirrorModeGroups(table, request, field.key, aggs, offset, size, abort.signal, rowSource).then( (data) => { if (live) setAnswer({ key, data }); }, (error: unknown) => { if (live) setAnswer({ key, error: error instanceof Error ? error.message : "Groups could not be read for this window." }); }); @@ -1462,21 +1464,23 @@ function useMirrorGroups(table: string, request: RowWindowRequest, field: Field, } function MirrorRecordWindow({ table, request, field, group, titleKey, fieldByKey, - cardKeys, size, calendar = false, onOpen }: { + cardKeys, size, calendar = false, onOpen, rowSource }: { table: string; request: RowWindowRequest; field: Field; group: ModeGroup; titleKey: string; fieldByKey: Map; cardKeys: string[]; size: number; calendar?: boolean; hosted?: boolean; onOpen: (record: Row) => void; + rowSource?: EmbeddedRowSource; }) { const [offset, setOffset] = useState(0); const [retry, setRetry] = useState(0); - const key = JSON.stringify([table, request, field.key, group.key, offset, size]); + const key = JSON.stringify([table, request, field.key, group.key, offset, size, rowSource?.key]); const [page, setPage] = useState<{ key: string; rows?: Row[]; total?: number; error?: string }>({ key: "" }); const seq = useRef(0); useEffect(() => { const current = ++seq.current; let live = true; setPage({ key }); - void fetchTableWindow(table, { ...request, offset, limit: size, + const readWindow = rowSource ? (_table: string, query: RowWindowRequest, _onFailure: (reason: string) => void) => rowSource.readWindow(query) : fetchTableWindow; + void readWindow(table, { ...request, offset, limit: size, group: { by: field.key, key: group.key } }, (reason) => { if (live && seq.current === current) console.debug("[CustomerGrid]", "record window failure", reason); }) @@ -1549,15 +1553,16 @@ function MirrorRecordWindow({ table, request, field, group, titleKey, fieldByKey } function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, size, - clamped, aggs, summaryMap, hosted = false, onOpen, onAddOption }: { + clamped, aggs, summaryMap, hosted = false, onOpen, onAddOption, rowSource }: { table: string; request: RowWindowRequest; field: Field; fieldByKey: Map; cardKeys: string[]; titleKey: string; size: number; clamped: boolean; aggs: ModeAgg[]; summaryMap: SummaryMap; hosted?: boolean; + rowSource?: EmbeddedRowSource; onOpen: (record: Row) => void; onAddOption: ((anchor: AnchorRect) => void) | null; }) { const [offset, setOffset] = useState(0); const columnsPerPage = 12; - const { data, error, retry } = useMirrorGroups(table, request, field, aggs, offset, columnsPerPage); + const { data, error, retry } = useMirrorGroups(table, request, field, aggs, offset, columnsPerPage, true, rowSource); useEffect(() => { if (data && offset >= data.totalGroups && offset > 0) setOffset(Math.max(0, Math.ceil(data.totalGroups / columnsPerPage) - 1) * columnsPerPage); @@ -1583,6 +1588,7 @@ function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, s : null; })} ; @@ -1597,9 +1603,10 @@ function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, s } function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, size, - summary, metrics, hosted = false, onOpen }: { + summary, metrics, hosted = false, onOpen, rowSource }: { table: string; request: RowWindowRequest; field: Field; fieldByKey: Map; titleKey: string; today?: string; size: number; summary: boolean; hosted?: boolean; + rowSource?: EmbeddedRowSource; metrics?: { id: string; field: string; agg: string }[]; onOpen: (record: Row) => void; }) { const [pickedMonth, setMonth] = useState(""); @@ -1613,7 +1620,7 @@ function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, si ] }; const aggs = summary ? (metrics ?? []).map((metric) => ({ colId: metric.field, agg: metric.agg === "avg" ? "average" : metric.agg })) : []; - const { data, error, retry } = useMirrorGroups(table, range, field, aggs, 0, 42, !!month); + const { data, error, retry } = useMirrorGroups(table, range, field, aggs, 0, 42, !!month, rowSource); const shift = (delta: number) => setMonth(new Date(Date.UTC(year, m - 1 + delta, 1)).toISOString().slice(0, 7)); const leading = month ? (new Date(Date.UTC(year, m - 1, 1)).getUTCDay() + 6) % 7 : 0; const cells = Array.from({ length: Math.ceil((leading + days) / 7) * 7 }, (_, i) => @@ -1648,6 +1655,7 @@ function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, si {value == null ? "Unavailable" : formatServerAgg(f, { agg: (metric.agg === "avg" ? "average" : metric.agg) as SummaryChoice["agg"] }, value)} : null; }) : } } @@ -2050,6 +2058,14 @@ type RecordIdInitialPage = { payload: CustomersPayload; }; +export interface EmbeddedRowSource { + /** Reader and query identity. Page IDs never define this source's population. */ + key: string; + readWindow: (request: RowWindowRequest) => Promise; + readAggregates: (request: AggregateRequest) => Promise; + readGroups: (request: RowWindowRequest, by: string, aggs: ModeAgg[], offset: number, size: number) => Promise; +} + interface CustomerGridProps { scope?: SurfaceScope; /** Shell-owned Data workspace location; grid resolves its view/field, never the global rail. */ @@ -2083,6 +2099,7 @@ interface CustomerGridProps { /** Exact linked pids to page on the server. */ embeddedRecordIds?: readonly number[]; embeddedInitialPage?: RecordIdInitialPage; + embeddedRowSource?: EmbeddedRowSource; /** A fresh drill is resolving ids; withhold the previous id page during that request. */ embeddedIdsPending?: boolean; onEmbeddedTotalChange?: (total: number | null) => void; @@ -2183,6 +2200,7 @@ interface LinkGridModalProps { table: SurfaceScope; recordIds: readonly number[]; initialPage?: RecordIdInitialPage; + rowSource?: EmbeddedRowSource; idsPending?: boolean; editable?: boolean; single?: boolean; @@ -2214,11 +2232,11 @@ interface LinkGridModalProps { // chrome slots and a `cellView` modifier while leaving existing callers and the focus trap intact. export function LinkGridModal({ label, table, recordIds, editable = false, single = false, excludeRecordId, selectionFilter, cellView = false, cellFilter, cellFields, onCellFilterChange, onCellDisplayChange, - headerActions, footer, initialPage, idsPending = false, + headerActions, footer, initialPage, rowSource, idsPending = false, onOpenRecord, onSave, onClose }: LinkGridModalProps) { const panelRef = useRef(null); const [selectedIds, setSelectedIds] = useState(() => [...recordIds]); - const countKey = `${table}:${recordIdsKey(recordIds)}`; + const countKey = `${table}:${rowSource?.key ?? recordIdsKey(recordIds)}`; const [serverCount, setServerCount] = useState<{ key: string; total: number | null }>(); const reportTotal = useCallback((total: number | null) => { setServerCount({ key: countKey, total }); @@ -2283,6 +2301,7 @@ export function LinkGridModal({ label, table, recordIds, editable = false, singl onCellDisplayChange={onCellDisplayChange} embeddedRecordIds={editable ? undefined : recordIds} embeddedInitialPage={editable ? undefined : initialPage} + embeddedRowSource={editable ? undefined : rowSource} embeddedIdsPending={!editable && idsPending} onEmbeddedTotalChange={editable ? undefined : reportTotal} embeddedExcludeRecordId={editable ? excludeRecordId : undefined} @@ -2489,9 +2508,9 @@ function linkSelectionPredicate(request: Pick ({ key, payload: prev?.key === key ? prev.payload : null, pending: true })); - void fetchTableWindow(scope, { ...req, + const readWindow = rowSource ? rowSource.readWindow : (request: RowWindowRequest) => fetchTableWindow(scope, request); + void readWindow({ ...req, ...(candidates ? linkSelectionPredicate(req, selectionFilter) : {}), - ...(ids !== undefined ? { ids } : {}), related }).then((payload) => { + ...(ids !== undefined && !rowSource ? { ids } : {}), related }).then((payload) => { if (seq !== sequence.current || currentKey.current !== key) return; if (candidates && selectionFilter && payload && unresolvedConditions(selectionFilter.nodes, { fieldByKey: new Map(payload.fields.map((field) => [field.key, field])), today: payload.today, @@ -2543,7 +2563,7 @@ function useRecordIdWindow(scope: SurfaceScope, ids: readonly number[] | undefin } setPage({ key, payload, pending: false }); }); - }, [scope, ids, related, key, seed, initialPage, idsPending, appliedCellFilters, candidates, selectionFilter]); + }, [scope, ids, related, key, seed, initialPage, idsPending, appliedCellFilters, candidates, selectionFilter, rowSource]); useEffect(() => { const refresh = () => { if (lastRequest.current) requestWindow(lastRequest.current, true); }; // React can replay effect setup after cancelling its first request in StrictMode. @@ -2580,6 +2600,7 @@ function CustomerGridSurface({ embeddedPayload, embeddedRecordIds, embeddedInitialPage, + embeddedRowSource, embeddedIdsPending = false, onEmbeddedTotalChange, embeddedSelectable = false, @@ -2643,7 +2664,7 @@ function CustomerGridSurface({ () => embeddedIdsKey ? JSON.parse(embeddedIdsKey) as number[] : undefined, [embeddedIdsKey]); const selectionWindow = embedded && !!embeddedSelectionFilter?.nodes?.length; const idWindow = useRecordIdWindow(scope, embeddedIds, relatedPin, embeddedInitialPage, embeddedIdsPending, cellView, - selectionWindow ? embeddedSelectionFilter : undefined); + selectionWindow ? embeddedSelectionFilter : undefined, embeddedRowSource); /** * ⭐⭐ W35-T23 (owner item 4 / R2) — **`previewReadOnly` IS GONE, AND THE SPLIT IS THE TICKET.** * @@ -4459,9 +4480,9 @@ function CustomerGridSurface({ const windowPredicate = useMemo( () => windowPredicateKey(config.filters, config.filterConj, config.sorts, search) + (embeddedIdsKey || selectionWindow - ? JSON.stringify([scope, embeddedIdsKey, relatedPinKey, cellView, embeddedSelectionFilter]) : ""), + ? JSON.stringify([scope, embeddedRowSource?.key ?? embeddedIdsKey, relatedPinKey, cellView, embeddedSelectionFilter]) : ""), [config.filters, config.filterConj, config.sorts, search, scope, embeddedIdsKey, relatedPinKey, cellView, - selectionWindow, embeddedSelectionFilter] + selectionWindow, embeddedSelectionFilter, embeddedRowSource] ); /* HOTFIX 2026-09-24 — ONE PAGE SIZE FOR EVERY DATABASE (50 / 100 / 200, default 100). */ const windowRequest = useCallback( @@ -4983,7 +5004,8 @@ function CustomerGridSurface({ return; } let live = true; - void fetchAggregates(topic.key, { + const readAggregates = embeddedRowSource ? (_table: string, request: AggregateRequest) => embeddedRowSource.readAggregates(request) : fetchAggregates; + void readAggregates(topic.key, { ...(embeddedIds !== undefined ? { ids: embeddedIds } : {}), ...linkSelectionPredicate({ filters: cellView && embeddedIds !== undefined ? withoutAppliedLeaves(config.filters) : config.filters, @@ -5020,7 +5042,7 @@ function CustomerGridSurface({ // is part of the question, so moving between two tabs on one target table must re-ask. Its // identity is stable (see `relatedPinKey`), so this cannot loop. }, [serverWindowed, topic.key, summaryRequestKey, windowPredicate, config.groupBy, relatedPin, embeddedIds, - serverAggRequestKey, cellView, embeddedSelectionFilter]); + serverAggRequestKey, cellView, embeddedSelectionFilter, embeddedRowSource]); /** * What the bar actually paints: C1's numbers when the server answered, this browser's fold * over the loaded window otherwise — and the two are never mixed within one paint, because a @@ -5179,7 +5201,8 @@ function CustomerGridSurface({ error, which replaced the whole grid with "Cannot reach the server." for one group's page; the failure belongs to the group that asked, so its cause stays with that group. */ let failure = ""; - void fetchTableWindow(scope, { + const readGroupWindow = embeddedRowSource ? (_scope: string, request: RowWindowRequest, _onFailure: (reason: string) => void) => embeddedRowSource.readWindow(request) : fetchTableWindow; + void readGroupWindow(scope, { ...windowRequest(offset), related: relatedPin, group: { by, key: groupKeyRaw.get(key) ?? null }, @@ -5199,7 +5222,7 @@ function CustomerGridSurface({ return next; }); }); - }, [config.groupBy, groupKeyRaw, scope, windowRequest, relatedPin]); + }, [config.groupBy, groupKeyRaw, scope, windowRequest, relatedPin, embeddedRowSource]); useEffect(() => { if (!serverWindowed || !config.groupBy || groupSummaryOn) return; for (const key of expanded) { @@ -12220,6 +12243,7 @@ function CustomerGridSurface({ {displayMode === "calendar" && (calendarField ? ( serverWindowed ? meta.group_limit! + || (expected !== undefined && expected !== total)) return null; + const next = offset + size; + if (next > total || (size === 0 && next < total) + || (meta.has_more !== undefined && meta.has_more !== (next < total))) return null; + return { total, done: next === total }; +} + /** * `POST /api/v1/odoo-tables//aggregates` — C1, over the WHOLE predicate. * @@ -1221,32 +1244,52 @@ export async function fetchAggregates( req: AggregateRequest ): Promise { if (!req.aggs.length && !req.groupBy) return null; - let res: Response; - try { - res = await fetch( - `${API_V1}/odoo-tables/${encodeURIComponent(tableKey)}/aggregates`, - { - method: "POST", - credentials: CREDENTIALS, - headers: JSON_HEADERS, - body: JSON.stringify({ - ...(req.ids !== undefined ? { ids: req.ids } : {}), - filters: req.filters ?? [], - filterConj: req.filterConj === "or" ? "or" : "and", - search: (req.search ?? "").trim(), - ...(req.groupBy ? { groupBy: req.groupBy } : {}), - /* ⛔ A BODY KEY HERE, A QUERY STRING ON `/rows` (`windowRowsPath`). Same object, two - transports, because this door is a POST and that one is a GET. */ - ...(req.related ? { related: req.related } : {}), - aggs: req.aggs, - }), - } - ); - } catch { - return null; - } - if (handledSession(res) || !res.ok) return null; - return aggregatePayload(await readJson(res)); + let offset = 0; + let answer: AggregateResult | null = null; + let expectedGroups: number | undefined; + do { + let res: Response; + try { + res = await fetch( + `${API_V1}/odoo-tables/${encodeURIComponent(tableKey)}/aggregates`, + { + method: "POST", + credentials: CREDENTIALS, + headers: JSON_HEADERS, + body: JSON.stringify({ + ...(req.ids !== undefined ? { ids: req.ids } : {}), + filters: req.filters ?? [], + filterConj: req.filterConj === "or" ? "or" : "and", + search: (req.search ?? "").trim(), + ...(req.groupBy ? { groupBy: req.groupBy } : {}), + ...(req.groupBy ? { groupOffset: offset, groupLimit: 1000 } : {}), + /* ⛔ A BODY KEY HERE, A QUERY STRING ON `/rows` (`windowRowsPath`). Same object, two + transports, because this door is a POST and that one is a GET. */ + ...(req.related ? { related: req.related } : {}), + aggs: req.aggs, + }), + } + ); + } catch { + return null; + } + if (handledSession(res) || !res.ok) return null; + const body = await readJson(res); + const page = aggregatePayload(body); + if (!page) return null; + if (!req.groupBy) return page; + const progress = aggregatePageProgress(body, offset, page.groups?.length, expectedGroups); + if (!progress) return null; + expectedGroups = progress.total; + if (!answer) answer = page; + else { + if (answer.totalRows !== page.totalRows + || JSON.stringify(answer.aggregates) !== JSON.stringify(page.aggregates)) return null; + answer.groups = [...(answer.groups ?? []), ...(page.groups ?? [])]; + } + offset += page.groups?.length ?? 0; + if (progress.done) return answer; + } while (true); } // ═══════════════════════════════════════════════════════════════════════════════════════════ diff --git a/web/src/sheet/DrillModal.tsx b/web/src/sheet/DrillModal.tsx index b6f0f4ce6d45980cca250315d5b1d609399d23ef..b626968e46824818c4db7929034bc8d08f2bfd6b 100644 --- a/web/src/sheet/DrillModal.tsx +++ b/web/src/sheet/DrillModal.tsx @@ -6,11 +6,9 @@ // popups that behave almost the same, and "almost" is where a focus trap or an escape key goes // missing on one of them. // -// ⛔⛔ THE IDS ARE FETCHED LIVE, EVERY TIME, AND THAT IS LOAD-BEARING. `LinkGridModal` runs no query -// of its own: when it is not editable it prints `recordIds.length` verbatim, so the count on screen -// is only as fresh as what the CALLER hands it. R13 requires the popup to show LIVE rows, so this -// component always calls `GET /sheets/{id}/cells/{addr}/drill` and never reads ids off the cell. -// Passing the cell's cached ids would be a stale count presented as a live one. +// R13 requires live rows and counts. The drill returns an exact count and page IDs. +// Its caller-owned row source replays structured queries under current permissions. +// Page IDs identify rows already read and never restrict the remaining population. // // ⭐ R13 ALSO FORBIDS DISCLOSING STALENESS HERE. A stale cell opens the same popup with the same // chrome and NO comparison text: no "was 24,920", no badge, nothing. The staleness is a property of @@ -55,10 +53,10 @@ import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { API_V1, CREDENTIALS } from "../apiContract"; -import { fetchWorkspace } from "../customer-grid/apiBridge"; +import { aggregatePageProgress, aggregatePayload, fetchWorkspace } from "../customer-grid/apiBridge"; import { dropAppliedFilterLimits } from "../customer-grid/drillLimits"; import type { SurfaceScope } from "../customer-grid/apiBridge"; -import type { CellDisplayChoice } from "../customer-grid/CustomerGrid"; +import type { CellDisplayChoice, EmbeddedRowSource } from "../customer-grid/CustomerGrid"; import { dataPathHash } from "../customer-grid/dataPath"; import { AnchoredOverlay, BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface"; import type { CustomersPayload, Field, FilterTree, GridWorkspace, Row } from "../customer-grid/types"; @@ -239,12 +237,9 @@ function DrillFilterBand( ); } -/** - * S66-T48 (standing rule 1): a drill whose ids were capped says so. A connector drill lists every - * id and this is empty; a typed database past its row ceiling says how many of how many are here. - */ -function drillCapText(drill: Pick | null): string { - if (!drill || drill.record_ids.length >= drill.count) return ""; +/** Explain a legacy server's ID cap while page-based responses use their ordinary pager. */ +function drillCapText(drill: Pick | null): string { + if (!drill || drill.record_ids_scope === "page" || drill.record_ids.length >= drill.count) return ""; return `Showing ${drill.record_ids.length.toLocaleString()} of ${drill.count.toLocaleString()} ` + "matching records. Narrow the cell's filter to list the rest."; } @@ -739,13 +734,77 @@ function DrillModal( today: drill.page.today, recordsMutable: drill.page.recordsMutable, limits: dropAppliedFilterLimits(drill.page.limits, source?.kind === "filter" ? source.tree : undefined), - counts: { shown: drill.page.rows.length, matched: drill.record_ids.length, - total: drill.record_ids.length, windowed: true }, + counts: { shown: drill.page.rows.length, matched: drill.count, + total: drill.page.totalUnfiltered ?? drill.count, windowed: true }, }; return { offset: drill.page.offset, limit: drill.page.limit, token: drill.token, payload, filter: source?.kind === "filter" ? source.tree : undefined }; }, [drill, fields]); + const rowSource = useMemo(() => { + if (!drill || drill.record_ids_scope !== "page") return undefined; + const source = scopeFromDrill(drill); + const filter = source?.kind === "filter" ? source.tree : undefined; + const read = async (query: import("./sheetApi").CellDrillQuery, offset = 0, limit = 100) => { + const got = await drillCell(workbookId, addr, tab, { query, offset, limit, filter }); + if (!got.ok) { setRowsError(got.message); return null; } + return got.value; + }; + return { + key: JSON.stringify([workbookId, addr, tab, drill.population, drill.filter, drill.view_id, reloadSeq]), + readGroups: async (request, by, aggs, offset, size) => { + const value = await read({ filters: request.filters, filterConj: request.filterConj, + search: request.search, groupBy: by, aggs, groupOffset: offset, groupLimit: size }); + const summary = value?.summary; + if (!summary || !Array.isArray(summary.groups) || typeof summary.total_groups !== "number") + throw new Error("Groups could not be read for this window."); + if (summary.limits.length) + throw new Error(summary.limits.map(limit => limit.cause).filter(Boolean).join(". ")); + return { groups: summary.groups as { key: string | number | null; label: string; count: number; + aggregates: Record }[], totalGroups: summary.total_groups, + totalRows: summary.total_rows, aggregates: summary.aggregates }; + }, + readWindow: async (request) => { + const value = await read({ filters: request.filters, filterConj: request.filterConj, + sorts: request.sorts, search: request.search, group: request.group }, request.offset, request.limit); + if (!value?.page) return null; + const page = value.page; + return { ...initialPage?.payload, fields: page.fields?.length ? page.fields : fields, + rows: page.rows, today: page.today, recordsMutable: page.recordsMutable, + limits: dropAppliedFilterLimits(page.limits, filter), + counts: { shown: page.rows.length, matched: value.count, + total: page.totalUnfiltered ?? value.count, windowed: true } }; + }, + readAggregates: async (request) => { + let offset = 0; + let answer: ReturnType = null; + let total: number | undefined; + let population: string | null | undefined; + do { + const value = await read({ filters: request.filters, filterConj: request.filterConj, + search: request.search, aggs: request.aggs, groupBy: request.groupBy, + groupOffset: offset, groupLimit: 1000 }); + const summary = value?.summary; + const page = summary ? aggregatePayload(summary) : null; + if (!page || !summary) return null; + if (!request.groupBy) return page; + const progress = aggregatePageProgress(summary, offset, page.groups?.length, total); + if (!progress) return null; + if (!progress.done && !value?.population) return null; + if (!answer) { answer = page; total = summary.total_groups; population = value?.population; } + else { + if (value?.population !== population || total !== summary.total_groups + || page.totalRows !== answer.totalRows + || JSON.stringify(page.aggregates) !== JSON.stringify(answer.aggregates)) return null; + answer.groups = [...(answer.groups ?? []), ...(page.groups ?? [])]; + } + offset += page.groups?.length ?? 0; + if (progress.done) return answer; + } while (true); + }, + }; + }, [drill, workbookId, addr, tab, reloadSeq, initialPage, fields]); + const identityField = fields.find((field) => field.pinned) ?? fields[0] ?? null; const savedView = drill?.view_id ? workspace?.views?.find((view) => view.id === drill.view_id) ?? null @@ -802,18 +861,33 @@ function DrillModal( setRowsError(""); try { const first = drill.page?.offset === 0 ? drill.page : undefined; - const recordRows: Row[] = [...(first?.rows ?? [])]; - let offset = first?.limit ?? 0; - while (offset < drill.record_ids.length) { - const got = await drillCell(workbookId, addr, tab, { offset, limit: 1000 }); + const recordRows: Row[] = []; + const recordIds = new Set(); + let offset = 0; + if (!drill.population) throw new Error("These records have no freshness stamp. Reopen View data and try Export again."); + if (!Number.isSafeInteger(drill.count) || drill.count < 0) + throw new Error("These records have no complete count. Reopen View data and try Export again."); + while (offset < drill.count) { + const got = await drillCell(workbookId, addr, tab, { offset, limit: 1000, + filter: drill.view_id ? undefined : drill.filter, query: drill.query }); if (!got.ok) throw new Error(got.message); if (!got.value.page) throw new Error("The records for Export did not include a page. Try again."); - if (got.value.token !== drill.token - || JSON.stringify(got.value.record_ids) !== JSON.stringify(drill.record_ids)) + if (got.value.population !== drill.population || got.value.count !== drill.count) throw new Error("These records changed while Export was preparing. Try again."); + if (got.value.page.offset !== offset || got.value.page.total !== drill.count + || got.value.page.rows.length > drill.count - offset) + throw new Error("Some records could not be read. Try Export again."); + if (!got.value.page.rows.length) + throw new Error("These records changed while Export was preparing. Try again."); + for (const row of got.value.page.rows) { + if (!Number.isSafeInteger(row.pid) || recordIds.has(row.pid)) + throw new Error("Some records could not be read. Try Export again."); + recordIds.add(row.pid); + } recordRows.push(...got.value.page.rows); - offset += got.value.page.limit; + offset += got.value.page.rows.length; } + if (recordRows.length !== drill.count) throw new Error("Some records could not be read. Try Export again."); const exporter = await import("../customer-grid/export"); exporter.runExport(format, `${label} records`, first?.today, visibleFields, recordRows); } catch (cause) { @@ -914,6 +988,7 @@ function DrillModal( label={label} table={scope} recordIds={drill.record_ids} + rowSource={rowSource} {...{ initialPage, idsPending }} editable={false} cellView @@ -940,7 +1015,7 @@ function DrillModal( className="sheet-cell-view-action" aria-haspopup="menu" aria-expanded={exportOpen} - disabled={!exportReady || exporting} + disabled={!exportReady || exporting || idsPending} onClick={() => setExportOpen((open) => !open)} >