Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +13 -6
- api/application_registry.py +6 -3
- api/assistant_stream.py +64 -5
- api/connectors_amazon.py +11 -5
- api/deps.py +61 -75
- api/main.py +44 -1
- api/odoo_relational.py +29 -11
- api/process_role.py +2 -4
- api/registry_source.py +765 -0
- api/request_class.py +24 -3
- api/request_timing.py +181 -1
- api/routes_admin.py +3 -2
- api/routes_connectors.py +12 -6
- api/routes_customers.py +55 -16
- api/routes_grid.py +94 -1
- api/routes_odoo_tables.py +621 -190
- api/routes_platform_admin.py +8 -8
- api/routes_products.py +27 -9
- api/routes_sheet.py +328 -110
- api/routes_tables.py +144 -55
- platform/core/grid_events.py +11 -0
- platform/core/measure_resolve.py +4 -2
- platform/core/odoo.py +219 -6
- platform/core/request_timing.py +221 -1
- platform/core/store_pg.py +660 -61
- platform/core/user_tables.py +61 -12
- platform/harness/canonical_source.py +383 -0
- platform/harness/datastore.py +165 -12
- platform/harness/filter_sql.py +7 -3
- platform/harness/measure_filter.py +23 -11
- platform/harness/mirror_tokens.py +3 -15
- platform/harness/semantic.py +40 -14
- platform/modules/ar.py +19 -13
- platform/modules/backorders.py +8 -3
- platform/modules/customer_data.py +129 -48
- platform/modules/customers.py +25 -8
- platform/modules/inventory.py +9 -5
- platform/modules/map.py +6 -3
- platform/modules/product_data.py +56 -8
- platform/modules/products.py +13 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/automation/AutomationBuilder.tsx +2 -1
- web/src/automation/AutomationDetail.tsx +2 -1
- web/src/automation/automationApi.ts +1 -1
- web/src/customer-grid/CustomerGrid.tsx +51 -26
- web/src/customer-grid/apiBridge.ts +69 -26
- web/src/sheet/DrillModal.tsx +100 -25
- web/src/sheet/sheetApi.ts +19 -2
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "9f34fefd92",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
9f34fefd92
|
api/amazon_relational.py
CHANGED
|
@@ -610,11 +610,15 @@ def grid_source(cur, key):
|
|
| 610 |
|
| 611 |
complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
|
| 612 |
peer = '"_amazon_identity_peer"'
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
#
|
| 616 |
-
occurrence = (
|
| 617 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 618 |
fallback_digest = digest(["'__amazon_missing_identity__'",
|
| 619 |
*(value(name) for name in fallback), occurrence])
|
| 620 |
digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
|
|
@@ -643,8 +647,11 @@ def grid_source(cur, key):
|
|
| 643 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 644 |
for lookup in lookups(cur, key, names):
|
| 645 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 646 |
-
|
| 647 |
"needs_excluded": False, "cols": cols}
|
|
|
|
|
|
|
|
|
|
| 648 |
|
| 649 |
|
| 650 |
def grid_sources(cur=None):
|
|
|
|
| 610 |
|
| 611 |
complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
|
| 612 |
peer = '"_amazon_identity_peer"'
|
| 613 |
+
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
|
| 614 |
+
partition = ", ".join(value(name, peer) for name in fallback)
|
| 615 |
+
# Rank only incomplete identities; earlier equal peers are exactly row_number minus one.
|
| 616 |
+
occurrence = (
|
| 617 |
+
'(SELECT CAST("_identity_occurrence" AS VARCHAR) FROM '
|
| 618 |
+
f'(SELECT {peer}.rowid AS "_identity_rowid", '
|
| 619 |
+
f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
|
| 620 |
+
f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
|
| 621 |
+
f'AS "_amazon_missing_identity" WHERE "_identity_rowid" = {source}.rowid)')
|
| 622 |
fallback_digest = digest(["'__amazon_missing_identity__'",
|
| 623 |
*(value(name) for name in fallback), occurrence])
|
| 624 |
digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
|
|
|
|
| 647 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 648 |
for lookup in lookups(cur, key, names):
|
| 649 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 650 |
+
spec = {"table": SOURCE[key], "where": "", "id": ident,
|
| 651 |
"needs_excluded": False, "cols": cols}
|
| 652 |
+
if table not in amazon_store.DERIVED_TABLES:
|
| 653 |
+
spec["page_key"] = f"{source}.rowid"
|
| 654 |
+
return spec
|
| 655 |
|
| 656 |
|
| 657 |
def grid_sources(cur=None):
|
api/application_registry.py
CHANGED
|
@@ -6,6 +6,7 @@ import copy
|
|
| 6 |
import hashlib
|
| 7 |
import json
|
| 8 |
from core import shares, write_execution
|
|
|
|
| 9 |
import app_rules
|
| 10 |
|
| 11 |
from modules.finance_apps import (finance_application as _finance_application,
|
|
@@ -114,10 +115,11 @@ def _brand_analytics_refused(runtime):
|
|
| 114 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 115 |
row = datastore.ro_cursor().execute(query, key).fetchone()
|
| 116 |
else:
|
| 117 |
-
with duckdb.connect(str(path), read_only=True) as cursor:
|
| 118 |
row = cursor.execute(query, key).fetchone()
|
| 119 |
return bool(row) and row[0] == "refused"
|
| 120 |
-
except Exception:
|
|
|
|
| 121 |
return False
|
| 122 |
|
| 123 |
|
|
@@ -133,7 +135,8 @@ def brand_analytics_offered(session):
|
|
| 133 |
and (enablement.get("databases") or {}).get(BRAND_ANALYTICS_DATABASE) is True)
|
| 134 |
refused = _brand_analytics_refused(session.runtime) if connected and enabled else None
|
| 135 |
return brand_analytics_holds(connected, enabled, refused)
|
| 136 |
-
except Exception:
|
|
|
|
| 137 |
return False
|
| 138 |
|
| 139 |
|
|
|
|
| 6 |
import hashlib
|
| 7 |
import json
|
| 8 |
from core import shares, write_execution
|
| 9 |
+
from core.request_timing import raise_if_resource_error
|
| 10 |
import app_rules
|
| 11 |
|
| 12 |
from modules.finance_apps import (finance_application as _finance_application,
|
|
|
|
| 115 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 116 |
row = datastore.ro_cursor().execute(query, key).fetchone()
|
| 117 |
else:
|
| 118 |
+
with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
|
| 119 |
row = cursor.execute(query, key).fetchone()
|
| 120 |
return bool(row) and row[0] == "refused"
|
| 121 |
+
except Exception as exc: # noqa: BLE001
|
| 122 |
+
raise_if_resource_error(exc)
|
| 123 |
return False
|
| 124 |
|
| 125 |
|
|
|
|
| 135 |
and (enablement.get("databases") or {}).get(BRAND_ANALYTICS_DATABASE) is True)
|
| 136 |
refused = _brand_analytics_refused(session.runtime) if connected and enabled else None
|
| 137 |
return brand_analytics_holds(connected, enabled, refused)
|
| 138 |
+
except Exception as exc: # noqa: BLE001
|
| 139 |
+
raise_if_resource_error(exc)
|
| 140 |
return False
|
| 141 |
|
| 142 |
|
api/assistant_stream.py
CHANGED
|
@@ -1,9 +1,13 @@
|
|
| 1 |
"""Expose one Query turn as server-sent events with same-turn error recovery."""
|
| 2 |
import asyncio
|
| 3 |
import contextvars
|
|
|
|
|
|
|
| 4 |
import json
|
| 5 |
import threading
|
| 6 |
|
|
|
|
|
|
|
| 7 |
from fastapi import APIRouter, Body, Depends
|
| 8 |
from starlette.responses import StreamingResponse
|
| 9 |
|
|
@@ -16,16 +20,65 @@ router = APIRouter(prefix="/api/v1", dependencies=[Depends(module_gate("analyst"
|
|
| 16 |
|
| 17 |
|
| 18 |
class StreamGZipMiddleware:
|
| 19 |
-
"""
|
| 20 |
|
| 21 |
def __init__(self, app, **kwargs):
|
| 22 |
-
from starlette.middleware.gzip import GZipMiddleware
|
| 23 |
self.app = app
|
| 24 |
-
self.
|
|
|
|
| 25 |
|
| 26 |
async def __call__(self, scope, receive, send):
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
def _event(name, data):
|
|
@@ -88,6 +141,12 @@ async def events(body, session, *, submit=None):
|
|
| 88 |
put("text_delta", {"text": content[len(text):]})
|
| 89 |
put("done", result)
|
| 90 |
except Exception as exc:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
detail = getattr(exc, "detail", None)
|
| 92 |
error = (detail or {}).get("error") if isinstance(detail, dict) else None
|
| 93 |
put("error", {"code": (error or {}).get("code", "stream_failed"),
|
|
|
|
| 1 |
"""Expose one Query turn as server-sent events with same-turn error recovery."""
|
| 2 |
import asyncio
|
| 3 |
import contextvars
|
| 4 |
+
import gzip
|
| 5 |
+
import io
|
| 6 |
import json
|
| 7 |
import threading
|
| 8 |
|
| 9 |
+
import anyio
|
| 10 |
+
|
| 11 |
from fastapi import APIRouter, Body, Depends
|
| 12 |
from starlette.responses import StreamingResponse
|
| 13 |
|
|
|
|
| 20 |
|
| 21 |
|
| 22 |
class StreamGZipMiddleware:
|
| 23 |
+
"""Compress response chunks in the bounded anyio pool while passing SSE through."""
|
| 24 |
|
| 25 |
def __init__(self, app, **kwargs):
|
|
|
|
| 26 |
self.app = app
|
| 27 |
+
self.minimum_size = kwargs.get("minimum_size", 500)
|
| 28 |
+
self.compresslevel = kwargs.get("compresslevel", 9)
|
| 29 |
|
| 30 |
async def __call__(self, scope, receive, send):
|
| 31 |
+
if scope.get("type") != "http" or scope.get("path") == PATH:
|
| 32 |
+
return await self.app(scope, receive, send)
|
| 33 |
+
from starlette.datastructures import Headers
|
| 34 |
+
from starlette.middleware.gzip import IdentityResponder
|
| 35 |
+
if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""):
|
| 36 |
+
responder = _OffLoopGZipResponder(self.app, self.minimum_size, self.compresslevel)
|
| 37 |
+
else:
|
| 38 |
+
responder = IdentityResponder(self.app, self.minimum_size)
|
| 39 |
+
await responder(scope, receive, send)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
from starlette.middleware.gzip import GZipResponder, IdentityResponder
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class _OffLoopGZipResponder(GZipResponder):
|
| 46 |
+
"""Retain Starlette's header and chunk behavior with compression outside the event loop."""
|
| 47 |
+
|
| 48 |
+
def __init__(self, app, minimum_size, compresslevel):
|
| 49 |
+
IdentityResponder.__init__(self, app, minimum_size)
|
| 50 |
+
self.compresslevel = compresslevel
|
| 51 |
+
self.gzip_file = None
|
| 52 |
+
|
| 53 |
+
async def __call__(self, scope, receive, send):
|
| 54 |
+
try:
|
| 55 |
+
await IdentityResponder.__call__(self, scope, receive, send)
|
| 56 |
+
finally:
|
| 57 |
+
if self.gzip_file is not None:
|
| 58 |
+
def close():
|
| 59 |
+
self.gzip_file.close()
|
| 60 |
+
self.gzip_buffer.close()
|
| 61 |
+
with anyio.CancelScope(shield=True):
|
| 62 |
+
await anyio.to_thread.run_sync(close)
|
| 63 |
+
|
| 64 |
+
def _compress(self, body, more):
|
| 65 |
+
if self.gzip_file is None:
|
| 66 |
+
self.gzip_buffer = io.BytesIO()
|
| 67 |
+
self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer,
|
| 68 |
+
compresslevel=self.compresslevel)
|
| 69 |
+
return GZipResponder.apply_compression(self, body, more_body=more)
|
| 70 |
+
|
| 71 |
+
async def send_with_compression(self, message):
|
| 72 |
+
if (message["type"] == "http.response.body"
|
| 73 |
+
and not self.content_encoding_set and not self.content_type_is_excluded):
|
| 74 |
+
body, more = message.get("body", b""), message.get("more_body", False)
|
| 75 |
+
if self.started or more or len(body) >= self.minimum_size:
|
| 76 |
+
self._compressed_body = await anyio.to_thread.run_sync(
|
| 77 |
+
lambda: self._compress(body, more))
|
| 78 |
+
await super().send_with_compression(message)
|
| 79 |
+
|
| 80 |
+
def apply_compression(self, body, *, more_body):
|
| 81 |
+
return self._compressed_body
|
| 82 |
|
| 83 |
|
| 84 |
def _event(name, data):
|
|
|
|
| 141 |
put("text_delta", {"text": content[len(text):]})
|
| 142 |
put("done", result)
|
| 143 |
except Exception as exc:
|
| 144 |
+
from request_timing import resource_cause
|
| 145 |
+
cause = resource_cause(exc)
|
| 146 |
+
if cause is not None:
|
| 147 |
+
put("error", {"code": "server_busy", "cause": cause,
|
| 148 |
+
"message": "The server is busy. Try again shortly.", "retry": False})
|
| 149 |
+
return
|
| 150 |
detail = getattr(exc, "detail", None)
|
| 151 |
error = (detail or {}).get("error") if isinstance(detail, dict) else None
|
| 152 |
put("error", {"code": (error or {}).get("code", "stream_failed"),
|
api/connectors_amazon.py
CHANGED
|
@@ -29,6 +29,8 @@ _PLATFORM = Path(__file__).resolve().parents[2] / "platform"
|
|
| 29 |
if str(_PLATFORM) not in sys.path:
|
| 30 |
sys.path.insert(0, str(_PLATFORM))
|
| 31 |
|
|
|
|
|
|
|
| 32 |
# ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
|
| 33 |
# map and the transport; this module adds the probe and the registration and nothing else.
|
| 34 |
from harness.amazon_client import ( # noqa: F401,E402
|
|
@@ -189,7 +191,8 @@ def _mirror_counts(cur, az):
|
|
| 189 |
try:
|
| 190 |
counts[table] = int(cur.execute(
|
| 191 |
f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
|
| 192 |
-
except Exception:
|
|
|
|
| 193 |
continue
|
| 194 |
return counts
|
| 195 |
|
|
@@ -337,7 +340,8 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 337 |
try:
|
| 338 |
purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
|
| 339 |
purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
|
| 340 |
-
except Exception:
|
|
|
|
| 341 |
purge_plan, purge_message, purge_unknown = None, "", True
|
| 342 |
|
| 343 |
def finish(status):
|
|
@@ -407,10 +411,11 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 407 |
entities = _sync_entities(_cur, cadence)
|
| 408 |
mirrored = _mirror_counts(_cur, az)
|
| 409 |
else:
|
| 410 |
-
with duckdb.connect(str(path), read_only=True) as cur:
|
| 411 |
entities = _sync_entities(cur, cadence)
|
| 412 |
mirrored = _mirror_counts(cur, az)
|
| 413 |
-
except Exception:
|
|
|
|
| 414 |
# No exception text: paths, provider payloads and credentials never become status copy.
|
| 415 |
return finish(unavailable)
|
| 416 |
if entities is None:
|
|
@@ -642,7 +647,8 @@ def read_database_enablement(rt):
|
|
| 642 |
keys = database_keys()
|
| 643 |
try:
|
| 644 |
current = rt.get(ENABLEMENT_KEY) if rt is not None else None
|
| 645 |
-
except Exception:
|
|
|
|
| 646 |
return {"available": False, "databases": {key: False for key in keys}}
|
| 647 |
# Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
|
| 648 |
# the migration default rather than malformed state.
|
|
|
|
| 29 |
if str(_PLATFORM) not in sys.path:
|
| 30 |
sys.path.insert(0, str(_PLATFORM))
|
| 31 |
|
| 32 |
+
from core.request_timing import raise_if_resource_error
|
| 33 |
+
|
| 34 |
# ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
|
| 35 |
# map and the transport; this module adds the probe and the registration and nothing else.
|
| 36 |
from harness.amazon_client import ( # noqa: F401,E402
|
|
|
|
| 191 |
try:
|
| 192 |
counts[table] = int(cur.execute(
|
| 193 |
f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
|
| 194 |
+
except Exception as exc: # noqa: BLE001
|
| 195 |
+
raise_if_resource_error(exc)
|
| 196 |
continue
|
| 197 |
return counts
|
| 198 |
|
|
|
|
| 340 |
try:
|
| 341 |
purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
|
| 342 |
purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
|
| 343 |
+
except Exception as exc:
|
| 344 |
+
raise_if_resource_error(exc)
|
| 345 |
purge_plan, purge_message, purge_unknown = None, "", True
|
| 346 |
|
| 347 |
def finish(status):
|
|
|
|
| 411 |
entities = _sync_entities(_cur, cadence)
|
| 412 |
mirrored = _mirror_counts(_cur, az)
|
| 413 |
else:
|
| 414 |
+
with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur:
|
| 415 |
entities = _sync_entities(cur, cadence)
|
| 416 |
mirrored = _mirror_counts(cur, az)
|
| 417 |
+
except Exception as exc: # noqa: BLE001
|
| 418 |
+
raise_if_resource_error(exc)
|
| 419 |
# No exception text: paths, provider payloads and credentials never become status copy.
|
| 420 |
return finish(unavailable)
|
| 421 |
if entities is None:
|
|
|
|
| 647 |
keys = database_keys()
|
| 648 |
try:
|
| 649 |
current = rt.get(ENABLEMENT_KEY) if rt is not None else None
|
| 650 |
+
except Exception as exc: # noqa: BLE001
|
| 651 |
+
raise_if_resource_error(exc)
|
| 652 |
return {"available": False, "databases": {key: False for key in keys}}
|
| 653 |
# Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
|
| 654 |
# the migration default rather than malformed state.
|
api/deps.py
CHANGED
|
@@ -16,8 +16,10 @@ import difflib
|
|
| 16 |
import os
|
| 17 |
import sys
|
| 18 |
import time
|
|
|
|
| 19 |
from collections import OrderedDict
|
| 20 |
from dataclasses import dataclass
|
|
|
|
| 21 |
from pathlib import Path
|
| 22 |
|
| 23 |
from fastapi import Depends, HTTPException, Request, Response
|
|
@@ -1429,47 +1431,10 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 1429 |
return out
|
| 1430 |
|
| 1431 |
|
| 1432 |
-
#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
|
| 1433 |
-
#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
|
| 1434 |
-
#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
|
| 1435 |
-
#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
|
| 1436 |
-
#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
|
| 1437 |
-
#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
|
| 1438 |
-
#: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
|
| 1439 |
-
#: answers 503 when the store is unreadable); a cached registry is never served in its place.
|
| 1440 |
-
#: The key carries the identity of the reader and the store it reads, so a swapped store or a
|
| 1441 |
-
#: patched `users.registry` / `store.get` misses instead of replaying another registry.
|
| 1442 |
-
_SESSION_REGISTRY = {"key": None, "reg": None}
|
| 1443 |
-
_SESSION_REGISTRY_LOCK = __import__("threading").Lock()
|
| 1444 |
-
|
| 1445 |
-
|
| 1446 |
def _session_registry():
|
| 1447 |
-
"""
|
| 1448 |
-
|
| 1449 |
-
|
| 1450 |
-
and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
|
| 1451 |
-
fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
|
| 1452 |
-
memo on HF), and its `get()` already holds the bucket for the life of the process, so the
|
| 1453 |
-
re-read this memo removes does not exist there: it reads straight through.
|
| 1454 |
-
"""
|
| 1455 |
-
key = None
|
| 1456 |
-
try:
|
| 1457 |
-
backend = store.backend()
|
| 1458 |
-
token = (store.revision("users") or {}).get("token") if backend == "pg" else None
|
| 1459 |
-
if token not in (None, ""):
|
| 1460 |
-
where = os.environ.get("AIOS_TENANT") or "royal-imports"
|
| 1461 |
-
key = (backend, where, id(users.registry), id(store.get), str(token))
|
| 1462 |
-
except Exception: # noqa: BLE001 -- fail-closed below
|
| 1463 |
-
key = None
|
| 1464 |
-
if key is not None:
|
| 1465 |
-
with _SESSION_REGISTRY_LOCK:
|
| 1466 |
-
if _SESSION_REGISTRY["key"] == key:
|
| 1467 |
-
return _SESSION_REGISTRY["reg"]
|
| 1468 |
-
reg = users.registry() or {}
|
| 1469 |
-
if key is not None:
|
| 1470 |
-
with _SESSION_REGISTRY_LOCK:
|
| 1471 |
-
_SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
|
| 1472 |
-
return reg
|
| 1473 |
|
| 1474 |
|
| 1475 |
def _user_for(claims):
|
|
@@ -1531,26 +1496,16 @@ def _user_for(claims):
|
|
| 1531 |
return None
|
| 1532 |
|
| 1533 |
|
| 1534 |
-
# ---
|
| 1535 |
-
#
|
| 1536 |
-
#
|
| 1537 |
-
#
|
| 1538 |
-
#
|
| 1539 |
-
#
|
| 1540 |
-
#
|
| 1541 |
-
# decide whether to write would be little better. So the staleness decision is made from a
|
| 1542 |
-
# PROCESS-LOCAL map: no I/O at all on the 99.97% of requests inside the window, and the write
|
| 1543 |
-
# itself (once an hour per account) happens on `core.users`' own background thread, so no request
|
| 1544 |
-
# ever waits for a hub round-trip. `core.users` explains why that write is SYNC and not the
|
| 1545 |
-
# coalesced async path — `users.json` is shared with the Streamlit host and an async flush rebases
|
| 1546 |
-
# on this process's cache, which would let a stamp revert somebody else's deactivation.
|
| 1547 |
-
#
|
| 1548 |
-
# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
|
| 1549 |
-
# write once per window (still bounded); a restart re-arms every user's first request; and the
|
| 1550 |
-
# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
|
| 1551 |
-
# renders it as. Bounded independently so it cannot grow into a leak on a deployment
|
| 1552 |
-
# with many accounts.
|
| 1553 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
|
|
|
|
|
|
| 1554 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
| 1555 |
_ACTIVE_MAX_TRACKED = 4096
|
| 1556 |
|
|
@@ -1572,30 +1527,59 @@ def _principal(tenant, username):
|
|
| 1572 |
|
| 1573 |
|
| 1574 |
def note_active(uname, when=None):
|
| 1575 |
-
"""
|
| 1576 |
-
has already stamped `last_active` itself; without it the very next request would find no entry
|
| 1577 |
-
and stamp again immediately, making the "at most once an hour" rule false by one write per
|
| 1578 |
-
sign-in."""
|
| 1579 |
key = (uname or "").strip().lower()
|
| 1580 |
if not key:
|
| 1581 |
return
|
| 1582 |
-
|
| 1583 |
-
|
| 1584 |
-
|
| 1585 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1586 |
|
| 1587 |
|
| 1588 |
def _touch_active(uname):
|
| 1589 |
-
"""
|
| 1590 |
try:
|
| 1591 |
key = (uname or "").strip().lower()
|
| 1592 |
if not key:
|
| 1593 |
return
|
| 1594 |
now = time.time()
|
| 1595 |
-
|
| 1596 |
-
|
| 1597 |
-
|
| 1598 |
-
|
|
|
|
| 1599 |
users.touch_active(key)
|
| 1600 |
except Exception: # noqa: BLE001 — telemetry may never break a request
|
| 1601 |
pass
|
|
@@ -1640,13 +1624,15 @@ def require_session(request: Request, response: Response) -> Session:
|
|
| 1640 |
# `session.tenant` read back from something the request could influence.
|
| 1641 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1642 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1643 |
-
#
|
| 1644 |
-
#
|
| 1645 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1646 |
# touch one through the default tenant-#0 users store.
|
| 1647 |
if (not user.get("qa_runner")
|
| 1648 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1649 |
-
|
|
|
|
|
|
|
| 1650 |
return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
|
| 1651 |
|
| 1652 |
|
|
|
|
| 16 |
import os
|
| 17 |
import sys
|
| 18 |
import time
|
| 19 |
+
import threading
|
| 20 |
from collections import OrderedDict
|
| 21 |
from dataclasses import dataclass
|
| 22 |
+
from datetime import datetime, timezone
|
| 23 |
from pathlib import Path
|
| 24 |
|
| 25 |
from fastapi import Depends, HTTPException, Request, Response
|
|
|
|
| 1431 |
return out
|
| 1432 |
|
| 1433 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1434 |
def _session_registry():
|
| 1435 |
+
"""Read the checked request registry through the store's revision-and-timestamp text cache."""
|
| 1436 |
+
# The same request snapshot serves roster hashing and people helpers through users.registry.
|
| 1437 |
+
return users.registry() or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1438 |
|
| 1439 |
|
| 1440 |
def _user_for(claims):
|
|
|
|
| 1496 |
return None
|
| 1497 |
|
| 1498 |
|
| 1499 |
+
# --- R4 activity: read observations and persisted-write throttling are separate ----------------
|
| 1500 |
+
# GET/HEAD/OPTIONS only observe admitted activity in bounded process memory. They never start a
|
| 1501 |
+
# store stamp, even after restart or window expiry. Admin projections may show these observations;
|
| 1502 |
+
# they disappear on restart and are not flushed. Passive notification polling remains excluded.
|
| 1503 |
+
# Login already persists both stamps. Other admitted mutating requests may launch core.users'
|
| 1504 |
+
# fresh-read background merge once per account/window/process. Reads cannot prime that throttle.
|
| 1505 |
+
# Both maps evict oldest entries at the bound; eviction or restart re-arms the next mutation.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1506 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
| 1507 |
+
_ACTIVE_OBSERVED: "OrderedDict[tuple[str, str], float]" = OrderedDict()
|
| 1508 |
+
_ACTIVE_LOCK = threading.Lock()
|
| 1509 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
| 1510 |
_ACTIVE_MAX_TRACKED = 4096
|
| 1511 |
|
|
|
|
| 1527 |
|
| 1528 |
|
| 1529 |
def note_active(uname, when=None):
|
| 1530 |
+
"""Prime the persisted-write throttle after login's stamp; read observations never call this."""
|
|
|
|
|
|
|
|
|
|
| 1531 |
key = (uname or "").strip().lower()
|
| 1532 |
if not key:
|
| 1533 |
return
|
| 1534 |
+
with _ACTIVE_LOCK:
|
| 1535 |
+
_remember_active(_ACTIVE_SEEN, key, when if when is not None else time.time())
|
| 1536 |
+
|
| 1537 |
+
|
| 1538 |
+
def _remember_active(target, key, when):
|
| 1539 |
+
"""Bound an activity map while its caller holds _ACTIVE_LOCK."""
|
| 1540 |
+
target[key] = when
|
| 1541 |
+
target.move_to_end(key)
|
| 1542 |
+
while len(target) > _ACTIVE_MAX_TRACKED:
|
| 1543 |
+
target.popitem(last=False)
|
| 1544 |
+
|
| 1545 |
+
|
| 1546 |
+
def _observe_active(tenant, uname):
|
| 1547 |
+
"""Observe admitted activity without I/O or changing the persisted-write throttle."""
|
| 1548 |
+
key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower())
|
| 1549 |
+
if all(key):
|
| 1550 |
+
with _ACTIVE_LOCK:
|
| 1551 |
+
_remember_active(_ACTIVE_OBSERVED, key, time.time())
|
| 1552 |
+
|
| 1553 |
+
|
| 1554 |
+
def observed_last_active(tenant, uname, persisted=None):
|
| 1555 |
+
"""Project the newer persisted or process-local observation for this exact tenant/account."""
|
| 1556 |
+
key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower())
|
| 1557 |
+
with _ACTIVE_LOCK:
|
| 1558 |
+
observed = _ACTIVE_OBSERVED.get(key)
|
| 1559 |
+
if observed is None:
|
| 1560 |
+
return persisted
|
| 1561 |
+
if persisted:
|
| 1562 |
+
try:
|
| 1563 |
+
saved = datetime.fromisoformat(str(persisted).replace("Z", "+00:00"))
|
| 1564 |
+
if saved.tzinfo is not None and saved.timestamp() >= observed:
|
| 1565 |
+
return persisted
|
| 1566 |
+
except (ValueError, OverflowError):
|
| 1567 |
+
pass
|
| 1568 |
+
return datetime.fromtimestamp(observed, timezone.utc).isoformat(timespec="seconds")
|
| 1569 |
|
| 1570 |
|
| 1571 |
def _touch_active(uname):
|
| 1572 |
+
"""Throttle an admitted mutation's persisted stamp per account/process. Never raises."""
|
| 1573 |
try:
|
| 1574 |
key = (uname or "").strip().lower()
|
| 1575 |
if not key:
|
| 1576 |
return
|
| 1577 |
now = time.time()
|
| 1578 |
+
with _ACTIVE_LOCK:
|
| 1579 |
+
last = _ACTIVE_SEEN.get(key)
|
| 1580 |
+
if last is not None and (now - last) < _ACTIVE_EVERY:
|
| 1581 |
+
return
|
| 1582 |
+
_remember_active(_ACTIVE_SEEN, key, now)
|
| 1583 |
users.touch_active(key)
|
| 1584 |
except Exception: # noqa: BLE001 — telemetry may never break a request
|
| 1585 |
pass
|
|
|
|
| 1624 |
# `session.tenant` read back from something the request could influence.
|
| 1625 |
response.headers[_TENANT_HEADER] = str(claims["t"])
|
| 1626 |
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
|
| 1627 |
+
# Observe only AFTER authentication admission. Read methods stay store-read-only; only an
|
| 1628 |
+
# admitted mutation can launch the persisted stamp, without moving it outside request timing.
|
| 1629 |
# qa-runner has no global account record. Never let the normal activity telemetry create or
|
| 1630 |
# touch one through the default tenant-#0 users store.
|
| 1631 |
if (not user.get("qa_runner")
|
| 1632 |
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
|
| 1633 |
+
_observe_active(claims["t"], user.get("username"))
|
| 1634 |
+
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
| 1635 |
+
_touch_active(user.get("username"))
|
| 1636 |
return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
|
| 1637 |
|
| 1638 |
|
api/main.py
CHANGED
|
@@ -168,6 +168,10 @@ def _error_shape(request: Request, exc: StarletteHTTPException):
|
|
| 168 |
422 from a malformed path param) is wrapped here so a client never has to branch on two
|
| 169 |
different error bodies.
|
| 170 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
detail = exc.detail
|
| 172 |
if isinstance(detail, dict) and "error" in detail:
|
| 173 |
body = detail
|
|
@@ -192,12 +196,51 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
|
| 192 |
adapter, which has no durable session dict to degrade into. A 200 over a write that
|
| 193 |
evaporated is the failure the whole rule exists to prevent.
|
| 194 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
return JSONResponse(
|
| 196 |
{"error": {"code": "store_unavailable",
|
| 197 |
"message": "the tenant store is unavailable. No change was saved"}},
|
| 198 |
status_code=503)
|
| 199 |
|
| 200 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
@app.exception_handler(freeze_notice.DataInputFrozen)
|
| 202 |
def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
|
| 203 |
"""Use the existing readable error envelope for every maintenance refusal."""
|
|
@@ -315,7 +358,7 @@ def public_freeze_notice():
|
|
| 315 |
|
| 316 |
|
| 317 |
@app.get("/api/health")
|
| 318 |
-
def health(challenge: str = ""):
|
| 319 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
| 320 |
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 321 |
no config: a health endpoint is the one URL every scanner finds first.
|
|
|
|
| 168 |
422 from a malformed path param) is wrapped here so a client never has to branch on two
|
| 169 |
different error bodies.
|
| 170 |
"""
|
| 171 |
+
from request_timing import resource_response
|
| 172 |
+
busy = resource_response(exc)
|
| 173 |
+
if busy is not None:
|
| 174 |
+
return busy
|
| 175 |
detail = exc.detail
|
| 176 |
if isinstance(detail, dict) and "error" in detail:
|
| 177 |
body = detail
|
|
|
|
| 196 |
adapter, which has no durable session dict to degrade into. A 200 over a write that
|
| 197 |
evaporated is the failure the whole rule exists to prevent.
|
| 198 |
"""
|
| 199 |
+
from request_timing import resource_response
|
| 200 |
+
busy = resource_response(exc)
|
| 201 |
+
if busy is not None:
|
| 202 |
+
return busy
|
| 203 |
return JSONResponse(
|
| 204 |
{"error": {"code": "store_unavailable",
|
| 205 |
"message": "the tenant store is unavailable. No change was saved"}},
|
| 206 |
status_code=503)
|
| 207 |
|
| 208 |
|
| 209 |
+
async def _resource_unavailable(request: Request, exc: Exception):
|
| 210 |
+
from request_timing import resource_response
|
| 211 |
+
return resource_response(exc)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
@app.exception_handler(Exception)
|
| 215 |
+
async def _unexpected_error(request: Request, exc: Exception):
|
| 216 |
+
from request_timing import resource_response
|
| 217 |
+
busy = resource_response(exc)
|
| 218 |
+
if busy is not None:
|
| 219 |
+
return busy
|
| 220 |
+
return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
|
| 221 |
+
status_code=500)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
from core.odoo import OdooPoolTimeout
|
| 225 |
+
from core.store_pg import SharedReadTimeout
|
| 226 |
+
from core.request_timing import RequestDeadlineExceeded
|
| 227 |
+
from request_timing import DeadlineCapacityExceeded
|
| 228 |
+
from duckdb import OutOfMemoryException
|
| 229 |
+
for _resource_error in (OdooPoolTimeout, SharedReadTimeout, RequestDeadlineExceeded,
|
| 230 |
+
DeadlineCapacityExceeded, OutOfMemoryException):
|
| 231 |
+
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 232 |
+
try:
|
| 233 |
+
from psycopg_pool import PoolTimeout, TooManyRequests
|
| 234 |
+
from psycopg import OperationalError, InterfaceError
|
| 235 |
+
from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
|
| 236 |
+
except ImportError:
|
| 237 |
+
pass
|
| 238 |
+
else:
|
| 239 |
+
for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
|
| 240 |
+
OperationalError, InterfaceError, IdleInTransactionSessionTimeout):
|
| 241 |
+
app.add_exception_handler(_resource_error, _resource_unavailable)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
@app.exception_handler(freeze_notice.DataInputFrozen)
|
| 245 |
def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
|
| 246 |
"""Use the existing readable error envelope for every maintenance refusal."""
|
|
|
|
| 358 |
|
| 359 |
|
| 360 |
@app.get("/api/health")
|
| 361 |
+
async def health(challenge: str = ""):
|
| 362 |
"""Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
|
| 363 |
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 364 |
no config: a health endpoint is the one URL every scanner finds first.
|
api/odoo_relational.py
CHANGED
|
@@ -3534,17 +3534,35 @@ def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False
|
|
| 3534 |
f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}'
|
| 3535 |
for k in keys)
|
| 3536 |
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})')
|
| 3537 |
-
|
| 3538 |
-
|
| 3539 |
-
|
| 3540 |
-
|
| 3541 |
-
|
| 3542 |
-
|
| 3543 |
-
|
| 3544 |
-
|
| 3545 |
-
|
| 3546 |
-
|
| 3547 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3548 |
return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
|
| 3549 |
|
| 3550 |
|
|
|
|
| 3534 |
f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}'
|
| 3535 |
for k in keys)
|
| 3536 |
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})')
|
| 3537 |
+
import itertools
|
| 3538 |
+
source = iter(rows if rows is not None else ())
|
| 3539 |
+
try:
|
| 3540 |
+
while True:
|
| 3541 |
+
batch = list(itertools.islice(source, 2048))
|
| 3542 |
+
if not batch:
|
| 3543 |
+
break
|
| 3544 |
+
values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
|
| 3545 |
+
for r in batch if isinstance(r, dict)]
|
| 3546 |
+
# Serialized values no longer need the dictionaries held by this batch.
|
| 3547 |
+
batch.clear()
|
| 3548 |
+
del batch
|
| 3549 |
+
if not values:
|
| 3550 |
+
del values
|
| 3551 |
+
continue
|
| 3552 |
+
if bulk:
|
| 3553 |
+
# Every insertion holds at most 2,048 rows, including column-array conversion.
|
| 3554 |
+
selects = ", ".join(
|
| 3555 |
+
f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
|
| 3556 |
+
cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
|
| 3557 |
+
else:
|
| 3558 |
+
placeholders = ", ".join(["?"] * len(keys))
|
| 3559 |
+
cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
|
| 3560 |
+
values.clear()
|
| 3561 |
+
del values
|
| 3562 |
+
finally:
|
| 3563 |
+
close = getattr(source, "close", None)
|
| 3564 |
+
if callable(close):
|
| 3565 |
+
close()
|
| 3566 |
return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
|
| 3567 |
|
| 3568 |
|
api/process_role.py
CHANGED
|
@@ -46,10 +46,8 @@ os.environ["AIOS_PROCESS_ROLE"] = ROLE
|
|
| 46 |
if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
|
| 47 |
print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
|
| 48 |
if ROLE == "web":
|
| 49 |
-
|
| 50 |
-
os.environ.setdefault("
|
| 51 |
-
if os.environ.get("AIOS_WEB_DUCKDB_THREADS"):
|
| 52 |
-
os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ["AIOS_WEB_DUCKDB_THREADS"])
|
| 53 |
|
| 54 |
_LOCK = threading.Lock()
|
| 55 |
_STARTED = {}
|
|
|
|
| 46 |
if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
|
| 47 |
print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
|
| 48 |
if ROLE == "web":
|
| 49 |
+
os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
|
| 50 |
+
os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
|
|
|
|
|
|
|
| 51 |
|
| 52 |
_LOCK = threading.Lock()
|
| 53 |
_STARTED = {}
|
api/registry_source.py
ADDED
|
@@ -0,0 +1,765 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bounded stored-source reads before canonical grid projection."""
|
| 2 |
+
import copy
|
| 3 |
+
from contextlib import closing
|
| 4 |
+
from dataclasses import is_dataclass, replace
|
| 5 |
+
|
| 6 |
+
from deps import err
|
| 7 |
+
|
| 8 |
+
BATCH_ROWS = 1024
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def with_runtime(session, runtime):
|
| 12 |
+
"""Retain ordinary lightweight host sessions beside the API dataclass."""
|
| 13 |
+
if is_dataclass(session):
|
| 14 |
+
return replace(session, runtime=runtime)
|
| 15 |
+
actor = copy.copy(session)
|
| 16 |
+
actor.runtime = runtime
|
| 17 |
+
return actor
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def backend(runtime):
|
| 21 |
+
"""Resolve the admitted runtime's physical store through its tenant guard."""
|
| 22 |
+
while hasattr(runtime, '_runtime'):
|
| 23 |
+
runtime = runtime._runtime
|
| 24 |
+
document_store = getattr(runtime, '_document_store', None)
|
| 25 |
+
if callable(document_store):
|
| 26 |
+
return document_store()
|
| 27 |
+
reader = runtime._store() if callable(getattr(runtime, '_store', None)) else runtime
|
| 28 |
+
from core import store
|
| 29 |
+
return store._d() if reader is store else reader
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class MetadataStore:
|
| 33 |
+
"""Keep per-record enrichment states outside the definition vocabulary."""
|
| 34 |
+
schema_only = True
|
| 35 |
+
|
| 36 |
+
def __init__(self, runtime):
|
| 37 |
+
self._runtime = runtime
|
| 38 |
+
|
| 39 |
+
def __getattr__(self, name):
|
| 40 |
+
return getattr(self._runtime, name)
|
| 41 |
+
|
| 42 |
+
def get_projection(self, name, drop=()):
|
| 43 |
+
if name == 'user_tables':
|
| 44 |
+
drop = tuple(dict.fromkeys([*drop, 'aiEnrich']))
|
| 45 |
+
return self._runtime.get_projection(name, drop=drop)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def paths(runtime, name, selected):
|
| 49 |
+
"""Read explicit values through the paged store door, or a borrowed offline document."""
|
| 50 |
+
if not selected:
|
| 51 |
+
return []
|
| 52 |
+
reader = backend(runtime)
|
| 53 |
+
method = getattr(reader, 'get_json_paths', None)
|
| 54 |
+
if callable(method):
|
| 55 |
+
return method(getattr(runtime, 'store_key', str)(name), selected)
|
| 56 |
+
doc = runtime.get(name) or {}
|
| 57 |
+
out = []
|
| 58 |
+
for path in selected:
|
| 59 |
+
value = doc
|
| 60 |
+
for part in path:
|
| 61 |
+
value = value.get(str(part)) if isinstance(value, dict) else None
|
| 62 |
+
out.append(copy.deepcopy(value))
|
| 63 |
+
return out
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def items(runtime, name, path=(), *, selected=None):
|
| 67 |
+
"""Yield bounded batches through a streaming store door or the legacy offline reader."""
|
| 68 |
+
reader = backend(runtime)
|
| 69 |
+
if selected is not None:
|
| 70 |
+
selected = iter(selected)
|
| 71 |
+
import itertools
|
| 72 |
+
while True:
|
| 73 |
+
keys = list(itertools.islice(selected, BATCH_ROWS))
|
| 74 |
+
if not keys:
|
| 75 |
+
return
|
| 76 |
+
values = paths(runtime, name, [[*path, str(key)] for key in keys])
|
| 77 |
+
yield [(str(key), value) for key, value in zip(keys, values) if value is not None]
|
| 78 |
+
else:
|
| 79 |
+
stream = getattr(reader, 'iter_json_items', None)
|
| 80 |
+
if callable(stream):
|
| 81 |
+
try:
|
| 82 |
+
yield from stream(getattr(runtime, 'store_key', str)(name), path, batch_size=BATCH_ROWS)
|
| 83 |
+
except RuntimeError as exc:
|
| 84 |
+
from core.request_timing import raise_if_resource_error
|
| 85 |
+
raise_if_resource_error(exc)
|
| 86 |
+
code = str(exc).split(':', 1)[0]
|
| 87 |
+
if code in {'canonical_source_unavailable', 'canonical_source_invalid'}:
|
| 88 |
+
raise err(503, code, str(exc).partition(':')[2].strip()) from exc
|
| 89 |
+
raise
|
| 90 |
+
return
|
| 91 |
+
else:
|
| 92 |
+
# Offline compatibility retains its existing whole-document read.
|
| 93 |
+
import itertools
|
| 94 |
+
doc = runtime.get(name) or {}
|
| 95 |
+
for part in path:
|
| 96 |
+
doc = doc.get(str(part)) if isinstance(doc, dict) else None
|
| 97 |
+
if doc is None:
|
| 98 |
+
return
|
| 99 |
+
if not isinstance(doc, dict):
|
| 100 |
+
raise err(409, 'canonical_source_invalid', f'{name} does not contain a row object.')
|
| 101 |
+
source = iter(doc.items())
|
| 102 |
+
while True:
|
| 103 |
+
batch = list(itertools.islice(source, BATCH_ROWS))
|
| 104 |
+
if not batch:
|
| 105 |
+
return
|
| 106 |
+
yield batch
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def array_values(runtime, name, path=(), *, drop=()):
|
| 110 |
+
"""Stream array entries through the bound store with an ordinary offline fallback."""
|
| 111 |
+
reader = backend(runtime)
|
| 112 |
+
method = getattr(reader, 'iter_json_values', None)
|
| 113 |
+
if callable(method):
|
| 114 |
+
yield from method(getattr(runtime, 'store_key', str)(name), path, drop=drop, batch_size=BATCH_ROWS)
|
| 115 |
+
return
|
| 116 |
+
value = runtime.get(name) or {}
|
| 117 |
+
for part in path:
|
| 118 |
+
value = value[int(part)] if isinstance(value, list) else value.get(str(part), {})
|
| 119 |
+
import itertools
|
| 120 |
+
source = enumerate(value if isinstance(value, list) else [])
|
| 121 |
+
while True:
|
| 122 |
+
batch = list(itertools.islice(source, BATCH_ROWS))
|
| 123 |
+
if not batch:
|
| 124 |
+
return
|
| 125 |
+
yield [(str(index), {k: v for k, v in entry.items() if k not in drop}
|
| 126 |
+
if isinstance(entry, dict) else entry) for index, entry in batch]
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def object_values(runtime, name, path=(), *, drop=(), omit=()):
|
| 130 |
+
"""Read nested definitions with record operands excluded at the backing store."""
|
| 131 |
+
reader = backend(runtime)
|
| 132 |
+
method = getattr(reader, 'get_json_object', None)
|
| 133 |
+
if callable(method):
|
| 134 |
+
return method(getattr(runtime, 'store_key', str)(name), path, drop=drop, omit=omit)
|
| 135 |
+
value = runtime.get(name) or {}
|
| 136 |
+
for part in path:
|
| 137 |
+
value = value.get(str(part), {}) if isinstance(value, dict) else {}
|
| 138 |
+
return {key: ({k: v for k, v in entry.items() if k not in drop} if isinstance(entry, dict) else entry)
|
| 139 |
+
for key, entry in value.items() if key not in omit} if isinstance(value, dict) else {}
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def members(runtime, name, path, ids, *, id_key=None, source_limit=None):
|
| 143 |
+
"""Read complete selected membership through the bound store's SQL predicate."""
|
| 144 |
+
reader = backend(runtime)
|
| 145 |
+
method = getattr(reader, 'get_json_members', None)
|
| 146 |
+
if callable(method):
|
| 147 |
+
return method(getattr(runtime, 'store_key', str)(name), path, ids,
|
| 148 |
+
id_key=id_key, source_limit=source_limit)
|
| 149 |
+
value = runtime.get(name) or {}
|
| 150 |
+
for part in path:
|
| 151 |
+
value = value[int(part)] if isinstance(value, list) else value.get(str(part), {})
|
| 152 |
+
selected, seen, out = {int(pid) for pid in ids}, set(), []
|
| 153 |
+
for index, item in enumerate(value if isinstance(value, list) else []):
|
| 154 |
+
if source_limit is not None and index >= source_limit:
|
| 155 |
+
break
|
| 156 |
+
raw = item.get(id_key) if id_key is not None and isinstance(item, dict) else item
|
| 157 |
+
try:
|
| 158 |
+
pid = int(raw)
|
| 159 |
+
except (TypeError, ValueError):
|
| 160 |
+
continue
|
| 161 |
+
if pid in selected and pid not in seen:
|
| 162 |
+
out.append(item)
|
| 163 |
+
seen.add(pid)
|
| 164 |
+
return out
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def generation(session, table_key):
|
| 168 |
+
"""Check population and projection buckets without reading their record values."""
|
| 169 |
+
from core import schema_overlay, shares, store_pg, user_tables
|
| 170 |
+
runtime, reader = session.runtime, backend(session.runtime)
|
| 171 |
+
names = [user_tables.STORE_KEY, table_key + '_table_workspace', table_key + '__shared',
|
| 172 |
+
table_key + '_docs', user_tables.ZONE_STORE_KEY, schema_overlay.STORE_KEY,
|
| 173 |
+
shares.SHARES_KEY, table_key + '_cohorts']
|
| 174 |
+
if table_key in {'customer_data', 'product_data'}:
|
| 175 |
+
workspace = 'customer_table_workspace' if table_key == 'customer_data' else 'product_table_workspace'
|
| 176 |
+
names += [workspace, workspace + '__shared', 'customer_ledger', 'ar_days_to_pay',
|
| 177 |
+
'geocoords', 'loopable_route_system', 'customer_docs', 'product_docs',
|
| 178 |
+
'customer_cohorts', 'product_cohorts']
|
| 179 |
+
if isinstance(reader, store_pg.PgStore):
|
| 180 |
+
key = getattr(runtime, 'store_key', str)
|
| 181 |
+
# A post-read check must cross the backing validation boundary, including updated_at.
|
| 182 |
+
tokens = store_pg._read_revisions([key(name) for name in names], reader.tenant_slug)
|
| 183 |
+
else:
|
| 184 |
+
revision = getattr(runtime, 'revision', None)
|
| 185 |
+
if not callable(revision):
|
| 186 |
+
return None # Ordinary offline readers retain their original snapshot semantics.
|
| 187 |
+
tokens = {name: revision(name) for name in names}
|
| 188 |
+
return copy.deepcopy((session.user, tokens))
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def require_generation(session, table_key, before):
|
| 192 |
+
if before is not None and generation(session, table_key) != before:
|
| 193 |
+
raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class BatchStore:
|
| 197 |
+
"""Lend canonical readers one row batch and its overlays, with every write refused."""
|
| 198 |
+
|
| 199 |
+
def __init__(self, runtime, table_key, definition, rows, *, metadata=None, uname=None, needed=None):
|
| 200 |
+
self._runtime, self.table_key = runtime, table_key
|
| 201 |
+
self.definition = dict(definition, rows=dict(rows))
|
| 202 |
+
self.row_ids = [str(key) for key, _ in rows]
|
| 203 |
+
self.docs = {}
|
| 204 |
+
self.metadata = metadata if metadata is not None else {}
|
| 205 |
+
self.uname = uname
|
| 206 |
+
self._canonical_needed = needed
|
| 207 |
+
self.measure_memo, self.mset_memo, self.pool_cache = {}, {}, {}
|
| 208 |
+
from core import user_tables
|
| 209 |
+
for field in definition.get('fields') or []:
|
| 210 |
+
if field.get('type') != 'ai_enrich':
|
| 211 |
+
continue
|
| 212 |
+
col = str(field['key'])
|
| 213 |
+
if needed is not None and col not in needed:
|
| 214 |
+
continue
|
| 215 |
+
values = paths(runtime, user_tables.STORE_KEY,
|
| 216 |
+
[[table_key, user_tables.AI_ENRICH_MARK_KEY, col, pid] for pid in self.row_ids])
|
| 217 |
+
self.definition.setdefault(user_tables.AI_ENRICH_MARK_KEY, {})[col] = {
|
| 218 |
+
pid: mark for pid, mark in zip(self.row_ids, values) if isinstance(mark, dict)}
|
| 219 |
+
|
| 220 |
+
def __getattr__(self, name):
|
| 221 |
+
return getattr(self._runtime, name)
|
| 222 |
+
|
| 223 |
+
def selected_values(self, name, path, ids):
|
| 224 |
+
ids = [str(pid) for pid in ids]
|
| 225 |
+
return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
|
| 226 |
+
[[*path, pid] for pid in ids])) if value is not None}
|
| 227 |
+
|
| 228 |
+
def update(self, *args, **kwargs):
|
| 229 |
+
raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
|
| 230 |
+
|
| 231 |
+
put = upload_bytes = delete_path = update
|
| 232 |
+
|
| 233 |
+
def _workspace(self, name):
|
| 234 |
+
reader = backend(self._runtime)
|
| 235 |
+
project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
|
| 236 |
+
if name not in self.metadata:
|
| 237 |
+
self.metadata[name] = (project(getattr(self._runtime, 'store_key', str)(name))
|
| 238 |
+
if callable(project) else self._runtime.get_projection(name, drop=('overlays',))) or {}
|
| 239 |
+
meta = copy.deepcopy(self.metadata[name])
|
| 240 |
+
reader = backend(self._runtime)
|
| 241 |
+
stream = getattr(reader, 'iter_view_members', None) or getattr(reader, '_iter_view_members', None)
|
| 242 |
+
selected = set(map(int, self.row_ids))
|
| 243 |
+
def invalid_membership():
|
| 244 |
+
return err(503, 'canonical_view_membership_invalid',
|
| 245 |
+
'A saved view contains membership entries but no valid record IDs. '
|
| 246 |
+
'Repair its membership before reading this database.')
|
| 247 |
+
for owner, ws in meta.items():
|
| 248 |
+
if not isinstance(ws, dict) or owner.startswith('_'):
|
| 249 |
+
continue
|
| 250 |
+
for vid, view in (ws.get('views') or {}).items():
|
| 251 |
+
cfg = view.get('config') if isinstance(view, dict) else None
|
| 252 |
+
if not isinstance(cfg, dict):
|
| 253 |
+
continue
|
| 254 |
+
if 'memberPids' in cfg:
|
| 255 |
+
authored = cfg.get('memberPids') or ()
|
| 256 |
+
if authored:
|
| 257 |
+
for raw in authored:
|
| 258 |
+
try:
|
| 259 |
+
int(raw)
|
| 260 |
+
except (TypeError, ValueError):
|
| 261 |
+
continue
|
| 262 |
+
break
|
| 263 |
+
else:
|
| 264 |
+
raise invalid_membership()
|
| 265 |
+
continue
|
| 266 |
+
if not callable(stream):
|
| 267 |
+
if callable(project):
|
| 268 |
+
raise err(503, 'canonical_source_unavailable', 'Saved view membership cannot be read safely.')
|
| 269 |
+
continue # An ordinary offline document did not strip memberships.
|
| 270 |
+
first = [None]
|
| 271 |
+
saw_any = [False]
|
| 272 |
+
def values():
|
| 273 |
+
parts = stream(getattr(self._runtime, 'store_key', str)(name),
|
| 274 |
+
[owner, 'views', vid, 'config', 'memberPids'],
|
| 275 |
+
getattr(self.metadata[name], 'source_revision', None), batch_size=BATCH_ROWS)
|
| 276 |
+
try:
|
| 277 |
+
with closing(parts):
|
| 278 |
+
for batch in parts:
|
| 279 |
+
for _, raw in batch:
|
| 280 |
+
saw_any[0] = True
|
| 281 |
+
try:
|
| 282 |
+
pid = int(raw)
|
| 283 |
+
except (TypeError, ValueError):
|
| 284 |
+
continue
|
| 285 |
+
if first[0] is None:
|
| 286 |
+
first[0] = pid
|
| 287 |
+
yield pid
|
| 288 |
+
except RuntimeError as exc:
|
| 289 |
+
from core.request_timing import raise_if_resource_error
|
| 290 |
+
raise_if_resource_error(exc)
|
| 291 |
+
raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc
|
| 292 |
+
cur = getattr(self, '_canonical_source_cursor', None)
|
| 293 |
+
if cur is not None and selected:
|
| 294 |
+
from routes_odoo_tables import _membership_relation
|
| 295 |
+
cache_key = ('view_members', name, owner, vid)
|
| 296 |
+
if cache_key not in self.metadata:
|
| 297 |
+
operand = _membership_relation(cur, values())
|
| 298 |
+
self.metadata[cache_key] = (operand, first[0], saw_any[0])
|
| 299 |
+
operand, first[0], saw_any[0] = self.metadata[cache_key]
|
| 300 |
+
if saw_any[0] and first[0] is None:
|
| 301 |
+
raise invalid_membership()
|
| 302 |
+
kept = [int(row[0]) for row in cur.execute(
|
| 303 |
+
f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) '
|
| 304 |
+
f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid',
|
| 305 |
+
[*operand.params, *selected]).fetchall()]
|
| 306 |
+
else:
|
| 307 |
+
# Metadata consumes no row memberships; retain only the curated witness.
|
| 308 |
+
with closing(values()) as source:
|
| 309 |
+
next(source, None)
|
| 310 |
+
kept = []
|
| 311 |
+
if saw_any[0] and first[0] is None:
|
| 312 |
+
# Config sanitization drops invalid entries; [] would mean all rows.
|
| 313 |
+
# Refuse instead of inventing an ID to stand for empty curated intent.
|
| 314 |
+
raise invalid_membership()
|
| 315 |
+
# Retain an actual authored witness when this batch has no member. An empty
|
| 316 |
+
# list would turn a curated view into all rows. Full SQL view resolution uses
|
| 317 |
+
# the original revision-bound backend, never this batch-local projection.
|
| 318 |
+
cfg['memberPids'] = kept if kept or first[0] is None else [first[0]]
|
| 319 |
+
for owner, ws in meta.items():
|
| 320 |
+
if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
|
| 321 |
+
continue
|
| 322 |
+
values = paths(self._runtime, name, [[owner, 'overlays', pid] for pid in self.row_ids])
|
| 323 |
+
# Only this owner's selected cells have been read, so admit a new bounded document.
|
| 324 |
+
meta[owner] = dict(ws, overlays={pid: value for pid, value in zip(self.row_ids, values)
|
| 325 |
+
if isinstance(value, dict)})
|
| 326 |
+
return meta
|
| 327 |
+
|
| 328 |
+
def _definitions(self):
|
| 329 |
+
from core import user_tables
|
| 330 |
+
if user_tables.STORE_KEY not in self.metadata:
|
| 331 |
+
self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection(
|
| 332 |
+
user_tables.STORE_KEY, drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {}
|
| 333 |
+
return {**self.metadata[user_tables.STORE_KEY], self.table_key: self.definition}
|
| 334 |
+
|
| 335 |
+
def get(self, name, fresh=False):
|
| 336 |
+
from core import user_tables
|
| 337 |
+
if name == user_tables.STORE_KEY:
|
| 338 |
+
return self._definitions()
|
| 339 |
+
if name not in self.docs or fresh:
|
| 340 |
+
if name.endswith('_table_workspace'):
|
| 341 |
+
value = self._workspace(name)
|
| 342 |
+
elif name.endswith('__shared'):
|
| 343 |
+
selected = paths(self._runtime, name, [['fields'],
|
| 344 |
+
*[['cells', pid] for pid in self.row_ids]])
|
| 345 |
+
fields, values = selected[0] or {}, selected[1:]
|
| 346 |
+
value = {'fields': fields, 'cells': {pid: cell for pid, cell in zip(self.row_ids, values)
|
| 347 |
+
if isinstance(cell, dict)}}
|
| 348 |
+
elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}:
|
| 349 |
+
values = (paths(self._runtime, name, [[pid] for pid in self.row_ids])
|
| 350 |
+
if self._canonical_needed is None else [None] * len(self.row_ids))
|
| 351 |
+
value = {pid: docs for pid, docs in zip(self.row_ids, values) if isinstance(docs, list)}
|
| 352 |
+
else:
|
| 353 |
+
if name not in self.metadata or fresh:
|
| 354 |
+
self.metadata[name] = self._runtime.get(name, fresh=fresh) if fresh else self._runtime.get(name)
|
| 355 |
+
value = self.metadata[name]
|
| 356 |
+
self.docs[name] = value
|
| 357 |
+
return copy.deepcopy(self.docs[name])
|
| 358 |
+
|
| 359 |
+
def _read_strict(self, name):
|
| 360 |
+
return self.get(name)
|
| 361 |
+
|
| 362 |
+
def get_entry(self, name, entry):
|
| 363 |
+
from core import user_tables
|
| 364 |
+
if name == user_tables.STORE_KEY:
|
| 365 |
+
return self.definition if str(entry) == self.table_key else self._runtime.get_entry(name, entry)
|
| 366 |
+
return (self.get(name) or {}).get(str(entry))
|
| 367 |
+
|
| 368 |
+
def get_projection(self, name, drop=()):
|
| 369 |
+
if name == 'user_tables':
|
| 370 |
+
from core import store
|
| 371 |
+
return store._project(self._definitions(), frozenset(drop))
|
| 372 |
+
return self._runtime.get_projection(name, drop=drop)
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
class CanonicalStore(BatchStore):
|
| 376 |
+
"""Lend the existing customer/product assemblies one canonical source batch."""
|
| 377 |
+
schema_only = False
|
| 378 |
+
_canonical_read_only = True
|
| 379 |
+
|
| 380 |
+
def __init__(self, session, table_key, rows, *, cur, metadata, needed):
|
| 381 |
+
super().__init__(session.runtime, table_key, {'fields': []},
|
| 382 |
+
[(str(row['pid']), {}) for row in rows], metadata=metadata,
|
| 383 |
+
uname=session.uname, needed=needed)
|
| 384 |
+
self._canonical_source_cursor = cur
|
| 385 |
+
self._canonical_rows = rows if table_key == 'customer_data' else None
|
| 386 |
+
self._canonical_product_rows = rows if table_key == 'product_data' else None
|
| 387 |
+
from core import perm_scope
|
| 388 |
+
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 389 |
+
import time
|
| 390 |
+
key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 391 |
+
self.pool_cache[key] = (time.time(), rows)
|
| 392 |
+
|
| 393 |
+
def get_projection(self, name, drop=()):
|
| 394 |
+
if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
|
| 395 |
+
return self.get(name)
|
| 396 |
+
if name.endswith('_table_workspace') or name.endswith('__shared'):
|
| 397 |
+
from core import store
|
| 398 |
+
return store._project(self.get(name), frozenset(drop))
|
| 399 |
+
return super().get_projection(name, drop=drop)
|
| 400 |
+
|
| 401 |
+
def get(self, name, fresh=False):
|
| 402 |
+
if name in self.docs and not fresh:
|
| 403 |
+
return copy.deepcopy(self.docs[name])
|
| 404 |
+
if name.endswith('_cohorts'):
|
| 405 |
+
from modules import cohort
|
| 406 |
+
definitions = object_values(self._runtime, name, [self.uname], drop=('members',))
|
| 407 |
+
value = {self.uname: {key: dict(meta, members=members(self._runtime, name,
|
| 408 |
+
[self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
|
| 409 |
+
for key, meta in definitions.items() if isinstance(meta, dict)}}
|
| 410 |
+
elif name == 'loopable_route_system':
|
| 411 |
+
value = object_values(self._runtime, name, omit=('routes', 'zones'))
|
| 412 |
+
value['routes'], value['zones'] = [], []
|
| 413 |
+
for batch in array_values(self._runtime, name, ['routes'], drop=('stops',)):
|
| 414 |
+
for index, route in batch:
|
| 415 |
+
if isinstance(route, dict):
|
| 416 |
+
route = dict(route, stops=members(self._runtime, name, ['routes', index, 'stops'],
|
| 417 |
+
self.row_ids, id_key='customerPid'))
|
| 418 |
+
value['routes'].append(route)
|
| 419 |
+
for batch in array_values(self._runtime, name, ['zones'], drop=(
|
| 420 |
+
'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
|
| 421 |
+
value['zones'].extend(meta for _, meta in batch)
|
| 422 |
+
elif name == 'table_zones':
|
| 423 |
+
value = {self.table_key: []}
|
| 424 |
+
for batch in array_values(self._runtime, name, [self.table_key], drop=(
|
| 425 |
+
'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
|
| 426 |
+
value[self.table_key].extend(meta for _, meta in batch)
|
| 427 |
+
else:
|
| 428 |
+
return super().get(name, fresh=fresh)
|
| 429 |
+
self.docs[name] = value
|
| 430 |
+
return copy.deepcopy(value)
|
| 431 |
+
|
| 432 |
+
def _definitions(self):
|
| 433 |
+
from core import user_tables
|
| 434 |
+
if user_tables.STORE_KEY not in self.metadata:
|
| 435 |
+
self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection(user_tables.STORE_KEY,
|
| 436 |
+
drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {}
|
| 437 |
+
return self.metadata[user_tables.STORE_KEY]
|
| 438 |
+
|
| 439 |
+
def get_entry(self, name, entry):
|
| 440 |
+
if name == 'user_tables':
|
| 441 |
+
return self._definitions().get(str(entry))
|
| 442 |
+
return super().get_entry(name, entry)
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def canonical_metadata(session, table_key):
|
| 446 |
+
"""Read the canonical permitted definitions without constructing source rows."""
|
| 447 |
+
import routes_customers as customers
|
| 448 |
+
import routes_products as products
|
| 449 |
+
session.require(table_key)
|
| 450 |
+
lent = CanonicalStore(session, table_key, [], cur=None, metadata={}, needed=frozenset())
|
| 451 |
+
actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
|
| 452 |
+
build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
|
| 453 |
+
return build(actor, scope='product' if table_key == 'product_data' else 'customer',
|
| 454 |
+
consume_corrections=False, with_rows=False, needed=frozenset())
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 458 |
+
"""Refuse a failed compiled source by name without publishing an empty population."""
|
| 459 |
+
from fastapi import HTTPException
|
| 460 |
+
from core.request_timing import raise_if_resource_error
|
| 461 |
+
try:
|
| 462 |
+
with closing(_canonical_batches(session, cur, table_key, ids=ids, needed=needed,
|
| 463 |
+
metric_window=metric_window, related=related)) as batches:
|
| 464 |
+
yield from batches
|
| 465 |
+
except HTTPException:
|
| 466 |
+
raise
|
| 467 |
+
except Exception as exc:
|
| 468 |
+
raise_if_resource_error(exc)
|
| 469 |
+
raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
|
| 473 |
+
"""Run canonical family, permission, overlay and measure projections over bounded identities."""
|
| 474 |
+
import aios_grid
|
| 475 |
+
import routes_customers as customers
|
| 476 |
+
import routes_products as products
|
| 477 |
+
import routes_sheet as sheet
|
| 478 |
+
from core import perm_scope, user_tables, route_system
|
| 479 |
+
from modules import customer_data, product_data
|
| 480 |
+
if getattr(session.runtime, 'key', None) != 'royal-imports':
|
| 481 |
+
raise err(503, 'canonical_source_unavailable', f'{table_key} has no compiled source for this tenant.')
|
| 482 |
+
session.require(table_key)
|
| 483 |
+
team, agent = perm_scope.derive_pool_scope(session.user, table_key)
|
| 484 |
+
before = generation(session, table_key)
|
| 485 |
+
metadata = {}
|
| 486 |
+
definition = canonical_metadata(session, table_key)
|
| 487 |
+
# Every permanent wall operand is populated before apply_row_scope judges the batch.
|
| 488 |
+
tree = perm_scope.entry(session.user, table_key) or {}
|
| 489 |
+
wall = tree.get('filter') or {}
|
| 490 |
+
def keys(node):
|
| 491 |
+
if isinstance(node, dict):
|
| 492 |
+
if node.get('colId'):
|
| 493 |
+
yield str(node['colId'])
|
| 494 |
+
for group in ('nodes', 'children', 'filters'):
|
| 495 |
+
for child in node.get(group) or []:
|
| 496 |
+
yield from keys(child)
|
| 497 |
+
required = None if needed is None else set(needed) | set(keys(wall))
|
| 498 |
+
if required is not None:
|
| 499 |
+
from core.if_field import references
|
| 500 |
+
vocabulary = {f['key']: f for f in [*definition.get('fields', []),
|
| 501 |
+
*(definition.get('_permission_scope') or {}).get('fields', [])] if isinstance(f, dict) and f.get('key')}
|
| 502 |
+
changed = True
|
| 503 |
+
while changed:
|
| 504 |
+
previous = set(required)
|
| 505 |
+
for key in list(required):
|
| 506 |
+
field = vocabulary.get(key) or {}
|
| 507 |
+
required.update(match.strip() for match in aios_grid._FORMULA_REF.findall(str(field.get('formula') or '')))
|
| 508 |
+
if field.get('type') == 'if':
|
| 509 |
+
required.update(references(field.get('if') or {}))
|
| 510 |
+
if field.get('type') == 'created_time':
|
| 511 |
+
required.add('_created')
|
| 512 |
+
changed = previous != required
|
| 513 |
+
# Zone and Route projections consume location and identity operands before display projection.
|
| 514 |
+
if required is not None and {'zone', 'routes'} & required:
|
| 515 |
+
required.update({'lat', 'lon'})
|
| 516 |
+
runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
|
| 517 |
+
all_keys = {f['key'] for f in definition['fields']} if required is None else required
|
| 518 |
+
raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
|
| 519 |
+
if required is not None:
|
| 520 |
+
raw = [field for field in raw if field.get('key') in required]
|
| 521 |
+
import routes_keychain
|
| 522 |
+
import time
|
| 523 |
+
paused = routes_keychain.odoo_paused(session.runtime)
|
| 524 |
+
pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 525 |
+
cached = session.runtime.pool_cache.get(pool_key)
|
| 526 |
+
if cached and (paused or time.time() - cached[0] < 900):
|
| 527 |
+
import itertools
|
| 528 |
+
def borrowed():
|
| 529 |
+
source = (r for r in cached[1] if ids is None or r['pid'] in ids)
|
| 530 |
+
while True:
|
| 531 |
+
batch = list(itertools.islice(source, BATCH_ROWS))
|
| 532 |
+
if not batch:
|
| 533 |
+
return
|
| 534 |
+
yield batch
|
| 535 |
+
batches = borrowed()
|
| 536 |
+
elif paused:
|
| 537 |
+
import itertools
|
| 538 |
+
key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
|
| 539 |
+
hit = session.runtime.pool_cache.get(key)
|
| 540 |
+
if hit:
|
| 541 |
+
def frozen():
|
| 542 |
+
source = (r for r in hit[1] if ids is None or r['pid'] in ids)
|
| 543 |
+
while True:
|
| 544 |
+
batch = list(itertools.islice(source, BATCH_ROWS))
|
| 545 |
+
if not batch:
|
| 546 |
+
return
|
| 547 |
+
yield batch
|
| 548 |
+
batches = frozen()
|
| 549 |
+
elif table_key == 'customer_data':
|
| 550 |
+
path = ['odoo_pool', routes_keychain._snap_scope_key(team, agent)]
|
| 551 |
+
stamp = paths(session.runtime, routes_keychain.SNAPSHOT_KEY, [[*path, 'ts']])[0]
|
| 552 |
+
if stamp is None:
|
| 553 |
+
raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
|
| 554 |
+
def frozen():
|
| 555 |
+
for batch in array_values(session.runtime, routes_keychain.SNAPSHOT_KEY, [*path, 'rows']):
|
| 556 |
+
yield [r for _, r in batch if isinstance(r, dict) and (ids is None or r.get('pid') in ids)]
|
| 557 |
+
batches = frozen()
|
| 558 |
+
else:
|
| 559 |
+
raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
|
| 560 |
+
elif table_key == 'product_data':
|
| 561 |
+
batches = product_data.source_batches(cur, st=runtime, team_id=team, ids=ids, needed=required,
|
| 562 |
+
raw_fields=[field['rawName'] for field in raw])
|
| 563 |
+
else:
|
| 564 |
+
batches = customer_data.source_batches(cur, st=runtime, agent_name=agent, team_id=team,
|
| 565 |
+
ids=ids, needed=required, ledger_batches=items(session.runtime, customer_data.LEDGER_KEY,
|
| 566 |
+
selected=ids) if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ())
|
| 567 |
+
with closing(batches):
|
| 568 |
+
for rows in batches:
|
| 569 |
+
runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required)
|
| 570 |
+
actor = with_runtime(session, runtime)
|
| 571 |
+
build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
|
| 572 |
+
g = build(actor, scope='product' if table_key == 'product_data' else 'customer',
|
| 573 |
+
consume_corrections=False, with_rows=True, needed=all_keys)
|
| 574 |
+
kept = g['fields'] if needed is None else [f for f in g['fields'] if f['key'] in needed]
|
| 575 |
+
derived, bounded, limits = sheet._metric_derived(actor, table_key,
|
| 576 |
+
dict(g, fields=kept), metric_window)
|
| 577 |
+
if table_key == 'product_data':
|
| 578 |
+
projected = products.product_rows(dict(g, fields=kept, derived=derived)) if kept else [
|
| 579 |
+
{'pid': r['pid']} for r in g['rows_src']]
|
| 580 |
+
projected = user_tables.zone_cells(table_key, projected, allowed_row_ids=g['pids'], st=runtime)
|
| 581 |
+
else:
|
| 582 |
+
projected = aios_grid.rows_from_pool(g['rows_src'], kept, g['ws'].get('overlays'), derived=derived) \
|
| 583 |
+
if kept else [{'pid': r['pid']} for r in g['rows_src']]
|
| 584 |
+
projected = route_system.apply_customer_projections(projected, g['routes'], g['zones'])
|
| 585 |
+
if related:
|
| 586 |
+
from routes_odoo_tables import narrow_related_rows
|
| 587 |
+
projected = narrow_related_rows(actor, table_key, projected, related)
|
| 588 |
+
from routes_tables import scope_linked_metrics
|
| 589 |
+
projected, _fields, _refusals = scope_linked_metrics(actor, table_key, projected,
|
| 590 |
+
kept, g['measures'], related)
|
| 591 |
+
from core import grid_events
|
| 592 |
+
docs = grid_events.docs_for(g['pids'], scope_key='customer', uname=session.uname,
|
| 593 |
+
admin=session.admin, st=runtime) if table_key == 'customer_data' and ids is not None and needed is None else {}
|
| 594 |
+
yield {'rows': projected, 'fields': g['fields'], 'today': g['today'], 'docs': docs,
|
| 595 |
+
'bounded': bounded, 'unbounded': limits,
|
| 596 |
+
'limits': list(g['workspace'].get('limits') or [])}
|
| 597 |
+
projected.clear()
|
| 598 |
+
runtime.pool_cache.clear()
|
| 599 |
+
runtime.measure_memo.clear()
|
| 600 |
+
require_generation(session, table_key, before)
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def canonical_grid(session, cur, table_key, *, needed=None, metric_window=None):
|
| 604 |
+
"""Materialize only required canonical source and wall columns into the isolated cursor."""
|
| 605 |
+
import odoo_relational as rel
|
| 606 |
+
import routes_odoo_tables as rot
|
| 607 |
+
from modules import sheet
|
| 608 |
+
g = canonical_metadata(session, table_key)
|
| 609 |
+
fields = g['fields']
|
| 610 |
+
agent_field = {'key': 'agent_id', 'type': 'int', 'label': 'Current agent ID'}
|
| 611 |
+
join_agent = False
|
| 612 |
+
if table_key == 'customer_data' and (needed is None or 'agent_id' in needed):
|
| 613 |
+
from core import perm_scope
|
| 614 |
+
hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime)
|
| 615 |
+
if not {'agent', 'agent_id'} & set(hidden):
|
| 616 |
+
if 'agent_id' in rel.columns(cur, 'res_partner'):
|
| 617 |
+
fields = [f for f in fields if f['key'] != 'agent_id'] + [agent_field]
|
| 618 |
+
join_agent = True
|
| 619 |
+
elif needed is not None and 'agent_id' in needed:
|
| 620 |
+
raise err(409, 'related_invalid', 'customer_data.agent_id is unavailable in the partner mirror.')
|
| 621 |
+
kept = fields if needed is None else [f for f in fields if f['key'] in needed]
|
| 622 |
+
unbounded, bounded = {}, set()
|
| 623 |
+
def rows():
|
| 624 |
+
seen = False
|
| 625 |
+
empty_limits = {}
|
| 626 |
+
with closing(canonical_batches(session, cur, table_key, needed=needed, metric_window=metric_window)) as batches:
|
| 627 |
+
for batch in batches:
|
| 628 |
+
if batch['rows']:
|
| 629 |
+
seen = True
|
| 630 |
+
unbounded.update(batch['unbounded'])
|
| 631 |
+
bounded.update(batch['bounded'])
|
| 632 |
+
else:
|
| 633 |
+
empty_limits.update(batch['unbounded'])
|
| 634 |
+
yield from batch['rows']
|
| 635 |
+
batch['rows'].clear()
|
| 636 |
+
if not seen:
|
| 637 |
+
unbounded.update(empty_limits)
|
| 638 |
+
typed = sheet.mirror_fields(kept)
|
| 639 |
+
import hashlib
|
| 640 |
+
import json
|
| 641 |
+
slot = hashlib.sha1(json.dumps([metric_window],
|
| 642 |
+
sort_keys=True, default=str).encode()).hexdigest()[:12]
|
| 643 |
+
spec = rel.registry_grid_spec(cur, f'{table_key}__at_{slot}', typed, rows(), bulk=True)
|
| 644 |
+
if join_agent:
|
| 645 |
+
cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id=partner.agent_id FROM res_partner partner WHERE grid.pid=partner.id')
|
| 646 |
+
return cur, dict(spec, fields=fields, bounded=tuple(bounded), unbounded=unbounded), rot._filter_columns(spec, typed)
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def canonical_page(session, cur, table_key, ids, *, related=None):
|
| 650 |
+
"""Hydrate a selected page through canonical projections without constructing a full payload."""
|
| 651 |
+
ids = list(dict.fromkeys(int(pid) for pid in ids))
|
| 652 |
+
found, docs, payload = {}, {}, None
|
| 653 |
+
with closing(canonical_batches(session, cur, table_key, ids=ids, related=related)) as batches:
|
| 654 |
+
for batch in batches:
|
| 655 |
+
payload = {key: value for key, value in batch.items()
|
| 656 |
+
if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
|
| 657 |
+
found.update((int(row['pid']), row) for row in batch['rows'])
|
| 658 |
+
docs.update(batch.get('docs') or {})
|
| 659 |
+
if payload is None:
|
| 660 |
+
g = canonical_metadata(session, table_key)
|
| 661 |
+
payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
|
| 662 |
+
return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
|
| 663 |
+
|
| 664 |
+
|
| 665 |
+
def stored_batches(session, table_key, *, ids=None, related=None, needed=None, cur=None):
|
| 666 |
+
"""Project bounded stored batches through the table's existing permission and overlay reader."""
|
| 667 |
+
import routes_tables as tables
|
| 668 |
+
from core import user_tables
|
| 669 |
+
metadata_actor = with_runtime(session, MetadataStore(session.runtime))
|
| 670 |
+
definition = tables._defn_or_refuse(metadata_actor, table_key, defs_only=True, scope_applied=True)
|
| 671 |
+
if definition.get('readThrough'):
|
| 672 |
+
raise err(409, 'canonical_source_unavailable', f'{table_key} requires its declared SQL source.')
|
| 673 |
+
metadata = {}
|
| 674 |
+
with closing(items(session.runtime, user_tables.STORE_KEY, [table_key, 'rows'], selected=ids)) as batches:
|
| 675 |
+
for rows in batches:
|
| 676 |
+
runtime = BatchStore(session.runtime, table_key, definition, rows,
|
| 677 |
+
metadata=metadata, uname=session.uname, needed=needed)
|
| 678 |
+
runtime._canonical_source_cursor = cur
|
| 679 |
+
actor = with_runtime(session, runtime)
|
| 680 |
+
# table_rows remains the canonical evaluator. Its document contains this batch only.
|
| 681 |
+
payload = tables.table_rows(table_key, session=actor, related=related)
|
| 682 |
+
if needed is not None:
|
| 683 |
+
payload = dict(payload, rows=[{k: v for k, v in row.items() if k == 'pid' or k in needed}
|
| 684 |
+
for row in payload['rows']])
|
| 685 |
+
yield payload
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def metadata_limits(session, table_key, metadata):
|
| 689 |
+
"""Retain the editable capacity receipt without asking for a population document."""
|
| 690 |
+
from core import user_tables
|
| 691 |
+
limits = list((metadata.get('workspace') or {}).get('limits') or [])
|
| 692 |
+
report = user_tables.limit_report(table_key, st=user_tables.lend(
|
| 693 |
+
session.runtime, user_tables={table_key: metadata['defn']}))
|
| 694 |
+
if report and report not in limits:
|
| 695 |
+
limits.append(report)
|
| 696 |
+
return limits
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def stored_grid(session, cur, table_key, *, needed=None):
|
| 700 |
+
"""Build an exact SQL population through bounded canonical table projections."""
|
| 701 |
+
import odoo_relational as rel
|
| 702 |
+
import routes_odoo_tables as rot
|
| 703 |
+
import routes_tables as tables
|
| 704 |
+
from modules import sheet
|
| 705 |
+
|
| 706 |
+
before = generation(session, table_key)
|
| 707 |
+
metadata_actor = with_runtime(session, MetadataStore(session.runtime))
|
| 708 |
+
metadata = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True,
|
| 709 |
+
consume_corrections=False)
|
| 710 |
+
fields = metadata['fields']
|
| 711 |
+
kept = fields if needed is None else [f for f in fields if f['key'] in needed]
|
| 712 |
+
|
| 713 |
+
def rows():
|
| 714 |
+
with closing(stored_batches(session, table_key, needed=needed)) as batches:
|
| 715 |
+
for payload in batches:
|
| 716 |
+
yield from payload['rows']
|
| 717 |
+
payload['rows'].clear()
|
| 718 |
+
|
| 719 |
+
spec_fields = sheet.mirror_fields(kept)
|
| 720 |
+
spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows(), bulk=True)
|
| 721 |
+
require_generation(session, table_key, before)
|
| 722 |
+
spec = dict(spec, fields=fields, scope_applied=True,
|
| 723 |
+
limits=metadata_limits(session, table_key, metadata))
|
| 724 |
+
return cur, spec, rot._filter_columns(spec, spec_fields)
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
def stored_page(session, table_key, ids, *, related=None):
|
| 728 |
+
"""Hydrate only selected identities through the canonical stored-row projection."""
|
| 729 |
+
before = generation(session, table_key)
|
| 730 |
+
ids = list(dict.fromkeys(int(pid) for pid in ids))
|
| 731 |
+
payload, found, docs, zones = None, {}, {}, {}
|
| 732 |
+
with closing(stored_batches(session, table_key, ids=ids, related=related)) as batches:
|
| 733 |
+
for batch in batches:
|
| 734 |
+
payload = {k: v for k, v in batch.items() if k != 'rows'}
|
| 735 |
+
found.update((int(row['pid']), row) for row in batch['rows'])
|
| 736 |
+
docs.update(batch.get('docs') or {})
|
| 737 |
+
for zone in batch.get('zones') or []:
|
| 738 |
+
key = str(zone.get('id') or zone.get('name'))
|
| 739 |
+
if key not in zones:
|
| 740 |
+
zones[key] = dict(zone, memberships=[])
|
| 741 |
+
zones[key]['memberships'].extend(zone.get('memberships') or [])
|
| 742 |
+
if payload is None:
|
| 743 |
+
import routes_tables as tables
|
| 744 |
+
metadata_actor = with_runtime(session, MetadataStore(session.runtime))
|
| 745 |
+
g = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True,
|
| 746 |
+
consume_corrections=False)
|
| 747 |
+
payload = {'fields': g['fields'], 'today': g['today'],
|
| 748 |
+
'limits': metadata_limits(session, table_key, g)}
|
| 749 |
+
rows = [found[pid] for pid in ids if pid in found]
|
| 750 |
+
require_generation(session, table_key, before)
|
| 751 |
+
return dict(payload, rows=rows, docs=docs, zones=list(zones.values()),
|
| 752 |
+
scope={**payload.get('scope', {}), 'rowCount': len(rows)})
|
| 753 |
+
|
| 754 |
+
|
| 755 |
+
def page_cells(runtime, table_key, pids):
|
| 756 |
+
"""Read a page's shared cells through the canonical parser and coherent row-bounded snapshots."""
|
| 757 |
+
from core import shared_overlay
|
| 758 |
+
import itertools
|
| 759 |
+
source, out = iter(pids), {}
|
| 760 |
+
while True:
|
| 761 |
+
batch = list(itertools.islice(source, BATCH_ROWS))
|
| 762 |
+
if not batch:
|
| 763 |
+
return out
|
| 764 |
+
reader = BatchStore(runtime, table_key, {'fields': []}, [(str(pid), {}) for pid in batch])
|
| 765 |
+
out.update(shared_overlay.cells(table_key, batch, st=reader))
|
api/request_class.py
CHANGED
|
@@ -147,8 +147,8 @@ def _env_float(name, default):
|
|
| 147 |
return default
|
| 148 |
|
| 149 |
|
| 150 |
-
async def _send_busy(send):
|
| 151 |
-
body = json.dumps({"error": {"code": BUSY_CODE, "message": BUSY_MESSAGE}}).encode()
|
| 152 |
await send({"type": "http.response.start", "status": 503,
|
| 153 |
"headers": [(b"content-type", b"application/json"),
|
| 154 |
(b"content-length", str(len(body)).encode()),
|
|
@@ -200,11 +200,22 @@ class HeavyGate:
|
|
| 200 |
sem = self._semaphore()
|
| 201 |
c = self.counters
|
| 202 |
c.add("queued", 1)
|
|
|
|
|
|
|
| 203 |
acquire = asyncio.ensure_future(sem.acquire())
|
| 204 |
gone = asyncio.ensure_future(_wait_disconnect(receive))
|
| 205 |
try:
|
| 206 |
-
|
|
|
|
| 207 |
return_when=asyncio.FIRST_COMPLETED)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
except BaseException:
|
| 209 |
# Cancellation can race a successful acquire before the handler owns the permit.
|
| 210 |
if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
|
|
@@ -220,8 +231,18 @@ class HeavyGate:
|
|
| 220 |
acquire.cancel()
|
| 221 |
if not gone.done():
|
| 222 |
gone.cancel()
|
|
|
|
| 223 |
# A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
|
| 224 |
got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
if acquire not in done or gone in done:
|
| 226 |
if got:
|
| 227 |
sem.release()
|
|
|
|
| 147 |
return default
|
| 148 |
|
| 149 |
|
| 150 |
+
async def _send_busy(send, cause="heavy_queue_timeout"):
|
| 151 |
+
body = json.dumps({"error": {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}}).encode()
|
| 152 |
await send({"type": "http.response.start", "status": 503,
|
| 153 |
"headers": [(b"content-type", b"application/json"),
|
| 154 |
(b"content-length", str(len(body)).encode()),
|
|
|
|
| 200 |
sem = self._semaphore()
|
| 201 |
c = self.counters
|
| 202 |
c.add("queued", 1)
|
| 203 |
+
timing = sys.modules.get("core.request_timing")
|
| 204 |
+
budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
|
| 205 |
acquire = asyncio.ensure_future(sem.acquire())
|
| 206 |
gone = asyncio.ensure_future(_wait_disconnect(receive))
|
| 207 |
try:
|
| 208 |
+
timeout = timing.remaining_budget(self.queue_seconds) if timing else self.queue_seconds
|
| 209 |
+
done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
|
| 210 |
return_when=asyncio.FIRST_COMPLETED)
|
| 211 |
+
if timing:
|
| 212 |
+
timing.remaining_budget()
|
| 213 |
+
except budget_error:
|
| 214 |
+
if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
|
| 215 |
+
sem.release()
|
| 216 |
+
c.add("busy_refusals", 1)
|
| 217 |
+
await _send_busy(send, "request_deadline")
|
| 218 |
+
return
|
| 219 |
except BaseException:
|
| 220 |
# Cancellation can race a successful acquire before the handler owns the permit.
|
| 221 |
if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
|
|
|
|
| 231 |
acquire.cancel()
|
| 232 |
if not gone.done():
|
| 233 |
gone.cancel()
|
| 234 |
+
await asyncio.gather(acquire, gone, return_exceptions=True)
|
| 235 |
# A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
|
| 236 |
got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
|
| 237 |
+
try:
|
| 238 |
+
if timing:
|
| 239 |
+
timing.remaining_budget()
|
| 240 |
+
except budget_error:
|
| 241 |
+
if got:
|
| 242 |
+
sem.release()
|
| 243 |
+
c.add("busy_refusals", 1)
|
| 244 |
+
await _send_busy(send, "request_deadline")
|
| 245 |
+
return
|
| 246 |
if acquire not in done or gone in done:
|
| 247 |
if got:
|
| 248 |
sem.release()
|
api/request_timing.py
CHANGED
|
@@ -20,6 +20,174 @@ _STATS = OrderedDict()
|
|
| 20 |
_ROUTE_LIMIT = 512
|
| 21 |
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
def slow_threshold_ms():
|
| 24 |
try:
|
| 25 |
value = float(os.environ.get("AIOS_SLOW_REQUEST_MS", "1000"))
|
|
@@ -185,6 +353,7 @@ class RequestTiming:
|
|
| 185 |
if scope.get("type") != "http":
|
| 186 |
return await self.app(scope, receive, send)
|
| 187 |
with timing.request_scope() as acc:
|
|
|
|
| 188 |
status, failed, disconnected, complete = None, False, False, False
|
| 189 |
threshold = slow_threshold_ms()
|
| 190 |
_resolve_route(scope)
|
|
@@ -192,9 +361,19 @@ class RequestTiming:
|
|
| 192 |
|
| 193 |
async def receive_timed():
|
| 194 |
nonlocal disconnected
|
| 195 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
if message["type"] == "http.disconnect":
|
| 197 |
disconnected = True
|
|
|
|
| 198 |
return message
|
| 199 |
|
| 200 |
async def send_timed(message):
|
|
@@ -227,6 +406,7 @@ class RequestTiming:
|
|
| 227 |
_SAMPLER.end(acc)
|
| 228 |
with acc.lock:
|
| 229 |
acc.closed = True
|
|
|
|
| 230 |
samples = sorted(acc.samples.items(), key=lambda item: -item[1])[:18]
|
| 231 |
_save_stats(scope, status, elapsed)
|
| 232 |
if elapsed >= threshold:
|
|
|
|
| 20 |
_ROUTE_LIMIT = 512
|
| 21 |
|
| 22 |
|
| 23 |
+
def request_timeout_s():
|
| 24 |
+
try:
|
| 25 |
+
value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", "45"))
|
| 26 |
+
return max(.01, value) if math.isfinite(value) else 45.0
|
| 27 |
+
except ValueError:
|
| 28 |
+
return 45.0
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
_BUSY_CAUSES = {
|
| 32 |
+
"PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
|
| 33 |
+
"QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
|
| 34 |
+
"SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
|
| 35 |
+
"RequestDeadlineExceeded": "request_deadline", "DeadlineCapacityExceeded": "deadline_capacity",
|
| 36 |
+
"OperationalError": "postgres_connection_unavailable", "InterfaceError": "postgres_connection_unavailable",
|
| 37 |
+
"ConnectionTimeout": "postgres_connection_timeout",
|
| 38 |
+
"IdleInTransactionSessionTimeout": "postgres_idle_transaction_timeout",
|
| 39 |
+
"OutOfMemoryException": "duckdb_memory_limit",
|
| 40 |
+
"PostgresOperationTimeout": "postgres_operation_timeout",
|
| 41 |
+
"OdooRequestTimeout": "odoo_rpc_timeout",
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def resource_cause(error):
|
| 46 |
+
"""Find a resource refusal through route wrappers without exposing database messages."""
|
| 47 |
+
seen = set()
|
| 48 |
+
while error is not None and id(error) not in seen:
|
| 49 |
+
seen.add(id(error))
|
| 50 |
+
cause = _BUSY_CAUSES.get(type(error).__name__)
|
| 51 |
+
if cause:
|
| 52 |
+
return cause
|
| 53 |
+
error = error.__cause__ or error.__context__
|
| 54 |
+
return None
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def resource_response(error):
|
| 58 |
+
from starlette.responses import JSONResponse
|
| 59 |
+
cause = resource_cause(error)
|
| 60 |
+
if cause is None:
|
| 61 |
+
return None
|
| 62 |
+
return JSONResponse({"error": {"code": "server_busy", "cause": cause,
|
| 63 |
+
"message": "The server is busy. Try again shortly."}},
|
| 64 |
+
status_code=503, headers={"Retry-After": "1"})
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class DeadlineCapacityExceeded(RuntimeError):
|
| 68 |
+
"""The bounded statement watchdog has no free registration slot."""
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class _StatementAlarm:
|
| 72 |
+
def __init__(self, owner, connection, acc):
|
| 73 |
+
self.owner, self.connection, self.acc = owner, connection, acc
|
| 74 |
+
self.lock = threading.Lock()
|
| 75 |
+
self.active, self.expired = True, False
|
| 76 |
+
|
| 77 |
+
def fire(self):
|
| 78 |
+
with self.lock:
|
| 79 |
+
if self.active and (self.acc.cancelled or time.monotonic() >= self.acc.deadline):
|
| 80 |
+
self.expired = True
|
| 81 |
+
self.active = False
|
| 82 |
+
self.connection.interrupt()
|
| 83 |
+
|
| 84 |
+
def close(self):
|
| 85 |
+
# Serialize disarm with interrupt before the cursor can execute another statement.
|
| 86 |
+
with self.lock:
|
| 87 |
+
self.active = False
|
| 88 |
+
expired = self.expired
|
| 89 |
+
with self.owner.lock:
|
| 90 |
+
self.owner.active.discard(self)
|
| 91 |
+
return expired
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class _DeadlineWatchdog:
|
| 95 |
+
INTERVAL = .01
|
| 96 |
+
CAPACITY = 128
|
| 97 |
+
|
| 98 |
+
def __init__(self):
|
| 99 |
+
self.lock = threading.Lock()
|
| 100 |
+
self.active = set()
|
| 101 |
+
self.thread = None
|
| 102 |
+
|
| 103 |
+
def arm(self, connection, acc):
|
| 104 |
+
import process_role
|
| 105 |
+
with self.lock:
|
| 106 |
+
if len(self.active) >= self.CAPACITY:
|
| 107 |
+
raise DeadlineCapacityExceeded("request statement watchdog is full")
|
| 108 |
+
alarm = _StatementAlarm(self, connection, acc)
|
| 109 |
+
self.active.add(alarm)
|
| 110 |
+
if self.thread is None:
|
| 111 |
+
job = process_role.thread(target=self._run, name="request-deadline", kind="light")
|
| 112 |
+
if not process_role.admitted(job):
|
| 113 |
+
self.active.remove(alarm)
|
| 114 |
+
raise DeadlineCapacityExceeded("request statement watchdog is unavailable")
|
| 115 |
+
self.thread = job
|
| 116 |
+
try:
|
| 117 |
+
job.start()
|
| 118 |
+
except BaseException:
|
| 119 |
+
self.thread = None
|
| 120 |
+
self.active.remove(alarm)
|
| 121 |
+
raise
|
| 122 |
+
return alarm
|
| 123 |
+
|
| 124 |
+
def _run(self):
|
| 125 |
+
while True:
|
| 126 |
+
time.sleep(self.INTERVAL)
|
| 127 |
+
with self.lock:
|
| 128 |
+
if not self.active:
|
| 129 |
+
self.thread = None
|
| 130 |
+
return
|
| 131 |
+
active = tuple(self.active)
|
| 132 |
+
for alarm in active:
|
| 133 |
+
try:
|
| 134 |
+
alarm.fire()
|
| 135 |
+
except Exception:
|
| 136 |
+
LOG.warning("request query interrupt failed", exc_info=False)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
_DEADLINES = _DeadlineWatchdog()
|
| 140 |
+
timing.configure_deadlines(_DEADLINES.arm)
|
| 141 |
+
|
| 142 |
+
_LOCK_PROBE_SLOTS = threading.BoundedSemaphore(8)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class _LockProbe:
|
| 146 |
+
def __init__(self, pid, diagnose):
|
| 147 |
+
self.pid, self.diagnose = pid, diagnose
|
| 148 |
+
self.lock = threading.Lock()
|
| 149 |
+
self.active, self.fired = True, False
|
| 150 |
+
self.job = None
|
| 151 |
+
|
| 152 |
+
def run(self):
|
| 153 |
+
with self.lock:
|
| 154 |
+
if not self.active:
|
| 155 |
+
return
|
| 156 |
+
self.fired = True
|
| 157 |
+
try:
|
| 158 |
+
self.diagnose(self.pid)
|
| 159 |
+
finally:
|
| 160 |
+
_LOCK_PROBE_SLOTS.release()
|
| 161 |
+
|
| 162 |
+
def close(self):
|
| 163 |
+
with self.lock:
|
| 164 |
+
if not self.active:
|
| 165 |
+
return
|
| 166 |
+
self.active = False
|
| 167 |
+
self.job.cancel()
|
| 168 |
+
if not self.fired:
|
| 169 |
+
_LOCK_PROBE_SLOTS.release()
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _arm_lock_probe(pid, wait_s, diagnose):
|
| 173 |
+
import process_role
|
| 174 |
+
if not _LOCK_PROBE_SLOTS.acquire(blocking=False):
|
| 175 |
+
return None
|
| 176 |
+
probe = _LockProbe(pid, diagnose)
|
| 177 |
+
try:
|
| 178 |
+
probe.job = process_role.thread(target=probe.run, name="store-lock-diagnosis",
|
| 179 |
+
kind="light", delay=max(.01, wait_s * .75))
|
| 180 |
+
probe.job.start()
|
| 181 |
+
except Exception:
|
| 182 |
+
_LOCK_PROBE_SLOTS.release()
|
| 183 |
+
LOG.warning("store lock diagnosis could not start", exc_info=False)
|
| 184 |
+
return None
|
| 185 |
+
return probe
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
timing.configure_lock_probes(_arm_lock_probe)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
def slow_threshold_ms():
|
| 192 |
try:
|
| 193 |
value = float(os.environ.get("AIOS_SLOW_REQUEST_MS", "1000"))
|
|
|
|
| 353 |
if scope.get("type") != "http":
|
| 354 |
return await self.app(scope, receive, send)
|
| 355 |
with timing.request_scope() as acc:
|
| 356 |
+
acc.deadline = time.monotonic() + request_timeout_s()
|
| 357 |
status, failed, disconnected, complete = None, False, False, False
|
| 358 |
threshold = slow_threshold_ms()
|
| 359 |
_resolve_route(scope)
|
|
|
|
| 361 |
|
| 362 |
async def receive_timed():
|
| 363 |
nonlocal disconnected
|
| 364 |
+
# Response streaming remains open after headers, including SSE disconnect waits.
|
| 365 |
+
if status is None:
|
| 366 |
+
remaining = timing.remaining_budget()
|
| 367 |
+
try:
|
| 368 |
+
message = await asyncio.wait_for(receive(), remaining)
|
| 369 |
+
except asyncio.TimeoutError as exc:
|
| 370 |
+
raise timing.RequestDeadlineExceeded("request body deadline exceeded") from exc
|
| 371 |
+
timing.remaining_budget()
|
| 372 |
+
else:
|
| 373 |
+
message = await receive()
|
| 374 |
if message["type"] == "http.disconnect":
|
| 375 |
disconnected = True
|
| 376 |
+
acc.cancelled = True
|
| 377 |
return message
|
| 378 |
|
| 379 |
async def send_timed(message):
|
|
|
|
| 406 |
_SAMPLER.end(acc)
|
| 407 |
with acc.lock:
|
| 408 |
acc.closed = True
|
| 409 |
+
acc.cancelled = True
|
| 410 |
samples = sorted(acc.samples.items(), key=lambda item: -item[1])[:18]
|
| 411 |
_save_stats(scope, status, elapsed)
|
| 412 |
if elapsed >= threshold:
|
api/routes_admin.py
CHANGED
|
@@ -40,7 +40,7 @@ import core.registry as registry
|
|
| 40 |
import core.schema_overlay as schema_overlay # W46-T36 / C1 - the overlay merge + R12 wall
|
| 41 |
import core.store as store
|
| 42 |
|
| 43 |
-
from deps import Session, err, perms, require_session, users
|
| 44 |
from routes_auth import _public_user
|
| 45 |
|
| 46 |
router = APIRouter(prefix="/api/v1")
|
|
@@ -112,6 +112,7 @@ def _registry():
|
|
| 112 |
def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
| 113 |
"""What an administrator may see about an account. NEVER `salt` or `hash` — this function is
|
| 114 |
the only projection these routes use, so there is one place that can leak and it does not.
|
|
|
|
| 115 |
|
| 116 |
⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both,
|
| 117 |
resolved once for the whole request; the single-record routes pass neither, because their
|
|
@@ -127,7 +128,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 127 |
"email": rec.get("email") or None,
|
| 128 |
"tenant": _tenant_of(rec),
|
| 129 |
"active": bool(rec.get("active", True)),
|
| 130 |
-
"last_active": rec.get("last_active"),
|
| 131 |
# The session-revocation handle. An admin needs it: it is the only visible evidence
|
| 132 |
# that "sign them out everywhere" actually happened.
|
| 133 |
"epoch": int(rec.get("epoch") or 0),
|
|
|
|
| 40 |
import core.schema_overlay as schema_overlay # W46-T36 / C1 - the overlay merge + R12 wall
|
| 41 |
import core.store as store
|
| 42 |
|
| 43 |
+
from deps import Session, err, observed_last_active, perms, require_session, users
|
| 44 |
from routes_auth import _public_user
|
| 45 |
|
| 46 |
router = APIRouter(prefix="/api/v1")
|
|
|
|
| 112 |
def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
| 113 |
"""What an administrator may see about an account. NEVER `salt` or `hash` — this function is
|
| 114 |
the only projection these routes use, so there is one place that can leak and it does not.
|
| 115 |
+
`last_active` includes this tenant/account's temporary read observations without saving them.
|
| 116 |
|
| 117 |
⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both,
|
| 118 |
resolved once for the whole request; the single-record routes pass neither, because their
|
|
|
|
| 128 |
"email": rec.get("email") or None,
|
| 129 |
"tenant": _tenant_of(rec),
|
| 130 |
"active": bool(rec.get("active", True)),
|
| 131 |
+
"last_active": observed_last_active(_tenant_of(rec), uname, rec.get("last_active")),
|
| 132 |
# The session-revocation handle. An admin needs it: it is the only visible evidence
|
| 133 |
# that "sign them out everywhere" actually happened.
|
| 134 |
"epoch": int(rec.get("epoch") or 0),
|
api/routes_connectors.py
CHANGED
|
@@ -51,6 +51,7 @@ from deps import Session, err, require_session, running_main
|
|
| 51 |
#: connector is in.
|
| 52 |
from core import connector_directory
|
| 53 |
from core import connector_sync
|
|
|
|
| 54 |
from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
|
| 55 |
from harness import amazon_client
|
| 56 |
#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
|
|
@@ -995,6 +996,7 @@ def _entry_types(rt, uname=""):
|
|
| 995 |
scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
|
| 996 |
return types, scopes, None
|
| 997 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 998 |
return set(), {}, f"{type(e).__name__}: {e}"
|
| 999 |
|
| 1000 |
|
|
@@ -2902,9 +2904,10 @@ def _amazon_completed_sync(rt):
|
|
| 2902 |
return None, False
|
| 2903 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 2904 |
return read(datastore.ro_cursor())
|
| 2905 |
-
with duckdb.connect(str(path), read_only=True) as cursor:
|
| 2906 |
return read(cursor)
|
| 2907 |
-
except Exception:
|
|
|
|
| 2908 |
return None, False
|
| 2909 |
|
| 2910 |
|
|
@@ -2931,9 +2934,10 @@ def _daily_completed_sync(rt, connector_key):
|
|
| 2931 |
return None, False
|
| 2932 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 2933 |
return read(datastore.ro_cursor())
|
| 2934 |
-
with duckdb.connect(str(path), read_only=True) as cursor:
|
| 2935 |
return read(cursor)
|
| 2936 |
-
except Exception:
|
|
|
|
| 2937 |
return None, False
|
| 2938 |
return None, False
|
| 2939 |
|
|
@@ -4642,12 +4646,13 @@ def _amazon_databases(rt, is_admin, spec=None):
|
|
| 4642 |
# trims an over-long cause itself; a credential shape still refuses, and the refusal carries
|
| 4643 |
# the guarded rows that passed, so only the refused tables read "Sync status is unavailable
|
| 4644 |
# for this database." (a missing status is exactly what `_amazon_database_sync_health`
|
| 4645 |
-
# degrades).
|
| 4646 |
try:
|
| 4647 |
states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
|
| 4648 |
except cx.AmazonStatusSafetyError as refused:
|
| 4649 |
states = dict(getattr(refused, "safe_status", None) or {})
|
| 4650 |
-
except Exception:
|
|
|
|
| 4651 |
states = {}
|
| 4652 |
|
| 4653 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
|
@@ -5401,6 +5406,7 @@ def _plain_read_failure(exc, code, sentence):
|
|
| 5401 |
`{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
|
| 5402 |
and no exception text (which can echo a payload) reaches the wire.
|
| 5403 |
"""
|
|
|
|
| 5404 |
import logging # noqa: PLC0415
|
| 5405 |
logging.getLogger(__name__).exception("connector read failed: %s", code)
|
| 5406 |
name = type(exc).__name__
|
|
|
|
| 51 |
#: connector is in.
|
| 52 |
from core import connector_directory
|
| 53 |
from core import connector_sync
|
| 54 |
+
from core.request_timing import raise_if_resource_error
|
| 55 |
from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
|
| 56 |
from harness import amazon_client
|
| 57 |
#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
|
|
|
|
| 996 |
scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
|
| 997 |
return types, scopes, None
|
| 998 |
except Exception as e: # noqa: BLE001
|
| 999 |
+
raise_if_resource_error(e)
|
| 1000 |
return set(), {}, f"{type(e).__name__}: {e}"
|
| 1001 |
|
| 1002 |
|
|
|
|
| 2904 |
return None, False
|
| 2905 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 2906 |
return read(datastore.ro_cursor())
|
| 2907 |
+
with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
|
| 2908 |
return read(cursor)
|
| 2909 |
+
except Exception as exc:
|
| 2910 |
+
raise_if_resource_error(exc)
|
| 2911 |
return None, False
|
| 2912 |
|
| 2913 |
|
|
|
|
| 2934 |
return None, False
|
| 2935 |
if path.resolve() == Path(datastore.DB_PATH).resolve():
|
| 2936 |
return read(datastore.ro_cursor())
|
| 2937 |
+
with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
|
| 2938 |
return read(cursor)
|
| 2939 |
+
except Exception as exc:
|
| 2940 |
+
raise_if_resource_error(exc)
|
| 2941 |
return None, False
|
| 2942 |
return None, False
|
| 2943 |
|
|
|
|
| 4646 |
# trims an over-long cause itself; a credential shape still refuses, and the refusal carries
|
| 4647 |
# the guarded rows that passed, so only the refused tables read "Sync status is unavailable
|
| 4648 |
# for this database." (a missing status is exactly what `_amazon_database_sync_health`
|
| 4649 |
+
# degrades). Resource failures propagate; other unreadable status degrades every row.
|
| 4650 |
try:
|
| 4651 |
states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
|
| 4652 |
except cx.AmazonStatusSafetyError as refused:
|
| 4653 |
states = dict(getattr(refused, "safe_status", None) or {})
|
| 4654 |
+
except Exception as exc: # noqa: BLE001
|
| 4655 |
+
raise_if_resource_error(exc)
|
| 4656 |
states = {}
|
| 4657 |
|
| 4658 |
# Which OTHER declared tables are filled from the SAME report document. Derived from the
|
|
|
|
| 5406 |
`{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
|
| 5407 |
and no exception text (which can echo a payload) reaches the wire.
|
| 5408 |
"""
|
| 5409 |
+
raise_if_resource_error(exc)
|
| 5410 |
import logging # noqa: PLC0415
|
| 5411 |
logging.getLogger(__name__).exception("connector read failed: %s", code)
|
| 5412 |
name = type(exc).__name__
|
api/routes_customers.py
CHANGED
|
@@ -121,15 +121,7 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r
|
|
| 121 |
found = {}
|
| 122 |
if requested:
|
| 123 |
try:
|
| 124 |
-
|
| 125 |
-
rows = con.execute(store_pg._q(
|
| 126 |
-
"SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])",
|
| 127 |
-
"store_kv", handle.tenant_slug), (sorted(requested),)).fetchall()
|
| 128 |
-
for name, raw_rev, stamp in rows:
|
| 129 |
-
rev = int(raw_rev) if raw_rev is not None else 0
|
| 130 |
-
found[name] = {"rev": rev,
|
| 131 |
-
"updated_at": stamp.timestamp() if hasattr(stamp, "timestamp") else None,
|
| 132 |
-
"token": f"pg:{rev}"}
|
| 133 |
except Exception as exc:
|
| 134 |
if type(exc).__name__ in {'PoolTimeout', 'OperationalError', 'InterfaceError', 'ConnectionTimeout'}:
|
| 135 |
from core.grid_events import StoreUnavailable
|
|
@@ -189,6 +181,9 @@ def _pool_for(rt, team_id, agent):
|
|
| 189 |
# capability result; empty is safe, another tenant's customer book is not.
|
| 190 |
if rt.key != "royal-imports":
|
| 191 |
return []
|
|
|
|
|
|
|
|
|
|
| 192 |
|
| 193 |
import modules.customer_data as cl
|
| 194 |
import routes_keychain
|
|
@@ -460,18 +455,44 @@ class _AssemblyStore:
|
|
| 460 |
self.tokens[name] = (self._validated_tokens[name] if name in self._validated_tokens
|
| 461 |
else _assembly_revision(self._runtime, name))
|
| 462 |
except AttributeError as exc:
|
|
|
|
|
|
|
|
|
|
| 463 |
self.cacheable = False
|
| 464 |
if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
|
| 465 |
print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
key = (name, shape)
|
| 467 |
if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
|
| 468 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
# TableStore readers can mutate nested projections; lend an independent read each time.
|
| 470 |
return deepcopy(self.docs[key])
|
| 471 |
|
| 472 |
def get(self, name, fresh=False):
|
|
|
|
|
|
|
|
|
|
| 473 |
if self.schema_only and fresh:
|
| 474 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 475 |
if self.schema_only and name.endswith("_table_workspace"):
|
| 476 |
return self._read(name, ("projection", ("overlays",)),
|
| 477 |
lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh)
|
|
@@ -482,10 +503,14 @@ class _AssemblyStore:
|
|
| 482 |
self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
|
| 483 |
|
| 484 |
def get_projection(self, name, drop=()):
|
|
|
|
|
|
|
| 485 |
return self._read(name, ("projection", tuple(drop)),
|
| 486 |
lambda: self._runtime.get_projection(name, drop=drop))
|
| 487 |
|
| 488 |
def get_entry(self, name, entry):
|
|
|
|
|
|
|
| 489 |
if self.schema_only:
|
| 490 |
backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime
|
| 491 |
if not callable(getattr(backend, "get_entry", None)):
|
|
@@ -504,6 +529,9 @@ class _AssemblyStore:
|
|
| 504 |
return self._read(name, "exists", lambda: self._runtime.exists(name))
|
| 505 |
|
| 506 |
def revision(self, name):
|
|
|
|
|
|
|
|
|
|
| 507 |
token = _assembly_revision(self._runtime, name)
|
| 508 |
self.tokens.setdefault(name, token)
|
| 509 |
return token
|
|
@@ -600,20 +628,30 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 600 |
result = ((*prefix, json.dumps(None)), {})
|
| 601 |
return result
|
| 602 |
names = {name for name, _token in revisions}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 603 |
with _grid_assembly_lock:
|
| 604 |
for memo_key in list(rt.pool_cache):
|
| 605 |
if memo_key[:-2] == prefix:
|
| 606 |
names.update(name for name, _token in memo_key[-1])
|
| 607 |
-
tokens, users_stamp = _assembly_revisions(rt, names, uses_roster,
|
| 608 |
getattr(session, "tenant", None), roster_memo)
|
| 609 |
result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
|
| 610 |
return result
|
| 611 |
|
| 612 |
def own_copy(result):
|
| 613 |
-
#
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
|
|
|
|
|
|
| 617 |
out["workspace"]["storageKey"] = storage_key
|
| 618 |
return out
|
| 619 |
|
|
@@ -653,7 +691,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
|
|
| 653 |
|
| 654 |
def valid(revisions, tokens):
|
| 655 |
if any(name not in tokens for name, _token in revisions):
|
| 656 |
-
tokens, _users = _assembly_revisions(
|
|
|
|
| 657 |
result = all(json.dumps(tokens[name], sort_keys=True, default=str) == token
|
| 658 |
for name, token in revisions)
|
| 659 |
return result
|
|
|
|
| 121 |
found = {}
|
| 122 |
if requested:
|
| 123 |
try:
|
| 124 |
+
found = store_pg._revision_many(sorted(requested), tenant_slug=handle.tenant_slug)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
except Exception as exc:
|
| 126 |
if type(exc).__name__ in {'PoolTimeout', 'OperationalError', 'InterfaceError', 'ConnectionTimeout'}:
|
| 127 |
from core.grid_events import StoreUnavailable
|
|
|
|
| 181 |
# capability result; empty is safe, another tenant's customer book is not.
|
| 182 |
if rt.key != "royal-imports":
|
| 183 |
return []
|
| 184 |
+
bounded = getattr(rt, '_canonical_rows', None)
|
| 185 |
+
if bounded is not None:
|
| 186 |
+
return bounded
|
| 187 |
|
| 188 |
import modules.customer_data as cl
|
| 189 |
import routes_keychain
|
|
|
|
| 455 |
self.tokens[name] = (self._validated_tokens[name] if name in self._validated_tokens
|
| 456 |
else _assembly_revision(self._runtime, name))
|
| 457 |
except AttributeError as exc:
|
| 458 |
+
if self.schema_only and callable(getattr(self._runtime, 'revision', None)):
|
| 459 |
+
from core.shared_overlay import SchemaUnavailable
|
| 460 |
+
raise SchemaUnavailable('table definition revisions are unavailable') from exc
|
| 461 |
self.cacheable = False
|
| 462 |
if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
|
| 463 |
print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
|
| 464 |
+
except Exception as exc:
|
| 465 |
+
if self.schema_only:
|
| 466 |
+
from core.request_timing import raise_if_resource_error
|
| 467 |
+
from core.shared_overlay import SchemaUnavailable
|
| 468 |
+
raise_if_resource_error(exc)
|
| 469 |
+
raise SchemaUnavailable('table definition revisions are unavailable') from exc
|
| 470 |
+
raise
|
| 471 |
key = (name, shape)
|
| 472 |
if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
|
| 473 |
+
try:
|
| 474 |
+
self.docs[key] = read()
|
| 475 |
+
except Exception as exc:
|
| 476 |
+
if self.schema_only:
|
| 477 |
+
from core.request_timing import raise_if_resource_error
|
| 478 |
+
from core.shared_overlay import SchemaUnavailable
|
| 479 |
+
raise_if_resource_error(exc)
|
| 480 |
+
raise SchemaUnavailable('table definitions are unavailable') from exc
|
| 481 |
+
raise
|
| 482 |
# TableStore readers can mutate nested projections; lend an independent read each time.
|
| 483 |
return deepcopy(self.docs[key])
|
| 484 |
|
| 485 |
def get(self, name, fresh=False):
|
| 486 |
+
if fresh:
|
| 487 |
+
from core.request_timing import clear_read_memo
|
| 488 |
+
clear_read_memo()
|
| 489 |
if self.schema_only and fresh:
|
| 490 |
self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
|
| 491 |
+
if self.schema_only and name == "user_tables":
|
| 492 |
+
from core.user_tables import DEF_ONLY_DROPS
|
| 493 |
+
drops = (*DEF_ONLY_DROPS, 'aiEnrich')
|
| 494 |
+
return self._read(name, ("projection", drops),
|
| 495 |
+
lambda: self._runtime.get_projection(name, drop=drops), fresh=fresh)
|
| 496 |
if self.schema_only and name.endswith("_table_workspace"):
|
| 497 |
return self._read(name, ("projection", ("overlays",)),
|
| 498 |
lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh)
|
|
|
|
| 503 |
self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
|
| 504 |
|
| 505 |
def get_projection(self, name, drop=()):
|
| 506 |
+
if self.schema_only and name == 'user_tables':
|
| 507 |
+
drop = tuple(dict.fromkeys([*drop, 'aiEnrich']))
|
| 508 |
return self._read(name, ("projection", tuple(drop)),
|
| 509 |
lambda: self._runtime.get_projection(name, drop=drop))
|
| 510 |
|
| 511 |
def get_entry(self, name, entry):
|
| 512 |
+
if self.schema_only and name == 'user_tables':
|
| 513 |
+
return (self.get(name) or {}).get(str(entry))
|
| 514 |
if self.schema_only:
|
| 515 |
backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime
|
| 516 |
if not callable(getattr(backend, "get_entry", None)):
|
|
|
|
| 529 |
return self._read(name, "exists", lambda: self._runtime.exists(name))
|
| 530 |
|
| 531 |
def revision(self, name):
|
| 532 |
+
# Revisionless schema reads cannot supply a token for a permission cache.
|
| 533 |
+
if self.schema_only and not callable(getattr(self._runtime, "revision", None)):
|
| 534 |
+
return None
|
| 535 |
token = _assembly_revision(self._runtime, name)
|
| 536 |
self.tokens.setdefault(name, token)
|
| 537 |
return token
|
|
|
|
| 628 |
result = ((*prefix, json.dumps(None)), {})
|
| 629 |
return result
|
| 630 |
names = {name for name, _token in revisions}
|
| 631 |
+
if (not with_rows and (scope.startswith('ut_') or scope in {'customer', 'product'})
|
| 632 |
+
and _assembly_pg_store(assembly_rt if assembly_rt is not None else rt) is not None):
|
| 633 |
+
# Check known metadata together before any borrowed read asks for its own token.
|
| 634 |
+
names.update({'user_tables', 'object_shares', 'automations', 'loopable_route_system',
|
| 635 |
+
'table_zones', 'ut_schema_overlay', 'database_setup', scope + '_cohorts',
|
| 636 |
+
scope + '_table_workspace', scope + '_table_workspace__shared'})
|
| 637 |
+
if scope.startswith('ut_'):
|
| 638 |
+
names.add(scope + '__shared')
|
| 639 |
with _grid_assembly_lock:
|
| 640 |
for memo_key in list(rt.pool_cache):
|
| 641 |
if memo_key[:-2] == prefix:
|
| 642 |
names.update(name for name, _token in memo_key[-1])
|
| 643 |
+
tokens, users_stamp = _assembly_revisions(assembly_rt if assembly_rt is not None else rt, names, uses_roster,
|
| 644 |
getattr(session, "tenant", None), roster_memo)
|
| 645 |
result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
|
| 646 |
return result
|
| 647 |
|
| 648 |
def own_copy(result):
|
| 649 |
+
# Both aliases lend immutable source rows. Copy the request-shaped metadata together.
|
| 650 |
+
borrowed = {id(result['rows_src']): result['rows_src']}
|
| 651 |
+
wall_rows = (result.get('_permission_scope') or {}).get('rows')
|
| 652 |
+
if wall_rows is not None:
|
| 653 |
+
borrowed[id(wall_rows)] = wall_rows
|
| 654 |
+
out = deepcopy(result, borrowed)
|
| 655 |
out["workspace"]["storageKey"] = storage_key
|
| 656 |
return out
|
| 657 |
|
|
|
|
| 691 |
|
| 692 |
def valid(revisions, tokens):
|
| 693 |
if any(name not in tokens for name, _token in revisions):
|
| 694 |
+
tokens, _users = _assembly_revisions(assembly_rt if assembly_rt is not None else rt,
|
| 695 |
+
[name for name, _token in revisions])
|
| 696 |
result = all(json.dumps(tokens[name], sort_keys=True, default=str) == token
|
| 697 |
for name, token in revisions)
|
| 698 |
return result
|
api/routes_grid.py
CHANGED
|
@@ -51,8 +51,9 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
|
|
| 51 |
if scope_key == "product":
|
| 52 |
import modules.product_data as pd
|
| 53 |
from routes_products import MODULE as _PMOD, pd_fields
|
|
|
|
| 54 |
|
| 55 |
-
module, canonical = _PMOD, pd_fields(consolidated=True, st=session.runtime)
|
| 56 |
kw.setdefault("table", pd.table_ops(session.runtime))
|
| 57 |
hidden = perm_scope.hidden_keys(session.user, module,
|
| 58 |
canonical if permission_fields is None else permission_fields,
|
|
@@ -1224,6 +1225,98 @@ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "custome
|
|
| 1224 |
return out
|
| 1225 |
|
| 1226 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1227 |
def _builtin_history_context(session, scope, *, restoring=False):
|
| 1228 |
"""Resolve history through the same assembly and write context as grid events."""
|
| 1229 |
from routes_customers import grid_assembly
|
|
|
|
| 51 |
if scope_key == "product":
|
| 52 |
import modules.product_data as pd
|
| 53 |
from routes_products import MODULE as _PMOD, pd_fields
|
| 54 |
+
from registry_source import MetadataStore
|
| 55 |
|
| 56 |
+
module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(session.runtime))
|
| 57 |
kw.setdefault("table", pd.table_ops(session.runtime))
|
| 58 |
hidden = perm_scope.hidden_keys(session.user, module,
|
| 59 |
canonical if permission_fields is None else permission_fields,
|
|
|
|
| 1225 |
return out
|
| 1226 |
|
| 1227 |
|
| 1228 |
+
def _builtin_activity_metadata(session, scope, *, st):
|
| 1229 |
+
"""Return detached visible fields, complete permission fields, hidden keys and no row identities for one Activity page."""
|
| 1230 |
+
from copy import copy, deepcopy
|
| 1231 |
+
import aios_grid
|
| 1232 |
+
import odoo_relational
|
| 1233 |
+
from core import perm_scope, route_system, schema_overlay
|
| 1234 |
+
from modules import cohort
|
| 1235 |
+
from routes_customers import _AssemblyStore, _merge_shared_fields, shared_fields
|
| 1236 |
+
|
| 1237 |
+
if scope not in ("customer", "product"):
|
| 1238 |
+
raise err(400, "bad_scope", "Activity requires a customer or product database.")
|
| 1239 |
+
reader = st if isinstance(st, _AssemblyStore) and st.schema_only else _AssemblyStore(st, schema_only=True)
|
| 1240 |
+
borrowed = copy(session)
|
| 1241 |
+
borrowed.runtime = reader
|
| 1242 |
+
module = "product_data" if scope == "product" else MODULE
|
| 1243 |
+
borrowed.require(module)
|
| 1244 |
+
if scope == "product":
|
| 1245 |
+
from routes_products import scoped_pool
|
| 1246 |
+
import modules.product_data as pd
|
| 1247 |
+
_, _, _, contract = scoped_pool(borrowed, with_rows=False)
|
| 1248 |
+
definitions = pd.shared_fields(st=reader)
|
| 1249 |
+
table_key = pd.TABLE_KEY
|
| 1250 |
+
else:
|
| 1251 |
+
contract = list(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE)
|
| 1252 |
+
definitions = shared_fields(st=reader)
|
| 1253 |
+
table_key = "customer_table_workspace"
|
| 1254 |
+
|
| 1255 |
+
if not reader.available():
|
| 1256 |
+
from core.grid_events import StoreUnavailable
|
| 1257 |
+
raise StoreUnavailable("Activity source fields require an available workspace.")
|
| 1258 |
+
document = reader.get(table_key) or {}
|
| 1259 |
+
personal = (document.get(session.uname) or {}).get("fields") or {}
|
| 1260 |
+
has_cohorts = bool(cohort.scoped(scope, st=reader).all_for(session.uname))
|
| 1261 |
+
fields = aios_grid.fields_from_workspace({"fields": personal}, cohorts=has_cohorts,
|
| 1262 |
+
scope_key=scope, fields_base=contract)
|
| 1263 |
+
if scope == "customer":
|
| 1264 |
+
# Shared definitions supersede personal collisions before the dependent-field wall.
|
| 1265 |
+
fields = _merge_shared_fields(fields, definitions, session=borrowed)
|
| 1266 |
+
else:
|
| 1267 |
+
fields = aios_grid.apply_choice_overrides(fields, definitions)
|
| 1268 |
+
from core import shares
|
| 1269 |
+
for field in fields:
|
| 1270 |
+
saved = definitions.get(field.get("key"))
|
| 1271 |
+
if isinstance(saved, dict) and saved.get("granted") is True:
|
| 1272 |
+
field["granted"] = True
|
| 1273 |
+
if isinstance(saved.get("createdBy"), str):
|
| 1274 |
+
field["createdBy"] = saved["createdBy"]
|
| 1275 |
+
have = {field["key"] for field in fields}
|
| 1276 |
+
fields += [dict(field, source="overlay", shared=True)
|
| 1277 |
+
for key, field in definitions.items() if key not in have]
|
| 1278 |
+
for field in fields:
|
| 1279 |
+
if field.get("granted") is True:
|
| 1280 |
+
role = shares.role_for("field", shares.field_oid(module, field["key"]),
|
| 1281 |
+
session.uname, is_admin=session.admin, st=reader)
|
| 1282 |
+
if role:
|
| 1283 |
+
field["sharedRole"] = role
|
| 1284 |
+
|
| 1285 |
+
# Keep projection field identities without loading Route/Zone objects or membership cells.
|
| 1286 |
+
projection = route_system.customer_projection_fields([], [])
|
| 1287 |
+
if scope == "product":
|
| 1288 |
+
projection = [field for field in projection if field["key"] == route_system.ZONE_FIELD_KEY]
|
| 1289 |
+
projected = {field["key"] for field in projection}
|
| 1290 |
+
fields = [field for field in fields if field["key"] not in projected] + projection
|
| 1291 |
+
corpus = deepcopy(fields)
|
| 1292 |
+
hidden = perm_scope.hidden_keys(session.user, module, corpus, st=reader)
|
| 1293 |
+
fields = [field for field in fields if field["key"] not in hidden]
|
| 1294 |
+
wall_contract = contract if scope == "product" else aios_grid.FIELDS
|
| 1295 |
+
fields, excluded = schema_overlay.for_grid(module, fields, st=reader, contract=wall_contract)
|
| 1296 |
+
return {"fields": deepcopy(fields), "pids": frozenset(),
|
| 1297 |
+
"defn": {"fields": corpus}, "hidden": hidden | excluded,
|
| 1298 |
+
"_permission_scope": {"row_fields": deepcopy(wall_contract)}}
|
| 1299 |
+
|
| 1300 |
+
|
| 1301 |
+
def _builtin_activity_pids(session, database, wanted, metadata, *, st):
|
| 1302 |
+
"""Authorize only requested builtin identities through the ordinary pool and permanent row walls."""
|
| 1303 |
+
from core import perm_scope
|
| 1304 |
+
team_id, agent = perm_scope.derive_pool_scope(session.user, database)
|
| 1305 |
+
if database == "customer_data":
|
| 1306 |
+
from routes_customers import _pool_for
|
| 1307 |
+
rows = _pool_for(st, team_id, agent)
|
| 1308 |
+
elif database == "product_data":
|
| 1309 |
+
from routes_products import _pool_for
|
| 1310 |
+
rows = _pool_for(st, team_id)
|
| 1311 |
+
else:
|
| 1312 |
+
raise err(400, "bad_scope", "Activity requires a customer or product database.")
|
| 1313 |
+
rows = [row for row in rows if str(row.get("pid")) in wanted]
|
| 1314 |
+
# The row wall needs real shared cells, never a schema projection with missing values.
|
| 1315 |
+
rows = perm_scope.apply_row_scope(rows, session.user, database,
|
| 1316 |
+
metadata["_permission_scope"]["row_fields"], st=session.runtime)
|
| 1317 |
+
return frozenset(row["pid"] for row in rows)
|
| 1318 |
+
|
| 1319 |
+
|
| 1320 |
def _builtin_history_context(session, scope, *, restoring=False):
|
| 1321 |
"""Resolve history through the same assembly and write context as grid events."""
|
| 1322 |
from routes_customers import grid_assembly
|
api/routes_odoo_tables.py
CHANGED
|
@@ -289,8 +289,7 @@ def odoo_tables_status(brief: bool = Query(False),
|
|
| 289 |
from harness import datastore as _ds
|
| 290 |
frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
|
| 291 |
else {"table": spec["table"]})
|
| 292 |
-
n = _ds.
|
| 293 |
-
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 294 |
else:
|
| 295 |
n = -1 # unknown, and never reported as zero
|
| 296 |
mirror_refreshed = mirror_stamp(
|
|
@@ -652,7 +651,9 @@ def _sync_entities(cur, cadence_seconds):
|
|
| 652 |
rows = cur.execute(
|
| 653 |
"SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
|
| 654 |
"FROM _sync_state ORDER BY entity").fetchall()
|
| 655 |
-
except Exception:
|
|
|
|
|
|
|
| 656 |
return None
|
| 657 |
out = {}
|
| 658 |
for entity, phase, last_id, cursor_wd, nrows, updated_at in rows:
|
|
@@ -1325,7 +1326,9 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
|
|
| 1325 |
if (spec is not None or definition.get("readThrough")
|
| 1326 |
or table_key in amazon.READ_THROUGH):
|
| 1327 |
# Check current table presence even if a binding was cached before a restart.
|
| 1328 |
-
current =
|
|
|
|
|
|
|
| 1329 |
if current.get("missing"):
|
| 1330 |
raise err(503, "amazon_mirror_not_loaded", current["missing"])
|
| 1331 |
# A dynamically oversized snapshot may have a durable stub before this process
|
|
@@ -1512,8 +1515,7 @@ def _unmaterialisable(cur=None):
|
|
| 1512 |
try:
|
| 1513 |
frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
|
| 1514 |
else {"table": spec["table"]})
|
| 1515 |
-
total = datastore.
|
| 1516 |
-
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
|
| 1517 |
except Exception: # noqa: BLE001
|
| 1518 |
total = None
|
| 1519 |
if total is not None and total > max_rows:
|
|
@@ -1600,12 +1602,9 @@ def _source_for(cur, table_key):
|
|
| 1600 |
if key.startswith("ut_amzn_"):
|
| 1601 |
import amazon_relational as amazon
|
| 1602 |
if key in amazon.SOURCE:
|
| 1603 |
-
|
| 1604 |
-
|
| 1605 |
-
|
| 1606 |
-
for lookup in amazon.lookups(cur, key)}
|
| 1607 |
-
spec = {**spec, "cols": {**{k: v for k, v in spec["cols"].items()
|
| 1608 |
-
if k not in amazon.LOOKUP_KEYS}, **derived}}
|
| 1609 |
from harness import datastore
|
| 1610 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1611 |
try:
|
|
@@ -1658,9 +1657,26 @@ def _source_for(cur, table_key):
|
|
| 1658 |
for key, (expr, cast) in spec["cols"].items():
|
| 1659 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1660 |
have_by_alias), cast)
|
|
|
|
|
|
|
| 1661 |
return {**spec, "cols": cols}
|
| 1662 |
|
| 1663 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1664 |
#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
|
| 1665 |
#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of
|
| 1666 |
#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of
|
|
@@ -1874,7 +1890,8 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 1874 |
if table_key in trail:
|
| 1875 |
raise ValueError("a relation has a circular access rule")
|
| 1876 |
trail = (*trail, table_key)
|
| 1877 |
-
handle = _ut().
|
|
|
|
| 1878 |
declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
|
| 1879 |
columns = _filter_columns(spec, fields)
|
| 1880 |
guards, resolved, targets = [], {}, {}
|
|
@@ -1895,7 +1912,7 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 1895 |
targets[key] = None
|
| 1896 |
return None
|
| 1897 |
definition = _ut().get(key, st=handle)
|
| 1898 |
-
target_spec =
|
| 1899 |
if target_spec is None and str(key).startswith("ut_amzn_"):
|
| 1900 |
# Small Amazon targets (notably listings) are stored snapshots, but
|
| 1901 |
# their raw mirror still answers the same automatic link in SQL.
|
|
@@ -2375,7 +2392,12 @@ def _related_source_value(session, entry, record):
|
|
| 2375 |
if not source:
|
| 2376 |
return None
|
| 2377 |
try:
|
| 2378 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2379 |
except Exception: # noqa: BLE001
|
| 2380 |
return None
|
| 2381 |
want = str(record)
|
|
@@ -2425,11 +2447,17 @@ _UNREAD = ...
|
|
| 2425 |
|
| 2426 |
|
| 2427 |
def _user_tables_token(rt):
|
| 2428 |
-
"""
|
| 2429 |
-
|
| 2430 |
-
|
| 2431 |
-
except Exception: # noqa: BLE001
|
| 2432 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2433 |
|
| 2434 |
|
| 2435 |
def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
|
@@ -2463,17 +2491,10 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
|
| 2463 |
|
| 2464 |
|
| 2465 |
def _page_defs(session):
|
| 2466 |
-
"""
|
| 2467 |
-
|
| 2468 |
-
|
| 2469 |
-
|
| 2470 |
-
target does not pay a second revision round trip.
|
| 2471 |
-
The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
|
| 2472 |
-
`user_tables` revision instead of once per question. Kept only when every definition really is
|
| 2473 |
-
a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
|
| 2474 |
-
28 MB document on a failed read and that must never be held, let alone copied per request.
|
| 2475 |
-
⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
|
| 2476 |
-
lends lazily live for this one request and are never carried to the next.
|
| 2477 |
"""
|
| 2478 |
import copy
|
| 2479 |
import core.store as store
|
|
@@ -2483,12 +2504,20 @@ def _page_defs(session):
|
|
| 2483 |
token = _user_tables_token(rt)
|
| 2484 |
|
| 2485 |
def keep(v):
|
| 2486 |
-
|
|
|
|
| 2487 |
return bool(tables) and all(store.is_projected(d) for d in tables)
|
| 2488 |
|
| 2489 |
-
|
| 2490 |
-
|
| 2491 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2492 |
|
| 2493 |
|
| 2494 |
def _stored_link_rows(session, table, token=_UNREAD):
|
|
@@ -2528,8 +2557,17 @@ def _traversal_defn_or_refuse(session, table_key, st=None):
|
|
| 2528 |
return defn
|
| 2529 |
|
| 2530 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2531 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2532 |
-
traversal_of=None, st=None, rev=..., normalize_match=False
|
|
|
|
| 2533 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2534 |
|
| 2535 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
@@ -2548,8 +2586,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2548 |
|
| 2549 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 2550 |
if not _materialised_table(table):
|
| 2551 |
-
read =
|
| 2552 |
-
|
|
|
|
| 2553 |
spec = read["spec"]
|
| 2554 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 2555 |
if columns - set(bindings):
|
|
@@ -2560,10 +2599,17 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2560 |
if values is None:
|
| 2561 |
continue
|
| 2562 |
expr, cast = bindings[key]
|
| 2563 |
-
values = list(values)
|
| 2564 |
if normalize_match and key != "pid":
|
| 2565 |
expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
|
| 2566 |
cast = lambda value: str(value).strip().lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2567 |
parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
|
| 2568 |
params.extend(cast(value) for value in values)
|
| 2569 |
if matches is not None:
|
|
@@ -2571,6 +2617,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2571 |
for key, values in matches.items():
|
| 2572 |
expr, cast = bindings[key]
|
| 2573 |
values = sorted(values)
|
|
|
|
|
|
|
|
|
|
| 2574 |
if values:
|
| 2575 |
alternatives.append(f"{expr} IN ({','.join('?' for _ in values)})")
|
| 2576 |
params.extend(cast(value) for value in values)
|
|
@@ -2579,8 +2628,23 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2579 |
projection = ", ".join(bindings[key][0] for key in names)
|
| 2580 |
source = spec.get("from_sql") or spec["table"]
|
| 2581 |
predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
|
| 2582 |
-
|
| 2583 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2584 |
|
| 2585 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 2586 |
# complete contract before taking any join value, including a join on a hidden column.
|
|
@@ -2615,8 +2679,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2615 |
wanted = {normalize(value) for value in values}
|
| 2616 |
rows = [row for row in rows if normalize(row.get(key)) in wanted]
|
| 2617 |
if matches is not None:
|
| 2618 |
-
|
| 2619 |
-
|
|
|
|
| 2620 |
for key, values in wanted.items())]
|
| 2621 |
return [{key: row.get(key) for key in columns} for row in rows]
|
| 2622 |
|
|
@@ -2627,7 +2692,48 @@ def _relation_cell_ids(value):
|
|
| 2627 |
return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
|
| 2628 |
|
| 2629 |
|
| 2630 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2631 |
"""Follow one declared leg, including manual, inverse and through links."""
|
| 2632 |
import core.relations as relations
|
| 2633 |
|
|
@@ -2645,6 +2751,9 @@ def _relation_leg_ids(session, source, source_ids, entry):
|
|
| 2645 |
source_col, target_col = ("pid", field) if reverse else (field, "pid")
|
| 2646 |
elif inverse:
|
| 2647 |
source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
|
|
|
|
|
|
|
|
|
|
| 2648 |
source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
|
| 2649 |
values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
|
| 2650 |
# A column-keyed edge cannot fall back to the wire record id. In particular,
|
|
@@ -2669,11 +2778,11 @@ def _relation_leg_ids(session, source, source_ids, entry):
|
|
| 2669 |
traversal_of=str(entry.get("key") or via.get("key") or ""))
|
| 2670 |
values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
|
| 2671 |
if (manual and reverse) or (inverse and not reverse):
|
| 2672 |
-
rows = _relation_rows(session, target, [target_col])
|
| 2673 |
-
rows =
|
| 2674 |
else:
|
| 2675 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2676 |
-
where=entry.get("where"))
|
| 2677 |
return {int(row["pid"]) for row in rows}
|
| 2678 |
|
| 2679 |
|
|
@@ -2686,7 +2795,7 @@ def _relation_label(key):
|
|
| 2686 |
if table == key and label), str(key))
|
| 2687 |
|
| 2688 |
|
| 2689 |
-
def _relation_target_ids(session, target, related):
|
| 2690 |
"""One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
|
| 2691 |
import core.relations as relations
|
| 2692 |
from routes_grid import _may_open, related_path_entries
|
|
@@ -2721,11 +2830,13 @@ def _relation_target_ids(session, target, related):
|
|
| 2721 |
if not _may_open(session, leg["target"]):
|
| 2722 |
raise err(403, "forbidden_table",
|
| 2723 |
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
|
| 2724 |
-
|
|
|
|
|
|
|
| 2725 |
# An earlier, authorised Amazon hop matched nothing. There is no next
|
| 2726 |
# source record to read, and that is an empty relation, not a missing key.
|
| 2727 |
continue
|
| 2728 |
-
ids = _relation_leg_ids(session, table, ids, leg)
|
| 2729 |
return ids
|
| 2730 |
|
| 2731 |
|
|
@@ -2961,7 +3072,9 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 2961 |
related = _json_arg(related, "related")
|
| 2962 |
if related in (None, "", {}):
|
| 2963 |
return None, [], []
|
| 2964 |
-
ids = _relation_target_ids(session, table_key, related)
|
|
|
|
|
|
|
| 2965 |
sql, params = relations.pin_sql(spec["id"], sorted(ids))
|
| 2966 |
notes = []
|
| 2967 |
if len(ids) > _PIN_REPORT_AT:
|
|
@@ -3020,6 +3133,119 @@ def _active_cohort_leaves(nodes):
|
|
| 3020 |
return out
|
| 3021 |
|
| 3022 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3023 |
def _view_store(session, table_key):
|
| 3024 |
"""The workspace document a saved view of THIS database lives in.
|
| 3025 |
|
|
@@ -3028,7 +3254,43 @@ def _view_store(session, table_key):
|
|
| 3028 |
to fail it: the client reads the views `/workspace` served out of this same document.
|
| 3029 |
"""
|
| 3030 |
import core.table_store as table_store
|
| 3031 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3032 |
|
| 3033 |
|
| 3034 |
def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits):
|
|
@@ -3052,8 +3314,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3052 |
* no filters and a curated `memberPids` is the client's `else if (curated)` branch: the
|
| 3053 |
membership IS the view;
|
| 3054 |
* no filters and no members means the view holds every row in scope;
|
| 3055 |
-
*
|
| 3056 |
-
way `_related_pin` does it, so the wall rides along and standing rule 1 is kept.
|
| 3057 |
|
| 3058 |
THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY:
|
| 3059 |
resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets`
|
|
@@ -3082,6 +3343,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3082 |
|
| 3083 |
store = _view_store(session, table_key)
|
| 3084 |
sets, memo, unresolved = {}, {}, []
|
|
|
|
| 3085 |
counter = [0]
|
| 3086 |
|
| 3087 |
def key_for():
|
|
@@ -3092,10 +3354,11 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3092 |
return "__view_%d__" % counter[0]
|
| 3093 |
|
| 3094 |
def resolve(vid, seen):
|
| 3095 |
-
"""
|
| 3096 |
if vid in memo:
|
| 3097 |
return memo[vid]
|
| 3098 |
if vid in seen:
|
|
|
|
| 3099 |
# `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left
|
| 3100 |
# ABSENT, so a stored loop degrades to an explained empty table, not a hung request.
|
| 3101 |
return None, "it refers to itself through another saved view"
|
|
@@ -3115,12 +3378,24 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3115 |
"own records rather than from this database's live connection")
|
| 3116 |
memo[vid] = got
|
| 3117 |
return got
|
| 3118 |
-
|
| 3119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3120 |
try:
|
| 3121 |
-
|
| 3122 |
-
|
| 3123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3124 |
raw_nodes = cfg.get("filters") or []
|
| 3125 |
if not isinstance(raw_nodes, list):
|
| 3126 |
raw_nodes = []
|
|
@@ -3129,10 +3404,14 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3129 |
# "anything active": with no nodes at all the curated set IS the view, and with nodes
|
| 3130 |
# that all happen to be half-built the view holds everything. Two different answers
|
| 3131 |
# from one list, so the emptiness is checked before the compiler sees it.
|
| 3132 |
-
got = ("ids", members) if
|
| 3133 |
memo[vid] = got
|
| 3134 |
return got
|
| 3135 |
nodes = rewrite(raw_nodes, seen + (vid,))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3136 |
unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols))
|
| 3137 |
if unknown:
|
| 3138 |
# ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it
|
|
@@ -3146,7 +3425,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3146 |
try:
|
| 3147 |
pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or"
|
| 3148 |
else "and"),
|
| 3149 |
-
columns=cols, member_ids=(members
|
| 3150 |
id_sql=spec["id"], today=_today(), cohort_sets=sets)
|
| 3151 |
except ValueError as exc:
|
| 3152 |
got = (None, "it uses a condition this database's live connection cannot evaluate ("
|
|
@@ -3167,9 +3446,9 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3167 |
return got
|
| 3168 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 3169 |
where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql
|
| 3170 |
-
ids =
|
| 3171 |
f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}',
|
| 3172 |
-
list(wall_params) + list(pred.params))
|
| 3173 |
got = ("ids", ids)
|
| 3174 |
memo[vid] = got
|
| 3175 |
return got
|
|
@@ -3200,7 +3479,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3200 |
if kind == "ids":
|
| 3201 |
sets[key] = got
|
| 3202 |
op = "noneOf" if not_in else "anyOf"
|
| 3203 |
-
if len(got) >= _PIN_REPORT_AT:
|
| 3204 |
limits.append({
|
| 3205 |
"subject": "view", "effect": "slow",
|
| 3206 |
"cause": f"the saved view this condition names holds {len(got):,} rows, "
|
|
@@ -3220,7 +3499,12 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
|
|
| 3220 |
out.append({"colId": "__cohort__", "op": op, "value": key})
|
| 3221 |
return out
|
| 3222 |
|
| 3223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3224 |
if unresolved:
|
| 3225 |
limits.append({
|
| 3226 |
"subject": ", ".join(vid for vid, _why in unresolved),
|
|
@@ -3240,9 +3524,9 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
|
|
| 3240 |
`__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
|
| 3241 |
|
| 3242 |
A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
|
| 3243 |
-
here
|
| 3244 |
-
|
| 3245 |
-
|
| 3246 |
"""
|
| 3247 |
fe = _fe()
|
| 3248 |
leaves = fe._rank_leaves(tree or [], cols)
|
|
@@ -3270,30 +3554,8 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
|
|
| 3270 |
params.extend(pred.params)
|
| 3271 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 3272 |
answers = {}
|
| 3273 |
-
python_leaves = []
|
| 3274 |
for n in leaves:
|
| 3275 |
-
|
| 3276 |
-
if got is None:
|
| 3277 |
-
python_leaves.append(n)
|
| 3278 |
-
else:
|
| 3279 |
-
answers[id(n)] = got
|
| 3280 |
-
if python_leaves:
|
| 3281 |
-
# A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and
|
| 3282 |
-
# the ranked columns once, uncapped, and let `resolve_rank_sets` answer it.
|
| 3283 |
-
ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols})
|
| 3284 |
-
select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked])
|
| 3285 |
-
sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "")
|
| 3286 |
-
rows = []
|
| 3287 |
-
for r in cur.execute(sql, params).fetchall():
|
| 3288 |
-
if not r or r[0] is None:
|
| 3289 |
-
continue
|
| 3290 |
-
row = {"pid": int(r[0])}
|
| 3291 |
-
for i, c in enumerate(ranked, start=1):
|
| 3292 |
-
row[c] = r[i]
|
| 3293 |
-
rows.append(row)
|
| 3294 |
-
resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols)
|
| 3295 |
-
for n in python_leaves:
|
| 3296 |
-
answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ())
|
| 3297 |
sets = dict(cohort_sets or {})
|
| 3298 |
counter = [0]
|
| 3299 |
wanted = {id(n) for n in leaves}
|
|
@@ -3322,66 +3584,101 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
|
|
| 3322 |
out.append(n)
|
| 3323 |
return out
|
| 3324 |
|
| 3325 |
-
|
|
|
|
|
|
|
|
|
|
| 3326 |
|
| 3327 |
|
| 3328 |
def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
|
| 3329 |
-
"""
|
| 3330 |
-
|
| 3331 |
-
Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a
|
| 3332 |
-
paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds,
|
| 3333 |
-
and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line
|
| 3334 |
-
for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values
|
| 3335 |
-
are ABSENT, ties break by ascending id in both directions, a bound outside the builder's
|
| 3336 |
-
whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than
|
| 3337 |
-
buckets is empty. The domain's count and mean are read once here and inlined as literals.
|
| 3338 |
-
"""
|
| 3339 |
fe = _fe()
|
| 3340 |
Sql = _fs().SqlMembership
|
| 3341 |
empty = Sql("SELECT NULL WHERE FALSE", [])
|
| 3342 |
-
scope,
|
| 3343 |
-
if scope
|
| 3344 |
-
return
|
| 3345 |
col = cols.get(leaf.get("colId")) or {}
|
| 3346 |
if col.get("type") not in fe.NUMERIC_TYPES:
|
| 3347 |
return empty
|
| 3348 |
v = f"TRY_CAST({col['sql']} AS DOUBLE)"
|
| 3349 |
-
|
| 3350 |
-
|
|
|
|
|
|
|
| 3351 |
params = list(params)
|
| 3352 |
op = leaf.get("op")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3353 |
if op in ("topN", "bottomN"):
|
| 3354 |
bound = fe._rank_bound(leaf.get("value"), 1, 10000)
|
| 3355 |
if bound is None:
|
| 3356 |
return empty
|
| 3357 |
-
|
| 3358 |
-
|
| 3359 |
-
params)
|
| 3360 |
-
count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone()
|
| 3361 |
-
count = int(count or 0)
|
| 3362 |
-
if not count:
|
| 3363 |
-
return empty
|
| 3364 |
-
if op in ("inTopPct", "inBottomPct"):
|
| 3365 |
pct = fe._rank_bound(leaf.get("value"), 1, 100)
|
| 3366 |
if pct is None:
|
| 3367 |
return empty
|
| 3368 |
-
|
| 3369 |
-
|
| 3370 |
-
return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}",
|
| 3371 |
-
params)
|
| 3372 |
-
if op in ("aboveAvg", "belowAvg"):
|
| 3373 |
-
mean = float(total) / count
|
| 3374 |
-
cmp = ">" if op == "aboveAvg" else "<"
|
| 3375 |
-
return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params)
|
| 3376 |
-
if op in ("inQuartile", "inDecile"):
|
| 3377 |
buckets = 4 if op == "inQuartile" else 10
|
| 3378 |
wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
|
| 3379 |
-
if wanted is None
|
| 3380 |
return empty
|
| 3381 |
-
|
| 3382 |
-
|
| 3383 |
-
|
| 3384 |
-
|
|
|
|
|
|
|
|
|
|
| 3385 |
|
| 3386 |
def _may_see_view(session, vid, store):
|
| 3387 |
"""May this caller open that saved view? The same predicate the view rail asks.
|
|
@@ -3509,7 +3806,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3509 |
if mismatch:
|
| 3510 |
raise err(409, "cross_tenant_store", mismatch)
|
| 3511 |
|
| 3512 |
-
spec =
|
| 3513 |
if spec is None:
|
| 3514 |
from core import perm_scope
|
| 3515 |
rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
|
|
@@ -3771,39 +4068,96 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3771 |
|
| 3772 |
|
| 3773 |
@contextmanager
|
| 3774 |
-
def _materialised_read(table_key, session, query, related, ids=None, *, payload=None):
|
| 3775 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 3776 |
|
| 3777 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
| 3778 |
closed before returning. It never registers data on the tenant's shared mirror connection.
|
| 3779 |
"""
|
| 3780 |
import duckdb
|
|
|
|
|
|
|
| 3781 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3782 |
if payload is not None:
|
| 3783 |
pass
|
| 3784 |
-
elif
|
| 3785 |
-
|
| 3786 |
-
session.require(table_key)
|
| 3787 |
-
payload = _payload(session, related=related)
|
| 3788 |
-
elif table_key == "product_data":
|
| 3789 |
-
from routes_products import products
|
| 3790 |
session.require(table_key)
|
| 3791 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3792 |
else:
|
| 3793 |
from routes_tables import table_rows
|
| 3794 |
payload = table_rows(table_key, session=session, related=related)
|
| 3795 |
-
if ids is not None:
|
| 3796 |
-
wanted = set(ids)
|
| 3797 |
-
payload = {**payload, "rows": [row for row in payload["rows"]
|
| 3798 |
-
if int(row["pid"]) in wanted]}
|
| 3799 |
fields = payload["fields"]
|
| 3800 |
-
|
| 3801 |
-
|
| 3802 |
-
|
| 3803 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3804 |
bindings = {}
|
| 3805 |
for field in fields:
|
| 3806 |
key = str(field["key"])
|
|
|
|
|
|
|
| 3807 |
pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
|
| 3808 |
expr = f"json_extract_string(payload, '{pointer}')"
|
| 3809 |
numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
|
|
@@ -3836,7 +4190,10 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
|
|
| 3836 |
where = f"({where}) AND {search.sql}" if where else search.sql
|
| 3837 |
params.extend(search.params)
|
| 3838 |
yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
|
| 3839 |
-
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload
|
|
|
|
|
|
|
|
|
|
| 3840 |
|
| 3841 |
|
| 3842 |
def _group_pin(cols, group_by, group_key):
|
|
@@ -3917,7 +4274,17 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
|
| 3917 |
'totalUnfiltered': len(payload['rows']), 'offset': offset, 'limit': limit,
|
| 3918 |
'scope': {'table': table_key, 'readThrough': True}}
|
| 3919 |
window_offset = 0
|
| 3920 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3921 |
spec = read["spec"]
|
| 3922 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 3923 |
where, params = read["where"], list(read["params"])
|
|
@@ -3929,15 +4296,33 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
|
| 3929 |
params=tuple(params), order_by=order or spec["id"],
|
| 3930 |
offset=window_offset, limit=limit or None, cur=read["cur"], **read["frm"])
|
| 3931 |
payload = read["payload"]
|
| 3932 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3933 |
if win["clamped"]:
|
| 3934 |
read["limits"].append({"subject": "limit", "effect": "window_clamped",
|
| 3935 |
"cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
|
| 3936 |
"recommendation": "Continue paging with offset. Every matching row remains available."})
|
|
|
|
|
|
|
| 3937 |
return {**payload, "ok": True, "rows": rows, "total": win["total"],
|
| 3938 |
-
"totalUnfiltered":
|
| 3939 |
"offset": offset if table_key == 'ut_activity' else win["offset"], "limit": win["limit"],
|
| 3940 |
-
"limits":
|
| 3941 |
|
| 3942 |
|
| 3943 |
def _activity_window_payload(session, offset, limit, related):
|
|
@@ -4002,7 +4387,7 @@ def odoo_table_rows(table_key: str,
|
|
| 4002 |
that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in
|
| 4003 |
`limits` with its cause and a recommendation — R6's second sentence — and never silently
|
| 4004 |
dropped, because an ignored condition widens.
|
| 4005 |
-
|
| 4006 |
⭐⭐ `related` (W51-T05 / C6) IS A **JSON QUERY STRING**, copying the `filters` precedent one
|
| 4007 |
argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
|
| 4008 |
window to the rows linked to that ONE record, ahead of `filters`, and `total` /
|
|
@@ -4068,7 +4453,8 @@ def odoo_table_rows(table_key: str,
|
|
| 4068 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 4069 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 4070 |
where=where, params=tuple(params), order_by=order,
|
| 4071 |
-
offset=offset, limit=(limit or None), cur=cur, page_first=True,
|
|
|
|
| 4072 |
if win["clamped"]:
|
| 4073 |
limits.append({
|
| 4074 |
"subject": "limit", "effect": "window_clamped",
|
|
@@ -4130,16 +4516,14 @@ def odoo_table_rows(table_key: str,
|
|
| 4130 |
# the wall and the predicate, so the argument it demands is the list we just fetched.
|
| 4131 |
# ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
|
| 4132 |
# answered "may this session open this surface".
|
| 4133 |
-
#
|
| 4134 |
-
# (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
|
| 4135 |
-
# Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
|
| 4136 |
shared_defs = _so().fields(table_key, st=page_st)
|
| 4137 |
if shared_defs:
|
| 4138 |
declared = {field["key"]: field for field in fields}
|
| 4139 |
declared.update({key: dict(field, source="overlay") for key, field in shared_defs.items()})
|
| 4140 |
fields = list(declared.values())
|
| 4141 |
-
|
| 4142 |
-
|
| 4143 |
overlays.setdefault(pid, {}).update(cells)
|
| 4144 |
if_keys = {field["key"] for field in fields if field.get("type") == "if"}
|
| 4145 |
for row in rows_src:
|
|
@@ -4162,9 +4546,8 @@ def odoo_table_rows(table_key: str,
|
|
| 4162 |
for pid, cells in overlays.items()}
|
| 4163 |
rows = aios_grid.rows_from_pool(rows_src, fields, overlays)
|
| 4164 |
|
| 4165 |
-
unfiltered = win["total"] if where == base_where else datastore.
|
| 4166 |
-
|
| 4167 |
-
limit=1, cur=cur, **frm)["total"]
|
| 4168 |
|
| 4169 |
from core import connector_spec
|
| 4170 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
|
@@ -4187,7 +4570,7 @@ def odoo_table_rows(table_key: str,
|
|
| 4187 |
# ══════════════════════════════════════════════════════════
|
| 4188 |
|
| 4189 |
#: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers
|
| 4190 |
-
#: that omit groupLimit receive
|
| 4191 |
MAX_GROUPS = 1_000
|
| 4192 |
|
| 4193 |
#: One request asks for at most this many summaries. Each is one more aggregate expression in a
|
|
@@ -4195,10 +4578,46 @@ MAX_GROUPS = 1_000
|
|
| 4195 |
MAX_AGGREGATES = 200
|
| 4196 |
|
| 4197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4198 |
def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
| 4199 |
-
"""Resolve
|
| 4200 |
from routes_grid import _may_open
|
| 4201 |
declared = {field["key"] for field in fields}
|
|
|
|
| 4202 |
for field in fields:
|
| 4203 |
bag = field.get("link") or {}
|
| 4204 |
if field.get("type") != "link" or not bag.get("on") or not bag.get("from"):
|
|
@@ -4206,19 +4625,38 @@ def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
|
| 4206 |
target, on, source = bag["table"], bag["on"], bag["from"]
|
| 4207 |
if _rel().base_table_key(target) not in _rel().READ_THROUGH_KEYS:
|
| 4208 |
continue
|
| 4209 |
-
|
| 4210 |
-
|
| 4211 |
-
|
| 4212 |
-
|
| 4213 |
-
|
| 4214 |
-
|
| 4215 |
-
|
| 4216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4217 |
for row in rows:
|
| 4218 |
-
|
| 4219 |
-
|
| 4220 |
-
|
| 4221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4222 |
|
| 4223 |
|
| 4224 |
def _agg_number(value):
|
|
@@ -4410,15 +4848,10 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 4410 |
f"THEN 1 ELSE 0 END"),
|
| 4411 |
where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
|
| 4412 |
projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
|
| 4413 |
-
if group_limit is not None
|
| 4414 |
-
|
| 4415 |
-
|
| 4416 |
-
|
| 4417 |
-
else:
|
| 4418 |
-
source = frm.get("from_sql") or frm["table"]
|
| 4419 |
-
tail = f" WHERE {where}" if where else ""
|
| 4420 |
-
rows = cur.execute(f"SELECT {projection} FROM {source}{tail} GROUP BY {gexpr} "
|
| 4421 |
-
"ORDER BY 2 DESC, 1 ASC", tuple(params)).fetchall()
|
| 4422 |
groups = []
|
| 4423 |
for r in rows:
|
| 4424 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
|
@@ -4427,7 +4860,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
|
|
| 4427 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 4428 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 4429 |
"total_groups": distinct, "group_offset": group_offset,
|
| 4430 |
-
"group_limit": group_limit, "
|
|
|
|
| 4431 |
|
| 4432 |
|
| 4433 |
@router.post("/odoo-tables/{table_key}/aggregates")
|
|
@@ -4476,8 +4910,6 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4476 |
raise err(400, "bad_selection", "groupOffset must be a nonnegative integer")
|
| 4477 |
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
|
| 4478 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 4479 |
-
if group_offset and group_limit is None:
|
| 4480 |
-
raise err(400, "bad_selection", "Send groupLimit when paging groups")
|
| 4481 |
|
| 4482 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4483 |
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
|
@@ -4533,7 +4965,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4533 |
"limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
|
| 4534 |
if out["groups"] is not None:
|
| 4535 |
body_out["groups"] = out["groups"]
|
| 4536 |
-
body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit")})
|
| 4537 |
return body_out
|
| 4538 |
|
| 4539 |
|
|
@@ -4663,8 +5095,7 @@ def population(table_key, cur=None, rt=None):
|
|
| 4663 |
if spec is None:
|
| 4664 |
return None
|
| 4665 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 4666 |
-
return datastore.
|
| 4667 |
-
limit=1, cur=cur, **frm)["total"]
|
| 4668 |
|
| 4669 |
|
| 4670 |
def whole_pool(table_key, cur=None, rt=None):
|
|
|
|
| 289 |
from harness import datastore as _ds
|
| 290 |
frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
|
| 291 |
else {"table": spec["table"]})
|
| 292 |
+
n = _ds.count_rows(where=spec.get("where") or "", cur=cur, **frm)
|
|
|
|
| 293 |
else:
|
| 294 |
n = -1 # unknown, and never reported as zero
|
| 295 |
mirror_refreshed = mirror_stamp(
|
|
|
|
| 651 |
rows = cur.execute(
|
| 652 |
"SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
|
| 653 |
"FROM _sync_state ORDER BY entity").fetchall()
|
| 654 |
+
except Exception as exc: # noqa: BLE001
|
| 655 |
+
from core.request_timing import raise_if_resource_error
|
| 656 |
+
raise_if_resource_error(exc)
|
| 657 |
return None
|
| 658 |
out = {}
|
| 659 |
for entity, phase, last_id, cursor_wd, nrows, updated_at in rows:
|
|
|
|
| 1326 |
if (spec is not None or definition.get("readThrough")
|
| 1327 |
or table_key in amazon.READ_THROUGH):
|
| 1328 |
# Check current table presence even if a binding was cached before a restart.
|
| 1329 |
+
current = _page_source_for(cur, table_key, st)
|
| 1330 |
+
if current is None:
|
| 1331 |
+
current = amazon.grid_source(cur, table_key)
|
| 1332 |
if current.get("missing"):
|
| 1333 |
raise err(503, "amazon_mirror_not_loaded", current["missing"])
|
| 1334 |
# A dynamically oversized snapshot may have a durable stub before this process
|
|
|
|
| 1515 |
try:
|
| 1516 |
frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
|
| 1517 |
else {"table": spec["table"]})
|
| 1518 |
+
total = datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm)
|
|
|
|
| 1519 |
except Exception: # noqa: BLE001
|
| 1520 |
total = None
|
| 1521 |
if total is not None and total > max_rows:
|
|
|
|
| 1602 |
if key.startswith("ut_amzn_"):
|
| 1603 |
import amazon_relational as amazon
|
| 1604 |
if key in amazon.SOURCE:
|
| 1605 |
+
spec = amazon.grid_source(cur, key)
|
| 1606 |
+
if spec.get("missing"):
|
| 1607 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
|
|
|
|
|
|
|
|
|
| 1608 |
from harness import datastore
|
| 1609 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1610 |
try:
|
|
|
|
| 1657 |
for key, (expr, cast) in spec["cols"].items():
|
| 1658 |
cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
|
| 1659 |
have_by_alias), cast)
|
| 1660 |
+
if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
|
| 1661 |
+
spec = {**spec, "page_key": spec["id"]}
|
| 1662 |
return {**spec, "cols": cols}
|
| 1663 |
|
| 1664 |
|
| 1665 |
+
def _page_source_for(cur, table_key, st=None):
|
| 1666 |
+
"""Reuse source resolution only inside the page's explicitly lent definition snapshot."""
|
| 1667 |
+
from harness import datastore
|
| 1668 |
+
plans = None
|
| 1669 |
+
if isinstance(st, _ut()._Lent):
|
| 1670 |
+
plans = st.__dict__.setdefault("_page_sources", {})
|
| 1671 |
+
key = (table_key, datastore.mirror_generation())
|
| 1672 |
+
if plans is not None and key in plans:
|
| 1673 |
+
return plans[key]
|
| 1674 |
+
spec = _source_for(cur, table_key)
|
| 1675 |
+
if plans is not None:
|
| 1676 |
+
plans[key] = spec
|
| 1677 |
+
return spec
|
| 1678 |
+
|
| 1679 |
+
|
| 1680 |
#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
|
| 1681 |
#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of
|
| 1682 |
#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of
|
|
|
|
| 1890 |
if table_key in trail:
|
| 1891 |
raise ValueError("a relation has a circular access rule")
|
| 1892 |
trail = (*trail, table_key)
|
| 1893 |
+
handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False)
|
| 1894 |
+
else _ut().lend_defs(st, schema_only=True))
|
| 1895 |
declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
|
| 1896 |
columns = _filter_columns(spec, fields)
|
| 1897 |
guards, resolved, targets = [], {}, {}
|
|
|
|
| 1912 |
targets[key] = None
|
| 1913 |
return None
|
| 1914 |
definition = _ut().get(key, st=handle)
|
| 1915 |
+
target_spec = _page_source_for(cur, key, handle)
|
| 1916 |
if target_spec is None and str(key).startswith("ut_amzn_"):
|
| 1917 |
# Small Amazon targets (notably listings) are stored snapshots, but
|
| 1918 |
# their raw mirror still answers the same automatic link in SQL.
|
|
|
|
| 2392 |
if not source:
|
| 2393 |
return None
|
| 2394 |
try:
|
| 2395 |
+
cursor = getattr(session.runtime, '_canonical_source_cursor', None)
|
| 2396 |
+
if cursor is not None and source in {'customer_data', 'product_data'}:
|
| 2397 |
+
import registry_source
|
| 2398 |
+
rows = registry_source.canonical_page(session, cursor, source, [record])['rows']
|
| 2399 |
+
else:
|
| 2400 |
+
rows = perm_scope.scoped_table(session.user, source, st=session.runtime) or []
|
| 2401 |
except Exception: # noqa: BLE001
|
| 2402 |
return None
|
| 2403 |
want = str(record)
|
|
|
|
| 2447 |
|
| 2448 |
|
| 2449 |
def _user_tables_token(rt):
|
| 2450 |
+
"""Return the durable revision and timestamp, or decline a revisionless reader."""
|
| 2451 |
+
reader = getattr(rt, "revision", None)
|
| 2452 |
+
if not callable(reader):
|
|
|
|
| 2453 |
return None
|
| 2454 |
+
try:
|
| 2455 |
+
revision = reader(_ut().STORE_KEY)
|
| 2456 |
+
return (revision["token"], revision.get("updated_at")) if revision else None
|
| 2457 |
+
except Exception as exc: # noqa: BLE001
|
| 2458 |
+
from core.request_timing import raise_if_resource_error
|
| 2459 |
+
raise_if_resource_error(exc)
|
| 2460 |
+
raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc
|
| 2461 |
|
| 2462 |
|
| 2463 |
def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
|
|
|
|
| 2491 |
|
| 2492 |
|
| 2493 |
def _page_defs(session):
|
| 2494 |
+
"""Return one schema-only `(snapshot, revision)` for a page and its scoped targets.
|
| 2495 |
+
|
| 2496 |
+
Shared metadata and source plans belong to the fresh lend, never to the revision memo.
|
| 2497 |
+
Projection and revision failures refuse instead of falling back to a whole document.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2498 |
"""
|
| 2499 |
import copy
|
| 2500 |
import core.store as store
|
|
|
|
| 2504 |
token = _user_tables_token(rt)
|
| 2505 |
|
| 2506 |
def keep(v):
|
| 2507 |
+
# Activity is a generated descriptor, not a stored row-bearing definition.
|
| 2508 |
+
tables = [d for key, d in (v or {}).items() if key != "ut_activity" and isinstance(d, dict)]
|
| 2509 |
return bool(tables) and all(store.is_projected(d) for d in tables)
|
| 2510 |
|
| 2511 |
+
try:
|
| 2512 |
+
defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True),
|
| 2513 |
+
keep=keep, clone=copy.deepcopy, token=token)
|
| 2514 |
+
except Exception as exc: # noqa: BLE001
|
| 2515 |
+
from core.request_timing import raise_if_resource_error
|
| 2516 |
+
raise_if_resource_error(exc)
|
| 2517 |
+
raise err(503, "schema_unavailable", "table definitions are unavailable") from exc
|
| 2518 |
+
lent = ut.lend(rt, **{ut.STORE_KEY: defs})
|
| 2519 |
+
lent.schema_only = True
|
| 2520 |
+
return lent, token
|
| 2521 |
|
| 2522 |
|
| 2523 |
def _stored_link_rows(session, table, token=_UNREAD):
|
|
|
|
| 2557 |
return defn
|
| 2558 |
|
| 2559 |
|
| 2560 |
+
class _RelationRows:
|
| 2561 |
+
def __init__(self, read):
|
| 2562 |
+
self.read = read
|
| 2563 |
+
|
| 2564 |
+
def __iter__(self):
|
| 2565 |
+
return self.read()
|
| 2566 |
+
|
| 2567 |
+
|
| 2568 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2569 |
+
traversal_of=None, st=None, rev=..., normalize_match=False,
|
| 2570 |
+
compiled=None, stream=False): # `...` IS `_UNREAD`
|
| 2571 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2572 |
|
| 2573 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
|
|
| 2586 |
|
| 2587 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 2588 |
if not _materialised_table(table):
|
| 2589 |
+
read = compiled if compiled is not None else (
|
| 2590 |
+
_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 2591 |
+
if traversal_of is not None else _compiled_read(table, session, st=st))
|
| 2592 |
spec = read["spec"]
|
| 2593 |
bindings = dict(spec["cols"], pid=(spec["id"], int))
|
| 2594 |
if columns - set(bindings):
|
|
|
|
| 2599 |
if values is None:
|
| 2600 |
continue
|
| 2601 |
expr, cast = bindings[key]
|
|
|
|
| 2602 |
if normalize_match and key != "pid":
|
| 2603 |
expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
|
| 2604 |
cast = lambda value: str(value).strip().lower()
|
| 2605 |
+
if isinstance(values, _fs().SqlMembership):
|
| 2606 |
+
operand = _membership_relation(read['cur'], _membership_values(values),
|
| 2607 |
+
normalize=cast, sql_type='HUGEINT' if cast is int else 'DOUBLE' if cast is float else 'VARCHAR',
|
| 2608 |
+
skip_invalid=False)
|
| 2609 |
+
parts.append(f'{expr} IN ({operand.sql})')
|
| 2610 |
+
params.extend(operand.params)
|
| 2611 |
+
continue
|
| 2612 |
+
values = list(values)
|
| 2613 |
parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
|
| 2614 |
params.extend(cast(value) for value in values)
|
| 2615 |
if matches is not None:
|
|
|
|
| 2617 |
for key, values in matches.items():
|
| 2618 |
expr, cast = bindings[key]
|
| 2619 |
values = sorted(values)
|
| 2620 |
+
if normalize_match:
|
| 2621 |
+
expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
|
| 2622 |
+
cast = lambda value: str(value).strip().lower()
|
| 2623 |
if values:
|
| 2624 |
alternatives.append(f"{expr} IN ({','.join('?' for _ in values)})")
|
| 2625 |
params.extend(cast(value) for value in values)
|
|
|
|
| 2628 |
projection = ", ".join(bindings[key][0] for key in names)
|
| 2629 |
source = spec.get("from_sql") or spec["table"]
|
| 2630 |
predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
|
| 2631 |
+
if spec.get("page_key") and not spec.get("from_sql"):
|
| 2632 |
+
key = spec["page_key"]
|
| 2633 |
+
sql = (f'WITH _relation_keys AS MATERIALIZED (SELECT {key} AS "_target_key" '
|
| 2634 |
+
f'FROM {source} WHERE {predicate}) SELECT {projection} FROM {source} '
|
| 2635 |
+
f'SEMI JOIN _relation_keys ON {key} = _relation_keys."_target_key"')
|
| 2636 |
+
else:
|
| 2637 |
+
sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
|
| 2638 |
+
def batches():
|
| 2639 |
+
cursor = read["cur"].execute(sql, params)
|
| 2640 |
+
while True:
|
| 2641 |
+
batch = cursor.fetchmany(2048)
|
| 2642 |
+
if not batch:
|
| 2643 |
+
return
|
| 2644 |
+
for row in batch:
|
| 2645 |
+
yield dict(zip(names, row))
|
| 2646 |
+
batch.clear()
|
| 2647 |
+
return _RelationRows(batches) if stream else list(batches())
|
| 2648 |
|
| 2649 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 2650 |
# complete contract before taking any join value, including a join on a hidden column.
|
|
|
|
| 2679 |
wanted = {normalize(value) for value in values}
|
| 2680 |
rows = [row for row in rows if normalize(row.get(key)) in wanted]
|
| 2681 |
if matches is not None:
|
| 2682 |
+
normalize = (lambda value: str(value).strip().lower()) if normalize_match else str
|
| 2683 |
+
wanted = {key: {normalize(value) for value in values} for key, values in matches.items()}
|
| 2684 |
+
rows = [row for row in rows if any(normalize(row.get(key)) in values
|
| 2685 |
for key, values in wanted.items())]
|
| 2686 |
return [{key: row.get(key) for key in columns} for row in rows]
|
| 2687 |
|
|
|
|
| 2692 |
return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
|
| 2693 |
|
| 2694 |
|
| 2695 |
+
def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_col,
|
| 2696 |
+
target_col, manual, inverse, reverse):
|
| 2697 |
+
"""Follow a scoped leg with SQL operands at every population-sized boundary."""
|
| 2698 |
+
import core.relations as relations
|
| 2699 |
+
target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {}
|
| 2700 |
+
observed = {'rows': 0, 'complete': True, 'available': False}
|
| 2701 |
+
def source_values():
|
| 2702 |
+
for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True):
|
| 2703 |
+
observed['rows'] += 1
|
| 2704 |
+
observed['complete'] &= source_col in row
|
| 2705 |
+
if row.get(source_col) not in (None, ''):
|
| 2706 |
+
observed['available'] = True
|
| 2707 |
+
if (manual and not reverse) or (inverse and reverse):
|
| 2708 |
+
yield from _relation_cell_ids(row[source_col])
|
| 2709 |
+
else:
|
| 2710 |
+
yield str(row[source_col])
|
| 2711 |
+
values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR')
|
| 2712 |
+
if source_col != relations.IDENTITY_COL and not observed['available']:
|
| 2713 |
+
if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_')
|
| 2714 |
+
and observed['rows'] and observed['complete']):
|
| 2715 |
+
return values
|
| 2716 |
+
raise err(400, 'unknown_relation', relations.unresolvable_source_refusal(source, entry))
|
| 2717 |
+
if (manual and not reverse) or (inverse and reverse):
|
| 2718 |
+
target_col = 'pid'
|
| 2719 |
+
through = relations.declared_through(link) if not manual else None
|
| 2720 |
+
if through:
|
| 2721 |
+
left = str(link['throughTo'] if reverse else link['throughOn'])
|
| 2722 |
+
right = str(link['throughOn'] if reverse else link['throughTo'])
|
| 2723 |
+
middle = _relation_rows(session, str(through), [left, right], match=(left, values),
|
| 2724 |
+
traversal_of=str(entry.get('key') or via.get('key') or ''), stream=True)
|
| 2725 |
+
values = _membership_relation(cur,
|
| 2726 |
+
(str(row[right]) for row in middle if row.get(right) not in (None, '')),
|
| 2727 |
+
normalize=str, sql_type='VARCHAR')
|
| 2728 |
+
if (manual and reverse) or (inverse and not reverse):
|
| 2729 |
+
rows = _relation_rows(session, target, [target_col], stream=True)
|
| 2730 |
+
return _relation_cell_membership(cur, rows, target_col, values)
|
| 2731 |
+
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2732 |
+
where=entry.get('where'), stream=True)
|
| 2733 |
+
return _membership_relation(cur, (row['pid'] for row in rows))
|
| 2734 |
+
|
| 2735 |
+
|
| 2736 |
+
def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None):
|
| 2737 |
"""Follow one declared leg, including manual, inverse and through links."""
|
| 2738 |
import core.relations as relations
|
| 2739 |
|
|
|
|
| 2751 |
source_col, target_col = ("pid", field) if reverse else (field, "pid")
|
| 2752 |
elif inverse:
|
| 2753 |
source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
|
| 2754 |
+
if membership_cur is not None:
|
| 2755 |
+
return _relation_leg_membership(session, source, source_ids, entry, membership_cur,
|
| 2756 |
+
source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse)
|
| 2757 |
source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
|
| 2758 |
values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
|
| 2759 |
# A column-keyed edge cannot fall back to the wire record id. In particular,
|
|
|
|
| 2778 |
traversal_of=str(entry.get("key") or via.get("key") or ""))
|
| 2779 |
values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
|
| 2780 |
if (manual and reverse) or (inverse and not reverse):
|
| 2781 |
+
rows = _relation_rows(session, target, [target_col], stream=True)
|
| 2782 |
+
rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values)
|
| 2783 |
else:
|
| 2784 |
rows = _relation_rows(session, target, [target_col], match=(target_col, values),
|
| 2785 |
+
where=entry.get("where"), stream=True)
|
| 2786 |
return {int(row["pid"]) for row in rows}
|
| 2787 |
|
| 2788 |
|
|
|
|
| 2795 |
if table == key and label), str(key))
|
| 2796 |
|
| 2797 |
|
| 2798 |
+
def _relation_target_ids(session, target, related, membership_cur=None):
|
| 2799 |
"""One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
|
| 2800 |
import core.relations as relations
|
| 2801 |
from routes_grid import _may_open, related_path_entries
|
|
|
|
| 2830 |
if not _may_open(session, leg["target"]):
|
| 2831 |
raise err(403, "forbidden_table",
|
| 2832 |
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
|
| 2833 |
+
empty_ids = (not ids.params if membership_cur is not None
|
| 2834 |
+
and isinstance(ids, _fs().SqlMembership) else not ids)
|
| 2835 |
+
if empty_ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
|
| 2836 |
# An earlier, authorised Amazon hop matched nothing. There is no next
|
| 2837 |
# source record to read, and that is an empty relation, not a missing key.
|
| 2838 |
continue
|
| 2839 |
+
ids = _relation_leg_ids(session, table, ids, leg, membership_cur)
|
| 2840 |
return ids
|
| 2841 |
|
| 2842 |
|
|
|
|
| 3072 |
related = _json_arg(related, "related")
|
| 3073 |
if related in (None, "", {}):
|
| 3074 |
return None, [], []
|
| 3075 |
+
ids = _relation_target_ids(session, table_key, related, membership_cur=cur)
|
| 3076 |
+
if isinstance(ids, _fs().SqlMembership):
|
| 3077 |
+
return f'COALESCE({spec["id"]} IN ({ids.sql}), FALSE)', list(ids.params), []
|
| 3078 |
sql, params = relations.pin_sql(spec["id"], sorted(ids))
|
| 3079 |
notes = []
|
| 3080 |
if len(ids) > _PIN_REPORT_AT:
|
|
|
|
| 3133 |
return out
|
| 3134 |
|
| 3135 |
|
| 3136 |
+
class _RelationLease(int):
|
| 3137 |
+
"""A bound scalar keeps its cursor-local relations alive through copied query params.
|
| 3138 |
+
|
| 3139 |
+
Every leaf gets its own lease. Releasing one query cannot invalidate another leaf or query
|
| 3140 |
+
on a retained cursor. No process/session map holds leases after their parameters disappear.
|
| 3141 |
+
"""
|
| 3142 |
+
def __new__(cls, cur):
|
| 3143 |
+
value = super().__new__(cls, 1)
|
| 3144 |
+
value.cur, value.names = cur, []
|
| 3145 |
+
return value
|
| 3146 |
+
|
| 3147 |
+
def table(self, suffix=""):
|
| 3148 |
+
import uuid
|
| 3149 |
+
name = '_request_members_' + uuid.uuid4().hex + suffix
|
| 3150 |
+
self.names.append(name)
|
| 3151 |
+
return name
|
| 3152 |
+
|
| 3153 |
+
def __reduce_ex__(self, protocol):
|
| 3154 |
+
raise TypeError('Request-owned membership cannot be persisted or cached')
|
| 3155 |
+
|
| 3156 |
+
def close(self):
|
| 3157 |
+
# Cleanup remains possible after the request's query deadline has fired.
|
| 3158 |
+
cur = getattr(self.cur, '_cursor', self.cur)
|
| 3159 |
+
while self.names:
|
| 3160 |
+
name = self.names[-1]
|
| 3161 |
+
cur.execute(f'DROP TABLE IF EXISTS "{name}"')
|
| 3162 |
+
self.names.pop()
|
| 3163 |
+
|
| 3164 |
+
def __del__(self):
|
| 3165 |
+
try:
|
| 3166 |
+
self.close()
|
| 3167 |
+
except Exception:
|
| 3168 |
+
pass # A closed cursor has already destroyed its TEMP relations.
|
| 3169 |
+
|
| 3170 |
+
|
| 3171 |
+
def _relation_cell_membership(cur, rows, column, values):
|
| 3172 |
+
"""Match complete manual/inverse cells in SQL without retaining the target population."""
|
| 3173 |
+
import csv
|
| 3174 |
+
import tempfile
|
| 3175 |
+
from pathlib import Path
|
| 3176 |
+
from core.request_timing import remaining_budget
|
| 3177 |
+
lease = _RelationLease(cur)
|
| 3178 |
+
name = lease.table('_cells')
|
| 3179 |
+
count = 0
|
| 3180 |
+
with tempfile.TemporaryDirectory(prefix='aios-related-') as directory:
|
| 3181 |
+
path = Path(directory) / 'cells.csv'
|
| 3182 |
+
with path.open('w', encoding='utf-8', newline='') as output:
|
| 3183 |
+
writer = csv.writer(output)
|
| 3184 |
+
for index, row in enumerate(rows):
|
| 3185 |
+
if index % 2048 == 0:
|
| 3186 |
+
remaining_budget()
|
| 3187 |
+
for token in _relation_cell_ids(row.get(column)):
|
| 3188 |
+
writer.writerow([int(row['pid']), token])
|
| 3189 |
+
count += 1
|
| 3190 |
+
if not count:
|
| 3191 |
+
return _fs().SqlMembership('SELECT NULL WHERE FALSE', [])
|
| 3192 |
+
cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT * FROM read_csv(?, header=false, '
|
| 3193 |
+
"columns={'pid':'BIGINT','token':'VARCHAR'})", [str(path)])
|
| 3194 |
+
return _fs().SqlMembership(f'SELECT pid FROM "{name}" WHERE token IN ({values.sql}) AND ?=1',
|
| 3195 |
+
[*values.params, lease])
|
| 3196 |
+
|
| 3197 |
+
|
| 3198 |
+
def _membership_values(membership):
|
| 3199 |
+
"""Replay SQL membership in bounded batches on its owning cursor."""
|
| 3200 |
+
if not membership.params:
|
| 3201 |
+
return
|
| 3202 |
+
cur = membership.params[-1].cur
|
| 3203 |
+
cur.execute(membership.sql, membership.params)
|
| 3204 |
+
while batch := cur.fetchmany(2048):
|
| 3205 |
+
for row in batch:
|
| 3206 |
+
yield row[0]
|
| 3207 |
+
batch.clear()
|
| 3208 |
+
|
| 3209 |
+
|
| 3210 |
+
def _membership_relation(cur, values, *, normalize=int, sql_type='HUGEINT', skip_invalid=True):
|
| 3211 |
+
"""Normalize a complete operand with bounded Python memory, then own its SQL lifetime."""
|
| 3212 |
+
import tempfile
|
| 3213 |
+
lease = _RelationLease(cur)
|
| 3214 |
+
name = lease.table()
|
| 3215 |
+
count = 0
|
| 3216 |
+
# Spooling also permits the producer to read this same cursor without an INSERT replacing
|
| 3217 |
+
# its active result. No population-sized list or Python set is constructed.
|
| 3218 |
+
import csv
|
| 3219 |
+
from pathlib import Path
|
| 3220 |
+
from core.request_timing import remaining_budget
|
| 3221 |
+
with tempfile.TemporaryDirectory(prefix='aios-members-') as directory:
|
| 3222 |
+
path = Path(directory) / 'members.csv'
|
| 3223 |
+
source = iter(values)
|
| 3224 |
+
try:
|
| 3225 |
+
with path.open('w', encoding='utf-8', newline='') as output:
|
| 3226 |
+
writer = csv.writer(output)
|
| 3227 |
+
for index, raw in enumerate(source):
|
| 3228 |
+
if index % 2048 == 0:
|
| 3229 |
+
remaining_budget()
|
| 3230 |
+
try:
|
| 3231 |
+
value = normalize(raw)
|
| 3232 |
+
except (TypeError, ValueError):
|
| 3233 |
+
if not skip_invalid or normalize is not int:
|
| 3234 |
+
raise
|
| 3235 |
+
continue
|
| 3236 |
+
writer.writerow([str(value)])
|
| 3237 |
+
count += 1
|
| 3238 |
+
finally:
|
| 3239 |
+
close = getattr(source, 'close', None)
|
| 3240 |
+
if close:
|
| 3241 |
+
close()
|
| 3242 |
+
if count:
|
| 3243 |
+
cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT pid FROM read_csv(?, '
|
| 3244 |
+
"header=false, columns={'pid':'VARCHAR'}, force_not_null=['pid'])", [str(path)])
|
| 3245 |
+
return _fs().SqlMembership(f'SELECT TRY_CAST(pid AS {sql_type}) FROM "{name}" WHERE ?=1',
|
| 3246 |
+
[lease]) if count else _fs().SqlMembership('SELECT NULL WHERE FALSE', [])
|
| 3247 |
+
|
| 3248 |
+
|
| 3249 |
def _view_store(session, table_key):
|
| 3250 |
"""The workspace document a saved view of THIS database lives in.
|
| 3251 |
|
|
|
|
| 3254 |
to fail it: the client reads the views `/workspace` served out of this same document.
|
| 3255 |
"""
|
| 3256 |
import core.table_store as table_store
|
| 3257 |
+
from routes_customers import _AssemblyStore
|
| 3258 |
+
runtime = session.runtime
|
| 3259 |
+
from registry_source import backend as document_backend
|
| 3260 |
+
backend = document_backend(runtime)
|
| 3261 |
+
project = getattr(backend, 'get_view_workspace', None) or getattr(backend, '_get_view_workspace', None)
|
| 3262 |
+
stream_values = getattr(backend, 'iter_json_values', None) or getattr(backend, '_iter_json_values', None)
|
| 3263 |
+
class ViewStore(_AssemblyStore):
|
| 3264 |
+
def get(self, name, fresh=False):
|
| 3265 |
+
if callable(project) and name == f'{table_key}_table_workspace':
|
| 3266 |
+
key = getattr(runtime, 'store_key', str)(name)
|
| 3267 |
+
return self._read(name, 'view_operands', lambda: project(key), fresh=fresh)
|
| 3268 |
+
return super().get(name, fresh=fresh)
|
| 3269 |
+
|
| 3270 |
+
def _read_strict(self, name):
|
| 3271 |
+
return self.get(name)
|
| 3272 |
+
result = table_store.make(f"{table_key}_table_workspace", st=ViewStore(runtime, schema_only=True))
|
| 3273 |
+
if callable(project):
|
| 3274 |
+
def members(owner, vid):
|
| 3275 |
+
name = f'{table_key}_table_workspace'
|
| 3276 |
+
key = getattr(runtime, 'store_key', str)(name)
|
| 3277 |
+
path = [owner, 'views', vid, 'config', 'memberPids']
|
| 3278 |
+
document = result.st.get(name)
|
| 3279 |
+
stable = getattr(backend, 'iter_view_members', None) or getattr(backend, '_iter_view_members', None)
|
| 3280 |
+
parts = (stable(key, path, getattr(document, 'source_revision', None), batch_size=2048)
|
| 3281 |
+
if callable(stable) else stream_values(key, path, batch_size=2048))
|
| 3282 |
+
from contextlib import closing
|
| 3283 |
+
try:
|
| 3284 |
+
with closing(parts):
|
| 3285 |
+
for batch in parts:
|
| 3286 |
+
for _, value in batch:
|
| 3287 |
+
yield value
|
| 3288 |
+
except Exception as exc:
|
| 3289 |
+
from core.request_timing import raise_if_resource_error
|
| 3290 |
+
raise_if_resource_error(exc)
|
| 3291 |
+
raise err(503, 'schema_unavailable', 'Saved view membership is unavailable. Try again.') from exc
|
| 3292 |
+
result.member_values = members
|
| 3293 |
+
return result
|
| 3294 |
|
| 3295 |
|
| 3296 |
def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits):
|
|
|
|
| 3314 |
* no filters and a curated `memberPids` is the client's `else if (curated)` branch: the
|
| 3315 |
membership IS the view;
|
| 3316 |
* no filters and no members means the view holds every row in scope;
|
| 3317 |
+
* membership stays a `SELECT DISTINCT` subquery under the wall, without a Python ID list.
|
|
|
|
| 3318 |
|
| 3319 |
THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY:
|
| 3320 |
resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets`
|
|
|
|
| 3343 |
|
| 3344 |
store = _view_store(session, table_key)
|
| 3345 |
sets, memo, unresolved = {}, {}, []
|
| 3346 |
+
cyclic = set()
|
| 3347 |
counter = [0]
|
| 3348 |
|
| 3349 |
def key_for():
|
|
|
|
| 3354 |
return "__view_%d__" % counter[0]
|
| 3355 |
|
| 3356 |
def resolve(vid, seen):
|
| 3357 |
+
"""Return SQL membership, authored IDs, all rows, or an unresolved reason."""
|
| 3358 |
if vid in memo:
|
| 3359 |
return memo[vid]
|
| 3360 |
if vid in seen:
|
| 3361 |
+
cyclic.update(seen[seen.index(vid):])
|
| 3362 |
# `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left
|
| 3363 |
# ABSENT, so a stored loop degrades to an explained empty table, not a hung request.
|
| 3364 |
return None, "it refers to itself through another saved view"
|
|
|
|
| 3378 |
"own records rather than from this database's live connection")
|
| 3379 |
memo[vid] = got
|
| 3380 |
return got
|
| 3381 |
+
source = (store.member_values(owner, vid) if 'memberPids' not in cfg
|
| 3382 |
+
and callable(getattr(store, 'member_values', None))
|
| 3383 |
+
else (cfg.get('memberPids') or ()))
|
| 3384 |
+
has_members = False
|
| 3385 |
+
def authored_values():
|
| 3386 |
+
nonlocal has_members
|
| 3387 |
+
values = iter(source)
|
| 3388 |
try:
|
| 3389 |
+
for raw in values:
|
| 3390 |
+
# A nonempty authored operand stays curated even if normalization
|
| 3391 |
+
# rejects every value. An actually empty operand means all rows.
|
| 3392 |
+
has_members = True
|
| 3393 |
+
yield raw
|
| 3394 |
+
finally:
|
| 3395 |
+
close = getattr(values, 'close', None)
|
| 3396 |
+
if close:
|
| 3397 |
+
close()
|
| 3398 |
+
members = _membership_relation(cur, authored_values())
|
| 3399 |
raw_nodes = cfg.get("filters") or []
|
| 3400 |
if not isinstance(raw_nodes, list):
|
| 3401 |
raw_nodes = []
|
|
|
|
| 3404 |
# "anything active": with no nodes at all the curated set IS the view, and with nodes
|
| 3405 |
# that all happen to be half-built the view holds everything. Two different answers
|
| 3406 |
# from one list, so the emptiness is checked before the compiler sees it.
|
| 3407 |
+
got = ("ids", members) if has_members else ("all", None)
|
| 3408 |
memo[vid] = got
|
| 3409 |
return got
|
| 3410 |
nodes = rewrite(raw_nodes, seen + (vid,))
|
| 3411 |
+
if vid in cyclic:
|
| 3412 |
+
got = (None, "it refers to itself through another saved view")
|
| 3413 |
+
memo[vid] = got
|
| 3414 |
+
return got
|
| 3415 |
unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols))
|
| 3416 |
if unknown:
|
| 3417 |
# ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it
|
|
|
|
| 3425 |
try:
|
| 3426 |
pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or"
|
| 3427 |
else "and"),
|
| 3428 |
+
columns=cols, member_ids=(members if has_members else None),
|
| 3429 |
id_sql=spec["id"], today=_today(), cohort_sets=sets)
|
| 3430 |
except ValueError as exc:
|
| 3431 |
got = (None, "it uses a condition this database's live connection cannot evaluate ("
|
|
|
|
| 3446 |
return got
|
| 3447 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 3448 |
where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql
|
| 3449 |
+
ids = _fs().SqlMembership(
|
| 3450 |
f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}',
|
| 3451 |
+
list(wall_params) + list(pred.params))
|
| 3452 |
got = ("ids", ids)
|
| 3453 |
memo[vid] = got
|
| 3454 |
return got
|
|
|
|
| 3479 |
if kind == "ids":
|
| 3480 |
sets[key] = got
|
| 3481 |
op = "noneOf" if not_in else "anyOf"
|
| 3482 |
+
if not isinstance(got, _fs().SqlMembership) and len(got) >= _PIN_REPORT_AT:
|
| 3483 |
limits.append({
|
| 3484 |
"subject": "view", "effect": "slow",
|
| 3485 |
"cause": f"the saved view this condition names holds {len(got):,} rows, "
|
|
|
|
| 3499 |
out.append({"colId": "__cohort__", "op": op, "value": key})
|
| 3500 |
return out
|
| 3501 |
|
| 3502 |
+
try:
|
| 3503 |
+
tree = rewrite(tree, ())
|
| 3504 |
+
finally:
|
| 3505 |
+
# Recursive closures must not postpone relation release until cyclic GC.
|
| 3506 |
+
memo.clear()
|
| 3507 |
+
resolve = rewrite = None
|
| 3508 |
if unresolved:
|
| 3509 |
limits.append({
|
| 3510 |
"subject": ", ".join(vid for vid, _why in unresolved),
|
|
|
|
| 3524 |
`__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
|
| 3525 |
|
| 3526 |
A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
|
| 3527 |
+
here as SQL over the non-rank residue under the wall. Finite values, group scopes and
|
| 3528 |
+
ascending-ID ties follow `filter_eval.resolve_rank_sets`. Averages stream sorted values
|
| 3529 |
+
through runtime Python sum with bounded batches. Other ranks remain entirely in SQL.
|
| 3530 |
"""
|
| 3531 |
fe = _fe()
|
| 3532 |
leaves = fe._rank_leaves(tree or [], cols)
|
|
|
|
| 3554 |
params.extend(pred.params)
|
| 3555 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 3556 |
answers = {}
|
|
|
|
| 3557 |
for n in leaves:
|
| 3558 |
+
answers[id(n)] = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3559 |
sets = dict(cohort_sets or {})
|
| 3560 |
counter = [0]
|
| 3561 |
wanted = {id(n) for n in leaves}
|
|
|
|
| 3584 |
out.append(n)
|
| 3585 |
return out
|
| 3586 |
|
| 3587 |
+
try:
|
| 3588 |
+
return rewrite(tree), sets
|
| 3589 |
+
finally:
|
| 3590 |
+
rewrite = None
|
| 3591 |
|
| 3592 |
|
| 3593 |
def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
|
| 3594 |
+
"""Resolve global and explicit group ranks as SQL membership over the walled residue."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3595 |
fe = _fe()
|
| 3596 |
Sql = _fs().SqlMembership
|
| 3597 |
empty = Sql("SELECT NULL WHERE FALSE", [])
|
| 3598 |
+
scope, field = fe._rank_scope(leaf, None)
|
| 3599 |
+
if scope == "group" and field not in cols:
|
| 3600 |
+
return empty
|
| 3601 |
col = cols.get(leaf.get("colId")) or {}
|
| 3602 |
if col.get("type") not in fe.NUMERIC_TYPES:
|
| 3603 |
return empty
|
| 3604 |
v = f"TRY_CAST({col['sql']} AS DOUBLE)"
|
| 3605 |
+
partition = cols[field]["sql"] if scope == "group" else "NULL"
|
| 3606 |
+
dom = (f"SELECT {id_sql} AS pid, {v} AS v, {partition} AS bucket FROM {frm_body} WHERE "
|
| 3607 |
+
+ (f"({where}) AND " if where else "")
|
| 3608 |
+
+ f"{id_sql} IS NOT NULL AND {v} IS NOT NULL AND isfinite({v})")
|
| 3609 |
params = list(params)
|
| 3610 |
op = leaf.get("op")
|
| 3611 |
+
if op in ("aboveAvg", "belowAvg"):
|
| 3612 |
+
import itertools
|
| 3613 |
+
import tempfile
|
| 3614 |
+
from pathlib import Path
|
| 3615 |
+
|
| 3616 |
+
# Python 3.12 sum uses compensation. SQL sum/fsum cannot replace its sorted oracle.
|
| 3617 |
+
domain = f"SELECT *, dense_rank() OVER (ORDER BY bucket) AS bucket_id FROM ({dom}) d"
|
| 3618 |
+
lease = _RelationLease(cur)
|
| 3619 |
+
name = lease.table('_mean')
|
| 3620 |
+
frozen = lease.table('_domain')
|
| 3621 |
+
# The mean and selected IDs must share one source snapshot across both statements.
|
| 3622 |
+
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{frozen}" AS {domain}', params)
|
| 3623 |
+
cur.execute(f'SELECT bucket_id, v FROM "{frozen}" ORDER BY bucket_id, v, pid')
|
| 3624 |
+
|
| 3625 |
+
def values():
|
| 3626 |
+
while True:
|
| 3627 |
+
batch = cur.fetchmany(4096)
|
| 3628 |
+
if not batch:
|
| 3629 |
+
return
|
| 3630 |
+
yield from batch
|
| 3631 |
+
|
| 3632 |
+
# Spill scalar group means, never a Python map of groups or a list of population rows.
|
| 3633 |
+
with tempfile.TemporaryDirectory(prefix="aios-rank-") as directory:
|
| 3634 |
+
path = Path(directory) / "means.csv"
|
| 3635 |
+
groups = 0
|
| 3636 |
+
with path.open("w", encoding="ascii") as output:
|
| 3637 |
+
for bucket, rows in itertools.groupby(values(), key=lambda row: row[0]):
|
| 3638 |
+
count = 0
|
| 3639 |
+
|
| 3640 |
+
def numbers():
|
| 3641 |
+
nonlocal count
|
| 3642 |
+
for _, number in rows:
|
| 3643 |
+
count += 1
|
| 3644 |
+
yield number
|
| 3645 |
+
|
| 3646 |
+
mean = sum(numbers()) / count
|
| 3647 |
+
output.write(f"{bucket},{mean!r}\n")
|
| 3648 |
+
groups += 1
|
| 3649 |
+
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{name}" (bucket_id BIGINT, mean DOUBLE)')
|
| 3650 |
+
if groups:
|
| 3651 |
+
cur.execute(f'INSERT INTO "{name}" SELECT * FROM read_csv(?, header=false, '
|
| 3652 |
+
"columns={'bucket_id':'BIGINT', 'mean':'DOUBLE'})", [str(path)])
|
| 3653 |
+
cmp = ">" if op == "aboveAvg" else "<"
|
| 3654 |
+
return Sql(f'SELECT d.pid FROM "{frozen}" d JOIN "{name}" m USING (bucket_id) '
|
| 3655 |
+
f"WHERE NOT isnan(m.mean) AND d.v {cmp} m.mean AND ?=1", [lease])
|
| 3656 |
+
direction = "DESC" if op in ("topN", "inTopPct") else "ASC"
|
| 3657 |
+
ranked = (f"SELECT pid, v, row_number() OVER (PARTITION BY bucket ORDER BY v {direction}, "
|
| 3658 |
+
"pid ASC) AS rn, count(*) OVER (PARTITION BY bucket) AS n "
|
| 3659 |
+
f"FROM ({dom}) d")
|
| 3660 |
if op in ("topN", "bottomN"):
|
| 3661 |
bound = fe._rank_bound(leaf.get("value"), 1, 10000)
|
| 3662 |
if bound is None:
|
| 3663 |
return empty
|
| 3664 |
+
predicate = f"rn <= {bound}"
|
| 3665 |
+
elif op in ("inTopPct", "inBottomPct"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3666 |
pct = fe._rank_bound(leaf.get("value"), 1, 100)
|
| 3667 |
if pct is None:
|
| 3668 |
return empty
|
| 3669 |
+
predicate = f"rn <= greatest(1, ceil(n * {pct} / 100.0))"
|
| 3670 |
+
elif op in ("inQuartile", "inDecile"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3671 |
buckets = 4 if op == "inQuartile" else 10
|
| 3672 |
wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
|
| 3673 |
+
if wanted is None:
|
| 3674 |
return empty
|
| 3675 |
+
# The Python resolver clears every group if any finite-value group is too small.
|
| 3676 |
+
ranked = f"SELECT *, min(n) OVER () AS smallest_group FROM ({ranked}) ranked_groups"
|
| 3677 |
+
predicate = (f"smallest_group >= {buckets} AND "
|
| 3678 |
+
f"least({buckets}, ((rn - 1) * {buckets}) // n + 1) = {wanted}")
|
| 3679 |
+
else:
|
| 3680 |
+
return empty
|
| 3681 |
+
return Sql(f"SELECT pid FROM ({ranked}) r WHERE {predicate}", params)
|
| 3682 |
|
| 3683 |
def _may_see_view(session, vid, store):
|
| 3684 |
"""May this caller open that saved view? The same predicate the view rail asks.
|
|
|
|
| 3806 |
if mismatch:
|
| 3807 |
raise err(409, "cross_tenant_store", mismatch)
|
| 3808 |
|
| 3809 |
+
spec = _page_source_for(cur, table_key, st)
|
| 3810 |
if spec is None:
|
| 3811 |
from core import perm_scope
|
| 3812 |
rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
|
|
|
|
| 4068 |
|
| 4069 |
|
| 4070 |
@contextmanager
|
| 4071 |
+
def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None):
|
| 4072 |
"""One compiler for native rows and their aggregate sibling, over the walled payload.
|
| 4073 |
|
| 4074 |
The temporary connection contains only this request's already stripped, pinned rows and is
|
| 4075 |
closed before returning. It never registers data on the tenant's shared mirror connection.
|
| 4076 |
"""
|
| 4077 |
import duckdb
|
| 4078 |
+
import itertools
|
| 4079 |
+
from harness import datastore
|
| 4080 |
|
| 4081 |
+
compiled = payload is None and table_key in {'customer_data', 'product_data'}
|
| 4082 |
+
streamed = payload is None and table_key != 'ut_activity'
|
| 4083 |
+
source_generation = None
|
| 4084 |
+
if streamed:
|
| 4085 |
+
import registry_source
|
| 4086 |
+
source_generation = registry_source.generation(session, table_key)
|
| 4087 |
if payload is not None:
|
| 4088 |
pass
|
| 4089 |
+
elif compiled:
|
| 4090 |
+
import registry_source
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4091 |
session.require(table_key)
|
| 4092 |
+
metadata = registry_source.canonical_metadata(session, table_key)
|
| 4093 |
+
payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [],
|
| 4094 |
+
'limits': list(metadata['workspace'].get('limits') or [])}
|
| 4095 |
+
elif streamed:
|
| 4096 |
+
import routes_tables
|
| 4097 |
+
from registry_source import MetadataStore, with_runtime, metadata_limits
|
| 4098 |
+
metadata_actor = with_runtime(session, MetadataStore(session.runtime))
|
| 4099 |
+
metadata = routes_tables.ut_assembly(metadata_actor, table_key, with_rows=False,
|
| 4100 |
+
fields_only=True, consume_corrections=False)
|
| 4101 |
+
payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [],
|
| 4102 |
+
'limits': metadata_limits(session, table_key, metadata)}
|
| 4103 |
else:
|
| 4104 |
from routes_tables import table_rows
|
| 4105 |
payload = table_rows(table_key, session=session, related=related)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4106 |
fields = payload["fields"]
|
| 4107 |
+
if compiled:
|
| 4108 |
+
source_cur, mismatch = _mirror_cur(session)
|
| 4109 |
+
if mismatch:
|
| 4110 |
+
raise err(409, 'cross_tenant_store', mismatch)
|
| 4111 |
+
if source_cur is None:
|
| 4112 |
+
raise err(503, 'canonical_source_unavailable', f'{table_key} requires its tenant mirror.')
|
| 4113 |
+
connection = source_cur.cursor()
|
| 4114 |
+
else:
|
| 4115 |
+
connection = datastore.bounded_connection(duckdb.connect(":memory:"))
|
| 4116 |
+
with connection as cur:
|
| 4117 |
+
cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
|
| 4118 |
+
wanted = set(ids) if ids is not None else None
|
| 4119 |
+
if streamed:
|
| 4120 |
+
import registry_source
|
| 4121 |
+
def source_rows():
|
| 4122 |
+
from contextlib import closing
|
| 4123 |
+
factory = registry_source.canonical_batches if compiled else registry_source.stored_batches
|
| 4124 |
+
options = {'ids': ids, 'related': related, 'needed': needed}
|
| 4125 |
+
if compiled:
|
| 4126 |
+
parts = factory(session, cur, table_key, **options)
|
| 4127 |
+
else:
|
| 4128 |
+
parts = factory(session, table_key, **options)
|
| 4129 |
+
with closing(parts) as parts:
|
| 4130 |
+
for part in parts:
|
| 4131 |
+
if compiled:
|
| 4132 |
+
payload.setdefault('docs', {}).update(part.get('docs') or {})
|
| 4133 |
+
yield from part['rows']
|
| 4134 |
+
part['rows'].clear()
|
| 4135 |
+
rows = source_rows()
|
| 4136 |
+
else:
|
| 4137 |
+
rows = payload['rows']
|
| 4138 |
+
source = ((index, row) for index, row in enumerate(rows)
|
| 4139 |
+
if wanted is None or int(row["pid"]) in wanted)
|
| 4140 |
+
source_count = 0
|
| 4141 |
+
try:
|
| 4142 |
+
while True:
|
| 4143 |
+
batch = list(itertools.islice(source, 2048))
|
| 4144 |
+
if not batch:
|
| 4145 |
+
break
|
| 4146 |
+
cur.execute("INSERT INTO pinned_rows SELECT unnest(?::BIGINT[]), unnest(?::JSON[])",
|
| 4147 |
+
[[index for index, _ in batch],
|
| 4148 |
+
[json.dumps(row, default=str) for _, row in batch]])
|
| 4149 |
+
source_count += len(batch)
|
| 4150 |
+
batch.clear()
|
| 4151 |
+
finally:
|
| 4152 |
+
source.close()
|
| 4153 |
+
close = getattr(rows, "close", None)
|
| 4154 |
+
if callable(close):
|
| 4155 |
+
close()
|
| 4156 |
bindings = {}
|
| 4157 |
for field in fields:
|
| 4158 |
key = str(field["key"])
|
| 4159 |
+
if streamed and needed is not None and key not in needed:
|
| 4160 |
+
continue
|
| 4161 |
pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
|
| 4162 |
expr = f"json_extract_string(payload, '{pointer}')"
|
| 4163 |
numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
|
|
|
|
| 4190 |
where = f"({where}) AND {search.sql}" if where else search.sql
|
| 4191 |
params.extend(search.params)
|
| 4192 |
yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
|
| 4193 |
+
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload,
|
| 4194 |
+
"source_count": source_count, 'streamed': streamed}
|
| 4195 |
+
if streamed:
|
| 4196 |
+
registry_source.require_generation(session, table_key, source_generation)
|
| 4197 |
|
| 4198 |
|
| 4199 |
def _group_pin(cols, group_by, group_key):
|
|
|
|
| 4274 |
'totalUnfiltered': len(payload['rows']), 'offset': offset, 'limit': limit,
|
| 4275 |
'scope': {'table': table_key, 'readThrough': True}}
|
| 4276 |
window_offset = 0
|
| 4277 |
+
needed = _leaf_cols(query['filters']) - _PSEUDO_COLS
|
| 4278 |
+
needed.update(sort['colId'] for sort in query['sorts'])
|
| 4279 |
+
if isinstance(groupBy, str) and groupBy.strip():
|
| 4280 |
+
needed.add(groupBy.strip())
|
| 4281 |
+
for leaf in _fe()._rank_leaves(query['filters'], {key: {} for key in needed}):
|
| 4282 |
+
_mode, group_field = _fe()._rank_scope(leaf, None)
|
| 4283 |
+
if group_field:
|
| 4284 |
+
needed.add(group_field)
|
| 4285 |
+
if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
|
| 4286 |
+
needed = None
|
| 4287 |
+
with _materialised_read(table_key, session, query, related, ids=ids, payload=payload, needed=needed) as read:
|
| 4288 |
spec = read["spec"]
|
| 4289 |
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
|
| 4290 |
where, params = read["where"], list(read["params"])
|
|
|
|
| 4296 |
params=tuple(params), order_by=order or spec["id"],
|
| 4297 |
offset=window_offset, limit=limit or None, cur=read["cur"], **read["frm"])
|
| 4298 |
payload = read["payload"]
|
| 4299 |
+
indexes = [int(row[0]) for row in win['rows']]
|
| 4300 |
+
value_sql = read['spec']['id'] if read['streamed'] else 'payload'
|
| 4301 |
+
values = (read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows '
|
| 4302 |
+
'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
|
| 4303 |
+
if indexes else [])
|
| 4304 |
+
if read['streamed']:
|
| 4305 |
+
import registry_source
|
| 4306 |
+
by_index = {int(index): int(value) for index, value in values}
|
| 4307 |
+
page_ids = [by_index[index] for index in indexes]
|
| 4308 |
+
if table_key in {'customer_data', 'product_data'}:
|
| 4309 |
+
payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
|
| 4310 |
+
else:
|
| 4311 |
+
payload = registry_source.stored_page(session, table_key, page_ids, related=related)
|
| 4312 |
+
rows = payload['rows']
|
| 4313 |
+
else:
|
| 4314 |
+
by_index = {int(index): json.loads(value) for index, value in values}
|
| 4315 |
+
rows = [by_index[index] for index in indexes]
|
| 4316 |
if win["clamped"]:
|
| 4317 |
read["limits"].append({"subject": "limit", "effect": "window_clamped",
|
| 4318 |
"cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
|
| 4319 |
"recommendation": "Continue paging with offset. Every matching row remains available."})
|
| 4320 |
+
limits = list(payload.get('limits') or [])
|
| 4321 |
+
limits.extend(report for report in read['limits'] if report not in limits)
|
| 4322 |
return {**payload, "ok": True, "rows": rows, "total": win["total"],
|
| 4323 |
+
"totalUnfiltered": read["source_count"],
|
| 4324 |
"offset": offset if table_key == 'ut_activity' else win["offset"], "limit": win["limit"],
|
| 4325 |
+
"limits": limits, "scope": {"table": table_key, "readThrough": True}}
|
| 4326 |
|
| 4327 |
|
| 4328 |
def _activity_window_payload(session, offset, limit, related):
|
|
|
|
| 4387 |
that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in
|
| 4388 |
`limits` with its cause and a recommendation — R6's second sentence — and never silently
|
| 4389 |
dropped, because an ignored condition widens.
|
| 4390 |
+
|
| 4391 |
⭐⭐ `related` (W51-T05 / C6) IS A **JSON QUERY STRING**, copying the `filters` precedent one
|
| 4392 |
argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
|
| 4393 |
window to the rows linked to that ONE record, ahead of `filters`, and `total` /
|
|
|
|
| 4453 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 4454 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 4455 |
where=where, params=tuple(params), order_by=order,
|
| 4456 |
+
offset=offset, limit=(limit or None), cur=cur, page_first=True,
|
| 4457 |
+
page_key=spec.get("page_key"), **frm)
|
| 4458 |
if win["clamped"]:
|
| 4459 |
limits.append({
|
| 4460 |
"subject": "limit", "effect": "window_clamped",
|
|
|
|
| 4516 |
# the wall and the predicate, so the argument it demands is the list we just fetched.
|
| 4517 |
# ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
|
| 4518 |
# answered "may this session open this surface".
|
| 4519 |
+
# Definitions use the projected reader. Values use coherent page-bound snapshots.
|
|
|
|
|
|
|
| 4520 |
shared_defs = _so().fields(table_key, st=page_st)
|
| 4521 |
if shared_defs:
|
| 4522 |
declared = {field["key"]: field for field in fields}
|
| 4523 |
declared.update({key: dict(field, source="overlay") for key, field in shared_defs.items()})
|
| 4524 |
fields = list(declared.values())
|
| 4525 |
+
from registry_source import page_cells
|
| 4526 |
+
for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
|
| 4527 |
overlays.setdefault(pid, {}).update(cells)
|
| 4528 |
if_keys = {field["key"] for field in fields if field.get("type") == "if"}
|
| 4529 |
for row in rows_src:
|
|
|
|
| 4546 |
for pid, cells in overlays.items()}
|
| 4547 |
rows = aios_grid.rows_from_pool(rows_src, fields, overlays)
|
| 4548 |
|
| 4549 |
+
unfiltered = win["total"] if where == base_where else datastore.count_rows(
|
| 4550 |
+
where=base_where, params=tuple(wall_params), cur=cur, **frm)
|
|
|
|
| 4551 |
|
| 4552 |
from core import connector_spec
|
| 4553 |
rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
|
|
|
|
| 4570 |
# ══════════════════════════════════════════════════════════
|
| 4571 |
|
| 4572 |
#: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers
|
| 4573 |
+
#: that omit groupLimit receive one page and exact traversal metadata.
|
| 4574 |
MAX_GROUPS = 1_000
|
| 4575 |
|
| 4576 |
#: One request asks for at most this many summaries. Each is one more aggregate expression in a
|
|
|
|
| 4578 |
MAX_AGGREGATES = 200
|
| 4579 |
|
| 4580 |
|
| 4581 |
+
LINK_VALUE_MAX_BYTES = 1_048_576
|
| 4582 |
+
|
| 4583 |
+
|
| 4584 |
+
def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
|
| 4585 |
+
"""Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
|
| 4586 |
+
read = read if read is not None else _compiled_read(target, session, st=st)
|
| 4587 |
+
spec, cur = read['spec'], read['cur']
|
| 4588 |
+
if on not in spec['cols']:
|
| 4589 |
+
raise err(409, 'relation_target_unavailable', 'A required relation column is unavailable.')
|
| 4590 |
+
expr = f'LOWER(TRIM(CAST({spec["cols"][on][0]} AS VARCHAR)))'
|
| 4591 |
+
wanted = _membership_relation(cur, values, normalize=str, sql_type='VARCHAR')
|
| 4592 |
+
lease = _RelationLease(cur)
|
| 4593 |
+
name = lease.table('_link_values')
|
| 4594 |
+
source = spec.get('from_sql') or spec['table']
|
| 4595 |
+
where = read['where'] or '1=1'
|
| 4596 |
+
try:
|
| 4597 |
+
cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT DISTINCT {expr} AS k, '
|
| 4598 |
+
f'{spec["id"]} AS pid FROM {source} WHERE ({where}) AND {expr} IN ({wanted.sql})',
|
| 4599 |
+
[*read['params'], *wanted.params])
|
| 4600 |
+
population = f'(SELECT k, min(pid) AS pid FROM "{name}" GROUP BY k)' if single else f'"{name}"'
|
| 4601 |
+
required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
|
| 4602 |
+
f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
|
| 4603 |
+
if required > remaining:
|
| 4604 |
+
raise err(503, 'relation_value_capacity',
|
| 4605 |
+
f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
|
| 4606 |
+
'Reduce the page size or open the related database with filters.')
|
| 4607 |
+
# The byte check bounds both the aggregate and its Python result. IDs are never a
|
| 4608 |
+
# Python population; DISTINCT and numeric ordering preserve the existing wire value.
|
| 4609 |
+
cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
|
| 4610 |
+
f'FROM {population} GROUP BY k').fetchall())
|
| 4611 |
+
return cells, required
|
| 4612 |
+
finally:
|
| 4613 |
+
lease.close()
|
| 4614 |
+
|
| 4615 |
+
|
| 4616 |
def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
| 4617 |
+
"""Resolve page link cells through scoped SQL and a complete-value byte budget."""
|
| 4618 |
from routes_grid import _may_open
|
| 4619 |
declared = {field["key"] for field in fields}
|
| 4620 |
+
edges, requested = [], {}
|
| 4621 |
for field in fields:
|
| 4622 |
bag = field.get("link") or {}
|
| 4623 |
if field.get("type") != "link" or not bag.get("on") or not bag.get("from"):
|
|
|
|
| 4625 |
target, on, source = bag["table"], bag["on"], bag["from"]
|
| 4626 |
if _rel().base_table_key(target) not in _rel().READ_THROUGH_KEYS:
|
| 4627 |
continue
|
| 4628 |
+
edges.append((field, target, on, source, bag))
|
| 4629 |
+
if source in declared:
|
| 4630 |
+
values = {str(row[source]).strip().lower() for row in rows if row.get(source) not in (None, "")}
|
| 4631 |
+
if values:
|
| 4632 |
+
requested.setdefault(target, {}).setdefault(on, set()).update(values)
|
| 4633 |
+
remaining = LINK_VALUE_MAX_BYTES
|
| 4634 |
+
target_reads, resolved = {}, {}
|
| 4635 |
+
for field, target, on, source, bag in edges:
|
| 4636 |
+
if source not in declared:
|
| 4637 |
+
for row in rows:
|
| 4638 |
+
row[field["key"]] = ""
|
| 4639 |
+
continue
|
| 4640 |
+
matches = requested.get(target, {}).get(on, ())
|
| 4641 |
+
if not matches or not _may_open(session, target, st=st):
|
| 4642 |
+
cells = {}
|
| 4643 |
+
else:
|
| 4644 |
+
if target not in target_reads:
|
| 4645 |
+
target_reads[target] = _compiled_read(target, session, st=st)
|
| 4646 |
+
cache_key = (target, on, bool(bag.get('single')))
|
| 4647 |
+
if cache_key not in resolved:
|
| 4648 |
+
resolved[cache_key], _required = _sql_link_cells(session, target, on, matches, st=st,
|
| 4649 |
+
single=cache_key[2], remaining=remaining, read=target_reads[target])
|
| 4650 |
+
cells = resolved[cache_key]
|
| 4651 |
for row in rows:
|
| 4652 |
+
value = cells.get(str(row.get(source) or '').strip().lower(), '')
|
| 4653 |
+
# Count repeated cells as well: several page rows may share one target key.
|
| 4654 |
+
remaining -= len(value) + (2 if value else 0)
|
| 4655 |
+
if remaining < 0:
|
| 4656 |
+
raise err(503, 'relation_value_capacity',
|
| 4657 |
+
f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
|
| 4658 |
+
'Reduce the page size or open the related database with filters.')
|
| 4659 |
+
row[field['key']] = value
|
| 4660 |
|
| 4661 |
|
| 4662 |
def _agg_number(value):
|
|
|
|
| 4848 |
f"THEN 1 ELSE 0 END"),
|
| 4849 |
where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
|
| 4850 |
projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
|
| 4851 |
+
group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
|
| 4852 |
+
rows = datastore.window(
|
| 4853 |
+
select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
|
| 4854 |
+
group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4855 |
groups = []
|
| 4856 |
for r in rows:
|
| 4857 |
vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
|
|
|
|
| 4860 |
"count": int(r[1] or 0), "aggregates": vals})
|
| 4861 |
return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
|
| 4862 |
"total_groups": distinct, "group_offset": group_offset,
|
| 4863 |
+
"group_limit": group_limit, "has_more": group_offset + len(groups) < distinct,
|
| 4864 |
+
"limits": limits}
|
| 4865 |
|
| 4866 |
|
| 4867 |
@router.post("/odoo-tables/{table_key}/aggregates")
|
|
|
|
| 4910 |
raise err(400, "bad_selection", "groupOffset must be a nonnegative integer")
|
| 4911 |
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
|
| 4912 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
|
|
|
|
|
|
| 4913 |
|
| 4914 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4915 |
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
|
|
|
| 4965 |
"limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
|
| 4966 |
if out["groups"] is not None:
|
| 4967 |
body_out["groups"] = out["groups"]
|
| 4968 |
+
body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit", "has_more")})
|
| 4969 |
return body_out
|
| 4970 |
|
| 4971 |
|
|
|
|
| 5095 |
if spec is None:
|
| 5096 |
return None
|
| 5097 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 5098 |
+
return datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm)
|
|
|
|
| 5099 |
|
| 5100 |
|
| 5101 |
def whole_pool(table_key, cur=None, rt=None):
|
api/routes_platform_admin.py
CHANGED
|
@@ -47,7 +47,7 @@ import core.do_usage as do_usage
|
|
| 47 |
import core.store as store
|
| 48 |
import operator_control
|
| 49 |
import voice_control_plane as voice
|
| 50 |
-
from deps import Session, err, require_session, users
|
| 51 |
|
| 52 |
@asynccontextmanager
|
| 53 |
async def _neon_lifespan(app):
|
|
@@ -329,7 +329,7 @@ def _user_rows(tenant=None):
|
|
| 329 |
`routes_admin._view`'s discipline: one function that can leak, and it does not.
|
| 330 |
|
| 331 |
R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
|
| 332 |
-
rendered as "never"
|
| 333 |
"""
|
| 334 |
try:
|
| 335 |
reg = users.registry() or {}
|
|
@@ -351,7 +351,7 @@ def _user_rows(tenant=None):
|
|
| 351 |
"role": rec.get("role", "user"),
|
| 352 |
"active": bool(rec.get("active", True)),
|
| 353 |
"lastLogin": rec.get("last_login") or "",
|
| 354 |
-
"lastActive": rec.get("last_active") or "",
|
| 355 |
"platformAdmin": rec.get("platform_admin") is True,
|
| 356 |
})
|
| 357 |
return rows
|
|
@@ -699,14 +699,14 @@ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)):
|
|
| 699 |
"""Every account on the platform, or one tenant's — the drill behind the Users count.
|
| 700 |
|
| 701 |
R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
|
| 702 |
-
`routes_auth.login`
|
| 703 |
-
|
| 704 |
"""
|
| 705 |
rows = _user_rows(tenant)
|
| 706 |
return {"users": rows, "count": len(rows),
|
| 707 |
-
"stampsNote": ("
|
| 708 |
-
"
|
| 709 |
-
"
|
| 710 |
|
| 711 |
|
| 712 |
@router.get("/databases")
|
|
|
|
| 47 |
import core.store as store
|
| 48 |
import operator_control
|
| 49 |
import voice_control_plane as voice
|
| 50 |
+
from deps import Session, err, observed_last_active, require_session, users
|
| 51 |
|
| 52 |
@asynccontextmanager
|
| 53 |
async def _neon_lifespan(app):
|
|
|
|
| 329 |
`routes_admin._view`'s discipline: one function that can leak, and it does not.
|
| 330 |
|
| 331 |
R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
|
| 332 |
+
rendered as "never" when neither a persisted stamp nor a process-local observation exists.
|
| 333 |
"""
|
| 334 |
try:
|
| 335 |
reg = users.registry() or {}
|
|
|
|
| 351 |
"role": rec.get("role", "user"),
|
| 352 |
"active": bool(rec.get("active", True)),
|
| 353 |
"lastLogin": rec.get("last_login") or "",
|
| 354 |
+
"lastActive": observed_last_active(slug, uname, rec.get("last_active")) or "",
|
| 355 |
"platformAdmin": rec.get("platform_admin") is True,
|
| 356 |
})
|
| 357 |
return rows
|
|
|
|
| 699 |
"""Every account on the platform, or one tenant's — the drill behind the Users count.
|
| 700 |
|
| 701 |
R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
|
| 702 |
+
`routes_auth.login`. `lastActive` includes bounded process-local read observations; persisted
|
| 703 |
+
activity comes from login and throttled admitted mutations. Absent means no recorded activity.
|
| 704 |
"""
|
| 705 |
rows = _user_rows(tenant)
|
| 706 |
return {"users": rows, "count": len(rows),
|
| 707 |
+
"stampsNote": ("Activity dates include recent visits observed by this server. "
|
| 708 |
+
"Visits are tracked temporarily. Saved dates come from sign-ins "
|
| 709 |
+
"and changes, and can be older after a server restart.")}
|
| 710 |
|
| 711 |
|
| 712 |
@router.get("/databases")
|
api/routes_products.py
CHANGED
|
@@ -53,6 +53,9 @@ def _pool_for(rt, team_id):
|
|
| 53 |
# fail-closed seam rather than widening it by accident.
|
| 54 |
if rt.key != "royal-imports":
|
| 55 |
return []
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
import modules.product_data as pd
|
| 58 |
from core import user_tables
|
|
@@ -128,13 +131,14 @@ def scoped_pool(session: Session, *, with_rows: bool = True):
|
|
| 128 |
except Exception as e:
|
| 129 |
raise err(503, "pool_unavailable",
|
| 130 |
f"the product catalogue could not be built; {str(e)[:160]}")
|
| 131 |
-
#
|
| 132 |
-
# the
|
| 133 |
-
# Leadership account owns the permission editor from the first request.
|
| 134 |
import modules.product_data as pd
|
|
|
|
|
|
|
| 135 |
if with_rows:
|
| 136 |
-
pd.image_field_definition(st=
|
| 137 |
-
fields_base = pd_fields(consolidated=team_id is None, st=
|
| 138 |
# The SAME wall the customer assembly applies, in the same order: rows first (before pids
|
| 139 |
# are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
|
| 140 |
# then the values stripped from the rows as well as the field list.
|
|
@@ -217,7 +221,7 @@ def product_measures(session):
|
|
| 217 |
return []
|
| 218 |
|
| 219 |
|
| 220 |
-
def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
|
| 221 |
"""`{pid: {field key: value}}` for every measure column on this grid.
|
| 222 |
|
| 223 |
⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's
|
|
@@ -236,6 +240,11 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
|
|
| 236 |
Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries.
|
| 237 |
"""
|
| 238 |
import aios_grid
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
|
| 240 |
mfields = aios_grid.measure_fields_of(fields)
|
| 241 |
if not mfields or not offer:
|
|
@@ -263,7 +272,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
|
|
| 263 |
out = {}
|
| 264 |
for (dfrom, dto), group in by_window.items():
|
| 265 |
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 266 |
-
mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys
|
|
|
|
| 267 |
if mkey not in memo:
|
| 268 |
try:
|
| 269 |
memo[mkey] = sem.entity_measure_values(
|
|
@@ -272,7 +282,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
|
|
| 272 |
# here. The service filter exists to stop Delivery Charges polluting a SKU
|
| 273 |
# RANKING; on a per-row column the row IS the service product, and excluding
|
| 274 |
# it would print a FALSE 0 — which under C1's rule asserts "sold nothing".
|
| 275 |
-
exclude_services=False, offer=offer)
|
|
|
|
| 276 |
except Exception as e: # noqa: BLE001
|
| 277 |
_measure_err("product-measure-column", e)
|
| 278 |
# Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure
|
|
@@ -441,8 +452,9 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
|
|
| 441 |
import aios_grid
|
| 442 |
import core.perm_scope as perm_scope
|
| 443 |
import modules.product_data as pd
|
|
|
|
| 444 |
pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows)
|
| 445 |
-
shared_defs = pd.shared_fields(st=session.runtime)
|
| 446 |
|
| 447 |
ctx = grid_events.EventCtx(
|
| 448 |
uname=session.uname, allowed_pids=pids, fields=[],
|
|
@@ -841,6 +853,12 @@ def pd_fields(consolidated=True, st=None):
|
|
| 841 |
# (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
|
| 842 |
if st is not None:
|
| 843 |
image = pd.image_field_definition(st=st)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 844 |
else:
|
| 845 |
# Contract checks and offline tooling have no tenant runtime. Keep a
|
| 846 |
# truthful local projection with the same created-field semantics.
|
|
|
|
| 53 |
# fail-closed seam rather than widening it by accident.
|
| 54 |
if rt.key != "royal-imports":
|
| 55 |
return []
|
| 56 |
+
bounded = getattr(rt, '_canonical_product_rows', None)
|
| 57 |
+
if bounded is not None:
|
| 58 |
+
return bounded
|
| 59 |
|
| 60 |
import modules.product_data as pd
|
| 61 |
from core import user_tables
|
|
|
|
| 131 |
except Exception as e:
|
| 132 |
raise err(503, "pool_unavailable",
|
| 133 |
f"the product catalogue could not be built; {str(e)[:160]}")
|
| 134 |
+
# Project detached definitions without seeding fields or replacing caller grants.
|
| 135 |
+
# Keep the original runtime for row/value hydration and authoritative mutations.
|
|
|
|
| 136 |
import modules.product_data as pd
|
| 137 |
+
from registry_source import MetadataStore
|
| 138 |
+
metadata = MetadataStore(session.runtime)
|
| 139 |
if with_rows:
|
| 140 |
+
pd.image_field_definition(st=metadata)
|
| 141 |
+
fields_base = pd_fields(consolidated=team_id is None, st=metadata)
|
| 142 |
# The SAME wall the customer assembly applies, in the same order: rows first (before pids
|
| 143 |
# are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
|
| 144 |
# then the values stripped from the rows as well as the field list.
|
|
|
|
| 221 |
return []
|
| 222 |
|
| 223 |
|
| 224 |
+
def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer, *, cur=None):
|
| 225 |
"""`{pid: {field key: value}}` for every measure column on this grid.
|
| 226 |
|
| 227 |
⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's
|
|
|
|
| 240 |
Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries.
|
| 241 |
"""
|
| 242 |
import aios_grid
|
| 243 |
+
if cur is None:
|
| 244 |
+
from harness.canonical_source import active
|
| 245 |
+
source = active()
|
| 246 |
+
if source is not None:
|
| 247 |
+
cur = source.cur
|
| 248 |
|
| 249 |
mfields = aios_grid.measure_fields_of(fields)
|
| 250 |
if not mfields or not offer:
|
|
|
|
| 272 |
out = {}
|
| 273 |
for (dfrom, dto), group in by_window.items():
|
| 274 |
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 275 |
+
mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys,
|
| 276 |
+
tuple(sorted(codes.values(), key=str)))
|
| 277 |
if mkey not in memo:
|
| 278 |
try:
|
| 279 |
memo[mkey] = sem.entity_measure_values(
|
|
|
|
| 282 |
# here. The service filter exists to stop Delivery Charges polluting a SKU
|
| 283 |
# RANKING; on a per-row column the row IS the service product, and excluding
|
| 284 |
# it would print a FALSE 0 — which under C1's rule asserts "sold nothing".
|
| 285 |
+
exclude_services=False, offer=offer, group_keys=list(codes.values()),
|
| 286 |
+
**({'cur': cur} if cur is not None else {}))
|
| 287 |
except Exception as e: # noqa: BLE001
|
| 288 |
_measure_err("product-measure-column", e)
|
| 289 |
# Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure
|
|
|
|
| 452 |
import aios_grid
|
| 453 |
import core.perm_scope as perm_scope
|
| 454 |
import modules.product_data as pd
|
| 455 |
+
from registry_source import MetadataStore
|
| 456 |
pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows)
|
| 457 |
+
shared_defs = pd.shared_fields(st=MetadataStore(session.runtime))
|
| 458 |
|
| 459 |
ctx = grid_events.EventCtx(
|
| 460 |
uname=session.uname, allowed_pids=pids, fields=[],
|
|
|
|
| 853 |
# (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
|
| 854 |
if st is not None:
|
| 855 |
image = pd.image_field_definition(st=st)
|
| 856 |
+
# An explicit grant still governs Image when its stored definition is missing
|
| 857 |
+
# or predates the grant marker. Let the existing permission closure decide access.
|
| 858 |
+
from core import shares
|
| 859 |
+
record = shares.grants("field", shares.field_oid(MODULE, PRODUCT_IMAGE_KEY), st=st)
|
| 860 |
+
if record.get("owner") or record.get("entries"):
|
| 861 |
+
image = dict(image, granted=True)
|
| 862 |
else:
|
| 863 |
# Contract checks and offline tooling have no tenant runtime. Keep a
|
| 864 |
# truthful local projection with the same created-field semantics.
|
api/routes_sheet.py
CHANGED
|
@@ -259,7 +259,7 @@ def _mirror(session: Session):
|
|
| 259 |
if connector_defn is None:
|
| 260 |
native = _native_table_definition(session, table_key)
|
| 261 |
if native is not None:
|
| 262 |
-
cur, spec, cols = _native_table_mirror(session, table_key)
|
| 263 |
return cur, {**spec, "declared_fields": native.get("fields") or []}, cols
|
| 264 |
raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
|
| 265 |
fields = list(connector_defn.get("fields") or [])
|
|
@@ -301,12 +301,12 @@ def _mirror(session: Session):
|
|
| 301 |
stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
|
| 302 |
# The store keeps records as {row id: row}. Iterating that dict as a list yields only
|
| 303 |
# the ids, which materialises an EMPTY table and answers every cell blank.
|
| 304 |
-
rows =
|
| 305 |
-
if isinstance(row, dict)
|
| 306 |
fields = rel.shipment_fields()
|
| 307 |
hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 308 |
fields = [field for field in fields if field["key"] not in hidden]
|
| 309 |
-
if not
|
| 310 |
# S75-T59: a full pass stores a rowless definition that reads through the mirror.
|
| 311 |
spec = rot._source_for(cur, table_key)
|
| 312 |
if spec is None:
|
|
@@ -419,23 +419,40 @@ def _native_table_definition(session, table_key):
|
|
| 419 |
return defn
|
| 420 |
|
| 421 |
|
| 422 |
-
def _native_table_mirror(session, table_key):
|
| 423 |
"""Reuse Data's scoped grid/spec reader, with a cursor owned by this Sheet request.
|
| 424 |
|
| 425 |
The payload has already applied permanent predicates, allowed IDs and field/BU walls.
|
| 426 |
Authored Sheet filters still compile below, against only the visible column vocabulary.
|
| 427 |
"""
|
| 428 |
-
import
|
|
|
|
|
|
|
| 429 |
memo = getattr(session, "_sheet_read_memo", None)
|
| 430 |
if not isinstance(memo, dict) or "sources" not in memo:
|
| 431 |
raise RuntimeError("A native Sheet source needs a request scope.")
|
| 432 |
slot = ("native_source", table_key)
|
| 433 |
-
if
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
return memo[slot]
|
| 440 |
|
| 441 |
|
|
@@ -535,10 +552,18 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr
|
|
| 535 |
with_rows=False, fields_only=not with_rows, needed=needed)
|
| 536 |
return memo[slot]
|
| 537 |
if table_key == "product_data":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 538 |
from routes_products import product_assembly
|
| 539 |
memo[slot] = product_assembly(session, scope="product",
|
| 540 |
consume_corrections=False, with_rows=with_rows, needed=needed)
|
| 541 |
return memo[slot]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
from routes_customers import grid_assembly
|
| 543 |
memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False,
|
| 544 |
with_rows=with_rows, needed=needed)
|
|
@@ -565,6 +590,9 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr
|
|
| 565 |
def _sheet_permission_reader(session):
|
| 566 |
"""Lend a captured schema/grant snapshot only inside the current Sheet request."""
|
| 567 |
memo = getattr(session, "_sheet_read_memo", None)
|
|
|
|
|
|
|
|
|
|
| 568 |
return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime
|
| 569 |
|
| 570 |
|
|
@@ -798,6 +826,22 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits):
|
|
| 798 |
spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0])
|
| 799 |
total_rows = int(cur.execute(count_sql, params).fetchone()[0])
|
| 800 |
tail = count_sql[len("SELECT COUNT(*)"):]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 801 |
sql = "SELECT " + ", ".join([*groups, *expressions]) + tail
|
| 802 |
if groups:
|
| 803 |
positions = ", ".join(str(index + 1) for index in range(len(groups)))
|
|
@@ -816,14 +860,6 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits):
|
|
| 816 |
row_keys.setdefault(row_key, len(row_keys))
|
| 817 |
column_keys.setdefault(column_key, len(column_keys))
|
| 818 |
values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]]
|
| 819 |
-
# The matrix below is dense, so its size is rows x columns x values whatever the data holds.
|
| 820 |
-
# Standing rule 1: a bound that cannot be lifted is reported with its cause and its fix.
|
| 821 |
-
cells_needed = len(row_keys) * len(column_keys) * len(pivot["values"])
|
| 822 |
-
if cells_needed > _PIVOT_MAX_CELLS:
|
| 823 |
-
raise err(409, "pivot_too_large",
|
| 824 |
-
f"This pivot needs {len(row_keys):,} rows by {len(column_keys):,} columns, "
|
| 825 |
-
f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. "
|
| 826 |
-
"Move a field to Filters, or use a field with fewer distinct values.")
|
| 827 |
# SQL orders the entire tuple, but a column tuple may first occur in a later row group.
|
| 828 |
# Sort each axis independently using its JSON spelling for deterministic mixed/null keys.
|
| 829 |
ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
|
|
@@ -1014,6 +1050,32 @@ def _linked_filter_fields(database, session):
|
|
| 1014 |
return {"fields": offered, "databases": databases, "unavailable": unavailable}
|
| 1015 |
|
| 1016 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1017 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
| 1018 |
"""`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
|
| 1019 |
|
|
@@ -1046,7 +1108,9 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 1046 |
column just because a filter named it would invent an empty column for a name this database
|
| 1047 |
does not have, and the wall would then compile against nothing and quietly stop restricting.
|
| 1048 |
"""
|
| 1049 |
-
|
|
|
|
|
|
|
| 1050 |
import odoo_relational as rel
|
| 1051 |
import routes_odoo_tables as rot
|
| 1052 |
from modules import sheet as sheet_mod
|
|
@@ -1058,8 +1122,8 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 1058 |
derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window)
|
| 1059 |
# Authorization has already scoped rows using the full wall vocabulary. Restrict only
|
| 1060 |
# cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default.
|
| 1061 |
-
rows = (
|
| 1062 |
-
|
| 1063 |
if table_key == "customer_data" and (needed is None or "agent_id" in needed):
|
| 1064 |
from core import perm_scope
|
| 1065 |
agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
|
|
@@ -1068,8 +1132,7 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 1068 |
# Join only to already-scoped customer rows. pid is the actual res.partner id.
|
| 1069 |
have = rel.columns(cur, "res_partner")
|
| 1070 |
if "agent_id" in have:
|
| 1071 |
-
|
| 1072 |
-
rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows]
|
| 1073 |
fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
|
| 1074 |
elif needed is not None and "agent_id" in needed:
|
| 1075 |
raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
|
|
@@ -1086,35 +1149,14 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 1086 |
spec_fields = sheet_mod.mirror_fields(kept)
|
| 1087 |
# The related customer side can contain thousands of permitted rows. Insert its column
|
| 1088 |
# arrays in one call; per-row execute round trips delayed the first drill page by seconds.
|
| 1089 |
-
spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows, bulk=True)
|
| 1090 |
-
# ⛔⛔ THE MATERIALISED TABLE IS GIVEN THE WINDOW'S OWN NAME, AND WITHOUT THIS THE MEMO ABOVE
|
| 1091 |
-
# SERVES ONE WINDOW'S ROWS UNDER ANOTHER WINDOW'S ANSWER (found by W53-T14, live).
|
| 1092 |
-
# `odoo_relational.registry_grid_spec` names its temp table `_registry_grid_<table_key>` and
|
| 1093 |
-
# `CREATE OR REPLACE`s it, so the name carries the DATABASE and not the PERIOD, while
|
| 1094 |
-
# `_mirror`'s memo is keyed by `(table_key, window)`. Resolve one customer_data cell at no
|
| 1095 |
-
# period and a second at `{month, 2026-08}` inside ONE request and the second build replaces
|
| 1096 |
-
# the first one's table underneath the spec the memo is still holding; a third cell reading
|
| 1097 |
-
# the first slot then runs its SQL against August's rows.
|
| 1098 |
-
# ⚠ MEASURED, and it is silent: `avg` over `measure_sales_the_last_12_months_6hlkf` answered
|
| 1099 |
-
# **138.3942281513749** in a batch that had already resolved that month (508,322.0 / 3,673)
|
| 1100 |
-
# and **1710.679825755513** on its own, which is the all-time number the preview door and the
|
| 1101 |
-
# grid both give. No refusal, no limits row, a plausible number under the wrong label.
|
| 1102 |
-
# ⛔ IT IS FIXED HERE RATHER THAN IN `registry_grid_spec` BECAUSE THAT FILE BELONGS TO ANOTHER
|
| 1103 |
-
# LANE THIS WAVE. The copy is one `CREATE TABLE AS SELECT` over rows already in memory, next
|
| 1104 |
-
# to an assembly that costs 20 to 130 seconds, so it is not a cost worth conditioning on.
|
| 1105 |
-
# ⚠ ALWAYS, INCLUDING THE `None` WINDOW: the unqualified name is REPLACED by every later
|
| 1106 |
-
# build, so leaving the no-period case pointing at it is exactly the case measured above.
|
| 1107 |
-
# ⛔⛔ W54-T36 — THE COLUMN SET IS PART OF THAT NAME FOR THE SAME REASON THE WINDOW IS. Two
|
| 1108 |
-
# cells on one database at one window may now be materialised over DIFFERENT columns, and a
|
| 1109 |
-
# narrower later build would otherwise replace a wider earlier one under a spec the memo is
|
| 1110 |
-
# still holding, so a query would address a column its own table no longer has.
|
| 1111 |
slot = hashlib.sha1(
|
| 1112 |
json.dumps([metric_window, None if needed is None else sorted(needed)],
|
| 1113 |
sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12]
|
| 1114 |
-
|
| 1115 |
-
|
| 1116 |
-
|
| 1117 |
-
|
|
|
|
| 1118 |
# ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal`
|
| 1119 |
# has to name WHY a column cannot be totalled, and the normalised copy has already thrown that
|
| 1120 |
# away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds
|
|
@@ -1123,6 +1165,22 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
|
|
| 1123 |
return cur, spec, rot._filter_columns(spec, spec_fields)
|
| 1124 |
|
| 1125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1126 |
def _metric_derived(session: Session, table_key: str, g, metric_window):
|
| 1127 |
"""`(derived, bounded_keys, unbounded)` for one assembly at the cell's period.
|
| 1128 |
|
|
@@ -1156,10 +1214,7 @@ def _metric_derived(session: Session, table_key: str, g, metric_window):
|
|
| 1156 |
seen.update(cells or ())
|
| 1157 |
live = [f for f in mfields if f.get("key") in seen]
|
| 1158 |
|
| 1159 |
-
out =
|
| 1160 |
-
for f in mfields: # every Metric column starts blank at a new window
|
| 1161 |
-
for cells in out.values():
|
| 1162 |
-
cells.pop(f.get("key"), None)
|
| 1163 |
|
| 1164 |
bounded, unbounded = [], {}
|
| 1165 |
for f in live:
|
|
@@ -1170,12 +1225,9 @@ def _metric_derived(session: Session, table_key: str, g, metric_window):
|
|
| 1170 |
cells = _measure_cells_at(session, table_key, g, rewritten)
|
| 1171 |
except Exception: # noqa: BLE001
|
| 1172 |
cells = {}
|
| 1173 |
-
got =
|
| 1174 |
-
for pid, values in (cells or {}).items():
|
| 1175 |
-
if key in values:
|
| 1176 |
-
out.setdefault(pid, {})[key] = values[key]
|
| 1177 |
-
got = True
|
| 1178 |
if got:
|
|
|
|
| 1179 |
bounded.append(key)
|
| 1180 |
else:
|
| 1181 |
unbounded[key] = sheet_mod.metric_window_limit(
|
|
@@ -1209,10 +1261,12 @@ def _measure_cells_at(session: Session, table_key: str, g, field):
|
|
| 1209 |
import routes_products as rp
|
| 1210 |
return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"),
|
| 1211 |
g.get("today"), "sheet-period", {},
|
| 1212 |
-
rp.product_measures(session)
|
|
|
|
| 1213 |
from core import measure_resolve
|
| 1214 |
return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(),
|
| 1215 |
-
g.get("today"), "sheet-period", {}
|
|
|
|
| 1216 |
|
| 1217 |
|
| 1218 |
def _offered_view(session: Session, database: str, view_id: str):
|
|
@@ -4798,7 +4852,7 @@ def _metric_column_aggregate(session, database, aggregate, window):
|
|
| 4798 |
# assembly for another database: that would create a field on the wrong database.
|
| 4799 |
if database not in ("customer_data", "product_data"):
|
| 4800 |
refuse("measure_not_offered")
|
| 4801 |
-
g = _registry_assembly(session, database)
|
| 4802 |
# Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue
|
| 4803 |
# with the served offer too: product has a narrower entity catalogue, and revoked Metric
|
| 4804 |
# capability makes the account's offer empty. A browser key can never widen either offer.
|
|
@@ -4858,7 +4912,7 @@ def _metric_column_aggregate(session, database, aggregate, window):
|
|
| 4858 |
if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys
|
| 4859 |
or raw["key"] in {f.get("key") for f in g["fields"]}):
|
| 4860 |
_forget_sheet_database(session, database)
|
| 4861 |
-
found = existing(_registry_assembly(session, database).get("fields"))
|
| 4862 |
if found is not None:
|
| 4863 |
return column(found)
|
| 4864 |
refuse("metric_column_conflict", label)
|
|
@@ -4910,6 +4964,7 @@ def _sheet_request_scope(session):
|
|
| 4910 |
previous = getattr(session, "_sheet_read_memo", None)
|
| 4911 |
previous_assemblies = getattr(session, "_sheet_assemblies", None)
|
| 4912 |
previous_context = getattr(session, "_sheet_scope_context", None)
|
|
|
|
| 4913 |
sources = contextlib.ExitStack()
|
| 4914 |
memo = {"today": _sheet_today(session), "sources": sources,
|
| 4915 |
"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
|
|
@@ -4929,6 +4984,7 @@ def _sheet_request_scope(session):
|
|
| 4929 |
session._sheet_read_memo = previous
|
| 4930 |
session._sheet_assemblies = previous_assemblies
|
| 4931 |
session._sheet_scope_context = previous_context
|
|
|
|
| 4932 |
|
| 4933 |
|
| 4934 |
def _request_scoped(route):
|
|
@@ -6320,7 +6376,7 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 6320 |
raise _refuse(exc)
|
| 6321 |
_require_reader(session, book)
|
| 6322 |
database = _require_database(session, binding["database"])
|
| 6323 |
-
fields = list(_registry_assembly(session, database).get("fields") or [])
|
| 6324 |
hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
|
| 6325 |
visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
|
| 6326 |
requested = binding["fields"]
|
|
@@ -6412,15 +6468,36 @@ _DRILL_MIRROR_LOCK = threading.RLock()
|
|
| 6412 |
_DRILL_MIRROR_MAX = 6
|
| 6413 |
|
| 6414 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6415 |
def _drill_tokens(session, databases):
|
| 6416 |
"""Unknown freshness disables reuse. No workspace documents are retained here."""
|
| 6417 |
from core import sheet_binding
|
|
|
|
| 6418 |
tokens = {}
|
| 6419 |
for db in sorted(databases):
|
| 6420 |
try:
|
| 6421 |
import harness.mirror_tokens # noqa: F401
|
| 6422 |
tokens[db] = sheet_binding.database_change_token(_store(session), db)
|
| 6423 |
except Exception as exc:
|
|
|
|
| 6424 |
_token_unread(session, db, exc)
|
| 6425 |
tokens[db] = None
|
| 6426 |
return tokens
|
|
@@ -6430,6 +6507,7 @@ def _drill_tokens(session, databases):
|
|
| 6430 |
def _drill_mirror(session, database, window, needed, databases):
|
| 6431 |
"""A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction."""
|
| 6432 |
from core import sheet_binding, perm_scope
|
|
|
|
| 6433 |
import datetime
|
| 6434 |
|
| 6435 |
if _native_table_definition(session, database) is not None:
|
|
@@ -6446,7 +6524,8 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6446 |
if callable(stamp):
|
| 6447 |
try:
|
| 6448 |
wall = stamp(session.tenant, session.uname, database, user=session.user)
|
| 6449 |
-
except Exception:
|
|
|
|
| 6450 |
pass
|
| 6451 |
if not wall:
|
| 6452 |
token = None
|
|
@@ -6459,7 +6538,7 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6459 |
"_sheet_drill_entries")
|
| 6460 |
absent = object()
|
| 6461 |
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 6462 |
-
with
|
| 6463 |
hit = _DRILL_MIRRORS.get(key) if token else None
|
| 6464 |
session._sheet_drill_isolated = True
|
| 6465 |
session._sheet_drill_cursor = hit["owned"] if hit else None
|
|
@@ -6505,16 +6584,23 @@ def _drill_mirror(session, database, window, needed, databases):
|
|
| 6505 |
setattr(session, name, value)
|
| 6506 |
|
| 6507 |
|
| 6508 |
-
def _drill_page(session, database, record_ids, offset, limit):
|
| 6509 |
"""Use fetchTableWindow's server door, including its row and field walls and wire shape."""
|
| 6510 |
import routes_odoo_tables as rot
|
| 6511 |
from harness import datastore
|
| 6512 |
|
| 6513 |
size = min(limit, datastore.WINDOW_MAX)
|
| 6514 |
-
ids = record_ids
|
| 6515 |
-
|
| 6516 |
-
|
| 6517 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6518 |
by_id = {int(row["pid"]): row for row in payload["rows"]}
|
| 6519 |
limits = list(payload.get("limits") or [])
|
| 6520 |
if size != limit:
|
|
@@ -6522,10 +6608,117 @@ def _drill_page(session, database, record_ids, offset, limit):
|
|
| 6522 |
"cause": f"One response carries at most {datastore.WINDOW_MAX} rows.",
|
| 6523 |
"recommendation": "Continue paging with offset. Every matching row remains available."})
|
| 6524 |
return {**payload, "offset": offset, "limit": size, "limits": limits,
|
| 6525 |
-
"total":
|
| 6526 |
"rows": [by_id[pid] for pid in ids if pid in by_id]}
|
| 6527 |
|
| 6528 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6529 |
def _drill_draft_binding(cell, binding, draft_filter):
|
| 6530 |
"""The stored binding with View data's UNSAVED filter in place of its own (S68-T23 finding).
|
| 6531 |
|
|
@@ -6559,35 +6752,20 @@ def _drill_draft_binding(cell, binding, draft_filter):
|
|
| 6559 |
def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
| 6560 |
session: Session = Depends(require_session),
|
| 6561 |
offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1),
|
| 6562 |
-
draft_filter: str | None = Query(default=None, alias="filter")
|
| 6563 |
-
|
| 6564 |
-
|
| 6565 |
-
|
| 6566 |
-
THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a
|
| 6567 |
-
Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old
|
| 6568 |
-
first-tab rule still answers, so an older client keeps working.
|
| 6569 |
-
|
| 6570 |
-
⭐⭐ S66-T48 — NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching
|
| 6571 |
-
id; the popup pages them and sends the set in a POST body. Only a database whose records a
|
| 6572 |
-
person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so
|
| 6573 |
-
the popup says "showing X of N" rather than presenting X as the whole.
|
| 6574 |
-
|
| 6575 |
-
⛔ A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
|
| 6576 |
-
would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
|
| 6577 |
-
clicked on.
|
| 6578 |
-
|
| 6579 |
-
⭐ THE IDS ARE READ LIVE, NOT TAKEN FROM THE CELL (R13). A stale cell opens the same popup with
|
| 6580 |
-
no comparison text, and the count beside those ids is the LIVE count, because `LinkGridModal`
|
| 6581 |
-
prints the length of what it is handed and runs no query of its own.
|
| 6582 |
-
"""
|
| 6583 |
-
from core import sheets
|
| 6584 |
from modules import sheet as sheet_mod
|
| 6585 |
import core.perm_scope as perm_scope
|
|
|
|
|
|
|
| 6586 |
|
| 6587 |
# Python callers omit FastAPI's Query sentinels. Offset has no maximum; only the page does.
|
| 6588 |
offset = offset if isinstance(offset, int) else 0
|
| 6589 |
limit = limit if isinstance(limit, int) else 100
|
| 6590 |
draft_filter = draft_filter if isinstance(draft_filter, str) else None
|
|
|
|
| 6591 |
if offset < 0 or limit < 1:
|
| 6592 |
raise err(400, "bad_drill_page", "Choose a nonnegative record offset and a positive page size.")
|
| 6593 |
|
|
@@ -6658,15 +6836,31 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
|
| 6658 |
raise err(400, exc.code, exc.message)
|
| 6659 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 6660 |
| sheet_mod.filter_columns_named(wall_nodes))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6661 |
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 6662 |
databases = {database}
|
| 6663 |
databases.update(_binding_related_entry(session, database, via)["target"]
|
| 6664 |
for via in sheet_mod.related_vias(scope_nodes))
|
|
|
|
| 6665 |
with _drill_mirror(session, database, window, needed, databases) as (made, token):
|
| 6666 |
cur, spec, cols = made
|
| 6667 |
cols = dict(cols)
|
| 6668 |
# Apply the aggregate's field wall before compiling either the count or the IDs.
|
| 6669 |
hidden = perm_scope.hidden_keys(session.user, database, spec.get("fields") or [], st=session.runtime)
|
|
|
|
| 6670 |
if spec.get("scope_applied"):
|
| 6671 |
wall_nodes, wall_conj = [], "and"
|
| 6672 |
unavailable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
|
|
@@ -6685,9 +6879,6 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
|
| 6685 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 6686 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 6687 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
| 6688 |
-
ids_sql, ids_params, _ = sheet_mod.build_ids_query(
|
| 6689 |
-
spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
|
| 6690 |
-
wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
| 6691 |
except sheet_mod.SheetRefusal as exc:
|
| 6692 |
raise err(400, exc.code, exc.message)
|
| 6693 |
except ValueError:
|
|
@@ -6696,25 +6887,52 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
|
| 6696 |
"connection. Edit the scope or ask an administrator to update the restriction.")
|
| 6697 |
if spec.get("scope_applied") and limits:
|
| 6698 |
raise err(409, "scope_not_applicable", "This cell's filter cannot be applied to this database.")
|
| 6699 |
-
|
| 6700 |
-
|
| 6701 |
-
|
| 6702 |
-
|
| 6703 |
-
|
| 6704 |
-
|
| 6705 |
-
|
| 6706 |
-
|
| 6707 |
-
|
| 6708 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6709 |
|
| 6710 |
limits = list(limits) + list(spec.get("limits") or [])
|
| 6711 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
| 6712 |
-
"
|
|
|
|
| 6713 |
if scope.get("view_id"):
|
| 6714 |
out["view_id"] = scope["view_id"]
|
| 6715 |
else:
|
| 6716 |
out["filter"] = scope.get("filter")
|
| 6717 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6718 |
out["token"] = token
|
| 6719 |
return out
|
| 6720 |
|
|
|
|
| 259 |
if connector_defn is None:
|
| 260 |
native = _native_table_definition(session, table_key)
|
| 261 |
if native is not None:
|
| 262 |
+
cur, spec, cols = _native_table_mirror(session, table_key, needed=needed)
|
| 263 |
return cur, {**spec, "declared_fields": native.get("fields") or []}, cols
|
| 264 |
raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
|
| 265 |
fields = list(connector_defn.get("fields") or [])
|
|
|
|
| 301 |
stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
|
| 302 |
# The store keeps records as {row id: row}. Iterating that dict as a list yields only
|
| 303 |
# the ids, which materialises an EMPTY table and answers every cell blank.
|
| 304 |
+
rows = (row for row in (stored.values() if isinstance(stored, dict) else stored)
|
| 305 |
+
if isinstance(row, dict))
|
| 306 |
fields = rel.shipment_fields()
|
| 307 |
hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
|
| 308 |
fields = [field for field in fields if field["key"] not in hidden]
|
| 309 |
+
if not stored:
|
| 310 |
# S75-T59: a full pass stores a rowless definition that reads through the mirror.
|
| 311 |
spec = rot._source_for(cur, table_key)
|
| 312 |
if spec is None:
|
|
|
|
| 419 |
return defn
|
| 420 |
|
| 421 |
|
| 422 |
+
def _native_table_mirror(session, table_key, *, needed=None):
|
| 423 |
"""Reuse Data's scoped grid/spec reader, with a cursor owned by this Sheet request.
|
| 424 |
|
| 425 |
The payload has already applied permanent predicates, allowed IDs and field/BU walls.
|
| 426 |
Authored Sheet filters still compile below, against only the visible column vocabulary.
|
| 427 |
"""
|
| 428 |
+
import duckdb
|
| 429 |
+
from harness import datastore
|
| 430 |
+
import registry_source
|
| 431 |
memo = getattr(session, "_sheet_read_memo", None)
|
| 432 |
if not isinstance(memo, dict) or "sources" not in memo:
|
| 433 |
raise RuntimeError("A native Sheet source needs a request scope.")
|
| 434 |
slot = ("native_source", table_key)
|
| 435 |
+
if table_key == 'ut_activity':
|
| 436 |
+
if slot not in memo:
|
| 437 |
+
import routes_odoo_tables as rot
|
| 438 |
+
read = memo['sources'].enter_context(rot._materialised_read(table_key, session,
|
| 439 |
+
{'filters': [], 'filterConj': 'and', 'search': ''}, None))
|
| 440 |
+
spec = dict(read['spec'], fields=read['payload']['fields'],
|
| 441 |
+
scope_applied=True, limits=list(read['limits']))
|
| 442 |
+
memo[slot] = (read['cur'], spec, read['cols'])
|
| 443 |
+
return memo[slot]
|
| 444 |
+
shape = ('native_columns', table_key)
|
| 445 |
+
built = memo.get(shape)
|
| 446 |
+
if slot not in memo or (built is not None and (needed is None or not set(needed) <= built)):
|
| 447 |
+
wanted = None if needed is None else set(needed) | (built or set())
|
| 448 |
+
owned = getattr(session, '_sheet_drill_cursor', None)
|
| 449 |
+
if owned is None:
|
| 450 |
+
cur = memo['sources'].enter_context(datastore.bounded_connection(duckdb.connect(':memory:')))
|
| 451 |
+
session._sheet_drill_cursor = (cur, None)
|
| 452 |
+
else:
|
| 453 |
+
cur = owned[0]
|
| 454 |
+
memo[slot] = registry_source.stored_grid(session, cur, table_key, needed=wanted)
|
| 455 |
+
memo[shape] = wanted
|
| 456 |
return memo[slot]
|
| 457 |
|
| 458 |
|
|
|
|
| 552 |
with_rows=False, fields_only=not with_rows, needed=needed)
|
| 553 |
return memo[slot]
|
| 554 |
if table_key == "product_data":
|
| 555 |
+
if not with_rows:
|
| 556 |
+
from registry_source import canonical_metadata
|
| 557 |
+
memo[slot] = canonical_metadata(session, table_key)
|
| 558 |
+
return memo[slot]
|
| 559 |
from routes_products import product_assembly
|
| 560 |
memo[slot] = product_assembly(session, scope="product",
|
| 561 |
consume_corrections=False, with_rows=with_rows, needed=needed)
|
| 562 |
return memo[slot]
|
| 563 |
+
if not with_rows:
|
| 564 |
+
from registry_source import canonical_metadata
|
| 565 |
+
memo[slot] = canonical_metadata(session, table_key)
|
| 566 |
+
return memo[slot]
|
| 567 |
from routes_customers import grid_assembly
|
| 568 |
memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False,
|
| 569 |
with_rows=with_rows, needed=needed)
|
|
|
|
| 590 |
def _sheet_permission_reader(session):
|
| 591 |
"""Lend a captured schema/grant snapshot only inside the current Sheet request."""
|
| 592 |
memo = getattr(session, "_sheet_read_memo", None)
|
| 593 |
+
if (isinstance(memo, dict) and 'permission_reader' not in memo
|
| 594 |
+
and callable(getattr(session.runtime, 'get_projection', None))):
|
| 595 |
+
_capture_sheet_permissions(session)
|
| 596 |
return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime
|
| 597 |
|
| 598 |
|
|
|
|
| 826 |
spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0])
|
| 827 |
total_rows = int(cur.execute(count_sql, params).fetchone()[0])
|
| 828 |
tail = count_sql[len("SELECT COUNT(*)"):]
|
| 829 |
+
# Count both dense axes before fetching any grouped records or allocating the matrix.
|
| 830 |
+
def axis_count(expressions):
|
| 831 |
+
if not expressions:
|
| 832 |
+
return 1
|
| 833 |
+
axis = "SELECT " + ", ".join(expressions) + tail
|
| 834 |
+
axis += " GROUP BY " + ", ".join(str(i + 1) for i in range(len(expressions)))
|
| 835 |
+
return int(cur.execute(f"SELECT count(*) FROM ({axis}) axis", params).fetchone()[0])
|
| 836 |
+
|
| 837 |
+
row_count = axis_count(groups[:len(row_fields)])
|
| 838 |
+
column_count = axis_count(groups[len(row_fields):])
|
| 839 |
+
cells_needed = row_count * column_count * len(pivot["values"])
|
| 840 |
+
if cells_needed > _PIVOT_MAX_CELLS:
|
| 841 |
+
raise err(409, "pivot_too_large",
|
| 842 |
+
f"This pivot needs {row_count:,} rows by {column_count:,} columns, "
|
| 843 |
+
f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. "
|
| 844 |
+
"Move a field to Filters, or use a field with fewer distinct values.")
|
| 845 |
sql = "SELECT " + ", ".join([*groups, *expressions]) + tail
|
| 846 |
if groups:
|
| 847 |
positions = ", ".join(str(index + 1) for index in range(len(groups)))
|
|
|
|
| 860 |
row_keys.setdefault(row_key, len(row_keys))
|
| 861 |
column_keys.setdefault(column_key, len(column_keys))
|
| 862 |
values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 863 |
# SQL orders the entire tuple, but a column tuple may first occur in a later row group.
|
| 864 |
# Sort each axis independently using its JSON spelling for deterministic mixed/null keys.
|
| 865 |
ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
|
|
|
|
| 1050 |
return {"fields": offered, "databases": databases, "unavailable": unavailable}
|
| 1051 |
|
| 1052 |
|
| 1053 |
+
def _registry_rows(source, fields, overlays, derived):
|
| 1054 |
+
"""Project borrowed scoped inputs through the grid's converter in bounded batches."""
|
| 1055 |
+
import itertools
|
| 1056 |
+
import aios_grid
|
| 1057 |
+
source = iter(source)
|
| 1058 |
+
while True:
|
| 1059 |
+
batch = list(itertools.islice(source, 2048))
|
| 1060 |
+
if not batch:
|
| 1061 |
+
return
|
| 1062 |
+
try:
|
| 1063 |
+
if fields:
|
| 1064 |
+
projected = aios_grid.rows_from_pool(batch, fields, overlays, derived=derived)
|
| 1065 |
+
# Release source rows before yielding their independent projections.
|
| 1066 |
+
batch.clear()
|
| 1067 |
+
try:
|
| 1068 |
+
yield from projected
|
| 1069 |
+
finally:
|
| 1070 |
+
projected.clear()
|
| 1071 |
+
del projected
|
| 1072 |
+
else:
|
| 1073 |
+
yield from ({"pid": row["pid"]} for row in batch)
|
| 1074 |
+
finally:
|
| 1075 |
+
batch.clear()
|
| 1076 |
+
del batch
|
| 1077 |
+
|
| 1078 |
+
|
| 1079 |
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
| 1080 |
"""`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
|
| 1081 |
|
|
|
|
| 1108 |
column just because a filter named it would invent an empty column for a name this database
|
| 1109 |
does not have, and the wall would then compile against nothing and quietly stop restricting.
|
| 1110 |
"""
|
| 1111 |
+
if table_key in {'customer_data', 'product_data'}:
|
| 1112 |
+
from registry_source import canonical_grid
|
| 1113 |
+
return canonical_grid(session, cur, table_key, needed=needed, metric_window=metric_window)
|
| 1114 |
import odoo_relational as rel
|
| 1115 |
import routes_odoo_tables as rot
|
| 1116 |
from modules import sheet as sheet_mod
|
|
|
|
| 1122 |
derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window)
|
| 1123 |
# Authorization has already scoped rows using the full wall vocabulary. Restrict only
|
| 1124 |
# cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default.
|
| 1125 |
+
rows = _registry_rows(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"), derived)
|
| 1126 |
+
join_agent = False
|
| 1127 |
if table_key == "customer_data" and (needed is None or "agent_id" in needed):
|
| 1128 |
from core import perm_scope
|
| 1129 |
agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
|
|
|
|
| 1132 |
# Join only to already-scoped customer rows. pid is the actual res.partner id.
|
| 1133 |
have = rel.columns(cur, "res_partner")
|
| 1134 |
if "agent_id" in have:
|
| 1135 |
+
join_agent = True
|
|
|
|
| 1136 |
fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
|
| 1137 |
elif needed is not None and "agent_id" in needed:
|
| 1138 |
raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
|
|
|
|
| 1149 |
spec_fields = sheet_mod.mirror_fields(kept)
|
| 1150 |
# The related customer side can contain thousands of permitted rows. Insert its column
|
| 1151 |
# arrays in one call; per-row execute round trips delayed the first drill page by seconds.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1152 |
slot = hashlib.sha1(
|
| 1153 |
json.dumps([metric_window, None if needed is None else sorted(needed)],
|
| 1154 |
sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12]
|
| 1155 |
+
spec = rel.registry_grid_spec(cur, f"{table_key}__at_{slot}", spec_fields, rows, bulk=True)
|
| 1156 |
+
if join_agent:
|
| 1157 |
+
cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id = partner.agent_id '
|
| 1158 |
+
'FROM res_partner AS partner WHERE grid.pid = partner.id')
|
| 1159 |
+
# Window and column identities name the table before insertion, preserving every memo slot.
|
| 1160 |
# ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal`
|
| 1161 |
# has to name WHY a column cannot be totalled, and the normalised copy has already thrown that
|
| 1162 |
# away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds
|
|
|
|
| 1165 |
return cur, spec, rot._filter_columns(spec, spec_fields)
|
| 1166 |
|
| 1167 |
|
| 1168 |
+
class _PeriodDerived:
|
| 1169 |
+
"""Read borrowed metric maps per row without copying their full population."""
|
| 1170 |
+
|
| 1171 |
+
def __init__(self, base, cleared):
|
| 1172 |
+
self.base, self.cleared, self.columns = base, cleared, []
|
| 1173 |
+
|
| 1174 |
+
def get(self, pid, default=None):
|
| 1175 |
+
cells = {key: value for key, value in (self.base.get(pid) or {}).items()
|
| 1176 |
+
if key not in self.cleared}
|
| 1177 |
+
for key, column in self.columns:
|
| 1178 |
+
values = column.get(pid) or {}
|
| 1179 |
+
if key in values:
|
| 1180 |
+
cells[key] = values[key]
|
| 1181 |
+
return cells or default
|
| 1182 |
+
|
| 1183 |
+
|
| 1184 |
def _metric_derived(session: Session, table_key: str, g, metric_window):
|
| 1185 |
"""`(derived, bounded_keys, unbounded)` for one assembly at the cell's period.
|
| 1186 |
|
|
|
|
| 1214 |
seen.update(cells or ())
|
| 1215 |
live = [f for f in mfields if f.get("key") in seen]
|
| 1216 |
|
| 1217 |
+
out = _PeriodDerived(derived, {f.get("key") for f in mfields})
|
|
|
|
|
|
|
|
|
|
| 1218 |
|
| 1219 |
bounded, unbounded = [], {}
|
| 1220 |
for f in live:
|
|
|
|
| 1225 |
cells = _measure_cells_at(session, table_key, g, rewritten)
|
| 1226 |
except Exception: # noqa: BLE001
|
| 1227 |
cells = {}
|
| 1228 |
+
got = any(key in values for values in (cells or {}).values())
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1229 |
if got:
|
| 1230 |
+
out.columns.append((key, cells))
|
| 1231 |
bounded.append(key)
|
| 1232 |
else:
|
| 1233 |
unbounded[key] = sheet_mod.metric_window_limit(
|
|
|
|
| 1261 |
import routes_products as rp
|
| 1262 |
return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"),
|
| 1263 |
g.get("today"), "sheet-period", {},
|
| 1264 |
+
rp.product_measures(session),
|
| 1265 |
+
cur=getattr(session.runtime, '_canonical_source_cursor', None))
|
| 1266 |
from core import measure_resolve
|
| 1267 |
return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(),
|
| 1268 |
+
g.get("today"), "sheet-period", {},
|
| 1269 |
+
cur=getattr(session.runtime, '_canonical_source_cursor', None))
|
| 1270 |
|
| 1271 |
|
| 1272 |
def _offered_view(session: Session, database: str, view_id: str):
|
|
|
|
| 4852 |
# assembly for another database: that would create a field on the wrong database.
|
| 4853 |
if database not in ("customer_data", "product_data"):
|
| 4854 |
refuse("measure_not_offered")
|
| 4855 |
+
g = _registry_assembly(session, database, with_rows=False)
|
| 4856 |
# Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue
|
| 4857 |
# with the served offer too: product has a narrower entity catalogue, and revoked Metric
|
| 4858 |
# capability makes the account's offer empty. A browser key can never widen either offer.
|
|
|
|
| 4912 |
if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys
|
| 4913 |
or raw["key"] in {f.get("key") for f in g["fields"]}):
|
| 4914 |
_forget_sheet_database(session, database)
|
| 4915 |
+
found = existing(_registry_assembly(session, database, with_rows=False).get("fields"))
|
| 4916 |
if found is not None:
|
| 4917 |
return column(found)
|
| 4918 |
refuse("metric_column_conflict", label)
|
|
|
|
| 4964 |
previous = getattr(session, "_sheet_read_memo", None)
|
| 4965 |
previous_assemblies = getattr(session, "_sheet_assemblies", None)
|
| 4966 |
previous_context = getattr(session, "_sheet_scope_context", None)
|
| 4967 |
+
previous_cursor = getattr(session, '_sheet_drill_cursor', None)
|
| 4968 |
sources = contextlib.ExitStack()
|
| 4969 |
memo = {"today": _sheet_today(session), "sources": sources,
|
| 4970 |
"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
|
|
|
|
| 4984 |
session._sheet_read_memo = previous
|
| 4985 |
session._sheet_assemblies = previous_assemblies
|
| 4986 |
session._sheet_scope_context = previous_context
|
| 4987 |
+
session._sheet_drill_cursor = previous_cursor
|
| 4988 |
|
| 4989 |
|
| 4990 |
def _request_scoped(route):
|
|
|
|
| 6376 |
raise _refuse(exc)
|
| 6377 |
_require_reader(session, book)
|
| 6378 |
database = _require_database(session, binding["database"])
|
| 6379 |
+
fields = list(_registry_assembly(session, database, with_rows=False).get("fields") or [])
|
| 6380 |
hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
|
| 6381 |
visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
|
| 6382 |
requested = binding["fields"]
|
|
|
|
| 6468 |
_DRILL_MIRROR_MAX = 6
|
| 6469 |
|
| 6470 |
|
| 6471 |
+
@contextlib.contextmanager
|
| 6472 |
+
def _drill_mirror_lock():
|
| 6473 |
+
"""Include shared-cursor contention in the caller's absolute request budget."""
|
| 6474 |
+
from core import request_timing
|
| 6475 |
+
timeout = request_timing.remaining_budget()
|
| 6476 |
+
acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
|
| 6477 |
+
else _DRILL_MIRROR_LOCK.acquire(timeout=timeout))
|
| 6478 |
+
if not acquired:
|
| 6479 |
+
request_timing.remaining_budget()
|
| 6480 |
+
budget = request_timing.current_budget()
|
| 6481 |
+
error = budget[1] if budget is not None else request_timing.RequestDeadlineExceeded
|
| 6482 |
+
raise error('drill mirror lock deadline exceeded')
|
| 6483 |
+
try:
|
| 6484 |
+
request_timing.remaining_budget()
|
| 6485 |
+
yield
|
| 6486 |
+
finally:
|
| 6487 |
+
_DRILL_MIRROR_LOCK.release()
|
| 6488 |
+
|
| 6489 |
+
|
| 6490 |
def _drill_tokens(session, databases):
|
| 6491 |
"""Unknown freshness disables reuse. No workspace documents are retained here."""
|
| 6492 |
from core import sheet_binding
|
| 6493 |
+
from core.request_timing import raise_if_resource_error
|
| 6494 |
tokens = {}
|
| 6495 |
for db in sorted(databases):
|
| 6496 |
try:
|
| 6497 |
import harness.mirror_tokens # noqa: F401
|
| 6498 |
tokens[db] = sheet_binding.database_change_token(_store(session), db)
|
| 6499 |
except Exception as exc:
|
| 6500 |
+
raise_if_resource_error(exc)
|
| 6501 |
_token_unread(session, db, exc)
|
| 6502 |
tokens[db] = None
|
| 6503 |
return tokens
|
|
|
|
| 6507 |
def _drill_mirror(session, database, window, needed, databases):
|
| 6508 |
"""A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction."""
|
| 6509 |
from core import sheet_binding, perm_scope
|
| 6510 |
+
from core.request_timing import raise_if_resource_error
|
| 6511 |
import datetime
|
| 6512 |
|
| 6513 |
if _native_table_definition(session, database) is not None:
|
|
|
|
| 6524 |
if callable(stamp):
|
| 6525 |
try:
|
| 6526 |
wall = stamp(session.tenant, session.uname, database, user=session.user)
|
| 6527 |
+
except Exception as exc:
|
| 6528 |
+
raise_if_resource_error(exc)
|
| 6529 |
pass
|
| 6530 |
if not wall:
|
| 6531 |
token = None
|
|
|
|
| 6538 |
"_sheet_drill_entries")
|
| 6539 |
absent = object()
|
| 6540 |
previous = {name: getattr(session, name, absent) for name in attrs}
|
| 6541 |
+
with _drill_mirror_lock():
|
| 6542 |
hit = _DRILL_MIRRORS.get(key) if token else None
|
| 6543 |
session._sheet_drill_isolated = True
|
| 6544 |
session._sheet_drill_cursor = hit["owned"] if hit else None
|
|
|
|
| 6584 |
setattr(session, name, value)
|
| 6585 |
|
| 6586 |
|
| 6587 |
+
def _drill_page(session, database, record_ids, offset, limit, total):
|
| 6588 |
"""Use fetchTableWindow's server door, including its row and field walls and wire shape."""
|
| 6589 |
import routes_odoo_tables as rot
|
| 6590 |
from harness import datastore
|
| 6591 |
|
| 6592 |
size = min(limit, datastore.WINDOW_MAX)
|
| 6593 |
+
ids = record_ids
|
| 6594 |
+
import odoo_relational as rel
|
| 6595 |
+
stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
|
| 6596 |
+
and not database.startswith('ut_amzn_') and rot._materialised_table(database))
|
| 6597 |
+
if ids and stored:
|
| 6598 |
+
import registry_source
|
| 6599 |
+
payload = registry_source.stored_page(session, database, ids)
|
| 6600 |
+
else:
|
| 6601 |
+
payload = rot.odoo_table_rows(database, offset=0, limit=size, filters=None,
|
| 6602 |
+
filterConj="and", sorts=None, search=None, related=None, groupBy=None, groupKey=None,
|
| 6603 |
+
session=session, body={"ids": ids}) if ids else {"rows": [], "fields": []}
|
| 6604 |
by_id = {int(row["pid"]): row for row in payload["rows"]}
|
| 6605 |
limits = list(payload.get("limits") or [])
|
| 6606 |
if size != limit:
|
|
|
|
| 6608 |
"cause": f"One response carries at most {datastore.WINDOW_MAX} rows.",
|
| 6609 |
"recommendation": "Continue paging with offset. Every matching row remains available."})
|
| 6610 |
return {**payload, "offset": offset, "limit": size, "limits": limits,
|
| 6611 |
+
"total": total, "totalUnfiltered": total,
|
| 6612 |
"rows": [by_id[pid] for pid in ids if pid in by_id]}
|
| 6613 |
|
| 6614 |
|
| 6615 |
+
def _drill_query(raw):
|
| 6616 |
+
"""Accept structured narrowing and presentation options, never caller SQL."""
|
| 6617 |
+
import routes_odoo_tables as rot
|
| 6618 |
+
import aios_grid
|
| 6619 |
+
try:
|
| 6620 |
+
value = json.loads(raw) if isinstance(raw, str) else {}
|
| 6621 |
+
except ValueError:
|
| 6622 |
+
raise err(400, "bad_drill_query", "The record query could not be read. Reopen View data.")
|
| 6623 |
+
allowed = {"filters", "filterConj", "sorts", "search", "ids", "group", "aggs",
|
| 6624 |
+
"groupBy", "groupOffset", "groupLimit"}
|
| 6625 |
+
if not isinstance(value, dict) or set(value) - allowed:
|
| 6626 |
+
raise err(400, "bad_drill_query", "Use record filters and paging options for this query.")
|
| 6627 |
+
query = {**value, **rot._selection_query(value)}
|
| 6628 |
+
budget = [0]
|
| 6629 |
+
|
| 6630 |
+
def check_nodes(nodes, depth=1):
|
| 6631 |
+
# Calendar range queries add one group and one date rule to an authored filter tree.
|
| 6632 |
+
if (depth > aios_grid.MAX_FILTER_DEPTH + 1 or not isinstance(nodes, list)
|
| 6633 |
+
or len(nodes) > aios_grid.MAX_FILTER_SIBLINGS):
|
| 6634 |
+
raise err(400, "bad_drill_query", "Use a valid record filter tree.")
|
| 6635 |
+
for node in nodes:
|
| 6636 |
+
budget[0] += 1
|
| 6637 |
+
if budget[0] > aios_grid.MAX_FILTER_NODES + 2 or not isinstance(node, dict):
|
| 6638 |
+
raise err(400, "bad_drill_query", "Use a valid record filter tree.")
|
| 6639 |
+
if "children" in node:
|
| 6640 |
+
if node.get("conj", "and") not in ("and", "or"):
|
| 6641 |
+
raise err(400, "bad_drill_query", "Choose all or any for this filter group.")
|
| 6642 |
+
check_nodes(node["children"], depth + 1)
|
| 6643 |
+
elif (not isinstance(node.get("colId"), str)
|
| 6644 |
+
or node.get("op") not in aios_grid.FILTER_OPS | aios_grid.COHORT_OPS | aios_grid.VIEW_OPS):
|
| 6645 |
+
raise err(400, "bad_drill_query", "Choose a field and condition for this filter.")
|
| 6646 |
+
|
| 6647 |
+
check_nodes(query["filters"])
|
| 6648 |
+
if "ids" in value:
|
| 6649 |
+
query["ids"] = rot._record_ids(value)
|
| 6650 |
+
if "aggs" in value and (not isinstance(value["aggs"], list)
|
| 6651 |
+
or len(value["aggs"]) > rot.MAX_AGGREGATES
|
| 6652 |
+
or any(not isinstance(agg, dict) or not isinstance(agg.get("colId"), str)
|
| 6653 |
+
or not isinstance(agg.get("agg"), str) for agg in value["aggs"])):
|
| 6654 |
+
raise err(400, "bad_drill_query", "Choose a list of field summaries.")
|
| 6655 |
+
if query.get("groupBy") is not None and not isinstance(query["groupBy"], str):
|
| 6656 |
+
raise err(400, "bad_drill_query", "Choose a grouping field.")
|
| 6657 |
+
for key, low, high in (("groupOffset", 0, None), ("groupLimit", 1, rot.MAX_GROUPS)):
|
| 6658 |
+
if key in query and (type(query[key]) is not int or query[key] < low
|
| 6659 |
+
or (high is not None and query[key] > high)):
|
| 6660 |
+
raise err(400, "bad_drill_query", "Choose a valid group offset and page size.")
|
| 6661 |
+
if query.get("group") is not None and (not isinstance(query["group"], dict)
|
| 6662 |
+
or set(query["group"]) != {"by", "key"}
|
| 6663 |
+
or not isinstance(query["group"]["by"], str)):
|
| 6664 |
+
raise err(400, "bad_drill_query", "Choose a grouping field and value.")
|
| 6665 |
+
if any(not isinstance(sort, dict) or not isinstance(sort.get("colId"), str)
|
| 6666 |
+
or sort.get("dir") not in ("asc", "desc") for sort in query["sorts"]):
|
| 6667 |
+
raise err(400, "bad_drill_query", "Choose a field and direction for each sort.")
|
| 6668 |
+
return query
|
| 6669 |
+
|
| 6670 |
+
|
| 6671 |
+
def _drill_read_sql(session, database, cur, spec, cols, count_sql, params, query, limits):
|
| 6672 |
+
"""Intersect grid options with the cell population before counting or paging."""
|
| 6673 |
+
import routes_odoo_tables as rot
|
| 6674 |
+
source = spec.get("from_sql") or spec["table"]
|
| 6675 |
+
prefix = f"SELECT COUNT(*) FROM {source}"
|
| 6676 |
+
if not count_sql.startswith(prefix):
|
| 6677 |
+
raise err(409, "scope_not_applicable", "This cell's record scope cannot be paged.")
|
| 6678 |
+
where = count_sql[len(prefix):].strip()
|
| 6679 |
+
where = where[len("WHERE "):] if where else "TRUE"
|
| 6680 |
+
where = f"({where})"
|
| 6681 |
+
params = list(params)
|
| 6682 |
+
tree = query["filters"]
|
| 6683 |
+
named = rot._leaf_cols(tree) - rot._PSEUDO_COLS
|
| 6684 |
+
named.update(sort["colId"] for sort in query["sorts"])
|
| 6685 |
+
if query.get("group"):
|
| 6686 |
+
named.add(query["group"]["by"])
|
| 6687 |
+
if named - set(cols):
|
| 6688 |
+
raise err(403, "field_unreadable", "This record query names an unavailable field.")
|
| 6689 |
+
try:
|
| 6690 |
+
had_cohort = bool(rot._active_cohort_leaves(tree))
|
| 6691 |
+
tree, sets = rot._views_as_cohorts(tree, database, session, spec, cols,
|
| 6692 |
+
where, params, cur, limits)
|
| 6693 |
+
tree, sets = rot._ranks_as_cohorts(tree, query["filterConj"], spec, cols,
|
| 6694 |
+
where, params, cur, sets, had_cohort, rot._today(), limits)
|
| 6695 |
+
pred = rot._fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols,
|
| 6696 |
+
id_sql=spec["id"], today=rot._today(), cohort_sets=sets)
|
| 6697 |
+
if pred is not None:
|
| 6698 |
+
if pred.uses_aggregate:
|
| 6699 |
+
raise ValueError("A record filter cannot use an aggregate.")
|
| 6700 |
+
where += f" AND ({pred.sql})"
|
| 6701 |
+
params.extend(pred.params)
|
| 6702 |
+
if query["search"].strip():
|
| 6703 |
+
search = rot._fs().compile_search(query["search"].strip(), cols)
|
| 6704 |
+
if search is not None:
|
| 6705 |
+
where += f" AND ({search.sql})"
|
| 6706 |
+
params.extend(search.params)
|
| 6707 |
+
if query.get("group"):
|
| 6708 |
+
group = query["group"]
|
| 6709 |
+
sql, values = rot._group_pin(cols, group["by"], json.dumps(group["key"]))
|
| 6710 |
+
where += f" AND ({sql})"
|
| 6711 |
+
params.extend(values)
|
| 6712 |
+
if query.get("ids") is not None:
|
| 6713 |
+
ids = query["ids"]
|
| 6714 |
+
where += f" AND {spec['id']} IN (SELECT unnest(?::BIGINT[]))"
|
| 6715 |
+
params.append(ids)
|
| 6716 |
+
order = rot._fs().compile_order_by(query["sorts"], cols, tiebreak_sql=spec["id"])
|
| 6717 |
+
except ValueError as exc:
|
| 6718 |
+
raise err(400, "filter_unsupported", str(exc))
|
| 6719 |
+
return source, f"({where})", params, order or spec["id"]
|
| 6720 |
+
|
| 6721 |
+
|
| 6722 |
def _drill_draft_binding(cell, binding, draft_filter):
|
| 6723 |
"""The stored binding with View data's UNSAVED filter in place of its own (S68-T23 finding).
|
| 6724 |
|
|
|
|
| 6752 |
def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
|
| 6753 |
session: Session = Depends(require_session),
|
| 6754 |
offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1),
|
| 6755 |
+
draft_filter: str | None = Query(default=None, alias="filter"),
|
| 6756 |
+
query: str | None = Query(default=None)):
|
| 6757 |
+
"""Return exact counts and one SQL page, with page IDs explicitly marked as incomplete."""
|
| 6758 |
+
from core import sheets, sheet_binding
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6759 |
from modules import sheet as sheet_mod
|
| 6760 |
import core.perm_scope as perm_scope
|
| 6761 |
+
import routes_odoo_tables as rot
|
| 6762 |
+
from harness import datastore
|
| 6763 |
|
| 6764 |
# Python callers omit FastAPI's Query sentinels. Offset has no maximum; only the page does.
|
| 6765 |
offset = offset if isinstance(offset, int) else 0
|
| 6766 |
limit = limit if isinstance(limit, int) else 100
|
| 6767 |
draft_filter = draft_filter if isinstance(draft_filter, str) else None
|
| 6768 |
+
query = _drill_query(query)
|
| 6769 |
if offset < 0 or limit < 1:
|
| 6770 |
raise err(400, "bad_drill_page", "Choose a nonnegative record offset and a positive page size.")
|
| 6771 |
|
|
|
|
| 6836 |
raise err(400, exc.code, exc.message)
|
| 6837 |
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 6838 |
| sheet_mod.filter_columns_named(wall_nodes))
|
| 6839 |
+
needed.update(rot._leaf_cols(query["filters"]) - rot._PSEUDO_COLS)
|
| 6840 |
+
needed.update(sort["colId"] for sort in query["sorts"])
|
| 6841 |
+
needed.update(agg.get("colId") for agg in query.get("aggs", []) if agg.get("colId"))
|
| 6842 |
+
if query.get("groupBy"):
|
| 6843 |
+
needed.add(query["groupBy"])
|
| 6844 |
+
if query.get("group"):
|
| 6845 |
+
needed.add(query["group"]["by"])
|
| 6846 |
+
for leaf in rot._fe()._rank_leaves(query["filters"], {key: {} for key in needed}):
|
| 6847 |
+
_mode, group_field = rot._fe()._rank_scope(leaf, None)
|
| 6848 |
+
if group_field:
|
| 6849 |
+
needed.add(group_field)
|
| 6850 |
+
if query["search"].strip():
|
| 6851 |
+
from routes_grid import _source_fields
|
| 6852 |
+
needed.update(field["key"] for field in _source_fields(session, database))
|
| 6853 |
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 6854 |
databases = {database}
|
| 6855 |
databases.update(_binding_related_entry(session, database, via)["target"]
|
| 6856 |
for via in sheet_mod.related_vias(scope_nodes))
|
| 6857 |
+
sources = _drill_tokens(session, databases)
|
| 6858 |
with _drill_mirror(session, database, window, needed, databases) as (made, token):
|
| 6859 |
cur, spec, cols = made
|
| 6860 |
cols = dict(cols)
|
| 6861 |
# Apply the aggregate's field wall before compiling either the count or the IDs.
|
| 6862 |
hidden = perm_scope.hidden_keys(session.user, database, spec.get("fields") or [], st=session.runtime)
|
| 6863 |
+
_require_aggregate_field(session, binding, hidden=hidden)
|
| 6864 |
if spec.get("scope_applied"):
|
| 6865 |
wall_nodes, wall_conj = [], "and"
|
| 6866 |
unavailable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
|
|
|
|
| 6879 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 6880 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 6881 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
|
|
|
|
|
|
|
|
|
|
| 6882 |
except sheet_mod.SheetRefusal as exc:
|
| 6883 |
raise err(400, exc.code, exc.message)
|
| 6884 |
except ValueError:
|
|
|
|
| 6887 |
"connection. Edit the scope or ask an administrator to update the restriction.")
|
| 6888 |
if spec.get("scope_applied") and limits:
|
| 6889 |
raise err(409, "scope_not_applicable", "This cell's filter cannot be applied to this database.")
|
| 6890 |
+
cols = {key: value for key, value in cols.items() if key not in hidden}
|
| 6891 |
+
population_total = int(cur.execute(count_sql, count_params).fetchone()[0])
|
| 6892 |
+
source, where, params, order = _drill_read_sql(
|
| 6893 |
+
session, database, cur, spec, cols, count_sql, count_params, query, limits)
|
| 6894 |
+
narrowed = bool(query["filters"] or query["search"].strip() or query.get("group")
|
| 6895 |
+
or query.get("ids") is not None)
|
| 6896 |
+
total = int(cur.execute(f"SELECT count(*) FROM {source} WHERE {where}", params).fetchone()[0]) \
|
| 6897 |
+
if narrowed else population_total
|
| 6898 |
+
size = min(limit, datastore.WINDOW_MAX)
|
| 6899 |
+
summary = None
|
| 6900 |
+
if "aggs" in query:
|
| 6901 |
+
summary = rot._aggregate_over(cur, spec, cols, where, params, {"from_sql": source},
|
| 6902 |
+
query["aggs"], group_by=query.get("groupBy"), limits=limits,
|
| 6903 |
+
group_offset=query.get("groupOffset", 0), group_limit=query.get("groupLimit"))
|
| 6904 |
+
record_ids = []
|
| 6905 |
+
else:
|
| 6906 |
+
ids_sql = (f"SELECT {spec['id']} FROM {source} WHERE {where} ORDER BY {order} "
|
| 6907 |
+
"LIMIT ? OFFSET ?")
|
| 6908 |
+
record_ids = [int(row[0]) for row in cur.execute(ids_sql, [*params, size, offset]).fetchall()]
|
| 6909 |
+
try:
|
| 6910 |
+
wall_stamp = perm_scope.wall_stamp(session.tenant, session.uname, database, user=session.user)
|
| 6911 |
+
except Exception as exc:
|
| 6912 |
+
from core.request_timing import raise_if_resource_error
|
| 6913 |
+
raise_if_resource_error(exc)
|
| 6914 |
+
wall_stamp = None
|
| 6915 |
+
population_query = {key: value for key, value in query.items()
|
| 6916 |
+
if key not in {"groupOffset", "groupLimit"}}
|
| 6917 |
+
population = sheet_binding.fingerprint([session.tenant, session.uname, session.user,
|
| 6918 |
+
binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
|
| 6919 |
+
if all(sources.values()) and wall_stamp else None
|
| 6920 |
|
| 6921 |
limits = list(limits) + list(spec.get("limits") or [])
|
| 6922 |
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
|
| 6923 |
+
"record_ids_scope": "page", "id_limit": None, "limits": limits,
|
| 6924 |
+
"population": population, "query": query}
|
| 6925 |
if scope.get("view_id"):
|
| 6926 |
out["view_id"] = scope["view_id"]
|
| 6927 |
else:
|
| 6928 |
out["filter"] = scope.get("filter")
|
| 6929 |
+
if summary is not None:
|
| 6930 |
+
out["summary"] = {"ok": True, **summary}
|
| 6931 |
+
else:
|
| 6932 |
+
out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
|
| 6933 |
+
out["page"]["totalUnfiltered"] = population_total
|
| 6934 |
+
if _drill_tokens(session, databases) != sources:
|
| 6935 |
+
raise err(409, "drill_population_changed", "These records changed while loading. Try again.")
|
| 6936 |
out["token"] = token
|
| 6937 |
return out
|
| 6938 |
|
api/routes_tables.py
CHANGED
|
@@ -252,7 +252,12 @@ def _table_lend(runtime, table_key, defs_only=False):
|
|
| 252 |
ut = _ut()
|
| 253 |
if defs_only:
|
| 254 |
# Metadata must enter through the store projection, even on entry-capable stores.
|
| 255 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
if _ut()._entry_reader(runtime) is None:
|
| 257 |
return ut.lend(runtime)
|
| 258 |
entry = _table_entry(runtime, table_key)
|
|
@@ -578,6 +583,8 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 578 |
if isinstance(documents, dict):
|
| 579 |
context_store = _ut().lend(context_store, **{
|
| 580 |
name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
|
|
|
|
|
|
|
| 581 |
from copy import copy
|
| 582 |
field_session = copy(session)
|
| 583 |
field_session.runtime = context_store
|
|
@@ -621,9 +628,10 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 621 |
if windowed:
|
| 622 |
connected.add(database)
|
| 623 |
elif database in ('customer_data', 'product_data'):
|
| 624 |
-
from routes_grid import
|
| 625 |
-
|
| 626 |
-
|
|
|
|
| 627 |
else:
|
| 628 |
g = None
|
| 629 |
contexts[database] = g
|
|
@@ -636,8 +644,26 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 636 |
contexts[database] = None
|
| 637 |
except perm_scope.Unresolvable:
|
| 638 |
contexts[database] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 639 |
return contexts[database]
|
| 640 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 641 |
def partner(obj):
|
| 642 |
nonlocal applications
|
| 643 |
identity = json.dumps(obj, sort_keys=True)
|
|
@@ -688,9 +714,14 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 688 |
if wanted:
|
| 689 |
from routes_odoo_tables import _relation_rows
|
| 690 |
try:
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 694 |
except HTTPException as exc:
|
| 695 |
if exc.status_code not in (403, 404, 409):
|
| 696 |
raise
|
|
@@ -725,7 +756,7 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 725 |
previous = pair.get('before')
|
| 726 |
declared = {f['key'] for f in (g.get('defn') or {}).get('fields', g['fields'])}
|
| 727 |
if field_key not in declared and isinstance(previous, dict):
|
| 728 |
-
corpus = list(g['fields']) + [previous]
|
| 729 |
hidden = _ut_hidden(session, database, corpus, st=context_store)
|
| 730 |
if field_key not in hidden and field_key not in g.get('hidden', ()):
|
| 731 |
visible = visible | {field_key}
|
|
@@ -1283,7 +1314,7 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None):
|
|
| 1283 |
def ut_write_ctx(session: Session, table_key: str):
|
| 1284 |
"""The events transport borrows the same rows-free assembly as workspace and Sheet."""
|
| 1285 |
return _compute_ut_assembly(session, table_key, consume_corrections=False,
|
| 1286 |
-
with_rows=False, include_usage=False)
|
| 1287 |
|
| 1288 |
|
| 1289 |
def _ut_hidden(session, table_key, fields, st=None):
|
|
@@ -1367,7 +1398,9 @@ def _ut_shared_cells(session, table_key, pids):
|
|
| 1367 |
from core import shared_overlay
|
| 1368 |
try:
|
| 1369 |
return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime)
|
| 1370 |
-
except Exception:
|
|
|
|
|
|
|
| 1371 |
return {}
|
| 1372 |
|
| 1373 |
|
|
@@ -1718,6 +1751,15 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 1718 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1719 |
*, fields_only: bool = False, needed=None):
|
| 1720 |
if not with_rows and st is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1721 |
from routes_customers import grid_assembly
|
| 1722 |
return grid_assembly(session, table_key, storage_key,
|
| 1723 |
build=lambda borrowed: _compute_ut_assembly(
|
|
@@ -1727,10 +1769,54 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
|
| 1727 |
fields_only=fields_only, needed=needed)
|
| 1728 |
|
| 1729 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1730 |
def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1731 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1732 |
*, fields_only: bool = False, needed=None, include_usage: bool = True,
|
| 1733 |
-
include_measures: bool = True, include_views: bool = True
|
|
|
|
| 1734 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 1735 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 1736 |
|
|
@@ -1750,9 +1836,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1750 |
* every materialised `ut_*` database stops allocating a dict per row and sorting them on a
|
| 1751 |
route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per
|
| 1752 |
database switch (owner item 7).
|
| 1753 |
-
|
| 1754 |
-
|
| 1755 |
-
on the write door.
|
| 1756 |
"""
|
| 1757 |
import aios_grid
|
| 1758 |
from core import grid_events
|
|
@@ -1765,6 +1850,7 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1765 |
# reading that bucket once per question asked of it.
|
| 1766 |
if st is None:
|
| 1767 |
st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
|
|
|
|
| 1768 |
if fields_only:
|
| 1769 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1770 |
fields_base = list(defn.get("fields") or [])
|
|
@@ -1774,7 +1860,14 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1774 |
# a READ route passes one — see `scoped_pool`'s own note and contract C5.
|
| 1775 |
pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
|
| 1776 |
else:
|
| 1777 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1778 |
rows_src = []
|
| 1779 |
|
| 1780 |
# ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it
|
|
@@ -1791,10 +1884,18 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1791 |
# that happen to carry a scoped `table`.
|
| 1792 |
st=session.runtime,
|
| 1793 |
scope_key=table_key, table=ops)
|
|
|
|
|
|
|
|
|
|
| 1794 |
ws = grid_events.table_workspace(ctx, allowed_pids=pids,
|
| 1795 |
consume_corrections=consume_corrections,
|
| 1796 |
capture_document=True,
|
| 1797 |
**({"include_views": False} if not include_views else {}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1798 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 1799 |
ws, session.uname, set(pids), defs={}, scope_key=table_key,
|
| 1800 |
storage_key=storage_key, fields_base=fields_base, st=st)
|
|
@@ -1932,7 +2033,9 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1932 |
derived = aios_grid.cohort_cells(lists) if not fields_only else {}
|
| 1933 |
materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
|
| 1934 |
for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures,
|
| 1935 |
-
getattr(session.runtime, 'measure_memo', {})
|
|
|
|
|
|
|
| 1936 |
if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
|
| 1937 |
derived.setdefault(_pid, {}).update(_cells)
|
| 1938 |
|
|
@@ -2026,7 +2129,7 @@ def ut_measures(table_key, session, *, include_offer=True):
|
|
| 2026 |
return []
|
| 2027 |
|
| 2028 |
|
| 2029 |
-
def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None):
|
| 2030 |
"""`{pid: {field key: value}}` for this database's Metric columns.
|
| 2031 |
|
| 2032 |
⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need
|
|
@@ -2059,11 +2162,12 @@ def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None)
|
|
| 2059 |
out = {}
|
| 2060 |
for (dfrom, dto), group in by_window.items():
|
| 2061 |
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 2062 |
-
mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys)
|
| 2063 |
if mkey not in memo:
|
| 2064 |
try:
|
| 2065 |
memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom,
|
| 2066 |
-
|
|
|
|
| 2067 |
except Exception as e: # noqa: BLE001
|
| 2068 |
_ut_measure_err("ut-measure-column", e)
|
| 2069 |
transient = False
|
|
@@ -2246,38 +2350,18 @@ def _module_link_targets(session, meta=None, st=None):
|
|
| 2246 |
@router.get("/tables")
|
| 2247 |
def list_tables(brief: bool = Query(False),
|
| 2248 |
session: Session = Depends(require_session)):
|
| 2249 |
-
"""
|
| 2250 |
-
|
| 2251 |
-
⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This
|
| 2252 |
-
route was never ticketed and has the same shape `/nav` and `/automations` were fixed for:
|
| 2253 |
-
`all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on
|
| 2254 |
-
tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant
|
| 2255 |
-
with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked
|
| 2256 |
-
about every table; it is handed the document this function already holds. See
|
| 2257 |
-
`user_tables.lend`'s own note for why inlining the predicate is the one fix that is not
|
| 2258 |
-
available.
|
| 2259 |
-
|
| 2260 |
-
`brief=1` is for a link-field picker. It needs the tables it may target and their schemas,
|
| 2261 |
-
not row counts or record mutability. Reading a full tenant document to decorate that one
|
| 2262 |
-
field editor would move every stored row through Neon for no visible result, so this branch
|
| 2263 |
-
keeps the server-side permission wall but lends its rows-free projection instead.
|
| 2264 |
-
|
| 2265 |
-
⭐⭐ W41-T15 / R9 / C5 (2026-08-24) — AND `brief=1` NOW OFFERS THE TWO REGISTRY MODULES TOO
|
| 2266 |
-
(`_module_link_targets`), because `_clean_link` accepts them as link targets and this list is
|
| 2267 |
-
the only producer of a link bag in the product.
|
| 2268 |
-
|
| 2269 |
-
⛔ THE BRIEF BRANCH ONLY, and the asymmetry is deliberate. The full listing is the "+ New
|
| 2270 |
-
database" management surface: it renders row counts, `createdBy` and a delete affordance over
|
| 2271 |
-
things a tenant OWNS, and `customer_data` is a compiled registry row with its own nav entry
|
| 2272 |
-
and no such lifecycle. Offering it there would put a database nobody can delete into the list
|
| 2273 |
-
of databases you delete from. Two callers, two questions, one route.
|
| 2274 |
-
"""
|
| 2275 |
from core.if_field import project_fields
|
| 2276 |
ut = _ut()
|
| 2277 |
meta = nav_meta(session)
|
| 2278 |
out = []
|
| 2279 |
-
|
| 2280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2281 |
lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables})
|
| 2282 |
for key, t in sorted(tables.items(),
|
| 2283 |
key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
|
|
@@ -2297,7 +2381,8 @@ def list_tables(brief: bool = Query(False),
|
|
| 2297 |
"recordsMutable": ut.records_mutable(key, st=lent),
|
| 2298 |
"createdBy": t.get("createdBy") or "",
|
| 2299 |
"created": t.get("created") or "",
|
| 2300 |
-
"rowCount":
|
|
|
|
| 2301 |
out.append(row)
|
| 2302 |
if brief:
|
| 2303 |
# ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
|
|
@@ -2710,6 +2795,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 2710 |
'recommendation': 'Use Next to read older events.'}]}
|
| 2711 |
|
| 2712 |
_ig_forward(session, table_key)
|
|
|
|
| 2713 |
# ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED.
|
| 2714 |
# This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE
|
| 2715 |
# (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy),
|
|
@@ -2721,17 +2807,17 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 2721 |
# value it replaced (contract C5).
|
| 2722 |
if _ut()._entry_reader(session.runtime) is None:
|
| 2723 |
_lent = _table_lend(session.runtime, table_key)
|
| 2724 |
-
g = ut_assembly(session, table_key, st=_lent, consume_corrections=False)
|
| 2725 |
else:
|
| 2726 |
-
from
|
| 2727 |
from routes_customers import _AssemblyStore
|
| 2728 |
# One request owns these borrowed documents. A cold memo build and its page share reads;
|
| 2729 |
# a warm page reads records/shared cells afresh and lends Zone documents to both readers.
|
| 2730 |
original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore)
|
| 2731 |
else session.runtime)
|
| 2732 |
if not isinstance(session.runtime, _AssemblyStore):
|
| 2733 |
-
session =
|
| 2734 |
-
g = ut_assembly(session, table_key, with_rows=False)
|
| 2735 |
assembly_rt = session.runtime
|
| 2736 |
entry_key = (_ut().STORE_KEY, ("entry", str(table_key)))
|
| 2737 |
if entry_key in assembly_rt.docs:
|
|
@@ -2743,7 +2829,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 2743 |
_lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {})
|
| 2744 |
# The page borrows the same envelope as workspace; only records and shared cells are fresh.
|
| 2745 |
pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent)
|
| 2746 |
-
g['pids']
|
| 2747 |
from core import perm_scope
|
| 2748 |
g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src
|
| 2749 |
if row['pid'] in g['pids']]
|
|
@@ -2805,8 +2891,11 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 2805 |
# honestly instead of showing half a truncated brace.
|
| 2806 |
# ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole,
|
| 2807 |
# so a tenant using `json` for a short config sees no change at all.
|
| 2808 |
-
|
| 2809 |
-
|
|
|
|
|
|
|
|
|
|
| 2810 |
rows = _ut().zone_cells(table_key, rows, allowed_row_ids=g["pids"], st=_lent)
|
| 2811 |
# ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
|
| 2812 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
|
|
|
| 252 |
ut = _ut()
|
| 253 |
if defs_only:
|
| 254 |
# Metadata must enter through the store projection, even on entry-capable stores.
|
| 255 |
+
try:
|
| 256 |
+
return ut.lend_defs(runtime, schema_only=True)
|
| 257 |
+
except Exception as exc:
|
| 258 |
+
from core.request_timing import raise_if_resource_error
|
| 259 |
+
raise_if_resource_error(exc)
|
| 260 |
+
raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc
|
| 261 |
if _ut()._entry_reader(runtime) is None:
|
| 262 |
return ut.lend(runtime)
|
| 263 |
entry = _table_entry(runtime, table_key)
|
|
|
|
| 583 |
if isinstance(documents, dict):
|
| 584 |
context_store = _ut().lend(context_store, **{
|
| 585 |
name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
|
| 586 |
+
from routes_customers import _AssemblyStore
|
| 587 |
+
context_store = _AssemblyStore(context_store, schema_only=True)
|
| 588 |
from copy import copy
|
| 589 |
field_session = copy(session)
|
| 590 |
field_session.runtime = context_store
|
|
|
|
| 628 |
if windowed:
|
| 629 |
connected.add(database)
|
| 630 |
elif database in ('customer_data', 'product_data'):
|
| 631 |
+
from routes_grid import _builtin_activity_metadata
|
| 632 |
+
g = _builtin_activity_metadata(field_session,
|
| 633 |
+
'product' if database == 'product_data' else 'customer', st=context_store)
|
| 634 |
+
page_sources.add(database)
|
| 635 |
else:
|
| 636 |
g = None
|
| 637 |
contexts[database] = g
|
|
|
|
| 644 |
contexts[database] = None
|
| 645 |
except perm_scope.Unresolvable:
|
| 646 |
contexts[database] = None
|
| 647 |
+
if database in ('customer_data', 'product_data'):
|
| 648 |
+
from copy import deepcopy
|
| 649 |
+
assemblies = getattr(session, '_sheet_assemblies', None)
|
| 650 |
+
if isinstance(assemblies, dict):
|
| 651 |
+
assemblies[(database, 'fields')] = deepcopy(contexts[database])
|
| 652 |
return contexts[database]
|
| 653 |
|
| 654 |
+
assemblies = getattr(session, '_sheet_assemblies', None)
|
| 655 |
+
if isinstance(assemblies, dict):
|
| 656 |
+
class ActivityBuiltinFields(dict):
|
| 657 |
+
def __contains__(self, slot):
|
| 658 |
+
# Sheet asks for a fields slot before reading any builtin binding values.
|
| 659 |
+
if (isinstance(slot, tuple) and len(slot) == 2 and slot[1] == 'fields'
|
| 660 |
+
and slot[0] in ('customer_data', 'product_data')
|
| 661 |
+
and not super().__contains__(slot)):
|
| 662 |
+
source(slot[0])
|
| 663 |
+
return super().__contains__(slot)
|
| 664 |
+
|
| 665 |
+
session._sheet_assemblies = ActivityBuiltinFields(assemblies)
|
| 666 |
+
|
| 667 |
def partner(obj):
|
| 668 |
nonlocal applications
|
| 669 |
identity = json.dumps(obj, sort_keys=True)
|
|
|
|
| 714 |
if wanted:
|
| 715 |
from routes_odoo_tables import _relation_rows
|
| 716 |
try:
|
| 717 |
+
if database in ('customer_data', 'product_data'):
|
| 718 |
+
from routes_grid import _builtin_activity_pids
|
| 719 |
+
contexts[database]['pids'] = _builtin_activity_pids(
|
| 720 |
+
session, database, wanted, contexts[database], st=context_store)
|
| 721 |
+
else:
|
| 722 |
+
contexts[database]['pids'] = frozenset(
|
| 723 |
+
row['pid'] for row in _relation_rows(
|
| 724 |
+
session, database, {'pid'}, ids=wanted, st=context_store))
|
| 725 |
except HTTPException as exc:
|
| 726 |
if exc.status_code not in (403, 404, 409):
|
| 727 |
raise
|
|
|
|
| 756 |
previous = pair.get('before')
|
| 757 |
declared = {f['key'] for f in (g.get('defn') or {}).get('fields', g['fields'])}
|
| 758 |
if field_key not in declared and isinstance(previous, dict):
|
| 759 |
+
corpus = list((g.get('defn') or {}).get('fields', g['fields'])) + [previous]
|
| 760 |
hidden = _ut_hidden(session, database, corpus, st=context_store)
|
| 761 |
if field_key not in hidden and field_key not in g.get('hidden', ()):
|
| 762 |
visible = visible | {field_key}
|
|
|
|
| 1314 |
def ut_write_ctx(session: Session, table_key: str):
|
| 1315 |
"""The events transport borrows the same rows-free assembly as workspace and Sheet."""
|
| 1316 |
return _compute_ut_assembly(session, table_key, consume_corrections=False,
|
| 1317 |
+
with_rows=False, include_usage=False, metadata_only=False)
|
| 1318 |
|
| 1319 |
|
| 1320 |
def _ut_hidden(session, table_key, fields, st=None):
|
|
|
|
| 1398 |
from core import shared_overlay
|
| 1399 |
try:
|
| 1400 |
return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime)
|
| 1401 |
+
except Exception as exc: # noqa: BLE001
|
| 1402 |
+
from core.request_timing import raise_if_resource_error
|
| 1403 |
+
raise_if_resource_error(exc)
|
| 1404 |
return {}
|
| 1405 |
|
| 1406 |
|
|
|
|
| 1751 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1752 |
*, fields_only: bool = False, needed=None):
|
| 1753 |
if not with_rows and st is None:
|
| 1754 |
+
if not fields_only:
|
| 1755 |
+
import routes_odoo_tables as connected
|
| 1756 |
+
# Reuse revision-validated schema bytes; classification must not reread native rows.
|
| 1757 |
+
definitions, _token = connected._page_defs(session)
|
| 1758 |
+
defn = _ut().get(table_key, st=definitions)
|
| 1759 |
+
if not _ut().materialises(table_key, st=definitions, defn=defn):
|
| 1760 |
+
# Source membership changes independently of stored and workspace revisions.
|
| 1761 |
+
return _compute_ut_assembly(session, table_key, storage_key, False, False,
|
| 1762 |
+
st=definitions, fields_only=False, needed=needed)
|
| 1763 |
from routes_customers import grid_assembly
|
| 1764 |
return grid_assembly(session, table_key, storage_key,
|
| 1765 |
build=lambda borrowed: _compute_ut_assembly(
|
|
|
|
| 1769 |
fields_only=fields_only, needed=needed)
|
| 1770 |
|
| 1771 |
|
| 1772 |
+
def _metadata_pids(session, table_key, ctx, st):
|
| 1773 |
+
"""Validate only saved membership IDs through the connected table's ordinary row wall."""
|
| 1774 |
+
from core import grid_events
|
| 1775 |
+
from modules import cohort
|
| 1776 |
+
import routes_odoo_tables as connected
|
| 1777 |
+
|
| 1778 |
+
document = ctx.table.st.get(ctx.table.table_key) or {}
|
| 1779 |
+
if isinstance(st, _ut()._Lent):
|
| 1780 |
+
st._docs[ctx.table.table_key] = document
|
| 1781 |
+
candidates = set()
|
| 1782 |
+
|
| 1783 |
+
def collect(value):
|
| 1784 |
+
if isinstance(value, dict):
|
| 1785 |
+
for key, item in value.items():
|
| 1786 |
+
if key in {"memberPids", "members"} and isinstance(item, (list, tuple)):
|
| 1787 |
+
candidates.update(pid for pid in item if type(pid) is int and pid > 0)
|
| 1788 |
+
elif isinstance(item, (dict, list, tuple)):
|
| 1789 |
+
collect(item)
|
| 1790 |
+
elif isinstance(value, (list, tuple)):
|
| 1791 |
+
for item in value:
|
| 1792 |
+
collect(item)
|
| 1793 |
+
|
| 1794 |
+
collect(document.get(session.uname) or {})
|
| 1795 |
+
from core.table_store import ASSIGNED_KEY
|
| 1796 |
+
assigned = ((document.get(ASSIGNED_KEY) or {}).get("views") or {}).get(session.uname) or {}
|
| 1797 |
+
collect(assigned)
|
| 1798 |
+
collect(grid_events._granted_views(ctx, session.uname))
|
| 1799 |
+
cohorts = cohort.scoped(table_key, st=st)
|
| 1800 |
+
if isinstance(st, _ut()._Lent):
|
| 1801 |
+
st._docs[cohorts.key] = st.get(cohorts.key) or {}
|
| 1802 |
+
collect(cohorts.all_for(session.uname))
|
| 1803 |
+
connected.require_amazon_mirror(table_key, session, st=st)
|
| 1804 |
+
if not candidates:
|
| 1805 |
+
return frozenset()
|
| 1806 |
+
read = connected._compiled_read(table_key, session, st=st)
|
| 1807 |
+
try:
|
| 1808 |
+
rows = connected._relation_rows(session, table_key, [], ids=sorted(candidates), st=st,
|
| 1809 |
+
compiled=read)
|
| 1810 |
+
return frozenset(int(row["pid"]) for row in rows)
|
| 1811 |
+
finally:
|
| 1812 |
+
read["cur"].close()
|
| 1813 |
+
|
| 1814 |
+
|
| 1815 |
def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 1816 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 1817 |
*, fields_only: bool = False, needed=None, include_usage: bool = True,
|
| 1818 |
+
include_measures: bool = True, include_views: bool = True,
|
| 1819 |
+
metadata_only: bool = True):
|
| 1820 |
"""The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
|
| 1821 |
`/workspace` and the events route consume any of the three interchangeably.
|
| 1822 |
|
|
|
|
| 1836 |
* every materialised `ut_*` database stops allocating a dict per row and sorting them on a
|
| 1837 |
route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per
|
| 1838 |
database switch (owner item 7).
|
| 1839 |
+
Connected metadata validates only explicitly saved membership IDs through the SQL row wall.
|
| 1840 |
+
Write contexts pass `metadata_only=False` and retain complete membership admission.
|
|
|
|
| 1841 |
"""
|
| 1842 |
import aios_grid
|
| 1843 |
from core import grid_events
|
|
|
|
| 1850 |
# reading that bucket once per question asked of it.
|
| 1851 |
if st is None:
|
| 1852 |
st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
|
| 1853 |
+
metadata_membership = False
|
| 1854 |
if fields_only:
|
| 1855 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1856 |
fields_base = list(defn.get("fields") or [])
|
|
|
|
| 1860 |
# a READ route passes one — see `scoped_pool`'s own note and contract C5.
|
| 1861 |
pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
|
| 1862 |
else:
|
| 1863 |
+
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1864 |
+
metadata_membership = (metadata_only and table_key != "ut_activity" and
|
| 1865 |
+
not _ut().materialises(table_key, st=session.runtime, defn=defn))
|
| 1866 |
+
if metadata_membership:
|
| 1867 |
+
fields_base = list(defn.get("fields") or [])
|
| 1868 |
+
pids = frozenset()
|
| 1869 |
+
else:
|
| 1870 |
+
pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st)
|
| 1871 |
rows_src = []
|
| 1872 |
|
| 1873 |
# ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it
|
|
|
|
| 1884 |
# that happen to carry a scoped `table`.
|
| 1885 |
st=session.runtime,
|
| 1886 |
scope_key=table_key, table=ops)
|
| 1887 |
+
if metadata_membership:
|
| 1888 |
+
pids = _metadata_pids(session, table_key, ctx, st)
|
| 1889 |
+
ctx.allowed_pids = pids
|
| 1890 |
ws = grid_events.table_workspace(ctx, allowed_pids=pids,
|
| 1891 |
consume_corrections=consume_corrections,
|
| 1892 |
capture_document=True,
|
| 1893 |
**({"include_views": False} if not include_views else {}))
|
| 1894 |
+
if metadata_membership or fields_only:
|
| 1895 |
+
ws["views"] = {vid: dict(view, config={**(view.get("config") or {}),
|
| 1896 |
+
"memberPids": [pid for pid in view["config"]["memberPids"] if pid in pids]})
|
| 1897 |
+
if isinstance(view, dict) and isinstance((view.get("config") or {}).get("memberPids"), list)
|
| 1898 |
+
else view for vid, view in (ws.get("views") or {}).items()}
|
| 1899 |
workspace, fields, views, lists = aios_grid.workspace_wire(
|
| 1900 |
ws, session.uname, set(pids), defs={}, scope_key=table_key,
|
| 1901 |
storage_key=storage_key, fields_base=fields_base, st=st)
|
|
|
|
| 2033 |
derived = aios_grid.cohort_cells(lists) if not fields_only else {}
|
| 2034 |
materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
|
| 2035 |
for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures,
|
| 2036 |
+
getattr(session.runtime, 'measure_memo', {}),
|
| 2037 |
+
**({'cur': session.runtime._canonical_source_cursor}
|
| 2038 |
+
if getattr(session.runtime, '_canonical_source_cursor', None) is not None else {}))
|
| 2039 |
if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
|
| 2040 |
derived.setdefault(_pid, {}).update(_cells)
|
| 2041 |
|
|
|
|
| 2129 |
return []
|
| 2130 |
|
| 2131 |
|
| 2132 |
+
def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None, *, cur=None):
|
| 2133 |
"""`{pid: {field key: value}}` for this database's Metric columns.
|
| 2134 |
|
| 2135 |
⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need
|
|
|
|
| 2162 |
out = {}
|
| 2163 |
for (dfrom, dto), group in by_window.items():
|
| 2164 |
keys = tuple(sorted({f["measure"]["key"] for f in group}))
|
| 2165 |
+
mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys, tuple(sorted(pids)))
|
| 2166 |
if mkey not in memo:
|
| 2167 |
try:
|
| 2168 |
memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom,
|
| 2169 |
+
date_to=dto, team_id=team_id, offer=offer, group_keys=pids,
|
| 2170 |
+
**({'cur': cur} if cur is not None else {}))
|
| 2171 |
except Exception as e: # noqa: BLE001
|
| 2172 |
_ut_measure_err("ut-measure-column", e)
|
| 2173 |
transient = False
|
|
|
|
| 2350 |
@router.get("/tables")
|
| 2351 |
def list_tables(brief: bool = Query(False),
|
| 2352 |
session: Session = Depends(require_session)):
|
| 2353 |
+
"""List permitted definitions with exact stored counts or explicit unknown source populations."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2354 |
from core.if_field import project_fields
|
| 2355 |
ut = _ut()
|
| 2356 |
meta = nav_meta(session)
|
| 2357 |
out = []
|
| 2358 |
+
try:
|
| 2359 |
+
tables = ut.all_defs(st=session.runtime, schema_only=True)
|
| 2360 |
+
counts = {} if brief else ut.definition_counts(session.runtime)
|
| 2361 |
+
except Exception as exc:
|
| 2362 |
+
from core.request_timing import raise_if_resource_error
|
| 2363 |
+
raise_if_resource_error(exc)
|
| 2364 |
+
raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc
|
| 2365 |
lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables})
|
| 2366 |
for key, t in sorted(tables.items(),
|
| 2367 |
key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
|
|
|
|
| 2381 |
"recordsMutable": ut.records_mutable(key, st=lent),
|
| 2382 |
"createdBy": t.get("createdBy") or "",
|
| 2383 |
"created": t.get("created") or "",
|
| 2384 |
+
"rowCount": (counts.get(key) if ut.materialises(key, defn=t)
|
| 2385 |
+
and type(counts.get(key)) is int and counts[key] >= 0 else None)})
|
| 2386 |
out.append(row)
|
| 2387 |
if brief:
|
| 2388 |
# ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
|
|
|
|
| 2795 |
'recommendation': 'Use Next to read older events.'}]}
|
| 2796 |
|
| 2797 |
_ig_forward(session, table_key)
|
| 2798 |
+
source_needed = getattr(session.runtime, '_canonical_needed', None)
|
| 2799 |
# ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED.
|
| 2800 |
# This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE
|
| 2801 |
# (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy),
|
|
|
|
| 2807 |
# value it replaced (contract C5).
|
| 2808 |
if _ut()._entry_reader(session.runtime) is None:
|
| 2809 |
_lent = _table_lend(session.runtime, table_key)
|
| 2810 |
+
g = ut_assembly(session, table_key, st=_lent, consume_corrections=False, needed=source_needed)
|
| 2811 |
else:
|
| 2812 |
+
from registry_source import with_runtime
|
| 2813 |
from routes_customers import _AssemblyStore
|
| 2814 |
# One request owns these borrowed documents. A cold memo build and its page share reads;
|
| 2815 |
# a warm page reads records/shared cells afresh and lends Zone documents to both readers.
|
| 2816 |
original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore)
|
| 2817 |
else session.runtime)
|
| 2818 |
if not isinstance(session.runtime, _AssemblyStore):
|
| 2819 |
+
session = with_runtime(session, _AssemblyStore(original_rt))
|
| 2820 |
+
g = ut_assembly(session, table_key, with_rows=False, needed=source_needed)
|
| 2821 |
assembly_rt = session.runtime
|
| 2822 |
entry_key = (_ut().STORE_KEY, ("entry", str(table_key)))
|
| 2823 |
if entry_key in assembly_rt.docs:
|
|
|
|
| 2829 |
_lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {})
|
| 2830 |
# The page borrows the same envelope as workspace; only records and shared cells are fresh.
|
| 2831 |
pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent)
|
| 2832 |
+
g['pids'] = pids
|
| 2833 |
from core import perm_scope
|
| 2834 |
g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src
|
| 2835 |
if row['pid'] in g['pids']]
|
|
|
|
| 2891 |
# honestly instead of showing half a truncated brace.
|
| 2892 |
# ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole,
|
| 2893 |
# so a tenant using `json` for a short config sees no change at all.
|
| 2894 |
+
projection_fields = (g['fields'] if source_needed is None
|
| 2895 |
+
else [field for field in g['fields'] if field['key'] in source_needed])
|
| 2896 |
+
rows = (aios_grid.rows_from_pool(
|
| 2897 |
+
g["rows_src"], projection_fields, _thin_json(projection_fields, merged, table_key), derived=g["derived"])
|
| 2898 |
+
if source_needed is None or projection_fields else [{'pid': row['pid']} for row in g['rows_src']])
|
| 2899 |
rows = _ut().zone_cells(table_key, rows, allowed_row_ids=g["pids"], st=_lent)
|
| 2900 |
# ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
|
| 2901 |
# `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
|
platform/core/grid_events.py
CHANGED
|
@@ -2083,6 +2083,10 @@ def handle_one(event, ctx):
|
|
| 2083 |
shared_prior = {**base_field, **stored_choice,
|
| 2084 |
'key': key, 'source': 'overlay', 'shared': True,
|
| 2085 |
'custom': True, 'type': base_field['type']}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2086 |
shared_field = True
|
| 2087 |
if shared_field:
|
| 2088 |
if not _may_edit_field_definition(ctx, key, shared_prior):
|
|
@@ -2095,6 +2099,13 @@ def handle_one(event, ctx):
|
|
| 2095 |
# cannot turn an image field into another schema type while saving
|
| 2096 |
# its permissions.
|
| 2097 |
field = dict(shared_prior)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2098 |
field['label'] = str(raw.get('label') or field.get('label') or 'Untitled')[:120]
|
| 2099 |
field['note'] = str(raw.get('note') or '')[:2000]
|
| 2100 |
fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
|
|
|
|
| 2083 |
shared_prior = {**base_field, **stored_choice,
|
| 2084 |
'key': key, 'source': 'overlay', 'shared': True,
|
| 2085 |
'custom': True, 'type': base_field['type']}
|
| 2086 |
+
# Registry admission comes from the server context, even when the native
|
| 2087 |
+
# definition stores granted=False. The stored marker is persistence only.
|
| 2088 |
+
if base_field.get('granted') is True:
|
| 2089 |
+
shared_prior['granted'] = True
|
| 2090 |
shared_field = True
|
| 2091 |
if shared_field:
|
| 2092 |
if not _may_edit_field_definition(ctx, key, shared_prior):
|
|
|
|
| 2099 |
# cannot turn an image field into another schema type while saving
|
| 2100 |
# its permissions.
|
| 2101 |
field = dict(shared_prior)
|
| 2102 |
+
# The Product choice projection may derive granted=True from the registry.
|
| 2103 |
+
# Keep it for admission above, but persist only the native marker state.
|
| 2104 |
+
if spreadsheet_choice:
|
| 2105 |
+
if 'granted' in stored_choice:
|
| 2106 |
+
field['granted'] = stored_choice['granted']
|
| 2107 |
+
else:
|
| 2108 |
+
field.pop('granted', None)
|
| 2109 |
field['label'] = str(raw.get('label') or field.get('label') or 'Untitled')[:120]
|
| 2110 |
field['note'] = str(raw.get('note') or '')[:2000]
|
| 2111 |
fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
|
platform/core/measure_resolve.py
CHANGED
|
@@ -142,7 +142,8 @@ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp,
|
|
| 142 |
return out
|
| 143 |
|
| 144 |
|
| 145 |
-
def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None
|
|
|
|
| 146 |
"""Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
|
| 147 |
handed to `rows_from_pool(derived=…)` beside the cohort cells.
|
| 148 |
|
|
@@ -171,7 +172,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
|
|
| 171 |
if key not in memo:
|
| 172 |
try:
|
| 173 |
memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
|
| 174 |
-
today, team_id, allowed_pids, related=related
|
|
|
|
| 175 |
except Exception as e:
|
| 176 |
if on_error:
|
| 177 |
on_error('measure-column', e)
|
|
|
|
| 142 |
return out
|
| 143 |
|
| 144 |
|
| 145 |
+
def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None,
|
| 146 |
+
*, cur=None):
|
| 147 |
"""Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
|
| 148 |
handed to `rows_from_pool(derived=…)` beside the cohort cells.
|
| 149 |
|
|
|
|
| 172 |
if key not in memo:
|
| 173 |
try:
|
| 174 |
memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
|
| 175 |
+
today, team_id, allowed_pids, related=related,
|
| 176 |
+
**({'cur': cur} if cur is not None else {}))
|
| 177 |
except Exception as e:
|
| 178 |
if on_error:
|
| 179 |
on_error('measure-column', e)
|
platform/core/odoo.py
CHANGED
|
@@ -10,9 +10,16 @@ a single source of truth for the connection. Adds:
|
|
| 10 |
NOTHING here writes to Odoo — the underlying client hard-blocks writes.
|
| 11 |
"""
|
| 12 |
import os
|
|
|
|
| 13 |
import sys
|
| 14 |
import queue as _queue
|
| 15 |
import threading
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
from concurrent.futures import ThreadPoolExecutor
|
| 17 |
from contextlib import contextmanager
|
| 18 |
from functools import lru_cache
|
|
@@ -23,7 +30,176 @@ from pathlib import Path
|
|
| 23 |
_ROOT = Path(__file__).resolve().parents[1] # platform/
|
| 24 |
if str(_ROOT) not in sys.path:
|
| 25 |
sys.path.insert(0, str(_ROOT))
|
| 26 |
-
from odoo_client import OdooClient
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
# ---- business constants (TENANT #0's scope, kept as module attributes for its modules) ----
|
| 29 |
# These are Royal Imports' config — the default a request gets when nothing binds a tenant.
|
|
@@ -151,16 +327,47 @@ _pool_n = 0
|
|
| 151 |
_pool_lock = threading.Lock()
|
| 152 |
|
| 153 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
def _checkout():
|
| 155 |
global _pool_n
|
|
|
|
| 156 |
try:
|
| 157 |
-
|
| 158 |
except _queue.Empty:
|
| 159 |
with _pool_lock:
|
| 160 |
grow = _pool_n < _POOL_MAX
|
| 161 |
if grow:
|
| 162 |
_pool_n += 1
|
| 163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
|
| 165 |
|
| 166 |
def _checkin(c):
|
|
@@ -205,7 +412,10 @@ def parallel(tasks):
|
|
| 205 |
if not tasks:
|
| 206 |
return []
|
| 207 |
if len(tasks) == 1:
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
caller_mode = doc_mode() # propagate the submitter's basis to each worker
|
| 211 |
caller_ctx = _ctx() # and the tenant binding — a worker thread must
|
|
@@ -216,14 +426,17 @@ def parallel(tasks):
|
|
| 216 |
_tlocal.doc_mode = caller_mode
|
| 217 |
_tlocal.tenant_ctx = caller_ctx
|
| 218 |
try:
|
| 219 |
-
|
|
|
|
|
|
|
| 220 |
finally:
|
| 221 |
_tlocal.client = None
|
| 222 |
_tlocal.doc_mode = 'order'
|
| 223 |
_tlocal.tenant_ctx = None
|
| 224 |
_checkin(c)
|
| 225 |
with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
|
| 226 |
-
|
|
|
|
| 227 |
|
| 228 |
|
| 229 |
def parallel_map(task_dict):
|
|
|
|
| 10 |
NOTHING here writes to Odoo — the underlying client hard-blocks writes.
|
| 11 |
"""
|
| 12 |
import os
|
| 13 |
+
import math
|
| 14 |
import sys
|
| 15 |
import queue as _queue
|
| 16 |
import threading
|
| 17 |
+
import contextvars
|
| 18 |
+
import http.client
|
| 19 |
+
import io
|
| 20 |
+
import socket
|
| 21 |
+
import time
|
| 22 |
+
import xmlrpc.client
|
| 23 |
from concurrent.futures import ThreadPoolExecutor
|
| 24 |
from contextlib import contextmanager
|
| 25 |
from functools import lru_cache
|
|
|
|
| 30 |
_ROOT = Path(__file__).resolve().parents[1] # platform/
|
| 31 |
if str(_ROOT) not in sys.path:
|
| 32 |
sys.path.insert(0, str(_ROOT))
|
| 33 |
+
from odoo_client import OdooClient as _BaseOdooClient, _RPC_TIMEOUT
|
| 34 |
+
from core import request_timing as _timing
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class OdooRequestTimeout(TimeoutError):
|
| 38 |
+
"""An Odoo request transport exhausted its absolute RPC budget."""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _rpc_remaining():
|
| 42 |
+
return _timing.remaining_budget(_RPC_TIMEOUT)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _create_rpc_connection(address, timeout=None, source_address=None):
|
| 46 |
+
"""Bound each address attempt while checking the budget around synchronous DNS resolution."""
|
| 47 |
+
_timing.remaining_budget()
|
| 48 |
+
addresses = socket.getaddrinfo(address[0], address[1], 0, socket.SOCK_STREAM)
|
| 49 |
+
_timing.remaining_budget()
|
| 50 |
+
error = None
|
| 51 |
+
for family, kind, protocol, _, target in addresses:
|
| 52 |
+
_timing.remaining_budget()
|
| 53 |
+
con = socket.socket(family, kind, protocol)
|
| 54 |
+
try:
|
| 55 |
+
con.settimeout(_rpc_remaining())
|
| 56 |
+
if source_address:
|
| 57 |
+
con.bind(source_address)
|
| 58 |
+
con.connect(target)
|
| 59 |
+
con.settimeout(_rpc_remaining())
|
| 60 |
+
return con
|
| 61 |
+
except BaseException as exc:
|
| 62 |
+
con.close()
|
| 63 |
+
if not isinstance(exc, OSError):
|
| 64 |
+
raise
|
| 65 |
+
error = exc
|
| 66 |
+
if error is not None:
|
| 67 |
+
raise error
|
| 68 |
+
raise OSError("Odoo DNS resolution returned no addresses")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class _SocketReader(io.RawIOBase):
|
| 72 |
+
def __init__(self, sock):
|
| 73 |
+
super().__init__()
|
| 74 |
+
self.sock = sock
|
| 75 |
+
|
| 76 |
+
def readable(self):
|
| 77 |
+
return True
|
| 78 |
+
|
| 79 |
+
def readinto(self, buffer):
|
| 80 |
+
return self.sock.recv_into(buffer)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class _BudgetSocket:
|
| 84 |
+
def __init__(self, sock):
|
| 85 |
+
self.sock = sock
|
| 86 |
+
|
| 87 |
+
def __getattr__(self, name):
|
| 88 |
+
return getattr(self.sock, name)
|
| 89 |
+
|
| 90 |
+
def _call(self, method, *args):
|
| 91 |
+
self.sock.settimeout(_rpc_remaining())
|
| 92 |
+
try:
|
| 93 |
+
result = getattr(self.sock, method)(*args)
|
| 94 |
+
except socket.timeout as exc:
|
| 95 |
+
if _timing.current_budget() is not None:
|
| 96 |
+
_timing.remaining_budget()
|
| 97 |
+
raise OdooRequestTimeout("Odoo RPC socket timed out") from exc
|
| 98 |
+
raise
|
| 99 |
+
_timing.remaining_budget()
|
| 100 |
+
return result
|
| 101 |
+
|
| 102 |
+
def recv_into(self, buffer):
|
| 103 |
+
return self._call("recv_into", buffer)
|
| 104 |
+
|
| 105 |
+
def sendall(self, data):
|
| 106 |
+
return self._call("sendall", data)
|
| 107 |
+
|
| 108 |
+
def makefile(self, mode, buffering=None):
|
| 109 |
+
if mode != "rb":
|
| 110 |
+
raise ValueError("Odoo transport only reads binary responses")
|
| 111 |
+
return io.BufferedReader(_SocketReader(self))
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class _BudgetHTTPConnection(http.client.HTTPConnection):
|
| 115 |
+
def __init__(self, *args, **kwargs):
|
| 116 |
+
super().__init__(*args, **kwargs)
|
| 117 |
+
self._create_connection = _create_rpc_connection
|
| 118 |
+
|
| 119 |
+
def connect(self):
|
| 120 |
+
self.timeout = _rpc_remaining()
|
| 121 |
+
super().connect()
|
| 122 |
+
self.sock = _BudgetSocket(self.sock)
|
| 123 |
+
_timing.remaining_budget()
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class _BudgetHTTPSConnection(http.client.HTTPSConnection):
|
| 127 |
+
def __init__(self, *args, **kwargs):
|
| 128 |
+
super().__init__(*args, **kwargs)
|
| 129 |
+
self._create_connection = _create_rpc_connection
|
| 130 |
+
|
| 131 |
+
def connect(self):
|
| 132 |
+
self.timeout = _rpc_remaining()
|
| 133 |
+
super().connect()
|
| 134 |
+
self.sock = _BudgetSocket(self.sock)
|
| 135 |
+
_timing.remaining_budget()
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class _BudgetTransport(xmlrpc.client.Transport):
|
| 139 |
+
connection_type = _BudgetHTTPConnection
|
| 140 |
+
|
| 141 |
+
def make_connection(self, host):
|
| 142 |
+
_timing.remaining_budget()
|
| 143 |
+
if self._connection and host == self._connection[0]:
|
| 144 |
+
return self._connection[1]
|
| 145 |
+
chost, self._extra_headers, _ = self.get_host_info(host)
|
| 146 |
+
con = self.connection_type(chost, timeout=_rpc_remaining())
|
| 147 |
+
self._connection = host, con
|
| 148 |
+
return con
|
| 149 |
+
|
| 150 |
+
def single_request(self, *args, **kwargs):
|
| 151 |
+
# A single deadline covers headers and every body read, including trickling responses.
|
| 152 |
+
deadline = time.monotonic() + _RPC_TIMEOUT
|
| 153 |
+
try:
|
| 154 |
+
with _timing.budget_scope(deadline, OdooRequestTimeout):
|
| 155 |
+
return super().single_request(*args, **kwargs)
|
| 156 |
+
except _timing.RequestDeadlineExceeded:
|
| 157 |
+
self.close()
|
| 158 |
+
raise
|
| 159 |
+
except socket.timeout as exc:
|
| 160 |
+
self.close()
|
| 161 |
+
_timing.remaining_budget()
|
| 162 |
+
raise OdooRequestTimeout("Odoo RPC socket timed out") from exc
|
| 163 |
+
except BaseException:
|
| 164 |
+
self.close()
|
| 165 |
+
raise
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
class _BudgetSafeTransport(_BudgetTransport, xmlrpc.client.SafeTransport):
|
| 169 |
+
connection_type = _BudgetHTTPSConnection
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _bounded_proxy(uri):
|
| 173 |
+
transport = _BudgetSafeTransport() if uri.startswith("https") else _BudgetTransport()
|
| 174 |
+
return xmlrpc.client.ServerProxy(uri, transport=transport)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
class OdooClient(_BaseOdooClient):
|
| 178 |
+
def _connect(self, _retried=False):
|
| 179 |
+
try:
|
| 180 |
+
with _bounded_proxy(f'{self.url}/xmlrpc/2/common') as common:
|
| 181 |
+
self.uid = common.authenticate(self.db, self.user, self.api_key, {})
|
| 182 |
+
except (http.client.HTTPException, ConnectionError, OSError):
|
| 183 |
+
if _retried or _timing.current_budget() is not None:
|
| 184 |
+
raise
|
| 185 |
+
time.sleep(3)
|
| 186 |
+
return self._connect(_retried=True)
|
| 187 |
+
if not self.uid:
|
| 188 |
+
raise RuntimeError("Odoo authentication failed; check credentials.")
|
| 189 |
+
self.models = _bounded_proxy(f'{self.url}/xmlrpc/2/object')
|
| 190 |
+
|
| 191 |
+
def execute_kw(self, model, method, args, kwargs=None):
|
| 192 |
+
if _timing.current_budget() is None:
|
| 193 |
+
return super().execute_kw(model, method, args, kwargs)
|
| 194 |
+
self._guard(method)
|
| 195 |
+
if not self._xlock.acquire(timeout=_rpc_remaining()):
|
| 196 |
+
_timing.remaining_budget()
|
| 197 |
+
raise OdooRequestTimeout("Odoo client lock timed out")
|
| 198 |
+
try:
|
| 199 |
+
_timing.remaining_budget()
|
| 200 |
+
return self.models.execute_kw(self.db, self.uid, self.api_key, model, method, args, kwargs or {})
|
| 201 |
+
finally:
|
| 202 |
+
self._xlock.release()
|
| 203 |
|
| 204 |
# ---- business constants (TENANT #0's scope, kept as module attributes for its modules) ----
|
| 205 |
# These are Royal Imports' config — the default a request gets when nothing binds a tenant.
|
|
|
|
| 327 |
_pool_lock = threading.Lock()
|
| 328 |
|
| 329 |
|
| 330 |
+
class OdooPoolTimeout(TimeoutError):
|
| 331 |
+
"""All read-only Odoo clients remained checked out past the acquisition budget."""
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def _checkout_timeout_s():
|
| 335 |
+
try:
|
| 336 |
+
value = float(os.environ.get("AIOS_ODOO_POOL_TIMEOUT_S", "10"))
|
| 337 |
+
return max(.01, value) if math.isfinite(value) else 10.0
|
| 338 |
+
except ValueError:
|
| 339 |
+
return 10.0
|
| 340 |
+
|
| 341 |
+
|
| 342 |
def _checkout():
|
| 343 |
global _pool_n
|
| 344 |
+
_timing.remaining_budget()
|
| 345 |
try:
|
| 346 |
+
client = _pool_q.get_nowait()
|
| 347 |
except _queue.Empty:
|
| 348 |
with _pool_lock:
|
| 349 |
grow = _pool_n < _POOL_MAX
|
| 350 |
if grow:
|
| 351 |
_pool_n += 1
|
| 352 |
+
if grow:
|
| 353 |
+
try:
|
| 354 |
+
client = OdooClient()
|
| 355 |
+
_timing.remaining_budget()
|
| 356 |
+
return client
|
| 357 |
+
except BaseException:
|
| 358 |
+
with _pool_lock:
|
| 359 |
+
_pool_n -= 1
|
| 360 |
+
raise
|
| 361 |
+
try:
|
| 362 |
+
client = _pool_q.get(timeout=_timing.remaining_budget(_checkout_timeout_s()))
|
| 363 |
+
except _queue.Empty:
|
| 364 |
+
raise OdooPoolTimeout("Odoo read pool acquisition timed out") from None
|
| 365 |
+
try:
|
| 366 |
+
_timing.remaining_budget()
|
| 367 |
+
except BaseException:
|
| 368 |
+
_checkin(client)
|
| 369 |
+
raise
|
| 370 |
+
return client
|
| 371 |
|
| 372 |
|
| 373 |
def _checkin(c):
|
|
|
|
| 412 |
if not tasks:
|
| 413 |
return []
|
| 414 |
if len(tasks) == 1:
|
| 415 |
+
_timing.remaining_budget()
|
| 416 |
+
result = tasks[0]()
|
| 417 |
+
_timing.remaining_budget()
|
| 418 |
+
return [result]
|
| 419 |
|
| 420 |
caller_mode = doc_mode() # propagate the submitter's basis to each worker
|
| 421 |
caller_ctx = _ctx() # and the tenant binding — a worker thread must
|
|
|
|
| 426 |
_tlocal.doc_mode = caller_mode
|
| 427 |
_tlocal.tenant_ctx = caller_ctx
|
| 428 |
try:
|
| 429 |
+
result = fn()
|
| 430 |
+
_timing.remaining_budget()
|
| 431 |
+
return result
|
| 432 |
finally:
|
| 433 |
_tlocal.client = None
|
| 434 |
_tlocal.doc_mode = 'order'
|
| 435 |
_tlocal.tenant_ctx = None
|
| 436 |
_checkin(c)
|
| 437 |
with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
|
| 438 |
+
contexts = [contextvars.copy_context() for _ in tasks]
|
| 439 |
+
return list(ex.map(lambda pair: pair[0].run(_run, pair[1]), zip(contexts, tasks)))
|
| 440 |
|
| 441 |
|
| 442 |
def parallel_map(task_dict):
|
platform/core/request_timing.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
| 3 |
|
| 4 |
import re
|
| 5 |
import hashlib
|
|
|
|
| 6 |
import threading
|
| 7 |
import time
|
| 8 |
from collections import Counter
|
|
@@ -11,9 +12,16 @@ from contextvars import ContextVar
|
|
| 11 |
from functools import wraps
|
| 12 |
|
| 13 |
CURRENT = ContextVar("aios_request_timing", default=None)
|
|
|
|
|
|
|
|
|
|
| 14 |
_STORE_SPAN = ContextVar("aios_store_timing_span", default=None)
|
|
|
|
| 15 |
_THREAD_LOCK = threading.Lock()
|
| 16 |
_THREADS = {}
|
|
|
|
|
|
|
|
|
|
| 17 |
_SQL_VALUES = re.compile(
|
| 18 |
r"--[^\n]*|/\*.*?\*/|\$([A-Za-z_][A-Za-z_0-9]*|)\$.*?\$\1\$|"
|
| 19 |
r"(?:[eEbBxX])?'(?:''|\\.|[^'])*'|\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b", re.S)
|
|
@@ -39,9 +47,18 @@ class Accumulator:
|
|
| 39 |
self.slowest_sql_ms = 0.0
|
| 40 |
self.response_bytes = 0
|
| 41 |
self.samples = Counter()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
def snapshot(self):
|
| 44 |
with self.lock:
|
|
|
|
|
|
|
| 45 |
return {"queue_ms": self.queue_ms, "store_ms": self.store_ms,
|
| 46 |
"store_reads": self.store_reads, "store_bytes": self.store_bytes,
|
| 47 |
"store_write_bytes": self.store_write_bytes,
|
|
@@ -60,9 +77,104 @@ def request_scope(accumulator=None):
|
|
| 60 |
try:
|
| 61 |
yield acc
|
| 62 |
finally:
|
|
|
|
| 63 |
CURRENT.reset(token)
|
| 64 |
|
| 65 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
@contextmanager
|
| 67 |
def thread_scope():
|
| 68 |
"""Associate only this operation's thread with the current request for sampling."""
|
|
@@ -89,6 +201,110 @@ def associated_threads():
|
|
| 89 |
return dict(_THREADS)
|
| 90 |
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
def text_bytes(value):
|
| 93 |
"""Measure document payload bytes without serializing decoded caller objects."""
|
| 94 |
if isinstance(value, str):
|
|
@@ -109,7 +325,8 @@ def record_store(operation, keys, milliseconds=0.0, *, bytes_read=0, bytes_writt
|
|
| 109 |
if acc.closed:
|
| 110 |
return
|
| 111 |
acc.store_ms += milliseconds
|
| 112 |
-
|
|
|
|
| 113 |
acc.store_reads += count if operation != "update" else 0
|
| 114 |
acc.store_bytes += bytes_read
|
| 115 |
acc.store_write_bytes += bytes_written
|
|
@@ -151,6 +368,9 @@ def store_operation(operation, *, many=False):
|
|
| 151 |
try:
|
| 152 |
with thread_scope():
|
| 153 |
return fn(*args, **kwargs)
|
|
|
|
|
|
|
|
|
|
| 154 |
finally:
|
| 155 |
elapsed = (time.perf_counter() - started) * 1000
|
| 156 |
_STORE_SPAN.reset(token)
|
|
|
|
| 3 |
|
| 4 |
import re
|
| 5 |
import hashlib
|
| 6 |
+
import sys
|
| 7 |
import threading
|
| 8 |
import time
|
| 9 |
from collections import Counter
|
|
|
|
| 12 |
from functools import wraps
|
| 13 |
|
| 14 |
CURRENT = ContextVar("aios_request_timing", default=None)
|
| 15 |
+
READ_MEMO_SECONDS = 2.0
|
| 16 |
+
READ_MEMO_ENTRIES = 64
|
| 17 |
+
READ_MEMO_BYTES = 8 * 1024 * 1024
|
| 18 |
_STORE_SPAN = ContextVar("aios_store_timing_span", default=None)
|
| 19 |
+
_MEMO_READ = ContextVar("aios_request_memo_read", default=None)
|
| 20 |
_THREAD_LOCK = threading.Lock()
|
| 21 |
_THREADS = {}
|
| 22 |
+
_DEADLINE_ARM = None
|
| 23 |
+
_LOCK_PROBE_ARM = None
|
| 24 |
+
_BUDGET = ContextVar("aios_operation_budget", default=None)
|
| 25 |
_SQL_VALUES = re.compile(
|
| 26 |
r"--[^\n]*|/\*.*?\*/|\$([A-Za-z_][A-Za-z_0-9]*|)\$.*?\$\1\$|"
|
| 27 |
r"(?:[eEbBxX])?'(?:''|\\.|[^'])*'|\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b", re.S)
|
|
|
|
| 47 |
self.slowest_sql_ms = 0.0
|
| 48 |
self.response_bytes = 0
|
| 49 |
self.samples = Counter()
|
| 50 |
+
self.deadline = None
|
| 51 |
+
self.cancelled = False
|
| 52 |
+
self.read_memo = {}
|
| 53 |
+
self.read_memo_bytes = 0
|
| 54 |
+
self.read_memo_epoch = 0
|
| 55 |
+
self.read_memo_invalidated_at = 0.0
|
| 56 |
+
self.read_memo_ended = False
|
| 57 |
|
| 58 |
def snapshot(self):
|
| 59 |
with self.lock:
|
| 60 |
+
# The middleware takes its first snapshot before sending response headers.
|
| 61 |
+
clear_read_memo(self, end=True)
|
| 62 |
return {"queue_ms": self.queue_ms, "store_ms": self.store_ms,
|
| 63 |
"store_reads": self.store_reads, "store_bytes": self.store_bytes,
|
| 64 |
"store_write_bytes": self.store_write_bytes,
|
|
|
|
| 77 |
try:
|
| 78 |
yield acc
|
| 79 |
finally:
|
| 80 |
+
clear_read_memo(acc, end=True)
|
| 81 |
CURRENT.reset(token)
|
| 82 |
|
| 83 |
|
| 84 |
+
def clear_read_memo(acc=None, *, end=False):
|
| 85 |
+
"""Invalidate in-flight memo fills as well as completed values in copied contexts."""
|
| 86 |
+
acc = CURRENT.get() if acc is None else acc
|
| 87 |
+
if acc is not None:
|
| 88 |
+
with acc.lock:
|
| 89 |
+
acc.read_memo.clear()
|
| 90 |
+
acc.read_memo_bytes = 0
|
| 91 |
+
acc.read_memo_epoch += 1
|
| 92 |
+
acc.read_memo_invalidated_at = time.monotonic()
|
| 93 |
+
acc.read_memo_ended |= end
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def memo_checkpoint():
|
| 97 |
+
"""Capture the active request epoch before a backing read or dependent fill."""
|
| 98 |
+
acc = CURRENT.get()
|
| 99 |
+
if acc is None:
|
| 100 |
+
return None
|
| 101 |
+
with acc.lock:
|
| 102 |
+
parent = _MEMO_READ.get()
|
| 103 |
+
if parent is not None and parent[0] is acc and parent[1] != acc.read_memo_epoch:
|
| 104 |
+
return None
|
| 105 |
+
if (acc.closed or acc.cancelled or acc.read_memo_ended
|
| 106 |
+
or time.perf_counter() - acc.started >= READ_MEMO_SECONDS):
|
| 107 |
+
clear_read_memo(acc, end=True)
|
| 108 |
+
return None
|
| 109 |
+
return acc, acc.read_memo_epoch
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def memo_read(fn):
|
| 113 |
+
"""Revoke request snapshots on failure and keep nested fills in their original epoch."""
|
| 114 |
+
@wraps(fn)
|
| 115 |
+
def guarded(*args, **kwargs):
|
| 116 |
+
acc = CURRENT.get()
|
| 117 |
+
if acc is None:
|
| 118 |
+
return fn(*args, **kwargs)
|
| 119 |
+
parent = _MEMO_READ.get()
|
| 120 |
+
checkpoint = parent if parent is not None and parent[0] is acc else memo_checkpoint()
|
| 121 |
+
token = _MEMO_READ.set(checkpoint)
|
| 122 |
+
try:
|
| 123 |
+
return fn(*args, **kwargs)
|
| 124 |
+
except BaseException:
|
| 125 |
+
clear_read_memo(acc)
|
| 126 |
+
raise
|
| 127 |
+
finally:
|
| 128 |
+
_MEMO_READ.reset(token)
|
| 129 |
+
return guarded
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def memo_peek(key):
|
| 133 |
+
"""Find immutable text only while this request's bounded memo remains active."""
|
| 134 |
+
checkpoint = memo_checkpoint()
|
| 135 |
+
if checkpoint is not None:
|
| 136 |
+
acc, epoch = checkpoint
|
| 137 |
+
with acc.lock:
|
| 138 |
+
if epoch == acc.read_memo_epoch and key in acc.read_memo:
|
| 139 |
+
return True, acc.read_memo[key]
|
| 140 |
+
return False, None
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def memo_seed(key, value, checkpoint):
|
| 144 |
+
"""Publish successful backing-read text without reviving an expired or invalidated epoch."""
|
| 145 |
+
if checkpoint is not None and (value is None or isinstance(value, str)):
|
| 146 |
+
acc, epoch = checkpoint
|
| 147 |
+
def key_size(part):
|
| 148 |
+
return sys.getsizeof(part) + (sum(key_size(item) for item in part)
|
| 149 |
+
if isinstance(part, tuple) else 0)
|
| 150 |
+
size = key_size(key) + sys.getsizeof(value)
|
| 151 |
+
with acc.lock:
|
| 152 |
+
if (CURRENT.get() is acc and epoch == acc.read_memo_epoch
|
| 153 |
+
and not acc.closed and not acc.cancelled
|
| 154 |
+
and not acc.read_memo_ended
|
| 155 |
+
and time.perf_counter() - acc.started < READ_MEMO_SECONDS
|
| 156 |
+
and key not in acc.read_memo
|
| 157 |
+
and len(acc.read_memo) < READ_MEMO_ENTRIES
|
| 158 |
+
and acc.read_memo_bytes + size <= READ_MEMO_BYTES):
|
| 159 |
+
acc.read_memo[key] = value
|
| 160 |
+
acc.read_memo_bytes += size
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
@memo_read
|
| 164 |
+
def memo_text(key, read, *, fresh=False):
|
| 165 |
+
"""Reuse bounded immutable backing-read text during the short pre-header request phase."""
|
| 166 |
+
if fresh:
|
| 167 |
+
clear_read_memo()
|
| 168 |
+
return read()
|
| 169 |
+
checkpoint = memo_checkpoint()
|
| 170 |
+
found, value = memo_peek(key)
|
| 171 |
+
if found:
|
| 172 |
+
return value
|
| 173 |
+
value = read()
|
| 174 |
+
memo_seed(key, value, checkpoint)
|
| 175 |
+
return value
|
| 176 |
+
|
| 177 |
+
|
| 178 |
@contextmanager
|
| 179 |
def thread_scope():
|
| 180 |
"""Associate only this operation's thread with the current request for sampling."""
|
|
|
|
| 201 |
return dict(_THREADS)
|
| 202 |
|
| 203 |
|
| 204 |
+
class RequestDeadlineExceeded(TimeoutError):
|
| 205 |
+
"""A request exhausted its analytical query budget."""
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def current_budget():
|
| 209 |
+
"""Return the earliest absolute deadline and its named failure."""
|
| 210 |
+
budget = _BUDGET.get()
|
| 211 |
+
acc = CURRENT.get()
|
| 212 |
+
if acc is not None:
|
| 213 |
+
if acc.cancelled:
|
| 214 |
+
raise RequestDeadlineExceeded("request cancelled")
|
| 215 |
+
if acc.deadline is not None and (budget is None or acc.deadline <= budget[0]):
|
| 216 |
+
budget = (acc.deadline, RequestDeadlineExceeded)
|
| 217 |
+
return budget
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def remaining_budget(limit=None):
|
| 221 |
+
"""Check the absolute budget before lending its remaining seconds to a blocking operation."""
|
| 222 |
+
budget = current_budget()
|
| 223 |
+
if budget is None:
|
| 224 |
+
return limit
|
| 225 |
+
remaining = budget[0] - time.monotonic()
|
| 226 |
+
if remaining <= 0:
|
| 227 |
+
raise budget[1]("absolute operation deadline exceeded")
|
| 228 |
+
return remaining if limit is None else min(limit, remaining)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
@contextmanager
|
| 232 |
+
def budget_scope(deadline, error=RequestDeadlineExceeded):
|
| 233 |
+
"""Narrow a controlled operation's budget without extending its caller's deadline."""
|
| 234 |
+
parent = current_budget()
|
| 235 |
+
token = _BUDGET.set(parent if parent is not None and parent[0] <= deadline else (deadline, error))
|
| 236 |
+
try:
|
| 237 |
+
remaining_budget()
|
| 238 |
+
yield
|
| 239 |
+
remaining_budget()
|
| 240 |
+
finally:
|
| 241 |
+
_BUDGET.reset(token)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def raise_if_resource_error(error):
|
| 245 |
+
"""Keep request resource refusals out of readiness fallbacks."""
|
| 246 |
+
if CURRENT.get() is None:
|
| 247 |
+
return
|
| 248 |
+
known = {
|
| 249 |
+
"RequestDeadlineExceeded", "DeadlineCapacityExceeded", "PoolTimeout", "TooManyRequests",
|
| 250 |
+
"QueryCanceled", "LockNotAvailable", "SharedReadTimeout", "OdooPoolTimeout",
|
| 251 |
+
"OperationalError", "InterfaceError", "IdleInTransactionSessionTimeout",
|
| 252 |
+
"OutOfMemoryException", "ConnectionTimeout", "PostgresOperationTimeout", "OdooRequestTimeout"}
|
| 253 |
+
seen, probe = set(), error
|
| 254 |
+
while probe is not None and id(probe) not in seen:
|
| 255 |
+
seen.add(id(probe))
|
| 256 |
+
if type(probe).__name__ in known:
|
| 257 |
+
raise error
|
| 258 |
+
probe = probe.__cause__ or probe.__context__
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def configure_deadlines(arm):
|
| 262 |
+
"""Install the runtime's registered statement watchdog without an upward import."""
|
| 263 |
+
global _DEADLINE_ARM
|
| 264 |
+
_DEADLINE_ARM = arm
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def configure_lock_probes(arm):
|
| 268 |
+
"""Install bounded lock diagnostics supplied by the runtime."""
|
| 269 |
+
global _LOCK_PROBE_ARM
|
| 270 |
+
_LOCK_PROBE_ARM = arm
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
@contextmanager
|
| 274 |
+
def lock_wait_probe(waiter_pid, wait_s, diagnose):
|
| 275 |
+
"""Sample blockers while the row transaction waits, using a separate connection."""
|
| 276 |
+
handle = _LOCK_PROBE_ARM(waiter_pid, wait_s, diagnose) if (
|
| 277 |
+
waiter_pid is not None and _LOCK_PROBE_ARM is not None) else None
|
| 278 |
+
try:
|
| 279 |
+
yield handle
|
| 280 |
+
finally:
|
| 281 |
+
if handle is not None:
|
| 282 |
+
handle.close()
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
@contextmanager
|
| 286 |
+
def duckdb_operation(connection):
|
| 287 |
+
"""Arm only the active cursor call and disarm before its connection can be reused."""
|
| 288 |
+
acc = CURRENT.get()
|
| 289 |
+
arm = _DEADLINE_ARM
|
| 290 |
+
if acc is None or acc.deadline is None or arm is None:
|
| 291 |
+
yield
|
| 292 |
+
return
|
| 293 |
+
if acc.cancelled or time.monotonic() >= acc.deadline:
|
| 294 |
+
raise RequestDeadlineExceeded("request query deadline exceeded")
|
| 295 |
+
handle = arm(connection, acc)
|
| 296 |
+
error = None
|
| 297 |
+
try:
|
| 298 |
+
yield
|
| 299 |
+
except BaseException as exc:
|
| 300 |
+
error = exc
|
| 301 |
+
raise
|
| 302 |
+
finally:
|
| 303 |
+
expired = handle.close()
|
| 304 |
+
if expired and (error is None or type(error).__name__ == "InterruptException"):
|
| 305 |
+
raise RequestDeadlineExceeded("request query deadline exceeded") from error
|
| 306 |
+
|
| 307 |
+
|
| 308 |
def text_bytes(value):
|
| 309 |
"""Measure document payload bytes without serializing decoded caller objects."""
|
| 310 |
if isinstance(value, str):
|
|
|
|
| 325 |
if acc.closed:
|
| 326 |
return
|
| 327 |
acc.store_ms += milliseconds
|
| 328 |
+
if count:
|
| 329 |
+
acc.store_operations[operation] += count
|
| 330 |
acc.store_reads += count if operation != "update" else 0
|
| 331 |
acc.store_bytes += bytes_read
|
| 332 |
acc.store_write_bytes += bytes_written
|
|
|
|
| 368 |
try:
|
| 369 |
with thread_scope():
|
| 370 |
return fn(*args, **kwargs)
|
| 371 |
+
except BaseException:
|
| 372 |
+
clear_read_memo()
|
| 373 |
+
raise
|
| 374 |
finally:
|
| 375 |
elapsed = (time.perf_counter() - started) * 1000
|
| 376 |
_STORE_SPAN.reset(token)
|
platform/core/store_pg.py
CHANGED
|
@@ -48,6 +48,7 @@ import tempfile
|
|
| 48 |
import threading
|
| 49 |
import time
|
| 50 |
from collections import OrderedDict
|
|
|
|
| 51 |
from functools import lru_cache
|
| 52 |
from pathlib import Path
|
| 53 |
from urllib.parse import urlparse
|
|
@@ -342,11 +343,18 @@ def _holder_pool_class(base, timeout_error):
|
|
| 342 |
super().__init__(*a, **kw)
|
| 343 |
|
| 344 |
def getconn(self, timeout=None):
|
|
|
|
|
|
|
| 345 |
try:
|
| 346 |
conn = super().getconn(timeout=timeout)
|
| 347 |
except timeout_error:
|
| 348 |
self._log_exhausted()
|
| 349 |
raise
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
label = _holder_label()
|
| 351 |
with self._holders_lock:
|
| 352 |
self._holders[id(conn)] = (time.monotonic(), label)
|
|
@@ -357,6 +365,19 @@ def _holder_pool_class(base, timeout_error):
|
|
| 357 |
self._holders.pop(id(conn), None)
|
| 358 |
super().putconn(conn)
|
| 359 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
def _log_exhausted(self):
|
| 361 |
now = time.monotonic()
|
| 362 |
with self._holders_lock:
|
|
@@ -382,6 +403,67 @@ def _lock_timeout_s():
|
|
| 382 |
return max(1.0, value)
|
| 383 |
|
| 384 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
def _pool():
|
| 386 |
"""The psycopg connection pool, opened once per process.
|
| 387 |
|
|
@@ -400,6 +482,7 @@ def _pool():
|
|
| 400 |
return _POOL['pool']
|
| 401 |
try:
|
| 402 |
from psycopg_pool import ConnectionPool, PoolTimeout
|
|
|
|
| 403 |
except ImportError as e: # pragma: no cover - depends on B-3
|
| 404 |
raise RuntimeError(
|
| 405 |
'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
|
|
@@ -416,11 +499,17 @@ def _pool():
|
|
| 416 |
# real requests, not be held awake by this process's idle pool for psycopg's 10-minute
|
| 417 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 418 |
# database client.
|
| 419 |
-
|
| 420 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
pool_cls = _holder_pool_class(ConnectionPool, PoolTimeout)
|
| 422 |
-
_POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=
|
| 423 |
-
max_idle=30.0,
|
|
|
|
| 424 |
_POOL['url'] = url
|
| 425 |
_POOL['sslrootcert'] = sslrootcert
|
| 426 |
return _POOL['pool']
|
|
@@ -435,6 +524,7 @@ def available():
|
|
| 435 |
"""
|
| 436 |
if not _url():
|
| 437 |
return False
|
|
|
|
| 438 |
if _POOL.get('ok'):
|
| 439 |
return True
|
| 440 |
try:
|
|
@@ -442,7 +532,8 @@ def available():
|
|
| 442 |
con.execute('SELECT 1')
|
| 443 |
_POOL['ok'] = True
|
| 444 |
return True
|
| 445 |
-
except Exception:
|
|
|
|
| 446 |
return False
|
| 447 |
|
| 448 |
|
|
@@ -478,7 +569,7 @@ def _q(template, kind, tenant_slug=None):
|
|
| 478 |
#: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
|
| 479 |
#: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
|
| 480 |
#: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
|
| 481 |
-
#:
|
| 482 |
#: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
|
| 483 |
#: finishes, and a read never joins a flight started under an older generation.
|
| 484 |
#: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
|
|
@@ -489,34 +580,57 @@ _READ_FLIGHT_LOCK = threading.Lock()
|
|
| 489 |
|
| 490 |
|
| 491 |
class _ReadFlight:
|
| 492 |
-
__slots__ = ('done', 'text', 'error', 'generation')
|
| 493 |
|
| 494 |
def __init__(self, generation):
|
| 495 |
self.done = threading.Event()
|
| 496 |
self.text = None
|
| 497 |
self.error = None
|
| 498 |
self.generation = generation
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 499 |
|
| 500 |
|
| 501 |
def _bump_write_generation(name, tenant_slug):
|
|
|
|
| 502 |
key = (_effective_slug(tenant_slug), str(name))
|
| 503 |
with _READ_FLIGHT_LOCK:
|
| 504 |
_WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
|
| 506 |
|
| 507 |
-
def _shared_read(name, drops, tenant_slug, fetch_text):
|
| 508 |
-
"""
|
| 509 |
doc = (_effective_slug(tenant_slug), str(name))
|
| 510 |
-
key = doc + (drops,)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
with _READ_FLIGHT_LOCK:
|
| 512 |
generation = _WRITE_GENERATION.get(doc, 0)
|
| 513 |
flight = _READ_FLIGHTS.get(key)
|
| 514 |
-
leader = flight is None or flight.generation != generation
|
|
|
|
| 515 |
if leader:
|
| 516 |
flight = _ReadFlight(generation)
|
| 517 |
_READ_FLIGHTS[key] = flight
|
| 518 |
if not leader:
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
| 520 |
if flight.error is not None:
|
| 521 |
# ⛔ THE LEADER'S OWN EXCEPTION, never a wrapper: routes and retries classify store
|
| 522 |
# failures by TYPE (`PoolTimeout`/`OperationalError` -> 503 "try again", a transient
|
|
@@ -525,7 +639,10 @@ def _shared_read(name, drops, tenant_slug, fetch_text):
|
|
| 525 |
raise flight.error
|
| 526 |
return flight.text
|
| 527 |
try:
|
| 528 |
-
|
|
|
|
|
|
|
|
|
|
| 529 |
except BaseException as exc:
|
| 530 |
flight.error = exc
|
| 531 |
raise
|
|
@@ -539,29 +656,172 @@ def _shared_read(name, drops, tenant_slug, fetch_text):
|
|
| 539 |
|
| 540 |
def _parsed(text):
|
| 541 |
"""The caller's own object from the shared JSON text (`None` = the key is absent)."""
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 546 |
|
| 547 |
|
| 548 |
-
@_request_timing.store_operation("get")
|
| 549 |
def get(name, fresh=False, tenant_slug=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 550 |
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 551 |
|
| 552 |
-
Ordinary reads
|
| 553 |
-
|
| 554 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 555 |
def fetch():
|
| 556 |
-
with
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 557 |
row = con.execute(
|
| 558 |
-
_q('SELECT
|
| 559 |
-
|
| 560 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 561 |
_request_timing.record_store("get", str(name), bytes_read=_request_timing.text_bytes(text), count=0)
|
| 562 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 563 |
|
| 564 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
|
| 566 |
|
| 567 |
_PROJECTION_TEXT = OrderedDict()
|
|
@@ -573,8 +833,10 @@ _PROJECTION_TEXT_LIMIT = 16 * 1024 * 1024
|
|
| 573 |
def _remember_projection(key, revision, stamp, text):
|
| 574 |
"""Retain bounded immutable content whose revision must pass the next database read."""
|
| 575 |
global _PROJECTION_TEXT_BYTES
|
|
|
|
| 576 |
size = len(text.encode('utf-8')) if isinstance(text, str) else 0
|
| 577 |
with _PROJECTION_TEXT_LOCK:
|
|
|
|
| 578 |
previous = _PROJECTION_TEXT.pop(key, None)
|
| 579 |
if previous is not None:
|
| 580 |
_PROJECTION_TEXT_BYTES -= previous[3]
|
|
@@ -587,8 +849,28 @@ def _remember_projection(key, revision, stamp, text):
|
|
| 587 |
_PROJECTION_TEXT_BYTES -= removed[3]
|
| 588 |
|
| 589 |
|
| 590 |
-
@_request_timing.store_operation("get_projection")
|
| 591 |
def get_projection(name, drop=(), tenant_slug=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 592 |
"""`core.store.get_projection` with the DROP performed inside Postgres.
|
| 593 |
|
| 594 |
`user_tables` is one JSONB value, and tenant #0's stored ``rows`` have measured at 28.6 MB
|
|
@@ -602,11 +884,11 @@ def get_projection(name, drop=(), tenant_slug=None):
|
|
| 602 |
so marking those dictionaries needs no second JSON serialization and parse. The ``CASE`` retains the generic store contract:
|
| 603 |
non-object JSON values are returned untouched rather than making ``jsonb_each`` error.
|
| 604 |
"""
|
| 605 |
-
import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
|
| 606 |
-
|
| 607 |
drops = tuple(str(d) for d in drop)
|
| 608 |
pool = _pool()
|
| 609 |
content_key = (pool, _effective_slug(tenant_slug), str(name), tuple(sorted(drops)))
|
|
|
|
|
|
|
| 610 |
|
| 611 |
def fetch():
|
| 612 |
with _PROJECTION_TEXT_LOCK:
|
|
@@ -617,12 +899,13 @@ def get_projection(name, drop=(), tenant_slug=None):
|
|
| 617 |
con.execute("SET LOCAL work_mem = '64MB'")
|
| 618 |
if not drops:
|
| 619 |
row = con.execute(
|
| 620 |
-
_q('SELECT value::text FROM {tbl} WHERE key = %s',
|
|
|
|
| 621 |
(str(name),)).fetchone()
|
| 622 |
-
text = row[
|
| 623 |
_request_timing.record_store("get_projection", str(name),
|
| 624 |
bytes_read=_request_timing.text_bytes(text), count=0)
|
| 625 |
-
return text
|
| 626 |
row = con.execute(
|
| 627 |
_q('SELECT kv.rev, kv.updated_at, CASE '
|
| 628 |
'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ELSE (CASE '
|
|
@@ -644,18 +927,14 @@ def get_projection(name, drop=(), tenant_slug=None):
|
|
| 644 |
raise RuntimeError('A projected read returned no content for its current revision')
|
| 645 |
text = cached[2]
|
| 646 |
_remember_projection(content_key, row[0] if row else None, row[1] if row else None, text)
|
| 647 |
-
return text
|
| 648 |
|
| 649 |
# D-2640: the projection is the read 14 concurrent backends were making; share it too.
|
| 650 |
-
text = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch)
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
if not isinstance(data, dict):
|
| 656 |
-
return data
|
| 657 |
-
return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped)
|
| 658 |
-
if isinstance(value, dict) else value for key, value in data.items()}
|
| 659 |
|
| 660 |
|
| 661 |
def _get_workbook_projection(name, *, metadata_only=False, tab_ids=None, cells=None,
|
|
@@ -793,7 +1072,10 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
|
|
| 793 |
if not keys:
|
| 794 |
return {}
|
| 795 |
drops = tuple(str(d) for d in drop)
|
| 796 |
-
|
|
|
|
|
|
|
|
|
|
| 797 |
if timeout_s is not None:
|
| 798 |
timeout_ms = max(1, int(float(timeout_s) * 1000))
|
| 799 |
# Transaction-local: a pooled connection must not lend this caller's budget to the
|
|
@@ -808,14 +1090,28 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
|
|
| 808 |
"CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
|
| 809 |
'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
|
| 810 |
'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
|
| 811 |
-
"'{{}}'::jsonb) ELSE kv.value END)::text FROM {tbl} AS kv "
|
| 812 |
'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
|
| 813 |
(list(drops), keys)).fetchall()
|
| 814 |
found = {}
|
| 815 |
-
for key, value in rows:
|
| 816 |
_request_timing.record_store("get_projections", str(key),
|
| 817 |
bytes_read=_request_timing.text_bytes(value), count=0)
|
| 818 |
found[str(key)] = _parsed(value)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 819 |
dropped = frozenset(drops)
|
| 820 |
return {
|
| 821 |
key: _hf._project(
|
|
@@ -847,17 +1143,221 @@ def _get_entry(name, entry, tenant_slug=None):
|
|
| 847 |
return dict(value) if isinstance(value, dict) else None
|
| 848 |
|
| 849 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 850 |
def _get_many(names, tenant_slug=None):
|
| 851 |
"""Several ordinary documents in one round trip, keyed by requested store key."""
|
| 852 |
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
| 853 |
if not keys:
|
| 854 |
return {}
|
| 855 |
-
|
|
|
|
|
|
|
|
|
|
| 856 |
rows = con.execute(
|
| 857 |
-
_q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])',
|
| 858 |
'store_kv', tenant_slug),
|
| 859 |
(keys,)).fetchall()
|
| 860 |
-
found = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 861 |
return {
|
| 862 |
key: (dict(found[key]) if isinstance(found.get(key), dict)
|
| 863 |
else (found[key] if key in found else {}))
|
|
@@ -1022,7 +1522,36 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=
|
|
| 1022 |
|
| 1023 |
|
| 1024 |
def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
|
| 1025 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1026 |
from psycopg.errors import LockNotAvailable
|
| 1027 |
wait_s = _lock_timeout_s()
|
| 1028 |
with _pool().connection() as con:
|
|
@@ -1031,15 +1560,20 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
|
|
| 1031 |
# lock gives its pooled connection back after `wait_s` instead of holding it forever.
|
| 1032 |
con.execute("SELECT set_config('lock_timeout', %s, true)",
|
| 1033 |
(f'{int(wait_s * 1000)}ms',))
|
| 1034 |
-
|
| 1035 |
-
|
| 1036 |
-
|
| 1037 |
-
|
| 1038 |
-
|
| 1039 |
-
|
| 1040 |
-
|
| 1041 |
-
|
| 1042 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1043 |
_request_timing.record_store("update", str(name),
|
| 1044 |
bytes_read=_request_timing.text_bytes(row[0] if row else None), count=0)
|
| 1045 |
if expect_rev is not None:
|
|
@@ -1115,8 +1649,47 @@ def delete_path(path_in_repo, tenant_slug=None):
|
|
| 1115 |
return True
|
| 1116 |
|
| 1117 |
|
| 1118 |
-
@_request_timing.store_operation("revision")
|
| 1119 |
def revision(name, tenant_slug=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1120 |
"""`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6.
|
| 1121 |
|
| 1122 |
⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and
|
|
@@ -1188,8 +1761,8 @@ class PgStore:
|
|
| 1188 |
"""`core.store.Store`'s interface for ONE tenant's schema.
|
| 1189 |
|
| 1190 |
Deliberately a thin binder over the module functions rather than a reimplementation: every
|
| 1191 |
-
behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`,
|
| 1192 |
-
|
| 1193 |
between the two entry points.
|
| 1194 |
|
| 1195 |
⭐⭐ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and
|
|
@@ -1239,6 +1812,27 @@ class PgStore:
|
|
| 1239 |
return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
|
| 1240 |
timeout_s=timeout_s)
|
| 1241 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1242 |
def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None,
|
| 1243 |
expected_rev=None):
|
| 1244 |
return _get_workbook_projection(name, metadata_only=metadata_only, tab_ids=tab_ids,
|
|
@@ -1269,6 +1863,11 @@ class PgStore:
|
|
| 1269 |
"""Bound multi-key read used by request-scoped read snapshots."""
|
| 1270 |
return _get_many(names, tenant_slug=self.tenant_slug)
|
| 1271 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1272 |
def get_entry(self, name, entry):
|
| 1273 |
"""Bound single-entry read; None means absent, and the caller falls back to `get`."""
|
| 1274 |
return _get_entry(name, entry, tenant_slug=self.tenant_slug)
|
|
|
|
| 48 |
import threading
|
| 49 |
import time
|
| 50 |
from collections import OrderedDict
|
| 51 |
+
from contextlib import contextmanager
|
| 52 |
from functools import lru_cache
|
| 53 |
from pathlib import Path
|
| 54 |
from urllib.parse import urlparse
|
|
|
|
| 343 |
super().__init__(*a, **kw)
|
| 344 |
|
| 345 |
def getconn(self, timeout=None):
|
| 346 |
+
timeout = _request_timing.remaining_budget(
|
| 347 |
+
timeout if timeout is not None else getattr(self, 'timeout', _pg_limits()['acquire']))
|
| 348 |
try:
|
| 349 |
conn = super().getconn(timeout=timeout)
|
| 350 |
except timeout_error:
|
| 351 |
self._log_exhausted()
|
| 352 |
raise
|
| 353 |
+
try:
|
| 354 |
+
_request_timing.remaining_budget()
|
| 355 |
+
except BaseException:
|
| 356 |
+
super().putconn(conn)
|
| 357 |
+
raise
|
| 358 |
label = _holder_label()
|
| 359 |
with self._holders_lock:
|
| 360 |
self._holders[id(conn)] = (time.monotonic(), label)
|
|
|
|
| 365 |
self._holders.pop(id(conn), None)
|
| 366 |
super().putconn(conn)
|
| 367 |
|
| 368 |
+
@contextmanager
|
| 369 |
+
def connection(self, timeout=None):
|
| 370 |
+
# SET LOCAL survives transaction pooling and rolls back with the caller's writes.
|
| 371 |
+
with super().connection(timeout=timeout) as con:
|
| 372 |
+
_request_timing.remaining_budget()
|
| 373 |
+
with con.transaction():
|
| 374 |
+
limits = _pg_limits()
|
| 375 |
+
con.execute("SELECT set_config('statement_timeout', %s, true), "
|
| 376 |
+
"set_config('idle_in_transaction_session_timeout', %s, true)",
|
| 377 |
+
(f"{int(limits['statement'] * 1000)}ms",
|
| 378 |
+
f"{int(limits['idle'] * 1000)}ms"))
|
| 379 |
+
yield con
|
| 380 |
+
|
| 381 |
def _log_exhausted(self):
|
| 382 |
now = time.monotonic()
|
| 383 |
with self._holders_lock:
|
|
|
|
| 403 |
return max(1.0, value)
|
| 404 |
|
| 405 |
|
| 406 |
+
def _bound_env(name, default, minimum=.01):
|
| 407 |
+
try:
|
| 408 |
+
value = float(os.environ.get(name, default))
|
| 409 |
+
return max(minimum, value) if math.isfinite(value) else float(default)
|
| 410 |
+
except ValueError:
|
| 411 |
+
return float(default)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def _pg_limits():
|
| 415 |
+
worker = os.environ.get('AIOS_PROCESS_ROLE') == 'worker'
|
| 416 |
+
return {
|
| 417 |
+
'connect': int(_bound_env('AIOS_PG_CONNECT_TIMEOUT_S', 5, 2)),
|
| 418 |
+
'acquire': _bound_env('AIOS_PG_POOL_TIMEOUT_S', 10),
|
| 419 |
+
'waiting': int(_bound_env('AIOS_PG_MAX_WAITING', 16, 1)),
|
| 420 |
+
'statement': _bound_env('AIOS_PG_STATEMENT_TIMEOUT_S', 300 if worker else 30),
|
| 421 |
+
'idle': _bound_env('AIOS_PG_IDLE_TRANSACTION_TIMEOUT_S', 300 if worker else 60),
|
| 422 |
+
'network_ms': int(_bound_env('AIOS_PG_TCP_USER_TIMEOUT_MS', 15000, 1)),
|
| 423 |
+
}
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def _read_timeout_s():
|
| 427 |
+
limits = _pg_limits()
|
| 428 |
+
return limits['acquire'] + limits['statement'] + limits['network_ms'] / 1000
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
class PostgresOperationTimeout(TimeoutError):
|
| 432 |
+
"""A PostgreSQL socket operation exhausted its client-side limit."""
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def _checked_pg_wait(connection, gen):
|
| 436 |
+
"""Check each libpq poll and discard expired connections without asynchronous cancellation."""
|
| 437 |
+
deadline = time.monotonic() + _pg_limits()['statement']
|
| 438 |
+
try:
|
| 439 |
+
with _request_timing.budget_scope(deadline, PostgresOperationTimeout):
|
| 440 |
+
ready = None
|
| 441 |
+
while True:
|
| 442 |
+
_request_timing.remaining_budget()
|
| 443 |
+
try:
|
| 444 |
+
state = gen.send(ready)
|
| 445 |
+
except StopIteration as result:
|
| 446 |
+
_request_timing.remaining_budget()
|
| 447 |
+
return result.value
|
| 448 |
+
ready = yield state
|
| 449 |
+
except (PostgresOperationTimeout, SharedReadTimeout, _request_timing.RequestDeadlineExceeded):
|
| 450 |
+
# No cancel packet can arrive after this socket is returned to the pool.
|
| 451 |
+
connection.close()
|
| 452 |
+
raise
|
| 453 |
+
finally:
|
| 454 |
+
gen.close()
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
@lru_cache(maxsize=None)
|
| 458 |
+
def _budget_connection_class(base):
|
| 459 |
+
class BudgetConnection(base):
|
| 460 |
+
def wait(self, gen, interval=.01):
|
| 461 |
+
# libpq yields on socket waits, including BEGIN, settings, fetch and COMMIT.
|
| 462 |
+
return super().wait(_checked_pg_wait(self, gen), interval=.01)
|
| 463 |
+
|
| 464 |
+
return BudgetConnection
|
| 465 |
+
|
| 466 |
+
|
| 467 |
def _pool():
|
| 468 |
"""The psycopg connection pool, opened once per process.
|
| 469 |
|
|
|
|
| 482 |
return _POOL['pool']
|
| 483 |
try:
|
| 484 |
from psycopg_pool import ConnectionPool, PoolTimeout
|
| 485 |
+
from psycopg import Connection
|
| 486 |
except ImportError as e: # pragma: no cover - depends on B-3
|
| 487 |
raise RuntimeError(
|
| 488 |
'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
|
|
|
|
| 499 |
# real requests, not be held awake by this process's idle pool for psycopg's 10-minute
|
| 500 |
# default. Thirty seconds still reuses a normal request burst without creating a resident
|
| 501 |
# database client.
|
| 502 |
+
limits = _pg_limits()
|
| 503 |
+
connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1,
|
| 504 |
+
keepalives_idle=30, keepalives_interval=10, keepalives_count=3,
|
| 505 |
+
tcp_user_timeout=limits['network_ms'])
|
| 506 |
+
if sslrootcert is not None:
|
| 507 |
+
connection_kwargs['sslrootcert'] = sslrootcert
|
| 508 |
+
# The pool applies transaction settings and lends connections with checked socket polls.
|
| 509 |
pool_cls = _holder_pool_class(ConnectionPool, PoolTimeout)
|
| 510 |
+
_POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=limits['acquire'],
|
| 511 |
+
max_idle=30.0, max_waiting=limits['waiting'], kwargs=connection_kwargs,
|
| 512 |
+
connection_class=_budget_connection_class(Connection))
|
| 513 |
_POOL['url'] = url
|
| 514 |
_POOL['sslrootcert'] = sslrootcert
|
| 515 |
return _POOL['pool']
|
|
|
|
| 524 |
"""
|
| 525 |
if not _url():
|
| 526 |
return False
|
| 527 |
+
_request_timing.remaining_budget()
|
| 528 |
if _POOL.get('ok'):
|
| 529 |
return True
|
| 530 |
try:
|
|
|
|
| 532 |
con.execute('SELECT 1')
|
| 533 |
_POOL['ok'] = True
|
| 534 |
return True
|
| 535 |
+
except Exception as exc:
|
| 536 |
+
_request_timing.raise_if_resource_error(exc)
|
| 537 |
return False
|
| 538 |
|
| 539 |
|
|
|
|
| 569 |
#: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
|
| 570 |
#: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
|
| 571 |
#: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
|
| 572 |
+
#: Completed reads may retain immutable text, validated by a new SELECT before reuse.
|
| 573 |
#: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
|
| 574 |
#: finishes, and a read never joins a flight started under an older generation.
|
| 575 |
#: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
|
|
|
|
| 580 |
|
| 581 |
|
| 582 |
class _ReadFlight:
|
| 583 |
+
__slots__ = ('done', 'text', 'error', 'generation', 'deadline', 'started')
|
| 584 |
|
| 585 |
def __init__(self, generation):
|
| 586 |
self.done = threading.Event()
|
| 587 |
self.text = None
|
| 588 |
self.error = None
|
| 589 |
self.generation = generation
|
| 590 |
+
self.started = time.monotonic()
|
| 591 |
+
self.deadline = self.started + _read_timeout_s()
|
| 592 |
+
budget = _request_timing.current_budget()
|
| 593 |
+
if budget is not None:
|
| 594 |
+
self.deadline = min(self.deadline, budget[0])
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
class SharedReadTimeout(TimeoutError):
|
| 598 |
+
"""A coalesced follower exhausted the original leader's read budget."""
|
| 599 |
|
| 600 |
|
| 601 |
def _bump_write_generation(name, tenant_slug):
|
| 602 |
+
_request_timing.clear_read_memo()
|
| 603 |
key = (_effective_slug(tenant_slug), str(name))
|
| 604 |
with _READ_FLIGHT_LOCK:
|
| 605 |
_WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
|
| 606 |
+
with _WHOLE_TEXT_LOCK:
|
| 607 |
+
for content_key in list(_WHOLE_TEXT):
|
| 608 |
+
if content_key[1:] == key:
|
| 609 |
+
_forget_whole(content_key)
|
| 610 |
|
| 611 |
|
| 612 |
+
def _shared_read(name, drops, tenant_slug, fetch_text, *, pool=None):
|
| 613 |
+
"""Share one immutable store-controlled result across concurrent identical reads."""
|
| 614 |
doc = (_effective_slug(tenant_slug), str(name))
|
| 615 |
+
key = doc + (drops,) if pool is None else (pool,) + doc + (drops,)
|
| 616 |
+
acc = _request_timing.CURRENT.get()
|
| 617 |
+
invalidated_at = 0.0
|
| 618 |
+
if acc is not None:
|
| 619 |
+
with acc.lock:
|
| 620 |
+
invalidated_at = acc.read_memo_invalidated_at
|
| 621 |
with _READ_FLIGHT_LOCK:
|
| 622 |
generation = _WRITE_GENERATION.get(doc, 0)
|
| 623 |
flight = _READ_FLIGHTS.get(key)
|
| 624 |
+
leader = (flight is None or flight.generation != generation
|
| 625 |
+
or flight.started <= invalidated_at)
|
| 626 |
if leader:
|
| 627 |
flight = _ReadFlight(generation)
|
| 628 |
_READ_FLIGHTS[key] = flight
|
| 629 |
if not leader:
|
| 630 |
+
with _request_timing.budget_scope(flight.deadline, SharedReadTimeout):
|
| 631 |
+
if not flight.done.wait(_request_timing.remaining_budget()):
|
| 632 |
+
_request_timing.remaining_budget()
|
| 633 |
+
raise SharedReadTimeout('shared store read deadline exceeded')
|
| 634 |
if flight.error is not None:
|
| 635 |
# ⛔ THE LEADER'S OWN EXCEPTION, never a wrapper: routes and retries classify store
|
| 636 |
# failures by TYPE (`PoolTimeout`/`OperationalError` -> 503 "try again", a transient
|
|
|
|
| 639 |
raise flight.error
|
| 640 |
return flight.text
|
| 641 |
try:
|
| 642 |
+
# Only store-controlled reads use this callback. Python decoding is cooperative.
|
| 643 |
+
with _request_timing.budget_scope(flight.deadline, SharedReadTimeout):
|
| 644 |
+
text = fetch_text()
|
| 645 |
+
flight.text = text
|
| 646 |
except BaseException as exc:
|
| 647 |
flight.error = exc
|
| 648 |
raise
|
|
|
|
| 656 |
|
| 657 |
def _parsed(text):
|
| 658 |
"""The caller's own object from the shared JSON text (`None` = the key is absent)."""
|
| 659 |
+
try:
|
| 660 |
+
_request_timing.remaining_budget()
|
| 661 |
+
value = {} if text is None else (json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text)
|
| 662 |
+
value = dict(value) if isinstance(value, dict) else value
|
| 663 |
+
_request_timing.remaining_budget()
|
| 664 |
+
return value
|
| 665 |
+
except BaseException:
|
| 666 |
+
_request_timing.clear_read_memo()
|
| 667 |
+
raise
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def _parse_read(name, operation, text):
|
| 671 |
+
"""Keep independent parsing in H1 store time without counting another backing read."""
|
| 672 |
+
if _request_timing.CURRENT.get() is None:
|
| 673 |
+
return _parsed(text)
|
| 674 |
+
started = time.perf_counter()
|
| 675 |
+
try:
|
| 676 |
+
return _parsed(text)
|
| 677 |
+
finally:
|
| 678 |
+
elapsed = (time.perf_counter() - started) * 1000
|
| 679 |
+
parent = _request_timing._STORE_SPAN.get()
|
| 680 |
+
if parent is not None:
|
| 681 |
+
parent[0] += elapsed
|
| 682 |
+
_request_timing.record_store(operation, str(name), elapsed, count=0)
|
| 683 |
+
|
| 684 |
+
|
| 685 |
+
def _request_key(name, shape, tenant_slug, pool=None):
|
| 686 |
+
"""Address a read shape by pool, tenant and local write generation."""
|
| 687 |
+
pool = _pool() if pool is None else pool
|
| 688 |
+
doc = (_effective_slug(tenant_slug), str(name))
|
| 689 |
+
with _READ_FLIGHT_LOCK:
|
| 690 |
+
generation = _WRITE_GENERATION.get(doc, 0)
|
| 691 |
+
return (pool,) + doc + (shape, generation)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
@_request_timing.memo_read
|
| 695 |
+
def _request_text(name, shape, tenant_slug, read, *, fresh=False):
|
| 696 |
+
"""Share only this request's validated backing read, including independently acquired handles."""
|
| 697 |
+
if _request_timing.CURRENT.get() is None:
|
| 698 |
+
return read()
|
| 699 |
+
return _request_timing.memo_text(_request_key(name, shape, tenant_slug), read, fresh=fresh)
|
| 700 |
+
|
| 701 |
+
|
| 702 |
+
def _revision_value(raw_rev, stamp):
|
| 703 |
+
rev = int(raw_rev) if raw_rev is not None else 0
|
| 704 |
+
return {'rev': rev, 'updated_at': stamp.timestamp() if hasattr(stamp, 'timestamp') else None,
|
| 705 |
+
'token': f'pg:{rev}'}
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
def _seed_read_token(key, checkpoint, revision, stamp):
|
| 709 |
+
"""A successful content read validates both revision columns for every request read shape."""
|
| 710 |
+
_request_timing.memo_seed(key, json.dumps(_revision_value(revision, stamp)), checkpoint)
|
| 711 |
+
|
| 712 |
+
|
| 713 |
+
def _validated_content(name, drops, tenant_slug):
|
| 714 |
+
"""Reuse retained content only against this request's checked revision AND timestamp."""
|
| 715 |
+
checkpoint = _request_timing.memo_checkpoint()
|
| 716 |
+
if checkpoint is None:
|
| 717 |
+
return False, None
|
| 718 |
+
key = _request_key(name, 'revision', tenant_slug)
|
| 719 |
+
found, text = _request_timing.memo_peek(key)
|
| 720 |
+
if not found:
|
| 721 |
+
return False, None
|
| 722 |
+
token = _parsed(text)
|
| 723 |
+
if token.get('updated_at') is None:
|
| 724 |
+
return False, None
|
| 725 |
+
cache_key = key[:3] if not drops else (*key[:3], drops)
|
| 726 |
+
cache, lock = ((_WHOLE_TEXT, _WHOLE_TEXT_LOCK) if not drops else
|
| 727 |
+
(_PROJECTION_TEXT, _PROJECTION_TEXT_LOCK))
|
| 728 |
+
with lock:
|
| 729 |
+
cached = cache.get(cache_key)
|
| 730 |
+
if cached is not None and _revision_value(*cached[:2]) == token:
|
| 731 |
+
shape = 'whole' if drops is None else ('projection', drops)
|
| 732 |
+
content_key = (*key[:3], shape, key[-1])
|
| 733 |
+
_request_timing.memo_seed(content_key, cached[2], checkpoint)
|
| 734 |
+
return _request_timing.memo_peek(content_key)
|
| 735 |
+
return False, None
|
| 736 |
|
| 737 |
|
|
|
|
| 738 |
def get(name, fresh=False, tenant_slug=None):
|
| 739 |
+
"""Parse an independent document from this request's backing read or a new validated read."""
|
| 740 |
+
def read():
|
| 741 |
+
found, text = (False, None) if fresh or str(name).endswith('user_tables') else (
|
| 742 |
+
_validated_content(name, None, tenant_slug))
|
| 743 |
+
return text if found else _get_text(name, fresh=fresh, tenant_slug=tenant_slug)
|
| 744 |
+
# Whole row documents never become request metadata snapshots.
|
| 745 |
+
text = read() if str(name).endswith('user_tables') and not fresh else _request_text(
|
| 746 |
+
name, 'whole', tenant_slug, read, fresh=fresh)
|
| 747 |
+
return _parse_read(name, 'get', text)
|
| 748 |
+
|
| 749 |
+
|
| 750 |
+
@_request_timing.store_operation("get")
|
| 751 |
+
def _get_text(name, fresh=False, tenant_slug=None):
|
| 752 |
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 753 |
|
| 754 |
+
Ordinary reads validate both live tokens before reusing immutable text and share concurrent
|
| 755 |
+
SELECTs. A fresh read bypasses both the content token and any existing flight.
|
| 756 |
"""
|
| 757 |
+
pool = _pool()
|
| 758 |
+
doc = (_effective_slug(tenant_slug), str(name))
|
| 759 |
+
content_key = (pool,) + doc
|
| 760 |
+
token_key = _request_key(name, 'revision', tenant_slug, pool)
|
| 761 |
+
checkpoint = None if fresh else _request_timing.memo_checkpoint()
|
| 762 |
+
|
| 763 |
def fetch():
|
| 764 |
+
with _READ_FLIGHT_LOCK:
|
| 765 |
+
generation = _WRITE_GENERATION.get(doc, 0)
|
| 766 |
+
with _WHOLE_TEXT_LOCK:
|
| 767 |
+
cached = _WHOLE_TEXT.get(content_key) if not fresh else None
|
| 768 |
+
if cached is not None:
|
| 769 |
+
_WHOLE_TEXT.move_to_end(content_key)
|
| 770 |
+
with pool.connection() as con:
|
| 771 |
row = con.execute(
|
| 772 |
+
_q('SELECT kv.rev, kv.updated_at, CASE '
|
| 773 |
+
'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL '
|
| 774 |
+
'ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s',
|
| 775 |
+
'store_kv', tenant_slug),
|
| 776 |
+
(cached[0] if cached else None, cached[1] if cached else None,
|
| 777 |
+
str(name))).fetchone()
|
| 778 |
+
text = row[2] if row else None
|
| 779 |
_request_timing.record_store("get", str(name), bytes_read=_request_timing.text_bytes(text), count=0)
|
| 780 |
+
if row and text is None:
|
| 781 |
+
if cached is None or row[:2] != cached[:2]:
|
| 782 |
+
raise RuntimeError('A whole read returned no content for its current revision')
|
| 783 |
+
text = cached[2]
|
| 784 |
+
_remember_whole(content_key, generation, row[0] if row else None,
|
| 785 |
+
row[1] if row else None, text)
|
| 786 |
+
return text, row[0] if row else None, row[1] if row else None
|
| 787 |
+
|
| 788 |
+
text, rev, stamp = fetch() if fresh else _shared_read(name, None, tenant_slug, fetch, pool=pool)
|
| 789 |
+
_seed_read_token(token_key, checkpoint, rev, stamp)
|
| 790 |
+
return text
|
| 791 |
+
|
| 792 |
+
|
| 793 |
+
_WHOLE_TEXT = OrderedDict()
|
| 794 |
+
_WHOLE_TEXT_LOCK = threading.Lock()
|
| 795 |
+
_WHOLE_TEXT_BYTES = 0
|
| 796 |
+
_WHOLE_TEXT_LIMIT = 128 * 1024 * 1024
|
| 797 |
+
_WHOLE_TEXT_ENTRIES = 64
|
| 798 |
+
|
| 799 |
|
| 800 |
+
def _forget_whole(key):
|
| 801 |
+
"""Remove one retained string while holding `_WHOLE_TEXT_LOCK`."""
|
| 802 |
+
global _WHOLE_TEXT_BYTES
|
| 803 |
+
previous = _WHOLE_TEXT.pop(key, None)
|
| 804 |
+
if previous is not None:
|
| 805 |
+
_WHOLE_TEXT_BYTES -= previous[3]
|
| 806 |
+
|
| 807 |
+
|
| 808 |
+
def _remember_whole(key, generation, revision, stamp, text):
|
| 809 |
+
"""Retain bounded immutable text only while its local write generation remains current."""
|
| 810 |
+
global _WHOLE_TEXT_BYTES
|
| 811 |
+
_request_timing.remaining_budget()
|
| 812 |
+
size = sys.getsizeof(text) if isinstance(text, str) else 0
|
| 813 |
+
with _READ_FLIGHT_LOCK:
|
| 814 |
+
if _WRITE_GENERATION.get(key[1:], 0) != generation:
|
| 815 |
+
return
|
| 816 |
+
with _WHOLE_TEXT_LOCK:
|
| 817 |
+
_request_timing.remaining_budget()
|
| 818 |
+
_forget_whole(key)
|
| 819 |
+
if revision is not None and stamp is not None and 0 < size <= _WHOLE_TEXT_LIMIT:
|
| 820 |
+
_WHOLE_TEXT[key] = (revision, stamp, text, size)
|
| 821 |
+
_WHOLE_TEXT_BYTES += size
|
| 822 |
+
while _WHOLE_TEXT and (len(_WHOLE_TEXT) > _WHOLE_TEXT_ENTRIES
|
| 823 |
+
or _WHOLE_TEXT_BYTES > _WHOLE_TEXT_LIMIT):
|
| 824 |
+
_forget_whole(next(iter(_WHOLE_TEXT)))
|
| 825 |
|
| 826 |
|
| 827 |
_PROJECTION_TEXT = OrderedDict()
|
|
|
|
| 833 |
def _remember_projection(key, revision, stamp, text):
|
| 834 |
"""Retain bounded immutable content whose revision must pass the next database read."""
|
| 835 |
global _PROJECTION_TEXT_BYTES
|
| 836 |
+
_request_timing.remaining_budget()
|
| 837 |
size = len(text.encode('utf-8')) if isinstance(text, str) else 0
|
| 838 |
with _PROJECTION_TEXT_LOCK:
|
| 839 |
+
_request_timing.remaining_budget()
|
| 840 |
previous = _PROJECTION_TEXT.pop(key, None)
|
| 841 |
if previous is not None:
|
| 842 |
_PROJECTION_TEXT_BYTES -= previous[3]
|
|
|
|
| 849 |
_PROJECTION_TEXT_BYTES -= removed[3]
|
| 850 |
|
| 851 |
|
|
|
|
| 852 |
def get_projection(name, drop=(), tenant_slug=None):
|
| 853 |
+
"""Parse independent projected definitions from the current request's backing read."""
|
| 854 |
+
import core.store as _hf
|
| 855 |
+
drops = tuple(sorted(set(str(d) for d in drop)))
|
| 856 |
+
can_memo = not str(name).endswith('user_tables') or 'rows' in drops
|
| 857 |
+
def read():
|
| 858 |
+
found, text = _validated_content(name, drops, tenant_slug) if can_memo else (False, None)
|
| 859 |
+
return text if found else _get_projection_text(name, drops, tenant_slug)
|
| 860 |
+
text = (read() if not can_memo else
|
| 861 |
+
_request_text(name, ('projection', drops), tenant_slug, read))
|
| 862 |
+
data = _parse_read(name, 'get_projection', text)
|
| 863 |
+
dropped = frozenset(drops)
|
| 864 |
+
if not isinstance(text, (str, bytes, bytearray)):
|
| 865 |
+
return _hf._project(data, dropped)
|
| 866 |
+
if not isinstance(data, dict):
|
| 867 |
+
return data
|
| 868 |
+
return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped)
|
| 869 |
+
if isinstance(value, dict) else value for key, value in data.items()}
|
| 870 |
+
|
| 871 |
+
|
| 872 |
+
@_request_timing.store_operation("get_projection")
|
| 873 |
+
def _get_projection_text(name, drop=(), tenant_slug=None):
|
| 874 |
"""`core.store.get_projection` with the DROP performed inside Postgres.
|
| 875 |
|
| 876 |
`user_tables` is one JSONB value, and tenant #0's stored ``rows`` have measured at 28.6 MB
|
|
|
|
| 884 |
so marking those dictionaries needs no second JSON serialization and parse. The ``CASE`` retains the generic store contract:
|
| 885 |
non-object JSON values are returned untouched rather than making ``jsonb_each`` error.
|
| 886 |
"""
|
|
|
|
|
|
|
| 887 |
drops = tuple(str(d) for d in drop)
|
| 888 |
pool = _pool()
|
| 889 |
content_key = (pool, _effective_slug(tenant_slug), str(name), tuple(sorted(drops)))
|
| 890 |
+
token_key = _request_key(name, 'revision', tenant_slug, pool)
|
| 891 |
+
checkpoint = _request_timing.memo_checkpoint()
|
| 892 |
|
| 893 |
def fetch():
|
| 894 |
with _PROJECTION_TEXT_LOCK:
|
|
|
|
| 899 |
con.execute("SET LOCAL work_mem = '64MB'")
|
| 900 |
if not drops:
|
| 901 |
row = con.execute(
|
| 902 |
+
_q('SELECT kv.rev, kv.updated_at, kv.value::text FROM {tbl} AS kv WHERE kv.key = %s',
|
| 903 |
+
'store_kv', tenant_slug),
|
| 904 |
(str(name),)).fetchone()
|
| 905 |
+
text = row[2] if row else None
|
| 906 |
_request_timing.record_store("get_projection", str(name),
|
| 907 |
bytes_read=_request_timing.text_bytes(text), count=0)
|
| 908 |
+
return text, row[0] if row else None, row[1] if row else None
|
| 909 |
row = con.execute(
|
| 910 |
_q('SELECT kv.rev, kv.updated_at, CASE '
|
| 911 |
'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ELSE (CASE '
|
|
|
|
| 927 |
raise RuntimeError('A projected read returned no content for its current revision')
|
| 928 |
text = cached[2]
|
| 929 |
_remember_projection(content_key, row[0] if row else None, row[1] if row else None, text)
|
| 930 |
+
return text, row[0] if row else None, row[1] if row else None
|
| 931 |
|
| 932 |
# D-2640: the projection is the read 14 concurrent backends were making; share it too.
|
| 933 |
+
text, rev, stamp = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch, pool=pool)
|
| 934 |
+
_seed_read_token(token_key, checkpoint, rev, stamp)
|
| 935 |
+
if not drops and not str(name).endswith('user_tables'):
|
| 936 |
+
_request_timing.memo_seed((*token_key[:3], 'whole', token_key[-1]), text, checkpoint)
|
| 937 |
+
return text
|
|
|
|
|
|
|
|
|
|
|
|
|
| 938 |
|
| 939 |
|
| 940 |
def _get_workbook_projection(name, *, metadata_only=False, tab_ids=None, cells=None,
|
|
|
|
| 1072 |
if not keys:
|
| 1073 |
return {}
|
| 1074 |
drops = tuple(str(d) for d in drop)
|
| 1075 |
+
pool = _pool()
|
| 1076 |
+
request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys}
|
| 1077 |
+
checkpoint = _request_timing.memo_checkpoint()
|
| 1078 |
+
with pool.connection() as con:
|
| 1079 |
if timeout_s is not None:
|
| 1080 |
timeout_ms = max(1, int(float(timeout_s) * 1000))
|
| 1081 |
# Transaction-local: a pooled connection must not lend this caller's budget to the
|
|
|
|
| 1090 |
"CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
|
| 1091 |
'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
|
| 1092 |
'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
|
| 1093 |
+
"'{{}}'::jsonb) ELSE kv.value END)::text, kv.rev, kv.updated_at FROM {tbl} AS kv "
|
| 1094 |
'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
|
| 1095 |
(list(drops), keys)).fetchall()
|
| 1096 |
found = {}
|
| 1097 |
+
for key, value, rev, stamp in rows:
|
| 1098 |
_request_timing.record_store("get_projections", str(key),
|
| 1099 |
bytes_read=_request_timing.text_bytes(value), count=0)
|
| 1100 |
found[str(key)] = _parsed(value)
|
| 1101 |
+
_seed_read_token(request_keys[str(key)], checkpoint, rev, stamp)
|
| 1102 |
+
if not str(key).endswith('user_tables') or 'rows' in drops:
|
| 1103 |
+
shape_key = (*request_keys[str(key)][:3], ('projection', tuple(sorted(set(drops)))),
|
| 1104 |
+
request_keys[str(key)][-1])
|
| 1105 |
+
_request_timing.memo_seed(shape_key, value, checkpoint)
|
| 1106 |
+
if not drops:
|
| 1107 |
+
_request_timing.memo_seed((*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1]),
|
| 1108 |
+
value, checkpoint)
|
| 1109 |
+
for key in keys:
|
| 1110 |
+
if key not in found:
|
| 1111 |
+
_seed_read_token(request_keys[key], checkpoint, None, None)
|
| 1112 |
+
if not key.endswith('user_tables') or 'rows' in drops:
|
| 1113 |
+
_request_timing.memo_seed((*request_keys[key][:3], ('projection', tuple(sorted(set(drops)))),
|
| 1114 |
+
request_keys[key][-1]), None, checkpoint)
|
| 1115 |
dropped = frozenset(drops)
|
| 1116 |
return {
|
| 1117 |
key: _hf._project(
|
|
|
|
| 1143 |
return dict(value) if isinstance(value, dict) else None
|
| 1144 |
|
| 1145 |
|
| 1146 |
+
@_request_timing.store_operation("json_paths")
|
| 1147 |
+
def _get_json_paths(name, paths, tenant_slug=None):
|
| 1148 |
+
"""Read explicit nested JSON values without transferring their containing document."""
|
| 1149 |
+
import itertools
|
| 1150 |
+
paths = [list(map(str, path)) for path in itertools.islice(iter(paths), 4097)]
|
| 1151 |
+
if len(paths) > 4096:
|
| 1152 |
+
raise ValueError("A JSON path read accepts at most 4096 paths per batch")
|
| 1153 |
+
if not paths:
|
| 1154 |
+
return []
|
| 1155 |
+
with _pool().connection() as con:
|
| 1156 |
+
row = con.execute(_q(
|
| 1157 |
+
"SELECT ARRAY(SELECT (kv.value #> ARRAY(SELECT jsonb_array_elements_text(p.path)))::text "
|
| 1158 |
+
"FROM jsonb_array_elements(%s::jsonb) WITH ORDINALITY AS p(path, n) ORDER BY p.n) "
|
| 1159 |
+
"FROM {tbl} AS kv WHERE kv.key = %s", 'store_kv', tenant_slug),
|
| 1160 |
+
(json.dumps(paths), str(name))).fetchone()
|
| 1161 |
+
return [json.loads(value) if value is not None else None for value in row[0]] if row else [None] * len(paths)
|
| 1162 |
+
|
| 1163 |
+
|
| 1164 |
+
def _iter_json_items(name, path=(), *, batch_size=1024, tenant_slug=None):
|
| 1165 |
+
"""Stream one JSON object through a server cursor within the existing checked connection."""
|
| 1166 |
+
import uuid
|
| 1167 |
+
size = int(batch_size)
|
| 1168 |
+
if not 1 <= size <= 4096:
|
| 1169 |
+
raise ValueError('A JSON object batch accepts between 1 and 4096 entries')
|
| 1170 |
+
with _pool().connection() as con:
|
| 1171 |
+
with con.cursor(name='registry_' + uuid.uuid4().hex) as cur:
|
| 1172 |
+
cur.execute(_q(
|
| 1173 |
+
'SELECT jsonb_typeof(kv.value #> %s::text[]), e.key, e.value::text FROM {tbl} AS kv '
|
| 1174 |
+
'LEFT JOIN LATERAL jsonb_each(CASE '
|
| 1175 |
+
"WHEN jsonb_typeof(kv.value #> %s::text[]) = 'object' "
|
| 1176 |
+
"THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) AS e ON TRUE WHERE kv.key = %s",
|
| 1177 |
+
'store_kv', tenant_slug), (list(map(str, path)), list(map(str, path)),
|
| 1178 |
+
list(map(str, path)), str(name)))
|
| 1179 |
+
found = False
|
| 1180 |
+
while True:
|
| 1181 |
+
_request_timing.remaining_budget()
|
| 1182 |
+
rows = cur.fetchmany(size)
|
| 1183 |
+
if not rows:
|
| 1184 |
+
if not found:
|
| 1185 |
+
raise RuntimeError('canonical_source_unavailable: the stored document disappeared')
|
| 1186 |
+
return
|
| 1187 |
+
found = True
|
| 1188 |
+
if any(kind not in (None, 'null', 'object') for kind, _key, _value in rows):
|
| 1189 |
+
raise RuntimeError('canonical_source_invalid: the stored rows are not an object')
|
| 1190 |
+
_request_timing.record_store('json_items', str(name),
|
| 1191 |
+
bytes_read=sum(len(value.encode('utf-8')) for _kind, _key, value in rows if value is not None))
|
| 1192 |
+
yield [(key, json.loads(value)) for _kind, key, value in rows if key is not None]
|
| 1193 |
+
rows.clear()
|
| 1194 |
+
|
| 1195 |
+
|
| 1196 |
+
def _iter_json_values(name, path=(), *, drop=(), batch_size=1024, tenant_slug=None):
|
| 1197 |
+
"""Stream an array with selected object operands removed before transport."""
|
| 1198 |
+
import uuid
|
| 1199 |
+
size = int(batch_size)
|
| 1200 |
+
if not 1 <= size <= 4096:
|
| 1201 |
+
raise ValueError('A JSON array batch accepts between 1 and 4096 values')
|
| 1202 |
+
with _pool().connection() as con:
|
| 1203 |
+
with con.cursor(name='canonical_' + uuid.uuid4().hex) as cur:
|
| 1204 |
+
cur.execute(_q("SELECT e.n-1, (CASE WHEN jsonb_typeof(e.v)='object' "
|
| 1205 |
+
"THEN e.v - %s::text[] ELSE e.v END)::text FROM {tbl} kv "
|
| 1206 |
+
"CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
|
| 1207 |
+
"THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) "
|
| 1208 |
+
'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug),
|
| 1209 |
+
(list(drop), list(map(str, path)), list(map(str, path)), str(name)))
|
| 1210 |
+
while True:
|
| 1211 |
+
_request_timing.remaining_budget()
|
| 1212 |
+
rows = cur.fetchmany(size)
|
| 1213 |
+
if not rows:
|
| 1214 |
+
return
|
| 1215 |
+
_request_timing.record_store('json_values', str(name),
|
| 1216 |
+
bytes_read=sum(len(value.encode('utf-8')) for _, value in rows))
|
| 1217 |
+
yield [(str(index), json.loads(value)) for index, value in rows]
|
| 1218 |
+
rows.clear()
|
| 1219 |
+
|
| 1220 |
+
|
| 1221 |
+
class _ViewWorkspace(dict):
|
| 1222 |
+
pass
|
| 1223 |
+
|
| 1224 |
+
|
| 1225 |
+
def _get_view_workspace(name, tenant_slug=None):
|
| 1226 |
+
"""Saved-view metadata without authored memberships crossing the JSON decoder."""
|
| 1227 |
+
with _pool().connection() as con:
|
| 1228 |
+
row = con.execute(_q("SELECT kv.rev, kv.updated_at, CASE WHEN jsonb_typeof(kv.value)='object' THEN "
|
| 1229 |
+
"coalesce((SELECT jsonb_object_agg(u.k, CASE WHEN jsonb_typeof(u.v)='object' THEN "
|
| 1230 |
+
"(u.v - 'overlays' - 'trash') || CASE WHEN jsonb_typeof(u.v->'views')='object' THEN "
|
| 1231 |
+
"jsonb_build_object('views', coalesce((SELECT jsonb_object_agg(v.k, CASE "
|
| 1232 |
+
"WHEN jsonb_typeof(v.v->'config')='object' THEN jsonb_set(v.v, '{{config}}', "
|
| 1233 |
+
"CASE WHEN jsonb_typeof(v.v->'config'->'memberPids')='array' THEN "
|
| 1234 |
+
"(v.v->'config') - 'memberPids' ELSE v.v->'config' END) ELSE v.v END) FROM jsonb_each(u.v->'views') v(k,v)), "
|
| 1235 |
+
"'{{}}'::jsonb)) ELSE '{{}}'::jsonb END ELSE u.v END) FROM jsonb_each(kv.value) u(k,v)), "
|
| 1236 |
+
"'{{}}'::jsonb) ELSE kv.value END::text FROM {tbl} kv WHERE kv.key=%s",
|
| 1237 |
+
'store_kv', tenant_slug), (str(name),)).fetchone()
|
| 1238 |
+
text = row[2] if row else None
|
| 1239 |
+
_request_timing.record_store('view_workspace', str(name), bytes_read=_request_timing.text_bytes(text), count=1)
|
| 1240 |
+
parsed = _parse_read(name, 'view_workspace', text)
|
| 1241 |
+
if not isinstance(parsed, dict):
|
| 1242 |
+
return parsed
|
| 1243 |
+
result = _ViewWorkspace(parsed)
|
| 1244 |
+
result.source_revision = row[:2] if row else None
|
| 1245 |
+
return result
|
| 1246 |
+
|
| 1247 |
+
|
| 1248 |
+
def _iter_view_members(name, path, revision, *, batch_size=2048, tenant_slug=None):
|
| 1249 |
+
"""Stream the exact metadata generation, refusing disappearance or concurrent edits."""
|
| 1250 |
+
import uuid
|
| 1251 |
+
size = int(batch_size)
|
| 1252 |
+
if not 1 <= size <= 4096:
|
| 1253 |
+
raise ValueError('A view membership batch accepts between 1 and 4096 values')
|
| 1254 |
+
with _pool().connection() as con:
|
| 1255 |
+
with con.cursor(name='view_members_' + uuid.uuid4().hex) as cur:
|
| 1256 |
+
cur.execute(_q("SELECT kv.rev, kv.updated_at, e.n, e.v::text FROM {tbl} kv "
|
| 1257 |
+
"LEFT JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
|
| 1258 |
+
"THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) ON TRUE "
|
| 1259 |
+
'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug),
|
| 1260 |
+
(list(map(str, path)), list(map(str, path)), str(name)))
|
| 1261 |
+
found = False
|
| 1262 |
+
while True:
|
| 1263 |
+
_request_timing.remaining_budget()
|
| 1264 |
+
rows = cur.fetchmany(size)
|
| 1265 |
+
if not rows:
|
| 1266 |
+
if not found:
|
| 1267 |
+
raise RuntimeError('canonical_source_unavailable: the saved-view document disappeared')
|
| 1268 |
+
return
|
| 1269 |
+
found = True
|
| 1270 |
+
if any(tuple(row[:2]) != tuple(revision or ()) for row in rows):
|
| 1271 |
+
raise RuntimeError('canonical_source_unavailable: the saved view changed during this read')
|
| 1272 |
+
_request_timing.record_store('view_members', str(name),
|
| 1273 |
+
bytes_read=sum(len(value.encode('utf-8')) for _, _, _, value in rows if value is not None))
|
| 1274 |
+
yield [(str(index - 1), json.loads(value)) for _, _, index, value in rows if index is not None]
|
| 1275 |
+
rows.clear()
|
| 1276 |
+
|
| 1277 |
+
|
| 1278 |
+
def _get_json_object(name, path=(), *, drop=(), omit=(), tenant_slug=None):
|
| 1279 |
+
"""Read nested object definitions while removing their population operands in SQL."""
|
| 1280 |
+
with _pool().connection() as con:
|
| 1281 |
+
row = con.execute(_q("SELECT coalesce(jsonb_object_agg(e.k, CASE WHEN jsonb_typeof(e.v)='object' "
|
| 1282 |
+
"THEN e.v - %s::text[] ELSE e.v END), '{{}}'::jsonb)::text FROM {tbl} kv "
|
| 1283 |
+
"CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='object' "
|
| 1284 |
+
"THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) e(k,v) WHERE kv.key=%s AND NOT e.k=ANY(%s::text[])",
|
| 1285 |
+
'store_kv', tenant_slug), (list(drop), list(map(str, path)), list(map(str, path)), str(name), list(omit))).fetchone()
|
| 1286 |
+
_request_timing.record_store('json_object', str(name), bytes_read=_request_timing.text_bytes(row[0]), count=1)
|
| 1287 |
+
return json.loads(row[0])
|
| 1288 |
+
|
| 1289 |
+
|
| 1290 |
+
def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant_slug=None):
|
| 1291 |
+
"""Read complete selected membership without transferring an entire stored array."""
|
| 1292 |
+
wanted = list(dict.fromkeys(str(int(pid)) for pid in ids))
|
| 1293 |
+
if len(wanted) > 4096:
|
| 1294 |
+
raise ValueError('A membership read accepts at most 4096 identities')
|
| 1295 |
+
if not wanted:
|
| 1296 |
+
return []
|
| 1297 |
+
with _pool().connection() as con:
|
| 1298 |
+
rows = con.execute(_q("WITH members AS (SELECT e.v, e.n, CASE WHEN %s::text IS NULL "
|
| 1299 |
+
"THEN e.v ELSE e.v -> %s END ident FROM {tbl} kv "
|
| 1300 |
+
"CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
|
| 1301 |
+
"THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) "
|
| 1302 |
+
'WHERE kv.key=%s AND (%s::bigint IS NULL OR e.n <= %s)), '
|
| 1303 |
+
"normalized AS (SELECT v,n,CASE WHEN jsonb_typeof(ident)='number' THEN trunc((ident #>> '{{}}')::numeric)::text "
|
| 1304 |
+
"WHEN ident='true'::jsonb THEN '1' WHEN ident='false'::jsonb THEN '0' ELSE trim(ident #>> '{{}}') END id FROM members), "
|
| 1305 |
+
"selected AS (SELECT v,n,CASE WHEN id ~ '^[+-]?[0-9]+$' THEN id::numeric::text END id FROM normalized), "
|
| 1306 |
+
'ranked AS (SELECT v,n,row_number() OVER (PARTITION BY id ORDER BY n) r FROM selected WHERE id=ANY(%s::text[])) '
|
| 1307 |
+
'SELECT v::text FROM ranked WHERE r=1 ORDER BY n', 'store_kv', tenant_slug),
|
| 1308 |
+
(id_key, id_key, list(map(str, path)), list(map(str, path)), str(name),
|
| 1309 |
+
source_limit, source_limit, wanted)).fetchall()
|
| 1310 |
+
_request_timing.record_store('json_members', str(name), bytes_read=sum(len(row[0]) for row in rows), count=1)
|
| 1311 |
+
return [json.loads(row[0]) for row in rows]
|
| 1312 |
+
|
| 1313 |
+
|
| 1314 |
+
@_request_timing.store_operation("definition_counts")
|
| 1315 |
+
def _definition_counts_text(name, tenant_slug=None):
|
| 1316 |
+
"""Count stored row keys in Postgres without returning row values."""
|
| 1317 |
+
with _pool().connection() as con:
|
| 1318 |
+
row = con.execute(_q(
|
| 1319 |
+
"SELECT COALESCE(json_object_agg(e.key, CASE "
|
| 1320 |
+
"WHEN jsonb_typeof(e.value->'rows') = 'object' THEN "
|
| 1321 |
+
"(SELECT count(*) FROM jsonb_object_keys(e.value->'rows')) "
|
| 1322 |
+
"WHEN NOT (e.value ? 'rows') OR e.value->'rows' = 'null'::jsonb THEN 0 "
|
| 1323 |
+
"ELSE NULL END), '{{}}'::json)::text FROM {tbl} AS kv "
|
| 1324 |
+
"CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value) = 'object' "
|
| 1325 |
+
"THEN kv.value ELSE '{{}}'::jsonb END) AS e WHERE kv.key = %s",
|
| 1326 |
+
'store_kv', tenant_slug), (str(name),)).fetchone()
|
| 1327 |
+
text = row[0] if row else None
|
| 1328 |
+
_request_timing.record_store('definition_counts', str(name),
|
| 1329 |
+
bytes_read=_request_timing.text_bytes(text), count=0)
|
| 1330 |
+
return text
|
| 1331 |
+
|
| 1332 |
+
|
| 1333 |
+
@_request_timing.store_operation('get_many', many=True)
|
| 1334 |
def _get_many(names, tenant_slug=None):
|
| 1335 |
"""Several ordinary documents in one round trip, keyed by requested store key."""
|
| 1336 |
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
|
| 1337 |
if not keys:
|
| 1338 |
return {}
|
| 1339 |
+
pool = _pool()
|
| 1340 |
+
request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys}
|
| 1341 |
+
checkpoint = _request_timing.memo_checkpoint()
|
| 1342 |
+
with pool.connection() as con:
|
| 1343 |
rows = con.execute(
|
| 1344 |
+
_q('SELECT key, value::text, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])',
|
| 1345 |
'store_kv', tenant_slug),
|
| 1346 |
(keys,)).fetchall()
|
| 1347 |
+
found = {}
|
| 1348 |
+
for key, text, rev, stamp in rows:
|
| 1349 |
+
found[str(key)] = _parsed(text)
|
| 1350 |
+
_request_timing.record_store('get_many', str(key), bytes_read=_request_timing.text_bytes(text), count=0)
|
| 1351 |
+
_seed_read_token(request_keys[str(key)], checkpoint, rev, stamp)
|
| 1352 |
+
if not str(key).endswith('user_tables'):
|
| 1353 |
+
shape_key = (*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1])
|
| 1354 |
+
_request_timing.memo_seed(shape_key, text, checkpoint)
|
| 1355 |
+
for key in keys:
|
| 1356 |
+
if key not in found:
|
| 1357 |
+
_seed_read_token(request_keys[key], checkpoint, None, None)
|
| 1358 |
+
if not key.endswith('user_tables'):
|
| 1359 |
+
_request_timing.memo_seed((*request_keys[key][:3], 'whole', request_keys[key][-1]),
|
| 1360 |
+
None, checkpoint)
|
| 1361 |
return {
|
| 1362 |
key: (dict(found[key]) if isinstance(found.get(key), dict)
|
| 1363 |
else (found[key] if key in found else {}))
|
|
|
|
| 1522 |
|
| 1523 |
|
| 1524 |
def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
|
| 1525 |
+
"""Diagnose lock refusal only after the aborted row transaction has rolled back."""
|
| 1526 |
+
from psycopg.errors import LockNotAvailable
|
| 1527 |
+
try:
|
| 1528 |
+
return _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row)
|
| 1529 |
+
except LockNotAvailable as exc:
|
| 1530 |
+
if not getattr(exc, '_aios_lock_diagnosed', False):
|
| 1531 |
+
_log_lock_blockers(getattr(exc, '_aios_waiter_pid', None))
|
| 1532 |
+
raise
|
| 1533 |
+
|
| 1534 |
+
|
| 1535 |
+
def _log_lock_blockers(waiter_pid):
|
| 1536 |
+
"""Read blocker identities on a separate transaction without logging statement values."""
|
| 1537 |
+
if waiter_pid is None:
|
| 1538 |
+
return
|
| 1539 |
+
try:
|
| 1540 |
+
with _pool().connection(timeout=1.0) as diagnostic:
|
| 1541 |
+
with diagnostic.transaction():
|
| 1542 |
+
diagnostic.execute("SELECT set_config('statement_timeout', '1000ms', true)")
|
| 1543 |
+
rows = diagnostic.execute(
|
| 1544 |
+
'SELECT a.pid, a.query FROM pg_stat_activity AS a '
|
| 1545 |
+
'WHERE a.pid = ANY(pg_blocking_pids(%s))', (int(waiter_pid),)).fetchall()
|
| 1546 |
+
blockers = [{'pid': int(pid), 'query': _request_timing.sql_text(query)} for pid, query in rows]
|
| 1547 |
+
print(f'[store-pg] lock blockers for pid {int(waiter_pid)}: '
|
| 1548 |
+
f'{json.dumps(blockers, ensure_ascii=True)}', flush=True)
|
| 1549 |
+
except Exception as exc:
|
| 1550 |
+
print(f'[store-pg] lock blocker diagnosis unavailable ({type(exc).__name__})', flush=True)
|
| 1551 |
+
|
| 1552 |
+
|
| 1553 |
+
def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row):
|
| 1554 |
+
"""Run the callback and write under the existing row lock and transaction."""
|
| 1555 |
from psycopg.errors import LockNotAvailable
|
| 1556 |
wait_s = _lock_timeout_s()
|
| 1557 |
with _pool().connection() as con:
|
|
|
|
| 1560 |
# lock gives its pooled connection back after `wait_s` instead of holding it forever.
|
| 1561 |
con.execute("SELECT set_config('lock_timeout', %s, true)",
|
| 1562 |
(f'{int(wait_s * 1000)}ms',))
|
| 1563 |
+
pid_row = con.execute('SELECT pg_backend_pid()').fetchone()
|
| 1564 |
+
waiter_pid = pid_row[0] if pid_row else None
|
| 1565 |
+
with _request_timing.lock_wait_probe(waiter_pid, wait_s, _log_lock_blockers) as probe:
|
| 1566 |
+
try:
|
| 1567 |
+
row = con.execute(
|
| 1568 |
+
_q('SELECT value::text, rev FROM {tbl} WHERE key = %s FOR UPDATE',
|
| 1569 |
+
'store_kv', tenant_slug),
|
| 1570 |
+
(str(name),)).fetchone()
|
| 1571 |
+
except LockNotAvailable as exc:
|
| 1572 |
+
exc._aios_waiter_pid = waiter_pid
|
| 1573 |
+
exc._aios_lock_diagnosed = probe is not None and probe.fired
|
| 1574 |
+
print(f'[store-pg] lock wait over {wait_s:g}s on '
|
| 1575 |
+
f'{_effective_slug(tenant_slug)}/{name}', flush=True)
|
| 1576 |
+
raise
|
| 1577 |
_request_timing.record_store("update", str(name),
|
| 1578 |
bytes_read=_request_timing.text_bytes(row[0] if row else None), count=0)
|
| 1579 |
if expect_rev is not None:
|
|
|
|
| 1649 |
return True
|
| 1650 |
|
| 1651 |
|
|
|
|
| 1652 |
def revision(name, tenant_slug=None):
|
| 1653 |
+
"""Return an independent revision object, reusing only this request's backing validation."""
|
| 1654 |
+
return _parsed(_request_text(name, 'revision', tenant_slug,
|
| 1655 |
+
lambda: json.dumps(_revision(name, tenant_slug))))
|
| 1656 |
+
|
| 1657 |
+
|
| 1658 |
+
@_request_timing.memo_read
|
| 1659 |
+
def _revision_many(names, tenant_slug=None):
|
| 1660 |
+
"""Batch only unchecked keys and lend complete tokens to every request read shape."""
|
| 1661 |
+
keys = tuple(sorted(set(map(str, names))))
|
| 1662 |
+
checkpoint = _request_timing.memo_checkpoint()
|
| 1663 |
+
request_keys = {name: _request_key(name, 'revision', tenant_slug) for name in keys}
|
| 1664 |
+
found, missing = {}, []
|
| 1665 |
+
for name, key in request_keys.items():
|
| 1666 |
+
hit, text = _request_timing.memo_peek(key)
|
| 1667 |
+
if hit:
|
| 1668 |
+
found[name] = _parsed(text)
|
| 1669 |
+
else:
|
| 1670 |
+
missing.append(name)
|
| 1671 |
+
if missing:
|
| 1672 |
+
read = _read_revisions(missing, tenant_slug)
|
| 1673 |
+
for name in missing:
|
| 1674 |
+
token = read.get(name, _revision_value(None, None))
|
| 1675 |
+
_request_timing.memo_seed(request_keys[name], json.dumps(token), checkpoint)
|
| 1676 |
+
found[name] = token
|
| 1677 |
+
return {name: found[name] for name in keys}
|
| 1678 |
+
|
| 1679 |
+
|
| 1680 |
+
@_request_timing.store_operation('revisions', many=True)
|
| 1681 |
+
def _read_revisions(names, tenant_slug=None):
|
| 1682 |
+
with _pool().connection() as con:
|
| 1683 |
+
rows = con.execute(_q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])',
|
| 1684 |
+
'store_kv', tenant_slug), (list(names),)).fetchall()
|
| 1685 |
+
found = {}
|
| 1686 |
+
for name, raw_rev, stamp in rows:
|
| 1687 |
+
found[name] = _revision_value(raw_rev, stamp)
|
| 1688 |
+
return found
|
| 1689 |
+
|
| 1690 |
+
|
| 1691 |
+
@_request_timing.store_operation("revision")
|
| 1692 |
+
def _revision(name, tenant_slug=None):
|
| 1693 |
"""`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6.
|
| 1694 |
|
| 1695 |
⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and
|
|
|
|
| 1761 |
"""`core.store.Store`'s interface for ONE tenant's schema.
|
| 1762 |
|
| 1763 |
Deliberately a thin binder over the module functions rather than a reimplementation: every
|
| 1764 |
+
behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, fresh reads
|
| 1765 |
+
bypassing text reuse) is stated once, up there, and cannot drift
|
| 1766 |
between the two entry points.
|
| 1767 |
|
| 1768 |
⭐⭐ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and
|
|
|
|
| 1812 |
return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
|
| 1813 |
timeout_s=timeout_s)
|
| 1814 |
|
| 1815 |
+
def get_json_paths(self, name, paths):
|
| 1816 |
+
return _get_json_paths(name, paths, tenant_slug=self.tenant_slug)
|
| 1817 |
+
|
| 1818 |
+
def iter_json_items(self, name, path=(), *, batch_size=1024):
|
| 1819 |
+
return _iter_json_items(name, path, batch_size=batch_size, tenant_slug=self.tenant_slug)
|
| 1820 |
+
|
| 1821 |
+
def iter_json_values(self, name, path=(), *, drop=(), batch_size=1024):
|
| 1822 |
+
return _iter_json_values(name, path, drop=drop, batch_size=batch_size, tenant_slug=self.tenant_slug)
|
| 1823 |
+
|
| 1824 |
+
def get_json_object(self, name, path=(), *, drop=(), omit=()):
|
| 1825 |
+
return _get_json_object(name, path, drop=drop, omit=omit, tenant_slug=self.tenant_slug)
|
| 1826 |
+
|
| 1827 |
+
def get_view_workspace(self, name):
|
| 1828 |
+
return _get_view_workspace(name, tenant_slug=self.tenant_slug)
|
| 1829 |
+
|
| 1830 |
+
def iter_view_members(self, name, path, revision, *, batch_size=2048):
|
| 1831 |
+
return _iter_view_members(name, path, revision, batch_size=batch_size, tenant_slug=self.tenant_slug)
|
| 1832 |
+
|
| 1833 |
+
def get_json_members(self, name, path, ids, *, id_key=None, source_limit=None):
|
| 1834 |
+
return _get_json_members(name, path, ids, id_key=id_key, source_limit=source_limit, tenant_slug=self.tenant_slug)
|
| 1835 |
+
|
| 1836 |
def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None,
|
| 1837 |
expected_rev=None):
|
| 1838 |
return _get_workbook_projection(name, metadata_only=metadata_only, tab_ids=tab_ids,
|
|
|
|
| 1863 |
"""Bound multi-key read used by request-scoped read snapshots."""
|
| 1864 |
return _get_many(names, tenant_slug=self.tenant_slug)
|
| 1865 |
|
| 1866 |
+
def get_definition_counts(self, name):
|
| 1867 |
+
"""Exact stored counts only; a read-through source's population is not measured here."""
|
| 1868 |
+
return _parsed(_request_text(name, 'definition_counts', self.tenant_slug,
|
| 1869 |
+
lambda: _definition_counts_text(name, self.tenant_slug)))
|
| 1870 |
+
|
| 1871 |
def get_entry(self, name, entry):
|
| 1872 |
"""Bound single-entry read; None means absent, and the caller falls back to `get`."""
|
| 1873 |
return _get_entry(name, entry, tenant_slug=self.tenant_slug)
|
platform/core/user_tables.py
CHANGED
|
@@ -1916,8 +1916,11 @@ def all_tables(st=None):
|
|
| 1916 |
⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that
|
| 1917 |
only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below.
|
| 1918 |
"""
|
|
|
|
|
|
|
|
|
|
| 1919 |
try:
|
| 1920 |
-
return _with_activity(
|
| 1921 |
except Exception:
|
| 1922 |
return {}
|
| 1923 |
|
|
@@ -1927,7 +1930,27 @@ def all_tables(st=None):
|
|
| 1927 |
DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit')
|
| 1928 |
|
| 1929 |
|
| 1930 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1931 |
"""{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row.
|
| 1932 |
|
| 1933 |
⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for
|
|
@@ -1940,8 +1963,9 @@ def all_defs(st=None):
|
|
| 1940 |
from a database that genuinely has no rows, and the caller least able to notice is the one that
|
| 1941 |
wanted rows. If you get that error you wanted `all_tables`.
|
| 1942 |
|
| 1943 |
-
|
| 1944 |
-
|
|
|
|
| 1945 |
fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for
|
| 1946 |
— which is the one direction a fallback here is allowed to be wrong in.
|
| 1947 |
|
|
@@ -1952,7 +1976,7 @@ def all_defs(st=None):
|
|
| 1952 |
databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]).
|
| 1953 |
Nothing had noticed because the projection does not fail on a healthy store — a fallback is
|
| 1954 |
dead code until the day it is the only code running. **Every failure now falls back to the
|
| 1955 |
-
whole read**, which keeps `all_tables`' own `except: return {}` as the single backstop instead
|
| 1956 |
of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on
|
| 1957 |
this function, so an empty answer here stops being a blank nav and becomes a 404 on every
|
| 1958 |
database in the tenant.
|
|
@@ -1960,13 +1984,22 @@ def all_defs(st=None):
|
|
| 1960 |
# Schema-only handles cannot use the historical whole-reader fallback described above:
|
| 1961 |
# their caller must distinguish an unavailable definition snapshot from an empty one.
|
| 1962 |
reader = _st(st)
|
|
|
|
|
|
|
| 1963 |
try:
|
| 1964 |
-
|
| 1965 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1966 |
# A schema-only request promises not to materialize rows, including on an outage.
|
| 1967 |
-
#
|
| 1968 |
-
|
| 1969 |
-
|
|
|
|
|
|
|
|
|
|
| 1970 |
raise
|
| 1971 |
# ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
|
| 1972 |
# only ever ONE of the ways this can fail, and singling it out is what made the other ways
|
|
@@ -1982,6 +2015,8 @@ def _entry_reader(st):
|
|
| 1982 |
"""Use an entry door only when it belongs to the same reader as the bound snapshot."""
|
| 1983 |
# The API's request reader defines get_entry even when its wrapped snapshot does not.
|
| 1984 |
# Judge that runtime's own static capability and owner, then retain the request read door.
|
|
|
|
|
|
|
| 1985 |
bound = st
|
| 1986 |
if type(st).__name__ == '_AssemblyStore':
|
| 1987 |
st = st._runtime
|
|
@@ -5430,6 +5465,15 @@ class _Lent:
|
|
| 5430 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 5431 |
|
| 5432 |
def get(self, name, fresh=False):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5433 |
if fresh or (name not in self._docs and name not in _LENDABLE
|
| 5434 |
and not str(name).endswith('__shared')):
|
| 5435 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
|
@@ -5470,7 +5514,7 @@ def lend(st=None, **docs):
|
|
| 5470 |
return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None})
|
| 5471 |
|
| 5472 |
|
| 5473 |
-
def lend_defs(st=None):
|
| 5474 |
"""`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows.
|
| 5475 |
|
| 5476 |
⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02
|
|
@@ -5489,7 +5533,12 @@ def lend_defs(st=None):
|
|
| 5489 |
⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY
|
| 5490 |
should take the whole read once rather than this plus that.
|
| 5491 |
"""
|
| 5492 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5493 |
|
| 5494 |
|
| 5495 |
def _utc_now():
|
|
|
|
| 1916 |
⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that
|
| 1917 |
only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below.
|
| 1918 |
"""
|
| 1919 |
+
reader = _st(st)
|
| 1920 |
+
if getattr(reader, 'schema_only', False):
|
| 1921 |
+
return all_defs(reader, schema_only=True)
|
| 1922 |
try:
|
| 1923 |
+
return _with_activity(reader.get(STORE_KEY))
|
| 1924 |
except Exception:
|
| 1925 |
return {}
|
| 1926 |
|
|
|
|
| 1930 |
DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit')
|
| 1931 |
|
| 1932 |
|
| 1933 |
+
def definition_counts(st=None):
|
| 1934 |
+
"""Exact stored counts from a bound capability, or unknown without reading rows."""
|
| 1935 |
+
reader = _st(st)
|
| 1936 |
+
# Snapshot wrappers cannot borrow live counts from a different document owner.
|
| 1937 |
+
if isinstance(reader, _Lent) or type(reader).__name__ == '_AssemblyStore':
|
| 1938 |
+
return {}
|
| 1939 |
+
backend = reader._store() if callable(getattr(reader, '_store', None)) else reader
|
| 1940 |
+
if callable(getattr(backend, '_d', None)):
|
| 1941 |
+
backend = backend._d()
|
| 1942 |
+
count = getattr(backend, 'get_definition_counts', None)
|
| 1943 |
+
if not callable(count):
|
| 1944 |
+
return {}
|
| 1945 |
+
name = reader.store_key(STORE_KEY) if callable(getattr(reader, 'store_key', None)) else STORE_KEY
|
| 1946 |
+
counts = count(name)
|
| 1947 |
+
if not isinstance(counts, dict):
|
| 1948 |
+
from core.shared_overlay import SchemaUnavailable
|
| 1949 |
+
raise SchemaUnavailable('stored row counts are unavailable')
|
| 1950 |
+
return counts
|
| 1951 |
+
|
| 1952 |
+
|
| 1953 |
+
def all_defs(st=None, *, schema_only=False):
|
| 1954 |
"""{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row.
|
| 1955 |
|
| 1956 |
⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for
|
|
|
|
| 1963 |
from a database that genuinely has no rows, and the caller least able to notice is the one that
|
| 1964 |
wanted rows. If you get that error you wanted `all_tables`.
|
| 1965 |
|
| 1966 |
+
Schema-only requests propagate read failures and never fall back to a whole read.
|
| 1967 |
+
Other readers fall back to `all_tables` on ANY failure, including a store handle too old to
|
| 1968 |
+
project (a fake store installed by a gate, a `_Lent` handle serving an already-read document). The
|
| 1969 |
fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for
|
| 1970 |
— which is the one direction a fallback here is allowed to be wrong in.
|
| 1971 |
|
|
|
|
| 1976 |
databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]).
|
| 1977 |
Nothing had noticed because the projection does not fail on a healthy store — a fallback is
|
| 1978 |
dead code until the day it is the only code running. **Every failure now falls back to the
|
| 1979 |
+
whole read for ordinary readers**, which keeps `all_tables`' own `except: return {}` as the single backstop instead
|
| 1980 |
of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on
|
| 1981 |
this function, so an empty answer here stops being a blank nav and becomes a 404 on every
|
| 1982 |
database in the tenant.
|
|
|
|
| 1984 |
# Schema-only handles cannot use the historical whole-reader fallback described above:
|
| 1985 |
# their caller must distinguish an unavailable definition snapshot from an empty one.
|
| 1986 |
reader = _st(st)
|
| 1987 |
+
from core import request_timing
|
| 1988 |
+
schema_only = schema_only or getattr(reader, 'schema_only', False) or request_timing.CURRENT.get() is not None
|
| 1989 |
try:
|
| 1990 |
+
project = getattr(reader, 'get_projection', None)
|
| 1991 |
+
if schema_only and not callable(project):
|
| 1992 |
+
from core.shared_overlay import SchemaUnavailable
|
| 1993 |
+
raise SchemaUnavailable('table definitions require a projected store reader')
|
| 1994 |
+
return _with_activity(project(STORE_KEY, drop=DEF_ONLY_DROPS))
|
| 1995 |
+
except Exception as exc:
|
| 1996 |
# A schema-only request promises not to materialize rows, including on an outage.
|
| 1997 |
+
# Preserve the original failure type for request callers to classify.
|
| 1998 |
+
if schema_only:
|
| 1999 |
+
request_timing.raise_if_resource_error(exc)
|
| 2000 |
+
if request_timing.CURRENT.get() is not None:
|
| 2001 |
+
from core.shared_overlay import SchemaUnavailable
|
| 2002 |
+
raise SchemaUnavailable('table definitions are unavailable') from exc
|
| 2003 |
raise
|
| 2004 |
# ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
|
| 2005 |
# only ever ONE of the ways this can fail, and singling it out is what made the other ways
|
|
|
|
| 2015 |
"""Use an entry door only when it belongs to the same reader as the bound snapshot."""
|
| 2016 |
# The API's request reader defines get_entry even when its wrapped snapshot does not.
|
| 2017 |
# Judge that runtime's own static capability and owner, then retain the request read door.
|
| 2018 |
+
if getattr(st, 'schema_only', False):
|
| 2019 |
+
return None
|
| 2020 |
bound = st
|
| 2021 |
if type(st).__name__ == '_AssemblyStore':
|
| 2022 |
st = st._runtime
|
|
|
|
| 5465 |
self._runtime, self._docs = runtime, dict(docs or {})
|
| 5466 |
|
| 5467 |
def get(self, name, fresh=False):
|
| 5468 |
+
if fresh:
|
| 5469 |
+
from core.request_timing import clear_read_memo
|
| 5470 |
+
clear_read_memo()
|
| 5471 |
+
if name == STORE_KEY and getattr(self, 'schema_only', False):
|
| 5472 |
+
if fresh:
|
| 5473 |
+
self._docs.pop(name, None)
|
| 5474 |
+
if fresh or name not in self._docs:
|
| 5475 |
+
self._docs[name] = all_defs(self._runtime, schema_only=True)
|
| 5476 |
+
return self._docs[name]
|
| 5477 |
if fresh or (name not in self._docs and name not in _LENDABLE
|
| 5478 |
and not str(name).endswith('__shared')):
|
| 5479 |
return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
|
|
|
|
| 5514 |
return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None})
|
| 5515 |
|
| 5516 |
|
| 5517 |
+
def lend_defs(st=None, *, schema_only=False):
|
| 5518 |
"""`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows.
|
| 5519 |
|
| 5520 |
⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02
|
|
|
|
| 5533 |
⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY
|
| 5534 |
should take the whole read once rather than this plus that.
|
| 5535 |
"""
|
| 5536 |
+
reader = _st(st)
|
| 5537 |
+
strict = schema_only or getattr(reader, 'schema_only', False)
|
| 5538 |
+
lent = lend(reader, **{STORE_KEY: all_defs(reader, schema_only=strict)})
|
| 5539 |
+
if strict:
|
| 5540 |
+
lent.schema_only = True
|
| 5541 |
+
return lent
|
| 5542 |
|
| 5543 |
|
| 5544 |
def _utc_now():
|
platform/harness/canonical_source.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bounded, read-only operands for the existing compiled customer and product projections."""
|
| 2 |
+
import contextvars
|
| 3 |
+
import itertools
|
| 4 |
+
import json
|
| 5 |
+
import uuid
|
| 6 |
+
from contextlib import contextmanager
|
| 7 |
+
|
| 8 |
+
from core import odoo as _odoo
|
| 9 |
+
|
| 10 |
+
BATCH = 1024
|
| 11 |
+
_ACTIVE = contextvars.ContextVar('canonical_source', default=None)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def code_sql(alias):
|
| 15 |
+
raw = f"json_extract_string({alias}.body, '/default_code')"
|
| 16 |
+
return (f"CASE WHEN {raw} IS NULL OR {raw} = '' OR json_type({alias}.body, '/default_code') = 'BOOLEAN' "
|
| 17 |
+
f"THEN 'pid:' || json_extract_string({alias}.body, '/id') ELSE trim({raw}) END")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class TierProjection(list):
|
| 21 |
+
"""Retain displayed tiers and the complete overflow receipt from SQL."""
|
| 22 |
+
def __init__(self):
|
| 23 |
+
super().__init__()
|
| 24 |
+
self.overflow = {}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Rows:
|
| 28 |
+
"""Reiterate a SQL snapshot without retaining its Python records."""
|
| 29 |
+
|
| 30 |
+
def __init__(self, owner, table):
|
| 31 |
+
self.owner, self.table = owner, table
|
| 32 |
+
|
| 33 |
+
def __iter__(self):
|
| 34 |
+
last = -1
|
| 35 |
+
while True:
|
| 36 |
+
batch = self.owner.cur.execute(
|
| 37 |
+
f'SELECT ordinal, body FROM "{self.table}" WHERE ordinal > ? ORDER BY ordinal LIMIT ?',
|
| 38 |
+
[last, BATCH]).fetchall()
|
| 39 |
+
if not batch:
|
| 40 |
+
return
|
| 41 |
+
for _, body in batch:
|
| 42 |
+
yield json.loads(body)
|
| 43 |
+
last = batch[-1][0]
|
| 44 |
+
batch.clear()
|
| 45 |
+
|
| 46 |
+
def __len__(self):
|
| 47 |
+
return self.owner.cur.execute(f'SELECT count(*) FROM "{self.table}"').fetchone()[0]
|
| 48 |
+
|
| 49 |
+
def __bool__(self):
|
| 50 |
+
return bool(len(self))
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class Source:
|
| 54 |
+
"""Own bounded source operands and release every temporary snapshot on exit."""
|
| 55 |
+
|
| 56 |
+
def __init__(self, cur, kind, identities, store=None):
|
| 57 |
+
self.cur, self.kind, self.identities, self.store = cur, kind, list(identities), store
|
| 58 |
+
self.tables = []
|
| 59 |
+
self.cache = {}
|
| 60 |
+
self.failure = None
|
| 61 |
+
|
| 62 |
+
def snapshot(self, records):
|
| 63 |
+
name = 'canonical_' + uuid.uuid4().hex
|
| 64 |
+
self.cur.execute(f'CREATE TEMP TABLE "{name}" (ordinal BIGINT, body JSON)')
|
| 65 |
+
self.tables.append(name)
|
| 66 |
+
records, offset = iter(records), 0
|
| 67 |
+
try:
|
| 68 |
+
while True:
|
| 69 |
+
batch = list(itertools.islice(records, BATCH))
|
| 70 |
+
if not batch:
|
| 71 |
+
break
|
| 72 |
+
self.cur.execute(f'INSERT INTO "{name}" SELECT unnest(?::BIGINT[]), unnest(?::JSON[])',
|
| 73 |
+
[list(range(offset, offset + len(batch))), [json.dumps(r, default=str) for r in batch]])
|
| 74 |
+
offset += len(batch)
|
| 75 |
+
batch.clear()
|
| 76 |
+
except BaseException as exc:
|
| 77 |
+
self.failure = exc
|
| 78 |
+
raise
|
| 79 |
+
finally:
|
| 80 |
+
close = getattr(records, 'close', None)
|
| 81 |
+
if close:
|
| 82 |
+
close()
|
| 83 |
+
return Rows(self, name)
|
| 84 |
+
|
| 85 |
+
def domain(self, model, domain):
|
| 86 |
+
domain = list(domain or [])
|
| 87 |
+
if self.kind == 'customer':
|
| 88 |
+
field = {'res.partner': 'id', 'sale.order': 'partner_id',
|
| 89 |
+
'sale.order.line': 'order_partner_id', 'account.move': 'partner_id'}.get(model)
|
| 90 |
+
# Explicit partner lookup domains also resolve agents and must retain their ids.
|
| 91 |
+
if model == 'res.partner' and any(isinstance(d, (tuple, list)) and d[0] in {'id', 'name'}
|
| 92 |
+
for d in domain):
|
| 93 |
+
field = None
|
| 94 |
+
if field:
|
| 95 |
+
domain.append((field, 'in', self.identities))
|
| 96 |
+
elif self.kind == 'product' and model == 'product.product':
|
| 97 |
+
# Match the canonical stripped code, including every duplicate and archived variant.
|
| 98 |
+
codes = list(self.identities)
|
| 99 |
+
ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()]
|
| 100 |
+
terms = [('default_code', '=like', '%' + code + '%') for code in codes]
|
| 101 |
+
terms.append(('id', 'in', ids))
|
| 102 |
+
domain += ['|'] * (len(terms) - 1) + terms
|
| 103 |
+
elif self.kind == 'product' and model in {'sale.order.line', 'stock.quant'}:
|
| 104 |
+
codes = list(self.identities)
|
| 105 |
+
ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()]
|
| 106 |
+
terms = [('product_id.default_code', '=like', '%' + code + '%') for code in codes]
|
| 107 |
+
terms.append(('product_id', 'in', ids))
|
| 108 |
+
domain += ['|'] * (len(terms) - 1) + terms
|
| 109 |
+
return domain
|
| 110 |
+
|
| 111 |
+
def products(self):
|
| 112 |
+
key = ('products',)
|
| 113 |
+
if key not in self.cache:
|
| 114 |
+
self.cache[key] = self.search_read('product.product', [('active', 'in', [True, False])],
|
| 115 |
+
['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
|
| 116 |
+
return self.cache[key]
|
| 117 |
+
|
| 118 |
+
def sku_revenue(self, groups):
|
| 119 |
+
products = self.products()
|
| 120 |
+
code = code_sql('p')
|
| 121 |
+
rows = self.cur.execute(f"""SELECT {code} code,
|
| 122 |
+
first(json_extract_string(g.body, '/product_id/1') ORDER BY g.ordinal),
|
| 123 |
+
sum(coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0)),
|
| 124 |
+
sum(coalesce(try_cast(json_extract_string(g.body, '/product_uom_qty') AS DOUBLE), 0)),
|
| 125 |
+
count(DISTINCT nullif(json_extract_string(g.body, '/order_id/0'), '0'))
|
| 126 |
+
FROM "{groups.table}" g JOIN "{products.table}" p
|
| 127 |
+
ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id')
|
| 128 |
+
WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL
|
| 129 |
+
GROUP BY code""").fetchall()
|
| 130 |
+
return {code: {'name': name, 'rev': rev, 'qty': qty, 'orders': orders}
|
| 131 |
+
for code, name, rev, qty, orders in rows}
|
| 132 |
+
|
| 133 |
+
def customer_mix(self, groups):
|
| 134 |
+
def products():
|
| 135 |
+
last = 0
|
| 136 |
+
while True:
|
| 137 |
+
ids = [r[0] for r in self.cur.execute(f"""SELECT DISTINCT
|
| 138 |
+
try_cast(json_extract_string(body, '/product_id/0') AS BIGINT) id
|
| 139 |
+
FROM "{groups.table}" WHERE id > ? ORDER BY id LIMIT ?""", [last, BATCH]).fetchall()]
|
| 140 |
+
if not ids:
|
| 141 |
+
return
|
| 142 |
+
yield from _odoo.search_read('product.product', [('id', 'in', ids)],
|
| 143 |
+
['id', 'name', 'categ_id'], limit=BATCH, order='id asc')
|
| 144 |
+
last = ids[-1]
|
| 145 |
+
prods = self.snapshot(products())
|
| 146 |
+
query = f"""WITH lines AS (
|
| 147 |
+
SELECT try_cast(json_extract_string(g.body, '/order_partner_id/0') AS BIGINT) pid,
|
| 148 |
+
json_extract_string(g.body, '/product_id/0') product,
|
| 149 |
+
coalesce(nullif(trim(list_last(string_split(json_extract_string(p.body, '/categ_id/1'), ' / '))), ''), '(none)') cat,
|
| 150 |
+
coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0) val,
|
| 151 |
+
g.ordinal, json_extract_string(p.body, '/name') AS product_name
|
| 152 |
+
FROM "{groups.table}" g LEFT JOIN "{prods.table}" p
|
| 153 |
+
ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id')
|
| 154 |
+
WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL),
|
| 155 |
+
cats AS (SELECT pid, cat, sum(val) val, min(ordinal) ordinal FROM lines GROUP BY pid, cat),
|
| 156 |
+
topcat AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM cats),
|
| 157 |
+
skus AS (SELECT pid, product, first(product_name ORDER BY ordinal) AS product_name, sum(val) val,
|
| 158 |
+
min(ordinal) ordinal FROM lines GROUP BY pid, product),
|
| 159 |
+
topsku AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM skus),
|
| 160 |
+
totals AS (SELECT pid, count(DISTINCT product) n, sum(val) total FROM lines GROUP BY pid)
|
| 161 |
+
SELECT t.pid, c.cat, CASE WHEN t.total != 0 THEN c.val / t.total END, t.n,
|
| 162 |
+
coalesce(s.product_name, '(none)') FROM totals t JOIN topcat c ON t.pid=c.pid AND c.rn=1
|
| 163 |
+
JOIN topsku s ON t.pid=s.pid AND s.rn=1"""
|
| 164 |
+
return {pid: {'top_category': cat, 'top_category_pct': pct, 'sku_count': n, 'top_sku': name}
|
| 165 |
+
for pid, cat, pct, n, name in self.cur.execute(query).fetchall()}
|
| 166 |
+
|
| 167 |
+
def categories(self):
|
| 168 |
+
rows = self.products()
|
| 169 |
+
cats = self.search_read('product.category', [], ['id', 'complete_name'])
|
| 170 |
+
return {code: category for code, category in self.cur.execute(f"""SELECT
|
| 171 |
+
{code_sql('p')} code,
|
| 172 |
+
first(coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''),
|
| 173 |
+
'(uncategorized)') ORDER BY p.ordinal)
|
| 174 |
+
FROM "{rows.table}" p LEFT JOIN "{cats.table}" c
|
| 175 |
+
ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id')
|
| 176 |
+
WHERE json_extract_string(p.body, '/active') = 'true'
|
| 177 |
+
AND json_type(p.body, '/default_code') = 'VARCHAR' AND json_extract_string(p.body, '/default_code') != ''
|
| 178 |
+
GROUP BY code""").fetchall()}
|
| 179 |
+
|
| 180 |
+
def backlog(self, groups):
|
| 181 |
+
products = self.products()
|
| 182 |
+
return {code: qty for code, qty in self.cur.execute(f"""WITH per AS (
|
| 183 |
+
SELECT json_extract_string(body, '/product_id/0') id,
|
| 184 |
+
greatest(0, coalesce(try_cast(json_extract_string(body, '/product_uom_qty') AS DOUBLE), 0)
|
| 185 |
+
- coalesce(try_cast(json_extract_string(body, '/qty_delivered') AS DOUBLE), 0)) qty
|
| 186 |
+
FROM "{groups.table}")
|
| 187 |
+
SELECT {code_sql('p')} code,
|
| 188 |
+
sum(per.qty) FROM per JOIN "{products.table}" p
|
| 189 |
+
ON per.id=json_extract_string(p.body, '/id')
|
| 190 |
+
WHERE per.id IS NOT NULL AND per.qty > 1e-9 GROUP BY code""").fetchall()}
|
| 191 |
+
|
| 192 |
+
def inventory_frame(self, t):
|
| 193 |
+
from core import periods
|
| 194 |
+
from modules import inventory
|
| 195 |
+
import pandas as pd
|
| 196 |
+
lf, lt = periods.ltm(t)
|
| 197 |
+
quant = self.read_group('stock.quant', [('location_id.usage', '=', 'internal')],
|
| 198 |
+
['product_id', 'quantity:sum'], ['product_id'], lazy=False)
|
| 199 |
+
sales = self.read_group('sale.order.line', _odoo.sale_line_domain(lf, lt, all_channels=True),
|
| 200 |
+
['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
|
| 201 |
+
products = self.search_read('product.product', [('default_code', '!=', False)],
|
| 202 |
+
['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
|
| 203 |
+
cats = self.search_read('product.category', [], ['id', 'complete_name'])
|
| 204 |
+
query = f"""WITH raw AS (SELECT p.ordinal,
|
| 205 |
+
trim(json_extract_string(p.body, '/default_code')) sku,
|
| 206 |
+
json_extract_string(p.body, '/name') AS product_name,
|
| 207 |
+
coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''), '(uncategorized)') category,
|
| 208 |
+
json_extract_string(p.body, '/type') = 'product' storable,
|
| 209 |
+
coalesce(try_cast(json_extract_string(p.body, '/standard_price') AS DOUBLE), 0) AS cost,
|
| 210 |
+
coalesce(try_cast(json_extract_string(q.body, '/quantity') AS DOUBLE), 0) on_hand,
|
| 211 |
+
coalesce(try_cast(json_extract_string(s.body, '/product_uom_qty') AS DOUBLE), 0) qty_ltm,
|
| 212 |
+
coalesce(try_cast(json_extract_string(s.body, '/price_subtotal') AS DOUBLE), 0) rev_ltm
|
| 213 |
+
FROM "{products.table}" p
|
| 214 |
+
LEFT JOIN "{quant.table}" q ON json_extract_string(p.body, '/id')=json_extract_string(q.body, '/product_id/0')
|
| 215 |
+
LEFT JOIN "{sales.table}" s ON json_extract_string(p.body, '/id')=json_extract_string(s.body, '/product_id/0')
|
| 216 |
+
LEFT JOIN "{cats.table}" c ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id'))
|
| 217 |
+
SELECT sku, first(product_name ORDER BY ordinal) AS name, first(category ORDER BY ordinal) category,
|
| 218 |
+
bool_or(storable) storable, sum(on_hand) on_hand, sum(on_hand*cost) inv_value,
|
| 219 |
+
sum(qty_ltm*cost) cogs_ltm, sum(qty_ltm) qty_ltm, sum(rev_ltm) rev_ltm
|
| 220 |
+
FROM raw GROUP BY sku"""
|
| 221 |
+
result = self.cur.execute(query)
|
| 222 |
+
names = [d[0] for d in result.description]
|
| 223 |
+
return pd.DataFrame(result.fetchall(), columns=names), (lf, lt)
|
| 224 |
+
|
| 225 |
+
def tiers(self, prods):
|
| 226 |
+
from core import periods
|
| 227 |
+
from modules import products
|
| 228 |
+
lists = self.search_read('product.pricelist', [], ['id', 'name'])
|
| 229 |
+
today = periods.today().isoformat()
|
| 230 |
+
rules = self.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'),
|
| 231 |
+
('applied_on', 'in', ['0_product_variant', '1_product']),
|
| 232 |
+
'|', ('date_start', '=', False), ('date_start', '<=', today),
|
| 233 |
+
'|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)],
|
| 234 |
+
['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
|
| 235 |
+
'fixed_price', 'packaging_id', 'packaging_price'])
|
| 236 |
+
normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
|
| 237 |
+
canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
|
| 238 |
+
prods = products.active_products() if prods is None else prods
|
| 239 |
+
declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
|
| 240 |
+
query = f"""WITH eligible AS (
|
| 241 |
+
SELECT {code_sql('p')} code, p.ordinal product_order, r.ordinal rule_order,
|
| 242 |
+
trim(json_extract_string(l.body, '/name')) pricelist,
|
| 243 |
+
try_cast(json_extract_string(r.body, '/canonical_price') AS DOUBLE) price,
|
| 244 |
+
json_extract_string(r.body, '/canonical_unit') unit
|
| 245 |
+
FROM "{prods.table}" p JOIN "{normalized.table}" r ON (
|
| 246 |
+
(json_extract_string(r.body, '/applied_on')='0_product_variant'
|
| 247 |
+
AND json_extract_string(r.body, '/product_id/0')=json_extract_string(p.body, '/id')) OR
|
| 248 |
+
(json_extract_string(r.body, '/applied_on')='1_product'
|
| 249 |
+
AND json_extract_string(r.body, '/product_tmpl_id/0')=json_extract_string(p.body, '/product_tmpl_id/0')
|
| 250 |
+
AND NOT EXISTS (SELECT 1 FROM "{normalized.table}" v
|
| 251 |
+
WHERE json_extract_string(v.body, '/applied_on')='0_product_variant'
|
| 252 |
+
AND json_extract_string(v.body, '/pricelist_id/0')=json_extract_string(r.body, '/pricelist_id/0')
|
| 253 |
+
AND json_extract_string(v.body, '/product_id/0')=json_extract_string(p.body, '/id'))))
|
| 254 |
+
JOIN "{lists.table}" l ON json_extract_string(l.body, '/id')=json_extract_string(r.body, '/pricelist_id/0')),
|
| 255 |
+
identified AS (SELECT *, min(product_order) OVER (PARTITION BY code) first_product,
|
| 256 |
+
row_number() OVER (PARTITION BY code, pricelist, price, unit ORDER BY product_order, rule_order) ident FROM eligible),
|
| 257 |
+
merged AS (SELECT * FROM identified WHERE product_order=first_product OR ident=1),
|
| 258 |
+
ranked AS (SELECT *, row_number() OVER (PARTITION BY code ORDER BY price, lower(unit), lower(pricelist), product_order, rule_order) pooled,
|
| 259 |
+
row_number() OVER (PARTITION BY code, pricelist ORDER BY price, lower(unit), product_order, rule_order) tier,
|
| 260 |
+
count(*) OVER (PARTITION BY code, pricelist) n FROM merged)
|
| 261 |
+
SELECT code, pricelist, price, unit, n FROM ranked
|
| 262 |
+
WHERE pooled <= 3 OR (pricelist IN (SELECT unnest(?::VARCHAR[])) AND tier <= ?)
|
| 263 |
+
ORDER BY code, price, lower(unit), lower(pricelist), product_order, rule_order"""
|
| 264 |
+
result = self.cur.execute(query, [declared, products.PRICELIST_TIER_DEPTH])
|
| 265 |
+
out = {}
|
| 266 |
+
while True:
|
| 267 |
+
batch = result.fetchmany(BATCH)
|
| 268 |
+
if not batch:
|
| 269 |
+
break
|
| 270 |
+
for code, name, price, unit, n in batch:
|
| 271 |
+
entry = out.setdefault(code, TierProjection())
|
| 272 |
+
entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
|
| 273 |
+
if name in declared and n > products.PRICELIST_TIER_DEPTH:
|
| 274 |
+
entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
|
| 275 |
+
return out, {'bounded': True}
|
| 276 |
+
|
| 277 |
+
def search_read(self, model, domain=None, fields=None, **kw):
|
| 278 |
+
domain = self.domain(model, domain)
|
| 279 |
+
# Keyset traversal avoids a connector's default limit and never silently caps connected rows.
|
| 280 |
+
kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset', 'order'}}
|
| 281 |
+
def records():
|
| 282 |
+
last = 0
|
| 283 |
+
while True:
|
| 284 |
+
page = _odoo.search_read(model, domain + [('id', '>', last)],
|
| 285 |
+
list(dict.fromkeys(['id', *(['default_code'] if self.kind == 'product' and model == 'product.product' else []),
|
| 286 |
+
*(fields or [])])), limit=BATCH, order='id asc', **kw)
|
| 287 |
+
if not page:
|
| 288 |
+
return
|
| 289 |
+
nxt = max(int(r['id']) for r in page)
|
| 290 |
+
if nxt <= last:
|
| 291 |
+
raise RuntimeError('canonical_source_invalid: source keyset did not advance')
|
| 292 |
+
if self.kind == 'product' and model == 'product.product':
|
| 293 |
+
selected = set(self.identities)
|
| 294 |
+
for row in page:
|
| 295 |
+
code = str(row['default_code']).strip() if row.get('default_code') else f"pid:{row['id']}"
|
| 296 |
+
if code in selected:
|
| 297 |
+
yield row
|
| 298 |
+
else:
|
| 299 |
+
yield from page
|
| 300 |
+
last = nxt
|
| 301 |
+
page.clear()
|
| 302 |
+
return self.snapshot(records())
|
| 303 |
+
|
| 304 |
+
def read_group(self, model, domain, fields, groupby, **kw):
|
| 305 |
+
domain = self.domain(model, domain)
|
| 306 |
+
kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset'}}
|
| 307 |
+
def records():
|
| 308 |
+
offset = 0
|
| 309 |
+
while True:
|
| 310 |
+
options = dict(kw, offset=offset, limit=BATCH)
|
| 311 |
+
if groupby:
|
| 312 |
+
options.setdefault('orderby', ','.join(groupby))
|
| 313 |
+
page = _odoo.get_odoo().execute_kw(model, 'read_group', [domain, fields, groupby], options)
|
| 314 |
+
if not page:
|
| 315 |
+
return
|
| 316 |
+
for row in page:
|
| 317 |
+
yield {key: value for key, value in row.items()
|
| 318 |
+
if not key.startswith('__') or key == '__count'}
|
| 319 |
+
offset += len(page)
|
| 320 |
+
page.clear()
|
| 321 |
+
return self.snapshot(records())
|
| 322 |
+
|
| 323 |
+
def search_count(self, model, domain=None, **kw):
|
| 324 |
+
if self.kind == 'product' and model == 'product.product':
|
| 325 |
+
return len(self.search_read(model, domain, ['id', 'default_code']))
|
| 326 |
+
return _odoo.get_odoo().search_count(model, self.domain(model, domain), **kw)
|
| 327 |
+
|
| 328 |
+
def agent_members(self, name):
|
| 329 |
+
if name == '(none)':
|
| 330 |
+
return self.search_read('res.partner', [('customer_rank', '>', 0),
|
| 331 |
+
('agent_ids', '=', False), ('active', 'in', [True, False])], ['id'])
|
| 332 |
+
agents = self.search_read('res.partner', [('name', '=', name),
|
| 333 |
+
('active', 'in', [True, False])], ['id'])
|
| 334 |
+
def records():
|
| 335 |
+
source = iter(agents)
|
| 336 |
+
while True:
|
| 337 |
+
batch = list(itertools.islice(source, BATCH))
|
| 338 |
+
if not batch:
|
| 339 |
+
return
|
| 340 |
+
rows = self.search_read('res.partner', [('agent_ids', 'in', [r['id'] for r in batch]),
|
| 341 |
+
('active', 'in', [True, False])], ['id'])
|
| 342 |
+
try:
|
| 343 |
+
yield from rows
|
| 344 |
+
finally:
|
| 345 |
+
self.cur.execute(f'DROP TABLE IF EXISTS "{rows.table}"')
|
| 346 |
+
self.tables.remove(rows.table)
|
| 347 |
+
return self.snapshot(records())
|
| 348 |
+
|
| 349 |
+
def close(self):
|
| 350 |
+
for table in reversed(self.tables):
|
| 351 |
+
self.cur.execute(f'DROP TABLE IF EXISTS "{table}"')
|
| 352 |
+
self.tables.clear()
|
| 353 |
+
self.cache.clear()
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def active():
|
| 357 |
+
return _ACTIVE.get()
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
@contextmanager
|
| 361 |
+
def bounded(cur, kind, identities, store=None):
|
| 362 |
+
source = Source(cur, kind, identities, store)
|
| 363 |
+
token = _ACTIVE.set(source)
|
| 364 |
+
try:
|
| 365 |
+
yield source
|
| 366 |
+
if source.failure is not None:
|
| 367 |
+
raise RuntimeError('canonical_source_unavailable: ' + str(source.failure)) from source.failure
|
| 368 |
+
finally:
|
| 369 |
+
_ACTIVE.reset(token)
|
| 370 |
+
source.close()
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
class _Odoo:
|
| 374 |
+
def __getattr__(self, name):
|
| 375 |
+
source = active()
|
| 376 |
+
if source is not None and name in {'search_read', 'read_group', 'search_count'}:
|
| 377 |
+
return getattr(source, name)
|
| 378 |
+
if source is not None and name == 'get_odoo':
|
| 379 |
+
return lambda: self
|
| 380 |
+
return getattr(_odoo, name)
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
odoo = _Odoo()
|
platform/harness/datastore.py
CHANGED
|
@@ -17,12 +17,16 @@ Design rules:
|
|
| 17 |
- Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
|
| 18 |
"""
|
| 19 |
import datetime as dt
|
|
|
|
| 20 |
import json
|
| 21 |
import math
|
| 22 |
import os
|
|
|
|
| 23 |
import re
|
| 24 |
import sys
|
| 25 |
import time
|
|
|
|
|
|
|
| 26 |
from contextlib import nullcontext
|
| 27 |
from pathlib import Path
|
| 28 |
|
|
@@ -137,16 +141,46 @@ def path_for(tenant_key):
|
|
| 137 |
|
| 138 |
|
| 139 |
def apply_limits(con):
|
| 140 |
-
"""
|
| 141 |
-
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
|
| 144 |
con.execute(f"SET memory_limit='{memory}'")
|
| 145 |
if threads.isdigit() and int(threads) > 0:
|
| 146 |
con.execute(f"SET threads={int(threads)}")
|
|
|
|
|
|
|
| 147 |
return con
|
| 148 |
|
| 149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
def use_path(path):
|
| 151 |
"""Repoint the store at `path`, CLOSING the process singleton first.
|
| 152 |
|
|
@@ -181,6 +215,7 @@ def use_path(path):
|
|
| 181 |
_READY["ok"] = False
|
| 182 |
_GENERATION[0] += 1 # every other thread's cached cursor is now stale
|
| 183 |
DB_PATH = Path(path)
|
|
|
|
| 184 |
return DB_PATH
|
| 185 |
|
| 186 |
# Entity specs: Odoo model → store table. m2o fields land as <name>_id + <name>_name.
|
|
@@ -506,7 +541,7 @@ def snapshot_status(tenant_key, source, *, rt=None):
|
|
| 506 |
result.update(saved)
|
| 507 |
if not path.exists():
|
| 508 |
return result
|
| 509 |
-
con = ro_cursor() if path.resolve() == DB_PATH.resolve() else _TimedCursor(duckdb.connect(str(path)))
|
| 510 |
try:
|
| 511 |
manifest = _publication(con, source)
|
| 512 |
if manifest:
|
|
@@ -827,11 +862,23 @@ _GENERATION = [0]
|
|
| 827 |
#: Bumped after every mirror snapshot install (`MirrorStore._install_snapshot`). D-2670: the
|
| 828 |
#: Sheet change token caches its table scan per generation, so an install must move this.
|
| 829 |
_MIRROR_INSTALLS = [0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 830 |
|
| 831 |
|
| 832 |
def note_mirror_install():
|
| 833 |
"""Mark that a mirror install committed or rolled back, so cached change tokens recompute."""
|
| 834 |
_MIRROR_INSTALLS[0] += 1
|
|
|
|
| 835 |
|
| 836 |
|
| 837 |
def mirror_generation():
|
|
@@ -880,21 +927,47 @@ class _TimedCursor:
|
|
| 880 |
self._statement = ""
|
| 881 |
self._statement_ms = 0.0
|
| 882 |
self._request = None
|
|
|
|
| 883 |
|
| 884 |
def _call(self, name, *args, **kwargs):
|
| 885 |
acc = _request_timing.CURRENT.get()
|
| 886 |
execute = name in {"execute", "executemany"}
|
|
|
|
|
|
|
| 887 |
if execute:
|
| 888 |
statement = args[0] if args else kwargs.get("query", "<statement>")
|
| 889 |
self._statement = _request_timing.sql_text(statement) if acc is not None else ""
|
| 890 |
self._statement_ms = 0.0
|
| 891 |
self._request = acc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 892 |
started = time.perf_counter()
|
| 893 |
try:
|
| 894 |
-
with _request_timing.thread_scope():
|
| 895 |
result = getattr(self._cursor, name)(*args, **kwargs)
|
|
|
|
|
|
|
| 896 |
return self if result is self._cursor else result
|
| 897 |
finally:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
if acc is not None and self._request is acc:
|
| 899 |
elapsed = (time.perf_counter() - started) * 1000
|
| 900 |
self._statement_ms += elapsed
|
|
@@ -955,7 +1028,8 @@ def ready():
|
|
| 955 |
cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
|
| 956 |
finally:
|
| 957 |
cur.close()
|
| 958 |
-
except Exception:
|
|
|
|
| 959 |
return False
|
| 960 |
# ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN
|
| 961 |
# OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every
|
|
@@ -1007,7 +1081,8 @@ def entity_live(key):
|
|
| 1007 |
columns_ready = _required_columns_ready(cur, key, ENTITIES.get(key) or {})
|
| 1008 |
finally:
|
| 1009 |
cur.close()
|
| 1010 |
-
except Exception:
|
|
|
|
| 1011 |
return False
|
| 1012 |
return bool(row) and row[0] == "live" and columns_ready
|
| 1013 |
|
|
@@ -1039,7 +1114,8 @@ def ro_con():
|
|
| 1039 |
try:
|
| 1040 |
cur.execute("SELECT 1") # cheap liveness probe
|
| 1041 |
return cur
|
| 1042 |
-
except Exception:
|
|
|
|
| 1043 |
_RO_TLS.cur = None # stale → reopen below
|
| 1044 |
if not ready():
|
| 1045 |
raise RuntimeError("the tenant data store is completing its FIRST sync on this "
|
|
@@ -1098,11 +1174,79 @@ _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
| 1098 |
|
| 1099 |
#: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
|
| 1100 |
_SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1101 |
|
| 1102 |
|
| 1103 |
def window(table=None, select=None, where=None, params=(), order_by=None,
|
| 1104 |
offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
|
| 1105 |
-
page_first=False):
|
| 1106 |
"""One page of a mirror table PLUS the true count of everything the same predicate matches.
|
| 1107 |
|
| 1108 |
Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
|
|
@@ -1129,6 +1273,8 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 1129 |
|
| 1130 |
`page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
|
| 1131 |
the page's rows before the projection runs, when only the projection holds a subquery.
|
|
|
|
|
|
|
| 1132 |
"""
|
| 1133 |
if bool(table) == bool(from_sql):
|
| 1134 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
@@ -1161,10 +1307,17 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 1161 |
# ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate:
|
| 1162 |
# if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than
|
| 1163 |
# quietly returning a good count beside a broken page.
|
| 1164 |
-
total =
|
| 1165 |
grp = str(group_by or "").strip()
|
| 1166 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 1167 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1168 |
and not _SUBQUERY.search(f"{pred} {order}")):
|
| 1169 |
# A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
|
| 1170 |
# of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
|
|
@@ -1400,7 +1553,7 @@ def _prepare_seed_mirrors(path, rt):
|
|
| 1400 |
if rt is None:
|
| 1401 |
return
|
| 1402 |
from harness import mirror_store
|
| 1403 |
-
con = duckdb.connect(str(path))
|
| 1404 |
try:
|
| 1405 |
_ensure_sync_state(con)
|
| 1406 |
for connector in tuple(_connector_spec.MIRRORS):
|
|
|
|
| 17 |
- Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
|
| 18 |
"""
|
| 19 |
import datetime as dt
|
| 20 |
+
import hashlib
|
| 21 |
import json
|
| 22 |
import math
|
| 23 |
import os
|
| 24 |
+
import pickle
|
| 25 |
import re
|
| 26 |
import sys
|
| 27 |
import time
|
| 28 |
+
import tempfile
|
| 29 |
+
from collections import OrderedDict
|
| 30 |
from contextlib import nullcontext
|
| 31 |
from pathlib import Path
|
| 32 |
|
|
|
|
| 141 |
|
| 142 |
|
| 143 |
def apply_limits(con):
|
| 144 |
+
"""Bound each instance's memory, execution threads and local spill space."""
|
| 145 |
+
web = os.environ.get("AIOS_PROCESS_ROLE") == "web"
|
| 146 |
+
memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or (
|
| 147 |
+
os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB" if web else "4GB")).strip()
|
| 148 |
+
threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or (
|
| 149 |
+
os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2" if web else "1")).strip()
|
| 150 |
+
spill = str(os.environ.get("AIOS_DUCKDB_MAX_TEMP_SIZE") or "4GB").strip()
|
| 151 |
+
temp = Path(os.environ.get("AIOS_DUCKDB_TEMP_DIR") or (
|
| 152 |
+
Path(tempfile.gettempdir()) / f"aios-duckdb-{os.getpid()}"))
|
| 153 |
+
rows = con.execute("PRAGMA database_list").fetchall()
|
| 154 |
+
identity = next((str(row[2]) for row in rows if row[2]), str(id(con)))
|
| 155 |
+
temp = temp / hashlib.sha256(identity.encode()).hexdigest()[:16]
|
| 156 |
+
temp.mkdir(parents=True, exist_ok=True, mode=0o700)
|
| 157 |
+
if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
|
| 158 |
+
raise ValueError("Invalid AIOS_DUCKDB_MEMORY")
|
| 159 |
+
if not threads.isdigit() or int(threads) < 1:
|
| 160 |
+
raise ValueError("Invalid AIOS_DUCKDB_THREADS")
|
| 161 |
+
if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", spill, re.IGNORECASE):
|
| 162 |
+
raise ValueError("Invalid AIOS_DUCKDB_MAX_TEMP_SIZE")
|
| 163 |
if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
|
| 164 |
con.execute(f"SET memory_limit='{memory}'")
|
| 165 |
if threads.isdigit() and int(threads) > 0:
|
| 166 |
con.execute(f"SET threads={int(threads)}")
|
| 167 |
+
con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'")
|
| 168 |
+
con.execute(f"SET max_temp_directory_size='{spill}'")
|
| 169 |
return con
|
| 170 |
|
| 171 |
|
| 172 |
+
def bounded_connection(con):
|
| 173 |
+
"""Apply instance budgets and request deadlines to a temporary analytical connection."""
|
| 174 |
+
try:
|
| 175 |
+
return _TimedCursor(apply_limits(con))
|
| 176 |
+
except BaseException:
|
| 177 |
+
try:
|
| 178 |
+
con.close()
|
| 179 |
+
except Exception:
|
| 180 |
+
pass
|
| 181 |
+
raise
|
| 182 |
+
|
| 183 |
+
|
| 184 |
def use_path(path):
|
| 185 |
"""Repoint the store at `path`, CLOSING the process singleton first.
|
| 186 |
|
|
|
|
| 215 |
_READY["ok"] = False
|
| 216 |
_GENERATION[0] += 1 # every other thread's cached cursor is now stale
|
| 217 |
DB_PATH = Path(path)
|
| 218 |
+
_invalidate_counts()
|
| 219 |
return DB_PATH
|
| 220 |
|
| 221 |
# Entity specs: Odoo model → store table. m2o fields land as <name>_id + <name>_name.
|
|
|
|
| 541 |
result.update(saved)
|
| 542 |
if not path.exists():
|
| 543 |
return result
|
| 544 |
+
con = ro_cursor() if path.resolve() == DB_PATH.resolve() else _TimedCursor(apply_limits(duckdb.connect(str(path))))
|
| 545 |
try:
|
| 546 |
manifest = _publication(con, source)
|
| 547 |
if manifest:
|
|
|
|
| 862 |
#: Bumped after every mirror snapshot install (`MirrorStore._install_snapshot`). D-2670: the
|
| 863 |
#: Sheet change token caches its table scan per generation, so an install must move this.
|
| 864 |
_MIRROR_INSTALLS = [0]
|
| 865 |
+
_MIRROR_WRITES = [0]
|
| 866 |
+
_ACTIVE_WRITES = [0]
|
| 867 |
+
_COUNT_CACHE = OrderedDict()
|
| 868 |
+
_COUNT_CACHE_MAX = 256
|
| 869 |
+
_COUNT_CACHE_LOCK = _thr.RLock()
|
| 870 |
+
|
| 871 |
+
|
| 872 |
+
def _invalidate_counts():
|
| 873 |
+
with _COUNT_CACHE_LOCK:
|
| 874 |
+
_MIRROR_WRITES[0] += 1
|
| 875 |
+
_COUNT_CACHE.clear()
|
| 876 |
|
| 877 |
|
| 878 |
def note_mirror_install():
|
| 879 |
"""Mark that a mirror install committed or rolled back, so cached change tokens recompute."""
|
| 880 |
_MIRROR_INSTALLS[0] += 1
|
| 881 |
+
_invalidate_counts()
|
| 882 |
|
| 883 |
|
| 884 |
def mirror_generation():
|
|
|
|
| 927 |
self._statement = ""
|
| 928 |
self._statement_ms = 0.0
|
| 929 |
self._request = None
|
| 930 |
+
self._in_transaction = False
|
| 931 |
|
| 932 |
def _call(self, name, *args, **kwargs):
|
| 933 |
acc = _request_timing.CURRENT.get()
|
| 934 |
execute = name in {"execute", "executemany"}
|
| 935 |
+
writes = False
|
| 936 |
+
transaction = None
|
| 937 |
if execute:
|
| 938 |
statement = args[0] if args else kwargs.get("query", "<statement>")
|
| 939 |
self._statement = _request_timing.sql_text(statement) if acc is not None else ""
|
| 940 |
self._statement_ms = 0.0
|
| 941 |
self._request = acc
|
| 942 |
+
try:
|
| 943 |
+
statements = self._cursor.extract_statements(statement)
|
| 944 |
+
writes = any(s.type.name not in {"SELECT", "TRANSACTION"} for s in statements)
|
| 945 |
+
for parsed in statements:
|
| 946 |
+
if parsed.type.name == "TRANSACTION":
|
| 947 |
+
transaction = not re.match(r"\s*(COMMIT|ROLLBACK|END)\b", parsed.query, re.I)
|
| 948 |
+
except Exception as exc:
|
| 949 |
+
_request_timing.raise_if_resource_error(exc)
|
| 950 |
+
# Unclassified statements cannot establish a reusable read snapshot.
|
| 951 |
+
writes, self._in_transaction = True, True
|
| 952 |
+
if transaction is not None:
|
| 953 |
+
# A failed statement batch may leave its BEGIN active.
|
| 954 |
+
self._in_transaction = True
|
| 955 |
+
if writes:
|
| 956 |
+
with _COUNT_CACHE_LOCK:
|
| 957 |
+
_ACTIVE_WRITES[0] += 1
|
| 958 |
+
_invalidate_counts()
|
| 959 |
started = time.perf_counter()
|
| 960 |
try:
|
| 961 |
+
with _request_timing.thread_scope(), _request_timing.duckdb_operation(self._cursor):
|
| 962 |
result = getattr(self._cursor, name)(*args, **kwargs)
|
| 963 |
+
if transaction is not None:
|
| 964 |
+
self._in_transaction = transaction
|
| 965 |
return self if result is self._cursor else result
|
| 966 |
finally:
|
| 967 |
+
if writes:
|
| 968 |
+
with _COUNT_CACHE_LOCK:
|
| 969 |
+
_ACTIVE_WRITES[0] -= 1
|
| 970 |
+
_invalidate_counts()
|
| 971 |
if acc is not None and self._request is acc:
|
| 972 |
elapsed = (time.perf_counter() - started) * 1000
|
| 973 |
self._statement_ms += elapsed
|
|
|
|
| 1028 |
cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
|
| 1029 |
finally:
|
| 1030 |
cur.close()
|
| 1031 |
+
except Exception as exc:
|
| 1032 |
+
_request_timing.raise_if_resource_error(exc)
|
| 1033 |
return False
|
| 1034 |
# ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN
|
| 1035 |
# OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every
|
|
|
|
| 1081 |
columns_ready = _required_columns_ready(cur, key, ENTITIES.get(key) or {})
|
| 1082 |
finally:
|
| 1083 |
cur.close()
|
| 1084 |
+
except Exception as exc: # noqa: BLE001
|
| 1085 |
+
_request_timing.raise_if_resource_error(exc)
|
| 1086 |
return False
|
| 1087 |
return bool(row) and row[0] == "live" and columns_ready
|
| 1088 |
|
|
|
|
| 1114 |
try:
|
| 1115 |
cur.execute("SELECT 1") # cheap liveness probe
|
| 1116 |
return cur
|
| 1117 |
+
except Exception as exc:
|
| 1118 |
+
_request_timing.raise_if_resource_error(exc)
|
| 1119 |
_RO_TLS.cur = None # stale → reopen below
|
| 1120 |
if not ready():
|
| 1121 |
raise RuntimeError("the tenant data store is completing its FIRST sync on this "
|
|
|
|
| 1174 |
|
| 1175 |
#: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
|
| 1176 |
_SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
|
| 1177 |
+
_ORDINAL_ORDER = re.compile(r"(?:^|,)\s*\d+\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?\s*(?:,|$)", re.I)
|
| 1178 |
+
|
| 1179 |
+
|
| 1180 |
+
def mirror_file_marks(cur):
|
| 1181 |
+
"""Identify the cursor's actual file and WAL, or decline an unnamed mirror."""
|
| 1182 |
+
row = cur.execute("SELECT path FROM duckdb_databases() "
|
| 1183 |
+
"WHERE database_name = current_database()").fetchone()
|
| 1184 |
+
path = row[0] if row else None
|
| 1185 |
+
if not path or path == ":memory:":
|
| 1186 |
+
return None
|
| 1187 |
+
marks = []
|
| 1188 |
+
for name in (path, path + ".wal"):
|
| 1189 |
+
try:
|
| 1190 |
+
stat = os.stat(name)
|
| 1191 |
+
marks.append((stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns))
|
| 1192 |
+
except OSError:
|
| 1193 |
+
marks.append(None)
|
| 1194 |
+
return (str(Path(path).resolve()), tuple(marks)) if marks[0] is not None else None
|
| 1195 |
+
|
| 1196 |
+
|
| 1197 |
+
def _count_revision(cur):
|
| 1198 |
+
"""Cache only managed, autocommit reads of one persistent mirror without local relations."""
|
| 1199 |
+
if not isinstance(cur, _TimedCursor) or cur._in_transaction or _ACTIVE_WRITES[0]:
|
| 1200 |
+
return None
|
| 1201 |
+
generation = mirror_generation(), _MIRROR_WRITES[0]
|
| 1202 |
+
marks = mirror_file_marks(cur)
|
| 1203 |
+
if marks is None:
|
| 1204 |
+
return None
|
| 1205 |
+
unsafe = cur.execute(
|
| 1206 |
+
"SELECT (SELECT count(*) FROM duckdb_databases() WHERE NOT internal) <> 1 "
|
| 1207 |
+
"OR EXISTS (SELECT 1 FROM duckdb_tables() WHERE temporary AND NOT internal) "
|
| 1208 |
+
"OR EXISTS (SELECT 1 FROM duckdb_views() WHERE temporary AND NOT internal)").fetchone()[0]
|
| 1209 |
+
return None if unsafe or generation != (mirror_generation(), _MIRROR_WRITES[0]) else (marks, generation)
|
| 1210 |
+
|
| 1211 |
+
|
| 1212 |
+
def count_rows(table=None, where=None, params=(), cur=None, from_sql=None):
|
| 1213 |
+
"""Return the exact matching row population, reusing only an unchanged mirror revision."""
|
| 1214 |
+
# Callers supply deterministic compiled SQL, as they do for window().
|
| 1215 |
+
if bool(table) == bool(from_sql):
|
| 1216 |
+
raise ValueError("datastore.count_rows: pass exactly one of `table` or `from_sql`")
|
| 1217 |
+
name = str(from_sql or table).strip()
|
| 1218 |
+
if table and not _IDENT_OK.fullmatch(name):
|
| 1219 |
+
raise ValueError(f"datastore.count_rows: {table!r} is not a table identifier")
|
| 1220 |
+
pred = str(where or "").strip()
|
| 1221 |
+
sql = f"SELECT count(*) FROM {name}" + (f" WHERE {pred}" if pred else "")
|
| 1222 |
+
params = tuple(params or ())
|
| 1223 |
+
cur = cur if cur is not None else ro_con()
|
| 1224 |
+
revision = _count_revision(cur)
|
| 1225 |
+
key = None
|
| 1226 |
+
if revision is not None:
|
| 1227 |
+
try:
|
| 1228 |
+
# SQL and typed bindings remain private, including large IN predicates.
|
| 1229 |
+
key = revision, hashlib.sha256(pickle.dumps((sql, params), protocol=5)).digest()
|
| 1230 |
+
except (TypeError, ValueError, AttributeError, OverflowError, pickle.PickleError):
|
| 1231 |
+
pass
|
| 1232 |
+
if key is not None:
|
| 1233 |
+
with _COUNT_CACHE_LOCK:
|
| 1234 |
+
if key in _COUNT_CACHE:
|
| 1235 |
+
_COUNT_CACHE.move_to_end(key)
|
| 1236 |
+
return _COUNT_CACHE[key]
|
| 1237 |
+
total = int(cur.execute(sql, params).fetchone()[0] or 0)
|
| 1238 |
+
if key is not None and _count_revision(cur) == revision:
|
| 1239 |
+
with _COUNT_CACHE_LOCK:
|
| 1240 |
+
_COUNT_CACHE[key] = total
|
| 1241 |
+
_COUNT_CACHE.move_to_end(key)
|
| 1242 |
+
while len(_COUNT_CACHE) > _COUNT_CACHE_MAX:
|
| 1243 |
+
_COUNT_CACHE.popitem(last=False)
|
| 1244 |
+
return total
|
| 1245 |
|
| 1246 |
|
| 1247 |
def window(table=None, select=None, where=None, params=(), order_by=None,
|
| 1248 |
offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
|
| 1249 |
+
page_first=False, page_key=None):
|
| 1250 |
"""One page of a mirror table PLUS the true count of everything the same predicate matches.
|
| 1251 |
|
| 1252 |
Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
|
|
|
|
| 1273 |
|
| 1274 |
`page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
|
| 1275 |
the page's rows before the projection runs, when only the projection holds a subquery.
|
| 1276 |
+
`page_key` opts a physical table into a narrow, materialised key page before projection.
|
| 1277 |
+
It must uniquely identify source rows, independently of the displayed record identity.
|
| 1278 |
"""
|
| 1279 |
if bool(table) == bool(from_sql):
|
| 1280 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
|
|
| 1307 |
# ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate:
|
| 1308 |
# if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than
|
| 1309 |
# quietly returning a good count beside a broken page.
|
| 1310 |
+
total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
|
| 1311 |
grp = str(group_by or "").strip()
|
| 1312 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 1313 |
+
if page_first and page_key and table and not grp and not _ORDINAL_ORDER.search(order):
|
| 1314 |
+
# Only keys cross the materialisation boundary; identity order still uses the full population.
|
| 1315 |
+
got = cur.execute(
|
| 1316 |
+
f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
|
| 1317 |
+
f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
|
| 1318 |
+
f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
|
| 1319 |
+
f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
|
| 1320 |
+
elif (page_first and not page_key and table and not grp and _SUBQUERY.search(proj)
|
| 1321 |
and not _SUBQUERY.search(f"{pred} {order}")):
|
| 1322 |
# A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
|
| 1323 |
# of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
|
|
|
|
| 1553 |
if rt is None:
|
| 1554 |
return
|
| 1555 |
from harness import mirror_store
|
| 1556 |
+
con = apply_limits(duckdb.connect(str(path)))
|
| 1557 |
try:
|
| 1558 |
_ensure_sync_state(con)
|
| 1559 |
for connector in tuple(_connector_spec.MIRRORS):
|
platform/harness/filter_sql.py
CHANGED
|
@@ -715,9 +715,13 @@ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql
|
|
| 715 |
if member_ids:
|
| 716 |
if not id_sql:
|
| 717 |
raise ValueError('member_ids requires id_sql (the row-identity column)')
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 721 |
return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
|
| 722 |
|
| 723 |
|
|
|
|
| 715 |
if member_ids:
|
| 716 |
if not id_sql:
|
| 717 |
raise ValueError('member_ids requires id_sql (the row-identity column)')
|
| 718 |
+
if isinstance(member_ids, SqlMembership):
|
| 719 |
+
sql = f"({sql} OR {id_sql} IN ({member_ids.sql}))"
|
| 720 |
+
params = list(params) + list(member_ids.params)
|
| 721 |
+
else:
|
| 722 |
+
ids = [int(x) for x in member_ids]
|
| 723 |
+
sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
|
| 724 |
+
params = list(params) + ids
|
| 725 |
return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
|
| 726 |
|
| 727 |
|
platform/harness/measure_filter.py
CHANGED
|
@@ -406,7 +406,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None, related=None):
|
|
| 406 |
"which would be a different question with an authoritative-looking answer")
|
| 407 |
con = _sem._store_con()
|
| 408 |
try:
|
| 409 |
-
totals = _totals(con, key, rule.get('window'), today, team_id, related)
|
| 410 |
finally:
|
| 411 |
con.close()
|
| 412 |
# ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision,
|
|
@@ -523,7 +523,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
|
|
| 523 |
# compare in Python — the same shape `resolve_pair` uses, and for the same reason.
|
| 524 |
con = _sem._store_con()
|
| 525 |
try:
|
| 526 |
-
totals = _totals(con, key, rule.get('window'), today, team_id, related)
|
| 527 |
finally:
|
| 528 |
con.close()
|
| 529 |
# No pool means we can only speak about customers a base has seen. WITH a pool, everyone
|
|
@@ -535,6 +535,9 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
|
|
| 535 |
|
| 536 |
frm, id_col, agg, where_sql, params = _query_parts(
|
| 537 |
key, rule.get('window'), today, team_id, related)
|
|
|
|
|
|
|
|
|
|
| 538 |
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
|
| 539 |
f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
|
| 540 |
|
|
@@ -559,7 +562,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
|
|
| 559 |
return _related_answer(matched, related)
|
| 560 |
|
| 561 |
|
| 562 |
-
def _totals(con, key, window, today, team_id, related=None):
|
| 563 |
"""`{pid: value}` for one (measure, window), COMPOSITES included.
|
| 564 |
|
| 565 |
For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
|
|
@@ -573,13 +576,16 @@ def _totals(con, key, window, today, team_id, related=None):
|
|
| 573 |
"""
|
| 574 |
if is_composite(key):
|
| 575 |
bases, _ = _sem._expand_measures([key])
|
| 576 |
-
parts = {b: _totals(con, b, window, today, team_id, related) for b in bases}
|
| 577 |
pids = set()
|
| 578 |
for got in parts.values():
|
| 579 |
pids |= set(got)
|
| 580 |
return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
|
| 581 |
for pid in pids}
|
| 582 |
frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related)
|
|
|
|
|
|
|
|
|
|
| 583 |
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
|
| 584 |
# `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
|
| 585 |
# COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and
|
|
@@ -620,8 +626,8 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None, related=None):
|
|
| 620 |
pool = _pool_of(allowed_pids)
|
| 621 |
con = _sem._store_con()
|
| 622 |
try:
|
| 623 |
-
left = _totals(con, lkey, rule.get('window'), today, team_id, related)
|
| 624 |
-
right = _totals(con, rkey, rhs.get('window'), today, team_id, related)
|
| 625 |
finally:
|
| 626 |
con.close()
|
| 627 |
# With no pool we can only speak about customers at least one side has seen — the zero-vs-
|
|
@@ -636,7 +642,7 @@ def _zero_satisfies(op, value):
|
|
| 636 |
return _CMP[op](0.0, value)
|
| 637 |
|
| 638 |
|
| 639 |
-
def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None):
|
| 640 |
"""`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN
|
| 641 |
version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
|
| 642 |
|
|
@@ -669,16 +675,22 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
|
|
| 669 |
f"unresolvable window {window!r} for the {key!r} column: refusing rather than "
|
| 670 |
f"defaulting to all time")
|
| 671 |
related = _related_scope(related, [key])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 672 |
pool = _pool_of(allowed_pids)
|
| 673 |
-
con = _sem._store_con()
|
| 674 |
try:
|
| 675 |
-
totals = _totals(con, key, window, today, team_id, related)
|
| 676 |
guard = None
|
| 677 |
metric = _sem.metrics().get(key, {})
|
| 678 |
if related is not None and not related.refusal and metric.get('agg') == 'ratio':
|
| 679 |
-
guard = _totals(con, metric['denominator'], window, today, team_id, related)
|
| 680 |
finally:
|
| 681 |
-
|
|
|
|
| 682 |
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 683 |
mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
|
| 684 |
nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
|
|
|
|
| 406 |
"which would be a different question with an authoritative-looking answer")
|
| 407 |
con = _sem._store_con()
|
| 408 |
try:
|
| 409 |
+
totals = _totals(con, key, rule.get('window'), today, team_id, related, pool)
|
| 410 |
finally:
|
| 411 |
con.close()
|
| 412 |
# ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision,
|
|
|
|
| 523 |
# compare in Python — the same shape `resolve_pair` uses, and for the same reason.
|
| 524 |
con = _sem._store_con()
|
| 525 |
try:
|
| 526 |
+
totals = _totals(con, key, rule.get('window'), today, team_id, related, pool)
|
| 527 |
finally:
|
| 528 |
con.close()
|
| 529 |
# No pool means we can only speak about customers a base has seen. WITH a pool, everyone
|
|
|
|
| 535 |
|
| 536 |
frm, id_col, agg, where_sql, params = _query_parts(
|
| 537 |
key, rule.get('window'), today, team_id, related)
|
| 538 |
+
if pool is not None:
|
| 539 |
+
where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))"
|
| 540 |
+
params = [*params, list(pool)]
|
| 541 |
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
|
| 542 |
f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
|
| 543 |
|
|
|
|
| 562 |
return _related_answer(matched, related)
|
| 563 |
|
| 564 |
|
| 565 |
+
def _totals(con, key, window, today, team_id, related=None, allowed_pids=None):
|
| 566 |
"""`{pid: value}` for one (measure, window), COMPOSITES included.
|
| 567 |
|
| 568 |
For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
|
|
|
|
| 576 |
"""
|
| 577 |
if is_composite(key):
|
| 578 |
bases, _ = _sem._expand_measures([key])
|
| 579 |
+
parts = {b: _totals(con, b, window, today, team_id, related, allowed_pids) for b in bases}
|
| 580 |
pids = set()
|
| 581 |
for got in parts.values():
|
| 582 |
pids |= set(got)
|
| 583 |
return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
|
| 584 |
for pid in pids}
|
| 585 |
frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related)
|
| 586 |
+
if allowed_pids is not None:
|
| 587 |
+
where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))"
|
| 588 |
+
params = [*params, list(allowed_pids)]
|
| 589 |
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
|
| 590 |
# `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
|
| 591 |
# COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and
|
|
|
|
| 626 |
pool = _pool_of(allowed_pids)
|
| 627 |
con = _sem._store_con()
|
| 628 |
try:
|
| 629 |
+
left = _totals(con, lkey, rule.get('window'), today, team_id, related, pool)
|
| 630 |
+
right = _totals(con, rkey, rhs.get('window'), today, team_id, related, pool)
|
| 631 |
finally:
|
| 632 |
con.close()
|
| 633 |
# With no pool we can only speak about customers at least one side has seen — the zero-vs-
|
|
|
|
| 642 |
return _CMP[op](0.0, value)
|
| 643 |
|
| 644 |
|
| 645 |
+
def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None, *, cur=None):
|
| 646 |
"""`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN
|
| 647 |
version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
|
| 648 |
|
|
|
|
| 675 |
f"unresolvable window {window!r} for the {key!r} column: refusing rather than "
|
| 676 |
f"defaulting to all time")
|
| 677 |
related = _related_scope(related, [key])
|
| 678 |
+
if cur is None:
|
| 679 |
+
from harness.canonical_source import active
|
| 680 |
+
source = active()
|
| 681 |
+
if source is not None:
|
| 682 |
+
cur = source.cur
|
| 683 |
pool = _pool_of(allowed_pids)
|
| 684 |
+
con = cur if cur is not None else _sem._store_con()
|
| 685 |
try:
|
| 686 |
+
totals = _totals(con, key, window, today, team_id, related, pool)
|
| 687 |
guard = None
|
| 688 |
metric = _sem.metrics().get(key, {})
|
| 689 |
if related is not None and not related.refusal and metric.get('agg') == 'ratio':
|
| 690 |
+
guard = _totals(con, metric['denominator'], window, today, team_id, related, pool)
|
| 691 |
finally:
|
| 692 |
+
if cur is None:
|
| 693 |
+
con.close()
|
| 694 |
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 695 |
mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
|
| 696 |
nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
|
platform/harness/mirror_tokens.py
CHANGED
|
@@ -13,7 +13,6 @@ does it. Until then that function RAISES, which `database_change_token` already
|
|
| 13 |
prove unchanged, therefore resolve" -- the safe direction.
|
| 14 |
"""
|
| 15 |
from __future__ import annotations
|
| 16 |
-
import os
|
| 17 |
import re
|
| 18 |
import threading
|
| 19 |
import time
|
|
@@ -28,20 +27,9 @@ _TABLE_ROWS_LOCK = threading.Lock()
|
|
| 28 |
|
| 29 |
|
| 30 |
def _file_marks(cur):
|
| 31 |
-
"""
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
path = row[0] if row else None
|
| 35 |
-
if not path or path == ":memory:":
|
| 36 |
-
return None
|
| 37 |
-
marks = []
|
| 38 |
-
for name in (path, path + ".wal"):
|
| 39 |
-
try:
|
| 40 |
-
stat = os.stat(name)
|
| 41 |
-
marks.append([stat.st_size, stat.st_mtime_ns])
|
| 42 |
-
except OSError:
|
| 43 |
-
marks.append(None)
|
| 44 |
-
return marks
|
| 45 |
|
| 46 |
|
| 47 |
def mirror_change_token(st, database):
|
|
|
|
| 13 |
prove unchanged, therefore resolve" -- the safe direction.
|
| 14 |
"""
|
| 15 |
from __future__ import annotations
|
|
|
|
| 16 |
import re
|
| 17 |
import threading
|
| 18 |
import time
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
def _file_marks(cur):
|
| 30 |
+
"""The shared actual-file and WAL revision, or None for an unnamed mirror."""
|
| 31 |
+
from harness import datastore
|
| 32 |
+
return datastore.mirror_file_marks(cur)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def mirror_change_token(st, database):
|
platform/harness/semantic.py
CHANGED
|
@@ -256,11 +256,17 @@ def render_scope(sql):
|
|
| 256 |
return out
|
| 257 |
|
| 258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
@functools.lru_cache(maxsize=1)
|
| 260 |
def _model():
|
| 261 |
topics, metrics = {}, {}
|
| 262 |
for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
|
| 263 |
-
d =
|
| 264 |
if not d or "key" not in d:
|
| 265 |
raise ModelError(f"topic file {f.name} lacks a 'key'")
|
| 266 |
if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
|
|
@@ -268,7 +274,7 @@ def _model():
|
|
| 268 |
# a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises
|
| 269 |
# a clear error if a live resolution is attempted on it
|
| 270 |
topics[d["key"]] = d
|
| 271 |
-
metric_docs = [(f.name,
|
| 272 |
for f in sorted((MODEL_DIR / "metrics").glob("*.yml"))]
|
| 273 |
# Metric libraries may own new fact topics without changing legacy topic definitions.
|
| 274 |
for name, doc in metric_docs:
|
|
@@ -1652,7 +1658,7 @@ def _check_shipped_links(date_from, date_to):
|
|
| 1652 |
def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
|
| 1653 |
team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
|
| 1654 |
filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None,
|
| 1655 |
-
*, runtime=None, user=None):
|
| 1656 |
"""The semantic query over the tenant store — the engine behind the Analyst's
|
| 1657 |
run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
|
| 1658 |
t = _model()["topics"].get(topic)
|
|
@@ -1810,6 +1816,16 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
|
|
| 1810 |
where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
|
| 1811 |
params.extend(ids)
|
| 1812 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1813 |
# The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
|
| 1814 |
# this is the nested, 11-operator contract the table UI emits. Compiled by
|
| 1815 |
# harness/filter_sql, which is held in lock-step with the TS engine and the validator by
|
|
@@ -1846,13 +1862,14 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
|
|
| 1846 |
# `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap).
|
| 1847 |
sql += f" LIMIT {limit + 1}"
|
| 1848 |
|
| 1849 |
-
con = _store_con()
|
| 1850 |
try:
|
| 1851 |
-
|
| 1852 |
-
cols = [d[0] for d in
|
| 1853 |
-
rows = [dict(zip(cols, r)) for r in
|
| 1854 |
finally:
|
| 1855 |
-
|
|
|
|
| 1856 |
for row in rows: # ratio/derived post-compute per group
|
| 1857 |
row.update(foreign_vals) # scalar cross-topic components
|
| 1858 |
for k in requested:
|
|
@@ -2391,7 +2408,8 @@ def _one_binding_offer(topic, b, strict=False, ready=None):
|
|
| 2391 |
|
| 2392 |
|
| 2393 |
def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None,
|
| 2394 |
-
exclude_services=False, offer=None, related=None
|
|
|
|
| 2395 |
"""`{group key: {measure key: value}}` for an entity topic's lookback columns.
|
| 2396 |
|
| 2397 |
ONE grouped query for all `keys` together — they share a topic, a scope and a window, so
|
|
@@ -2421,6 +2439,10 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
|
|
| 2421 |
`_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin
|
| 2422 |
nobody measured, under a row the user has no reason to doubt.
|
| 2423 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2424 |
offer = offer if offer is not None else entity_measures(topic)
|
| 2425 |
by_key = {m["key"]: m for m in offer}
|
| 2426 |
want = [k for k in keys if k in by_key]
|
|
@@ -2439,7 +2461,8 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
|
|
| 2439 |
for (_src, _dim, _variant), ks in groups.items():
|
| 2440 |
answer = entity_measure_values(
|
| 2441 |
topic, ks, date_from=date_from, date_to=date_to, team_id=team_id,
|
| 2442 |
-
exclude_services=exclude_services, offer=offer, related=related
|
|
|
|
| 2443 |
if related is not None:
|
| 2444 |
merged.refusals.extend(getattr(answer, 'refusals', ()))
|
| 2445 |
for gk, cell in answer.items():
|
|
@@ -2458,7 +2481,9 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
|
|
| 2458 |
base_of = {k: (by_key[k].get("base_key") or k) for k in want}
|
| 2459 |
res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from,
|
| 2460 |
date_to=date_to, team_id=team_id, exclude_services=exclude_services,
|
| 2461 |
-
limit=MAX_GROUPS, scope_variant=scope_variant, related=related
|
|
|
|
|
|
|
| 2462 |
if res.get("refusals"):
|
| 2463 |
from core import relations
|
| 2464 |
return relations.MeasureAnswer(refusals=[dict(marker, measure=k)
|
|
@@ -2532,7 +2557,7 @@ def entity_measure_leaves(nodes, admitted):
|
|
| 2532 |
|
| 2533 |
|
| 2534 |
def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None,
|
| 2535 |
-
exclude_services=False):
|
| 2536 |
"""`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row.
|
| 2537 |
|
| 2538 |
⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty
|
|
@@ -2578,7 +2603,8 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe
|
|
| 2578 |
else:
|
| 2579 |
vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1],
|
| 2580 |
team_id=team_id, exclude_services=exclude_services,
|
| 2581 |
-
offer=offer
|
|
|
|
| 2582 |
cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c}
|
| 2583 |
return cache[sig]
|
| 2584 |
|
|
@@ -2595,7 +2621,7 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe
|
|
| 2595 |
left = values_for(mkey, rule.get("window"))
|
| 2596 |
if left is None:
|
| 2597 |
continue
|
| 2598 |
-
pool = list(keys_by_id
|
| 2599 |
zero = m["empty"] == "zero"
|
| 2600 |
rhs = rule.get("rhs")
|
| 2601 |
try:
|
|
|
|
| 256 |
return out
|
| 257 |
|
| 258 |
|
| 259 |
+
def _load_model_yaml(text):
|
| 260 |
+
"""Parse model documents with the safe C loader when available."""
|
| 261 |
+
loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
|
| 262 |
+
return yaml.load(text, Loader=loader)
|
| 263 |
+
|
| 264 |
+
|
| 265 |
@functools.lru_cache(maxsize=1)
|
| 266 |
def _model():
|
| 267 |
topics, metrics = {}, {}
|
| 268 |
for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
|
| 269 |
+
d = _load_model_yaml(f.read_text(encoding="utf-8"))
|
| 270 |
if not d or "key" not in d:
|
| 271 |
raise ModelError(f"topic file {f.name} lacks a 'key'")
|
| 272 |
if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
|
|
|
|
| 274 |
# a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises
|
| 275 |
# a clear error if a live resolution is attempted on it
|
| 276 |
topics[d["key"]] = d
|
| 277 |
+
metric_docs = [(f.name, _load_model_yaml(f.read_text(encoding="utf-8")))
|
| 278 |
for f in sorted((MODEL_DIR / "metrics").glob("*.yml"))]
|
| 279 |
# Metric libraries may own new fact topics without changing legacy topic definitions.
|
| 280 |
for name, doc in metric_docs:
|
|
|
|
| 1658 |
def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
|
| 1659 |
team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
|
| 1660 |
filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None,
|
| 1661 |
+
*, runtime=None, user=None, cur=None, group_keys=None):
|
| 1662 |
"""The semantic query over the tenant store — the engine behind the Analyst's
|
| 1663 |
run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
|
| 1664 |
t = _model()["topics"].get(topic)
|
|
|
|
| 1816 |
where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
|
| 1817 |
params.extend(ids)
|
| 1818 |
|
| 1819 |
+
if group_keys is not None:
|
| 1820 |
+
if len(gb) != 1:
|
| 1821 |
+
raise ModelError('Selected entity keys require exactly one grouping dimension')
|
| 1822 |
+
values = list(group_keys)
|
| 1823 |
+
if values:
|
| 1824 |
+
where.append(f"{dims[gb[0]]['col']} IN ({','.join('?' for _ in values)})")
|
| 1825 |
+
params.extend(values)
|
| 1826 |
+
else:
|
| 1827 |
+
where.append('FALSE')
|
| 1828 |
+
|
| 1829 |
# The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
|
| 1830 |
# this is the nested, 11-operator contract the table UI emits. Compiled by
|
| 1831 |
# harness/filter_sql, which is held in lock-step with the TS engine and the validator by
|
|
|
|
| 1862 |
# `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap).
|
| 1863 |
sql += f" LIMIT {limit + 1}"
|
| 1864 |
|
| 1865 |
+
con = cur if cur is not None else _store_con()
|
| 1866 |
try:
|
| 1867 |
+
result = con.execute(sql, params)
|
| 1868 |
+
cols = [d[0] for d in result.description]
|
| 1869 |
+
rows = [dict(zip(cols, r)) for r in result.fetchall()]
|
| 1870 |
finally:
|
| 1871 |
+
if cur is None:
|
| 1872 |
+
con.close()
|
| 1873 |
for row in rows: # ratio/derived post-compute per group
|
| 1874 |
row.update(foreign_vals) # scalar cross-topic components
|
| 1875 |
for k in requested:
|
|
|
|
| 2408 |
|
| 2409 |
|
| 2410 |
def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None,
|
| 2411 |
+
exclude_services=False, offer=None, related=None, *, group_keys=None,
|
| 2412 |
+
cur=None):
|
| 2413 |
"""`{group key: {measure key: value}}` for an entity topic's lookback columns.
|
| 2414 |
|
| 2415 |
ONE grouped query for all `keys` together — they share a topic, a scope and a window, so
|
|
|
|
| 2439 |
`_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin
|
| 2440 |
nobody measured, under a row the user has no reason to doubt.
|
| 2441 |
"""
|
| 2442 |
+
if group_keys is not None:
|
| 2443 |
+
group_keys = tuple(group_keys)
|
| 2444 |
+
if not group_keys:
|
| 2445 |
+
return {}
|
| 2446 |
offer = offer if offer is not None else entity_measures(topic)
|
| 2447 |
by_key = {m["key"]: m for m in offer}
|
| 2448 |
want = [k for k in keys if k in by_key]
|
|
|
|
| 2461 |
for (_src, _dim, _variant), ks in groups.items():
|
| 2462 |
answer = entity_measure_values(
|
| 2463 |
topic, ks, date_from=date_from, date_to=date_to, team_id=team_id,
|
| 2464 |
+
exclude_services=exclude_services, offer=offer, related=related,
|
| 2465 |
+
group_keys=group_keys, cur=cur)
|
| 2466 |
if related is not None:
|
| 2467 |
merged.refusals.extend(getattr(answer, 'refusals', ()))
|
| 2468 |
for gk, cell in answer.items():
|
|
|
|
| 2481 |
base_of = {k: (by_key[k].get("base_key") or k) for k in want}
|
| 2482 |
res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from,
|
| 2483 |
date_to=date_to, team_id=team_id, exclude_services=exclude_services,
|
| 2484 |
+
limit=MAX_GROUPS, scope_variant=scope_variant, related=related,
|
| 2485 |
+
**({'group_keys': group_keys} if group_keys is not None else {}),
|
| 2486 |
+
**({'cur': cur} if cur is not None else {}))
|
| 2487 |
if res.get("refusals"):
|
| 2488 |
from core import relations
|
| 2489 |
return relations.MeasureAnswer(refusals=[dict(marker, measure=k)
|
|
|
|
| 2557 |
|
| 2558 |
|
| 2559 |
def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None,
|
| 2560 |
+
exclude_services=False, *, cur=None):
|
| 2561 |
"""`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row.
|
| 2562 |
|
| 2563 |
⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty
|
|
|
|
| 2603 |
else:
|
| 2604 |
vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1],
|
| 2605 |
team_id=team_id, exclude_services=exclude_services,
|
| 2606 |
+
offer=offer, group_keys=keys_by_id,
|
| 2607 |
+
**({'cur': cur} if cur is not None else {}))
|
| 2608 |
cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c}
|
| 2609 |
return cache[sig]
|
| 2610 |
|
|
|
|
| 2621 |
left = values_for(mkey, rule.get("window"))
|
| 2622 |
if left is None:
|
| 2623 |
continue
|
| 2624 |
+
pool = list(left if keys_by_id is None else keys_by_id)
|
| 2625 |
zero = m["empty"] == "zero"
|
| 2626 |
rhs = rule.get("rhs")
|
| 2627 |
try:
|
platform/modules/ar.py
CHANGED
|
@@ -10,7 +10,7 @@ import math
|
|
| 10 |
import datetime as dt
|
| 11 |
from pathlib import Path
|
| 12 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 13 |
-
|
| 14 |
import core.periods as P
|
| 15 |
import core.store as store
|
| 16 |
|
|
@@ -32,7 +32,7 @@ DTP_KEY = 'ar_days_to_pay'
|
|
| 32 |
DTP_MAX_AGE_HOURS = 20
|
| 33 |
|
| 34 |
|
| 35 |
-
def days_to_pay_snapshot():
|
| 36 |
"""`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises.
|
| 37 |
|
| 38 |
Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the
|
|
@@ -40,8 +40,11 @@ def days_to_pay_snapshot():
|
|
| 40 |
derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about.
|
| 41 |
"""
|
| 42 |
try:
|
| 43 |
-
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
| 45 |
out = {}
|
| 46 |
for k, v in days.items():
|
| 47 |
try:
|
|
@@ -123,15 +126,18 @@ def _open_docs(t=None):
|
|
| 123 |
['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
|
| 124 |
'amount_total', 'amount_residual_signed'])
|
| 125 |
today = t or P.today()
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
def _ltm_invoiced(t=None):
|
|
|
|
| 10 |
import datetime as dt
|
| 11 |
from pathlib import Path
|
| 12 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 13 |
+
from harness.canonical_source import odoo as O
|
| 14 |
import core.periods as P
|
| 15 |
import core.store as store
|
| 16 |
|
|
|
|
| 32 |
DTP_MAX_AGE_HOURS = 20
|
| 33 |
|
| 34 |
|
| 35 |
+
def days_to_pay_snapshot(*, pids=None, st=None):
|
| 36 |
"""`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises.
|
| 37 |
|
| 38 |
Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the
|
|
|
|
| 40 |
derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about.
|
| 41 |
"""
|
| 42 |
try:
|
| 43 |
+
if pids is not None and st is not None:
|
| 44 |
+
days = st.selected_values(DTP_KEY, ['days'], pids)
|
| 45 |
+
else:
|
| 46 |
+
snap = store.get(DTP_KEY) or {}
|
| 47 |
+
days = snap.get('days') or {}
|
| 48 |
out = {}
|
| 49 |
for k, v in days.items():
|
| 50 |
try:
|
|
|
|
| 126 |
['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
|
| 127 |
'amount_total', 'amount_residual_signed'])
|
| 128 |
today = t or P.today()
|
| 129 |
+
def enriched():
|
| 130 |
+
for d in docs:
|
| 131 |
+
due = d.get('invoice_date_due')
|
| 132 |
+
try:
|
| 133 |
+
d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0
|
| 134 |
+
except Exception:
|
| 135 |
+
d['days_overdue'] = 0
|
| 136 |
+
d['bucket'] = _bucket(d['days_overdue'])
|
| 137 |
+
d['open'] = d.get('amount_residual_signed') or 0.0
|
| 138 |
+
yield d
|
| 139 |
+
from harness.canonical_source import active
|
| 140 |
+
return enriched() if active() is not None else list(enriched())
|
| 141 |
|
| 142 |
|
| 143 |
def _ltm_invoiced(t=None):
|
platform/modules/backorders.py
CHANGED
|
@@ -11,7 +11,7 @@ to dealers. BU-filterable.
|
|
| 11 |
"""
|
| 12 |
import datetime as dt
|
| 13 |
|
| 14 |
-
|
| 15 |
import core.periods as P
|
| 16 |
import modules.procurement as proc
|
| 17 |
|
|
@@ -72,6 +72,9 @@ def sku_backlog(t=None, team_id=None):
|
|
| 72 |
groups = O.read_group('sale.order.line', domain,
|
| 73 |
['product_uom_qty:sum', 'qty_delivered:sum'], ['product_id'],
|
| 74 |
lazy=False)
|
|
|
|
|
|
|
|
|
|
| 75 |
out = {}
|
| 76 |
for group in groups:
|
| 77 |
pid = O.m2o_id(group.get('product_id'))
|
|
@@ -85,7 +88,8 @@ def sku_backlog(t=None, team_id=None):
|
|
| 85 |
|
| 86 |
open_by_pid = _by_product(list(common))
|
| 87 |
pre_by_pid = _by_product(list(common) + [('order_id.commitment_date', '>', end_of_day)])
|
| 88 |
-
|
|
|
|
| 89 |
codes = {}
|
| 90 |
for ch in _chunk(pids, 500):
|
| 91 |
for product in O.search_read('product.product',
|
|
@@ -101,7 +105,8 @@ def sku_backlog(t=None, team_id=None):
|
|
| 101 |
out[code] = out.get(code, 0.0) + qty
|
| 102 |
return out
|
| 103 |
|
| 104 |
-
pre_by_code, open_by_code =
|
|
|
|
| 105 |
report = {
|
| 106 |
'as_of': t.isoformat(),
|
| 107 |
'team_ids': teams,
|
|
|
|
| 11 |
"""
|
| 12 |
import datetime as dt
|
| 13 |
|
| 14 |
+
from harness.canonical_source import odoo as O
|
| 15 |
import core.periods as P
|
| 16 |
import modules.procurement as proc
|
| 17 |
|
|
|
|
| 72 |
groups = O.read_group('sale.order.line', domain,
|
| 73 |
['product_uom_qty:sum', 'qty_delivered:sum'], ['product_id'],
|
| 74 |
lazy=False)
|
| 75 |
+
from harness.canonical_source import active
|
| 76 |
+
if active() is not None:
|
| 77 |
+
return active().backlog(groups)
|
| 78 |
out = {}
|
| 79 |
for group in groups:
|
| 80 |
pid = O.m2o_id(group.get('product_id'))
|
|
|
|
| 88 |
|
| 89 |
open_by_pid = _by_product(list(common))
|
| 90 |
pre_by_pid = _by_product(list(common) + [('order_id.commitment_date', '>', end_of_day)])
|
| 91 |
+
from harness.canonical_source import active
|
| 92 |
+
pids = [] if active() is not None else sorted(set(open_by_pid) | set(pre_by_pid))
|
| 93 |
codes = {}
|
| 94 |
for ch in _chunk(pids, 500):
|
| 95 |
for product in O.search_read('product.product',
|
|
|
|
| 105 |
out[code] = out.get(code, 0.0) + qty
|
| 106 |
return out
|
| 107 |
|
| 108 |
+
pre_by_code, open_by_code = ((pre_by_pid, open_by_pid) if active() is not None else
|
| 109 |
+
(_by_code(pre_by_pid), _by_code(open_by_pid)))
|
| 110 |
report = {
|
| 111 |
'as_of': t.isoformat(),
|
| 112 |
'team_ids': teams,
|
platform/modules/customer_data.py
CHANGED
|
@@ -16,7 +16,7 @@ no link (owner/CFO/admin) see the whole book.
|
|
| 16 |
import re
|
| 17 |
from urllib.parse import quote_plus
|
| 18 |
|
| 19 |
-
|
| 20 |
import core.periods as P
|
| 21 |
import core.store as store
|
| 22 |
import core.table_store as table_store
|
|
@@ -117,7 +117,7 @@ AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60',
|
|
| 117 |
'61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
|
| 118 |
|
| 119 |
|
| 120 |
-
def _ar_attrs(t=None):
|
| 121 |
"""{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
|
| 122 |
|
| 123 |
Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
|
|
@@ -160,7 +160,9 @@ def _ar_attrs(t=None):
|
|
| 160 |
}
|
| 161 |
except Exception:
|
| 162 |
pass
|
| 163 |
-
|
|
|
|
|
|
|
| 164 |
# A customer with an AR row but no settled invoice in the window keeps `None` — blank,
|
| 165 |
# not 0. "Pays in 0 days" is a claim, and a false one.
|
| 166 |
out.setdefault(pid, {})['days_to_pay'] = avg
|
|
@@ -197,6 +199,10 @@ def _mix_attrs(pids=None, t=None):
|
|
| 197 |
keep = set(pids) if pids else None
|
| 198 |
rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
|
| 199 |
['order_partner_id', 'product_id'], lazy=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
|
| 201 |
prod_ids.discard(None)
|
| 202 |
cats = {}
|
|
@@ -502,6 +508,28 @@ def _dba_attrs(pids, t):
|
|
| 502 |
return {}
|
| 503 |
|
| 504 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
def _book_pids(team_id=None, agent_pids=None):
|
| 506 |
"""⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
|
| 507 |
|
|
@@ -548,36 +576,7 @@ def _book_pids(team_id=None, agent_pids=None):
|
|
| 548 |
the whole permissioning model rests on.
|
| 549 |
"""
|
| 550 |
try:
|
| 551 |
-
|
| 552 |
-
# Scoped: partners with confirmed orders in THIS BU, ever. `read_group` on partner_id
|
| 553 |
-
# rather than a search_read of orders — the group is the distinct set, and it is one
|
| 554 |
-
# round trip instead of paging tens of thousands of order rows.
|
| 555 |
-
rows = O.read_group('sale.order',
|
| 556 |
-
[('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
|
| 557 |
-
['partner_id'], ['partner_id'], lazy=False)
|
| 558 |
-
book = {O.m2o_id(r.get('partner_id')) for r in rows}
|
| 559 |
-
book.discard(None)
|
| 560 |
-
else:
|
| 561 |
-
rank = O.search_read('res.partner',
|
| 562 |
-
[('customer_rank', '>', 0),
|
| 563 |
-
('active', 'in', [True, False])],
|
| 564 |
-
['id'], limit=200000)
|
| 565 |
-
# `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or
|
| 566 |
-
# have had its rank reset, and an account that demonstrably bought from us belongs in
|
| 567 |
-
# the book whatever its flags say now. An ORDER is a fact; a rank is a setting.
|
| 568 |
-
rows = O.read_group('sale.order',
|
| 569 |
-
[('state', 'in', ('sale', 'done'))],
|
| 570 |
-
['partner_id'], ['partner_id'], lazy=False)
|
| 571 |
-
ever = {O.m2o_id(r.get('partner_id')) for r in rows}
|
| 572 |
-
ever.discard(None)
|
| 573 |
-
docs = O.read_group(
|
| 574 |
-
'account.move',
|
| 575 |
-
[('state', '=', 'posted'),
|
| 576 |
-
('move_type', 'in', ('out_invoice', 'out_refund'))],
|
| 577 |
-
['partner_id'], ['partner_id'], lazy=False)
|
| 578 |
-
documented = {O.m2o_id(r.get('partner_id')) for r in docs}
|
| 579 |
-
documented.discard(None)
|
| 580 |
-
book = {r['id'] for r in rank} | ever | documented
|
| 581 |
# An agent filter NARROWS the book to that agent's partners — never widens it. The
|
| 582 |
# intersection is what keeps an agent-login user inside their own book while still
|
| 583 |
# gaining every no-order account assigned to them, which is the whole point of item 27.
|
|
@@ -630,8 +629,85 @@ def identity_pids(agent_name=None, team_id=None, t=None):
|
|
| 630 |
return set(_pool_identity(agent_name, team_id, t)["pids"])
|
| 631 |
|
| 632 |
|
| 633 |
-
def
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
t = identity["today"]
|
| 636 |
pids = identity["agentPids"]
|
| 637 |
this, last, ltm = identity["this"], identity["last"], identity["ltm"]
|
|
@@ -644,8 +720,9 @@ def _pool_build(agent_name, team_id, limit, fast=False):
|
|
| 644 |
key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
|
| 645 |
+ (last.get(p) or {}).get('rev', 0.0)))
|
| 646 |
all_pids = set(ranked[:limit])
|
| 647 |
-
attrs = cust._partner_attrs(list(all_pids))
|
| 648 |
-
invoice_days = {} if fast
|
|
|
|
| 649 |
if fast:
|
| 650 |
# C1 fast phase: the slow families stay empty — their columns render blank behind the
|
| 651 |
# toolbar's partial indicator until the full build swaps in.
|
|
@@ -654,21 +731,25 @@ def _pool_build(agent_name, team_id, limit, fast=False):
|
|
| 654 |
# The three families the owner asked for (2026-07-27). Each degrades to {} rather than
|
| 655 |
# raising: a customer table that cannot render because AR is momentarily unreachable is
|
| 656 |
# a worse failure than one with a blank column.
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
|
|
|
|
|
|
|
|
|
| 660 |
# W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
|
| 661 |
# read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
|
| 662 |
# client's map view discloses the count.
|
| 663 |
-
geo_a = map_mod.coords_for(list(all_pids))
|
|
|
|
| 664 |
# C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
|
| 665 |
-
dba_a = _dba_attrs(all_pids, t)
|
| 666 |
# customers whose only activity sits in the older half of the 24-month cadence window
|
| 667 |
# appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
|
| 668 |
# the win-back targets this union exists for, would render as '?')
|
| 669 |
unnamed = [pid for pid in all_pids
|
| 670 |
if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
|
| 671 |
-
or (ltm.get(pid) or {}).get('name'))]
|
| 672 |
extra_names = ({r['id']: r['name'] for r in
|
| 673 |
O.search_read('res.partner', [('id', 'in', unnamed),
|
| 674 |
('active', 'in', [True, False])], ['name'])}
|
|
@@ -763,7 +844,7 @@ def _pool_build(agent_name, team_id, limit, fast=False):
|
|
| 763 |
# I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
|
| 764 |
# Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
|
| 765 |
# so letting it write would mark ~1,450 living customers archived on every cold load.
|
| 766 |
-
if not fast:
|
| 767 |
rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
|
| 768 |
rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
|
| 769 |
return {'rows': rows, 'total': total}
|
|
@@ -772,7 +853,7 @@ def _pool_build(agent_name, team_id, limit, fast=False):
|
|
| 772 |
LEDGER_KEY = 'customer_ledger'
|
| 773 |
|
| 774 |
|
| 775 |
-
def _reconcile_ledger(rows, total, agent_name, team_id):
|
| 776 |
"""Remember every customer we have ever seen, and keep serving the ones Odoo stopped
|
| 777 |
returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
|
| 778 |
so stored in our platform even if deleted in Odoo").
|
|
@@ -791,14 +872,14 @@ def _reconcile_ledger(rows, total, agent_name, team_id):
|
|
| 791 |
they cannot prove they should see.
|
| 792 |
"""
|
| 793 |
try:
|
| 794 |
-
ledger = dict(store.get(LEDGER_KEY) or {})
|
| 795 |
except Exception:
|
| 796 |
return rows, total # a ledger we cannot read must not break the table
|
| 797 |
today = str(P.today())
|
| 798 |
live = {r['pid'] for r in rows}
|
| 799 |
unscoped = agent_name is None and team_id is None
|
| 800 |
|
| 801 |
-
if unscoped:
|
| 802 |
for r in rows:
|
| 803 |
e = ledger.get(str(r['pid'])) or {}
|
| 804 |
e.update({'name': r['customer'], 'last_seen': today})
|
|
@@ -849,7 +930,7 @@ def _reconcile_ledger(rows, total, agent_name, team_id):
|
|
| 849 |
'days_since': None, 'days_since_invoice': None,
|
| 850 |
'typical_gap_days': None, 'overdue_days': None,
|
| 851 |
})
|
| 852 |
-
if store.available():
|
| 853 |
try:
|
| 854 |
store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
|
| 855 |
except Exception:
|
|
|
|
| 16 |
import re
|
| 17 |
from urllib.parse import quote_plus
|
| 18 |
|
| 19 |
+
from harness.canonical_source import odoo as O
|
| 20 |
import core.periods as P
|
| 21 |
import core.store as store
|
| 22 |
import core.table_store as table_store
|
|
|
|
| 117 |
'61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
|
| 118 |
|
| 119 |
|
| 120 |
+
def _ar_attrs(t=None, *, pids=None, st=None):
|
| 121 |
"""{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
|
| 122 |
|
| 123 |
Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
|
|
|
|
| 160 |
}
|
| 161 |
except Exception:
|
| 162 |
pass
|
| 163 |
+
snapshot = (ar.days_to_pay_snapshot() if pids is None and st is None else
|
| 164 |
+
ar.days_to_pay_snapshot(pids=pids, st=st))
|
| 165 |
+
for pid, avg in snapshot.items():
|
| 166 |
# A customer with an AR row but no settled invoice in the window keeps `None` — blank,
|
| 167 |
# not 0. "Pays in 0 days" is a claim, and a false one.
|
| 168 |
out.setdefault(pid, {})['days_to_pay'] = avg
|
|
|
|
| 199 |
keep = set(pids) if pids else None
|
| 200 |
rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
|
| 201 |
['order_partner_id', 'product_id'], lazy=False)
|
| 202 |
+
from harness.canonical_source import active
|
| 203 |
+
source = active()
|
| 204 |
+
if source is not None:
|
| 205 |
+
return source.customer_mix(rows)
|
| 206 |
prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
|
| 207 |
prod_ids.discard(None)
|
| 208 |
cats = {}
|
|
|
|
| 508 |
return {}
|
| 509 |
|
| 510 |
|
| 511 |
+
def _book_records(team_id=None):
|
| 512 |
+
"""Yield the canonical rank, confirmed-order and posted-document identity union."""
|
| 513 |
+
if team_id:
|
| 514 |
+
groups = O.read_group('sale.order',
|
| 515 |
+
[('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
|
| 516 |
+
['partner_id'], ['partner_id'], lazy=False)
|
| 517 |
+
for row in groups:
|
| 518 |
+
pid = O.m2o_id(row.get('partner_id'))
|
| 519 |
+
if pid is not None:
|
| 520 |
+
yield {'id': pid}
|
| 521 |
+
return
|
| 522 |
+
yield from O.search_read('res.partner', [('customer_rank', '>', 0),
|
| 523 |
+
('active', 'in', [True, False])], ['id'], limit=200000)
|
| 524 |
+
for model, domain in [
|
| 525 |
+
('sale.order', [('state', 'in', ('sale', 'done'))]),
|
| 526 |
+
('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))])]:
|
| 527 |
+
for row in O.read_group(model, domain, ['partner_id'], ['partner_id'], lazy=False):
|
| 528 |
+
pid = O.m2o_id(row.get('partner_id'))
|
| 529 |
+
if pid is not None:
|
| 530 |
+
yield {'id': pid}
|
| 531 |
+
|
| 532 |
+
|
| 533 |
def _book_pids(team_id=None, agent_pids=None):
|
| 534 |
"""⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
|
| 535 |
|
|
|
|
| 576 |
the whole permissioning model rests on.
|
| 577 |
"""
|
| 578 |
try:
|
| 579 |
+
book = {r['id'] for r in _book_records(team_id)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
# An agent filter NARROWS the book to that agent's partners — never widens it. The
|
| 581 |
# intersection is what keeps an agent-login user inside their own book while still
|
| 582 |
# gaining every no-order account assigned to them, which is the whole point of item 27.
|
|
|
|
| 629 |
return set(_pool_identity(agent_name, team_id, t)["pids"])
|
| 630 |
|
| 631 |
|
| 632 |
+
def source_batches(cur, *, st, agent_name=None, team_id=None, ids=None, needed=None,
|
| 633 |
+
ledger_batches=()):
|
| 634 |
+
"""Stream the canonical customer identity union and project only one batch at a time."""
|
| 635 |
+
from harness.canonical_source import bounded, BATCH
|
| 636 |
+
with bounded(cur, 'identity' if ids is None else 'customer', ids or [], st) as identity:
|
| 637 |
+
live = identity.snapshot(_book_records(team_id))
|
| 638 |
+
where = ''
|
| 639 |
+
if agent_name and agent_name not in {'All', 'All agents'}:
|
| 640 |
+
members = identity.agent_members(agent_name)
|
| 641 |
+
where = (f"WHERE json_extract_string(body, '/id') IN "
|
| 642 |
+
f"(SELECT json_extract_string(body, '/id') FROM \"{members.table}\")")
|
| 643 |
+
population = identity.snapshot({'id': r[0], 'live': True} for r in ())
|
| 644 |
+
cur.execute(f'INSERT INTO "{population.table}" SELECT row_number() OVER (), '
|
| 645 |
+
f"json_object('id', try_cast(json_extract_string(body, '/id') AS BIGINT), 'live', true) "
|
| 646 |
+
f'FROM "{live.table}" {where} GROUP BY json_extract_string(body, \'/id\')')
|
| 647 |
+
if not agent_name and team_id is None:
|
| 648 |
+
for batch in ledger_batches:
|
| 649 |
+
records = []
|
| 650 |
+
for key, entry in batch:
|
| 651 |
+
try:
|
| 652 |
+
pid = int(key)
|
| 653 |
+
except (TypeError, ValueError):
|
| 654 |
+
continue
|
| 655 |
+
if ids is None or pid in ids:
|
| 656 |
+
records.append({'id': pid, 'live': False, 'entry': entry})
|
| 657 |
+
if records:
|
| 658 |
+
cur.execute(f'INSERT INTO "{population.table}" SELECT 0, j FROM unnest(?::JSON[]) r(j) '
|
| 659 |
+
f"WHERE try_cast(json_extract_string(j, '/id') AS BIGINT) NOT IN "
|
| 660 |
+
f"(SELECT try_cast(json_extract_string(body, '/id') AS BIGINT) FROM \"{live.table}\")",
|
| 661 |
+
[[__import__('json').dumps(r) for r in records]])
|
| 662 |
+
records.clear()
|
| 663 |
+
# Sort the complete live/retained union once; pages seek a physical key, not JSON.
|
| 664 |
+
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
|
| 665 |
+
f"SELECT ordinal, body, try_cast(json_extract_string(body, '/id') AS BIGINT) pid "
|
| 666 |
+
f'FROM "{population.table}" ORDER BY pid')
|
| 667 |
+
last = None
|
| 668 |
+
while True:
|
| 669 |
+
rows = cur.execute(f"SELECT pid, "
|
| 670 |
+
f"json_extract_string(body, '/live') = 'true', json_extract(body, '/entry') "
|
| 671 |
+
f'FROM "{population.table}" WHERE (?::BIGINT IS NULL OR pid > ?) ORDER BY pid LIMIT ?',
|
| 672 |
+
[last, last, BATCH]).fetchall()
|
| 673 |
+
if not rows:
|
| 674 |
+
return
|
| 675 |
+
live_ids = [pid for pid, present, _ in rows if present]
|
| 676 |
+
ledger = {str(pid): __import__('json').loads(entry) for pid, present, entry in rows if not present}
|
| 677 |
+
with bounded(cur, 'customer', live_ids, st):
|
| 678 |
+
if needed is not None and not needed:
|
| 679 |
+
projected = [dict(cust.audit_attr_defaults(), pid=pid) for pid in live_ids]
|
| 680 |
+
else:
|
| 681 |
+
projected = _pool_build(agent_name, team_id, None, selected=live_ids,
|
| 682 |
+
needed=needed, st=st)['rows'] if live_ids else []
|
| 683 |
+
projected, _ = _reconcile_ledger(projected, len(projected), agent_name, team_id,
|
| 684 |
+
ledger=ledger, write=False)
|
| 685 |
+
yield projected
|
| 686 |
+
last = rows[-1][0]
|
| 687 |
+
projected.clear()
|
| 688 |
+
rows.clear()
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def _pool_build(agent_name, team_id, limit, fast=False, *, selected=None, needed=None, st=None):
|
| 692 |
+
from harness.canonical_source import active
|
| 693 |
+
source = active()
|
| 694 |
+
if selected is None:
|
| 695 |
+
identity = _pool_identity(agent_name, team_id)
|
| 696 |
+
else:
|
| 697 |
+
t = P.today()
|
| 698 |
+
pids = set(selected)
|
| 699 |
+
wants = lambda keys: needed is None or bool(set(keys) & set(needed))
|
| 700 |
+
yf, yt = P.ytd(t)
|
| 701 |
+
lf, lt = P.ytd_last_year(t)
|
| 702 |
+
mf, mt = P.ltm(t)
|
| 703 |
+
identity = {'today': t, 'agentPids': pids, 'pids': pids,
|
| 704 |
+
'this': cust._cust_rev(yf, yt, team_id, pids) if wants(
|
| 705 |
+
['customer', 'status', 'revenue_ytd', 'yoy_pct', 'at_risk']) else {},
|
| 706 |
+
'last': cust._cust_rev(lf, lt, team_id, pids) if wants(
|
| 707 |
+
['customer', 'status', 'revenue_ly', 'yoy_pct', 'at_risk']) else {},
|
| 708 |
+
'ltm': cust._cust_rev(mf, mt, team_id, pids) if wants(['customer', 'ltm_rev']) else {},
|
| 709 |
+
'cadence': cust._cadence_bulk(t, team_id, agent_pids=pids) if wants([
|
| 710 |
+
'orders_24m', 'aov', 'last_order', 'days_since', 'typical_gap_days', 'overdue_days']) else {}}
|
| 711 |
t = identity["today"]
|
| 712 |
pids = identity["agentPids"]
|
| 713 |
this, last, ltm = identity["this"], identity["last"], identity["ltm"]
|
|
|
|
| 720 |
key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
|
| 721 |
+ (last.get(p) or {}).get('rev', 0.0)))
|
| 722 |
all_pids = set(ranked[:limit])
|
| 723 |
+
attrs = cust._partner_attrs(list(all_pids), **({'needed': needed} if selected is not None else {}))
|
| 724 |
+
invoice_days = {} if fast or (needed is not None and 'days_since_invoice' not in needed) \
|
| 725 |
+
else cust._invoice_cadence_bulk(t, partner_ids=all_pids)
|
| 726 |
if fast:
|
| 727 |
# C1 fast phase: the slow families stay empty — their columns render blank behind the
|
| 728 |
# toolbar's partial indicator until the full build swaps in.
|
|
|
|
| 731 |
# The three families the owner asked for (2026-07-27). Each degrades to {} rather than
|
| 732 |
# raising: a customer table that cannot render because AR is momentarily unreachable is
|
| 733 |
# a worse failure than one with a blank column.
|
| 734 |
+
wants = lambda keys: needed is None or bool(set(keys) & set(needed))
|
| 735 |
+
ar_a = (_ar_attrs(t, pids=all_pids, st=st) if source is not None else _ar_attrs(t)) \
|
| 736 |
+
if wants(['ar_open', 'ar_overdue', 'ar_outstanding', 'ar_exposure', 'days_to_pay',
|
| 737 |
+
*AR_BUCKET_FIELDS.values()]) else {}
|
| 738 |
+
mix_a = _mix_attrs(all_pids, t) if wants(['top_category', 'top_category_pct', 'sku_count', 'top_sku']) else {}
|
| 739 |
+
sp_a = _salesperson_attrs(all_pids, t) if wants(['salesperson']) else {}
|
| 740 |
# W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
|
| 741 |
# read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
|
| 742 |
# client's map view discloses the count.
|
| 743 |
+
geo_a = map_mod.coords_for(list(all_pids), **({'st': st} if source is not None else {})) \
|
| 744 |
+
if wants(['lat', 'lon']) else {}
|
| 745 |
# C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
|
| 746 |
+
dba_a = _dba_attrs(all_pids, t) if wants(['dba']) else {}
|
| 747 |
# customers whose only activity sits in the older half of the 24-month cadence window
|
| 748 |
# appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
|
| 749 |
# the win-back targets this union exists for, would render as '?')
|
| 750 |
unnamed = [pid for pid in all_pids
|
| 751 |
if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
|
| 752 |
+
or (ltm.get(pid) or {}).get('name'))] if needed is None or 'customer' in needed else []
|
| 753 |
extra_names = ({r['id']: r['name'] for r in
|
| 754 |
O.search_read('res.partner', [('id', 'in', unnamed),
|
| 755 |
('active', 'in', [True, False])], ['name'])}
|
|
|
|
| 844 |
# I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
|
| 845 |
# Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
|
| 846 |
# so letting it write would mark ~1,450 living customers archived on every cold load.
|
| 847 |
+
if not fast and selected is None:
|
| 848 |
rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
|
| 849 |
rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
|
| 850 |
return {'rows': rows, 'total': total}
|
|
|
|
| 853 |
LEDGER_KEY = 'customer_ledger'
|
| 854 |
|
| 855 |
|
| 856 |
+
def _reconcile_ledger(rows, total, agent_name, team_id, *, ledger=None, write=True):
|
| 857 |
"""Remember every customer we have ever seen, and keep serving the ones Odoo stopped
|
| 858 |
returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
|
| 859 |
so stored in our platform even if deleted in Odoo").
|
|
|
|
| 872 |
they cannot prove they should see.
|
| 873 |
"""
|
| 874 |
try:
|
| 875 |
+
ledger = dict(store.get(LEDGER_KEY) or {}) if ledger is None else dict(ledger)
|
| 876 |
except Exception:
|
| 877 |
return rows, total # a ledger we cannot read must not break the table
|
| 878 |
today = str(P.today())
|
| 879 |
live = {r['pid'] for r in rows}
|
| 880 |
unscoped = agent_name is None and team_id is None
|
| 881 |
|
| 882 |
+
if unscoped and write:
|
| 883 |
for r in rows:
|
| 884 |
e = ledger.get(str(r['pid'])) or {}
|
| 885 |
e.update({'name': r['customer'], 'last_seen': today})
|
|
|
|
| 930 |
'days_since': None, 'days_since_invoice': None,
|
| 931 |
'typical_gap_days': None, 'overdue_days': None,
|
| 932 |
})
|
| 933 |
+
if write and store.available():
|
| 934 |
try:
|
| 935 |
store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
|
| 936 |
except Exception:
|
platform/modules/customers.py
CHANGED
|
@@ -14,7 +14,7 @@ import datetime as dt
|
|
| 14 |
import statistics
|
| 15 |
from pathlib import Path
|
| 16 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 17 |
-
|
| 18 |
import core.periods as P
|
| 19 |
import modules.sales as sales_mod
|
| 20 |
|
|
@@ -49,7 +49,10 @@ def _cust_group_store(date_from, date_to, team_id, agent_pids, select):
|
|
| 49 |
params += list(agent_pids)
|
| 50 |
if O.doc_mode() == 'invoice':
|
| 51 |
w.append("invoice_status = 'invoiced'")
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
| 53 |
f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} "
|
| 54 |
"FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id "
|
| 55 |
"WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall()
|
|
@@ -534,10 +537,13 @@ def _invoice_cadence_bulk(t=None, partner_ids=None):
|
|
| 534 |
if USE_STORE:
|
| 535 |
try:
|
| 536 |
from harness import semantic as sem
|
|
|
|
|
|
|
| 537 |
result = sem.store_query(
|
| 538 |
'customer_invoices', ['customer_days_since_invoice'], group_by=['partner'],
|
| 539 |
filters={'partner': partner_ids} if partner_ids is not None else None,
|
| 540 |
-
limit=sem.MAX_GROUPS, today=t.isoformat()
|
|
|
|
| 541 |
if result.get('truncated'):
|
| 542 |
raise sem.ModelError('The invoice cadence group set was truncated.')
|
| 543 |
return {int(row['partner_id']): row['customer_days_since_invoice']
|
|
@@ -650,7 +656,7 @@ def _audit_attrs(record):
|
|
| 650 |
return out
|
| 651 |
|
| 652 |
|
| 653 |
-
def _partner_attrs(pids):
|
| 654 |
"""{pid: {street, street2, city, state, country, agent, zip, payment_terms, customer_since,
|
| 655 |
tags, pricelist}} — the customer attributes the sum-level rollups slice by and the Customer
|
| 656 |
table displays.
|
|
@@ -667,15 +673,26 @@ def _partner_attrs(pids):
|
|
| 667 |
pids = list(pids)
|
| 668 |
if not pids:
|
| 669 |
return {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 670 |
# Odoo's active_test context otherwise hides archived partners even when their ids are named.
|
| 671 |
# The explicit two-state domain makes `archived` an ordinary source attribute rather than a
|
| 672 |
# conclusion inferred from a missing row.
|
| 673 |
rows = O.search_read('res.partner', [('id', 'in', pids),
|
| 674 |
('active', 'in', [True, False])],
|
| 675 |
-
|
| 676 |
-
'zip', 'property_payment_term_id', 'create_date', 'category_id',
|
| 677 |
-
'property_product_pricelist', 'active']
|
| 678 |
-
+ [field for field, _kind in AUDIT_ATTRS.values()])
|
| 679 |
aids = {a for r in rows for a in (r.get('agent_ids') or [])}
|
| 680 |
anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])}
|
| 681 |
if aids else {})
|
|
|
|
| 14 |
import statistics
|
| 15 |
from pathlib import Path
|
| 16 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 17 |
+
from harness.canonical_source import odoo as O
|
| 18 |
import core.periods as P
|
| 19 |
import modules.sales as sales_mod
|
| 20 |
|
|
|
|
| 49 |
params += list(agent_pids)
|
| 50 |
if O.doc_mode() == 'invoice':
|
| 51 |
w.append("invoice_status = 'invoiced'")
|
| 52 |
+
from harness.canonical_source import active
|
| 53 |
+
source = active()
|
| 54 |
+
con = source.cur if source is not None else DS.ro_con()
|
| 55 |
+
return con.execute(
|
| 56 |
f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} "
|
| 57 |
"FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id "
|
| 58 |
"WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall()
|
|
|
|
| 537 |
if USE_STORE:
|
| 538 |
try:
|
| 539 |
from harness import semantic as sem
|
| 540 |
+
from harness.canonical_source import active
|
| 541 |
+
source = active()
|
| 542 |
result = sem.store_query(
|
| 543 |
'customer_invoices', ['customer_days_since_invoice'], group_by=['partner'],
|
| 544 |
filters={'partner': partner_ids} if partner_ids is not None else None,
|
| 545 |
+
limit=sem.MAX_GROUPS, today=t.isoformat(),
|
| 546 |
+
**({'cur': source.cur} if source is not None else {}))
|
| 547 |
if result.get('truncated'):
|
| 548 |
raise sem.ModelError('The invoice cadence group set was truncated.')
|
| 549 |
return {int(row['partner_id']): row['customer_days_since_invoice']
|
|
|
|
| 656 |
return out
|
| 657 |
|
| 658 |
|
| 659 |
+
def _partner_attrs(pids, *, needed=None):
|
| 660 |
"""{pid: {street, street2, city, state, country, agent, zip, payment_terms, customer_since,
|
| 661 |
tags, pricelist}} — the customer attributes the sum-level rollups slice by and the Customer
|
| 662 |
table displays.
|
|
|
|
| 673 |
pids = list(pids)
|
| 674 |
if not pids:
|
| 675 |
return {}
|
| 676 |
+
columns = ['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids',
|
| 677 |
+
'zip', 'property_payment_term_id', 'create_date', 'category_id',
|
| 678 |
+
'property_product_pricelist', 'active'] + [field for field, _kind in AUDIT_ATTRS.values()]
|
| 679 |
+
if needed is not None:
|
| 680 |
+
operands = {'state': ['state_id'], 'country': ['country_id'], 'agent': ['agent_ids'],
|
| 681 |
+
'payment_terms': ['property_payment_term_id'], 'customer_since': ['create_date'],
|
| 682 |
+
'_created': ['create_date'], 'created_on': ['create_date'], 'tags': ['category_id'],
|
| 683 |
+
'pricelist': ['property_product_pricelist'], 'archived': ['active'],
|
| 684 |
+
'odoo_status': ['active'], 'maps_url': ['street', 'street2', 'city', 'state_id', 'country_id', 'zip']}
|
| 685 |
+
operands.update({key: [field] for key, (field, _kind) in AUDIT_ATTRS.items()})
|
| 686 |
+
wanted = {field for key in needed for field in operands.get(key, [key])}
|
| 687 |
+
columns = [field for field in columns if field in wanted]
|
| 688 |
+
if not columns:
|
| 689 |
+
return {}
|
| 690 |
# Odoo's active_test context otherwise hides archived partners even when their ids are named.
|
| 691 |
# The explicit two-state domain makes `archived` an ordinary source attribute rather than a
|
| 692 |
# conclusion inferred from a missing row.
|
| 693 |
rows = O.search_read('res.partner', [('id', 'in', pids),
|
| 694 |
('active', 'in', [True, False])],
|
| 695 |
+
columns)
|
|
|
|
|
|
|
|
|
|
| 696 |
aids = {a for r in rows for a in (r.get('agent_ids') or [])}
|
| 697 |
anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])}
|
| 698 |
if aids else {})
|
platform/modules/inventory.py
CHANGED
|
@@ -9,7 +9,7 @@ import sys
|
|
| 9 |
from pathlib import Path
|
| 10 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 11 |
import pandas as pd
|
| 12 |
-
|
| 13 |
import core.periods as P
|
| 14 |
|
| 15 |
|
|
@@ -273,13 +273,17 @@ def sku_inventory(target_days=180, carry_pct=_CARRY_PCT, t=None):
|
|
| 273 |
brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand,
|
| 274 |
inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value,
|
| 275 |
carry_cost, turns, target_units, target_days}}."""
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'),
|
| 280 |
on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'),
|
| 281 |
cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'),
|
| 282 |
-
|
| 283 |
out = {}
|
| 284 |
for r in g.itertuples(index=False):
|
| 285 |
daily = r.qty_ltm / 365.0
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 11 |
import pandas as pd
|
| 12 |
+
from harness.canonical_source import odoo as O
|
| 13 |
import core.periods as P
|
| 14 |
|
| 15 |
|
|
|
|
| 273 |
brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand,
|
| 274 |
inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value,
|
| 275 |
carry_cost, turns, target_units, target_days}}."""
|
| 276 |
+
from harness.canonical_source import active
|
| 277 |
+
if active() is not None:
|
| 278 |
+
g, _ = active().inventory_frame(t or P.today())
|
| 279 |
+
else:
|
| 280 |
+
df, _ = _build(t)
|
| 281 |
+
df = _enrich(df, target_days)
|
| 282 |
+
g = df.groupby('sku').agg(
|
| 283 |
name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'),
|
| 284 |
on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'),
|
| 285 |
cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'),
|
| 286 |
+
).reset_index()
|
| 287 |
out = {}
|
| 288 |
for r in g.itertuples(index=False):
|
| 289 |
daily = r.qty_ltm / 365.0
|
platform/modules/map.py
CHANGED
|
@@ -18,7 +18,7 @@ import csv
|
|
| 18 |
import io
|
| 19 |
import hashlib
|
| 20 |
|
| 21 |
-
|
| 22 |
import core.periods as P
|
| 23 |
import core.store as store
|
| 24 |
import modules.sales as sales_mod
|
|
@@ -174,7 +174,7 @@ def customer_locations(t=None, team_id=None, agent_pids=None):
|
|
| 174 |
bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])}
|
| 175 |
|
| 176 |
|
| 177 |
-
def coords_for(pids):
|
| 178 |
"""{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode
|
| 179 |
CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map
|
| 180 |
VIEW reads these off the pool rows; the standalone Map page retired the same day, and
|
|
@@ -182,7 +182,10 @@ def coords_for(pids):
|
|
| 182 |
if not pids:
|
| 183 |
return {}
|
| 184 |
try:
|
| 185 |
-
|
|
|
|
|
|
|
|
|
|
| 186 |
parts = O.search_read(
|
| 187 |
'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])],
|
| 188 |
['street', 'street2', 'city', 'state_id', 'zip',
|
|
|
|
| 18 |
import io
|
| 19 |
import hashlib
|
| 20 |
|
| 21 |
+
from harness.canonical_source import odoo as O
|
| 22 |
import core.periods as P
|
| 23 |
import core.store as store
|
| 24 |
import modules.sales as sales_mod
|
|
|
|
| 174 |
bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])}
|
| 175 |
|
| 176 |
|
| 177 |
+
def coords_for(pids, *, st=None):
|
| 178 |
"""{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode
|
| 179 |
CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map
|
| 180 |
VIEW reads these off the pool rows; the standalone Map page retired the same day, and
|
|
|
|
| 182 |
if not pids:
|
| 183 |
return {}
|
| 184 |
try:
|
| 185 |
+
if st is None:
|
| 186 |
+
cache = store.get(_GEO_NS) if store.available() else {}
|
| 187 |
+
else:
|
| 188 |
+
cache = st.selected_values(_GEO_NS, [], pids)
|
| 189 |
parts = O.search_read(
|
| 190 |
'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])],
|
| 191 |
['street', 'street2', 'city', 'state_id', 'zip',
|
platform/modules/product_data.py
CHANGED
|
@@ -622,6 +622,45 @@ def identity_rows():
|
|
| 622 |
return rows
|
| 623 |
|
| 624 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
def _fba_by_code(cat):
|
| 626 |
"""`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows.
|
| 627 |
|
|
@@ -667,7 +706,7 @@ def _fba_by_code(cat):
|
|
| 667 |
return out, report
|
| 668 |
|
| 669 |
|
| 670 |
-
def pool(team_id=None, t=None, raw_fields=()):
|
| 671 |
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 672 |
|
| 673 |
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
|
@@ -683,10 +722,14 @@ def pool(team_id=None, t=None, raw_fields=()):
|
|
| 683 |
consolidated = team_id is None
|
| 684 |
# ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 685 |
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 686 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 687 |
inventory_source_available = isinstance(inventory_snapshot, dict)
|
| 688 |
inv = inventory_snapshot if inventory_source_available else {}
|
| 689 |
-
bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
|
| 690 |
sup = supplier_master()
|
| 691 |
# ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build:
|
| 692 |
# tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the
|
|
@@ -697,15 +740,19 @@ def pool(team_id=None, t=None, raw_fields=()):
|
|
| 697 |
_prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both
|
| 698 |
except Exception: # noqa: BLE001
|
| 699 |
_prods = None
|
| 700 |
-
tiers, _tier_report = products.tier_prices_by_code(_prods)
|
| 701 |
-
|
|
|
|
|
|
|
|
|
|
| 702 |
cat = _catalogue_by_code()
|
| 703 |
# ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
|
| 704 |
# same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
|
| 705 |
# will sell. Degrades to `{}` on a read failure, matching every other column source here — a
|
| 706 |
# blank cover gap is honest, a product grid that will not render is not.
|
| 707 |
try:
|
| 708 |
-
fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id)
|
|
|
|
| 709 |
except Exception: # noqa: BLE001
|
| 710 |
fwd_demand, _fwd_report = {}, {"error": "forward demand unavailable"}
|
| 711 |
# The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
|
|
@@ -716,8 +763,9 @@ def pool(team_id=None, t=None, raw_fields=()):
|
|
| 716 |
# because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds
|
| 717 |
# units, not brands, and pro-rating them onto a BU would print a number no Odoo query could
|
| 718 |
# reproduce. Blank, never zero, when the tenant has no FBA warehouse.
|
| 719 |
-
fba_by_code, _fba_report = _fba_by_code(cat)
|
| 720 |
-
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
|
|
|
|
| 721 |
raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p
|
| 722 |
for p in (_prods or []) if p.get("default_code") or p.get("id")}
|
| 723 |
|
|
|
|
| 622 |
return rows
|
| 623 |
|
| 624 |
|
| 625 |
+
def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
|
| 626 |
+
"""Stream catalogue identities, refuse global hash collisions and project bounded code batches."""
|
| 627 |
+
from harness.canonical_source import bounded, BATCH
|
| 628 |
+
with bounded(cur, 'identity', [], st) as identity:
|
| 629 |
+
def codes():
|
| 630 |
+
for product in identity.search_read('product.product', [('active', '=', True)], ['id', 'default_code']):
|
| 631 |
+
code = str(product['default_code']).strip() if product.get('default_code') else f"pid:{product['id']}"
|
| 632 |
+
yield {'code': code, 'pid': sku_pid(code)}
|
| 633 |
+
raw = identity.snapshot(codes())
|
| 634 |
+
collision = cur.execute(f"SELECT json_extract_string(body, '/pid') pid "
|
| 635 |
+
f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
|
| 636 |
+
if collision:
|
| 637 |
+
raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
|
| 638 |
+
selected = '' if ids is None else " WHERE try_cast(json_extract_string(body, '/pid') AS BIGINT) IN (SELECT unnest(?::BIGINT[]))"
|
| 639 |
+
params = [] if ids is None else [list(ids)]
|
| 640 |
+
population = identity.snapshot(())
|
| 641 |
+
cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
|
| 642 |
+
f'{selected} GROUP BY body', params)
|
| 643 |
+
# Keep the tracked snapshot name while materialising its sorted, typed seek key.
|
| 644 |
+
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
|
| 645 |
+
f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
|
| 646 |
+
f'FROM "{population.table}" ORDER BY pid')
|
| 647 |
+
last = -1
|
| 648 |
+
while True:
|
| 649 |
+
codes = cur.execute(f"SELECT pid, "
|
| 650 |
+
f"json_extract_string(body, '/code') FROM \"{population.table}\" "
|
| 651 |
+
'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
|
| 652 |
+
if not codes:
|
| 653 |
+
return
|
| 654 |
+
with bounded(cur, 'product', [code for _, code in codes], st):
|
| 655 |
+
rows = ([{'pid': pid, 'code': code} for pid, code in codes]
|
| 656 |
+
if needed is not None and not needed else
|
| 657 |
+
pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
|
| 658 |
+
yield rows
|
| 659 |
+
last = codes[-1][0]
|
| 660 |
+
rows.clear()
|
| 661 |
+
codes.clear()
|
| 662 |
+
|
| 663 |
+
|
| 664 |
def _fba_by_code(cat):
|
| 665 |
"""`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows.
|
| 666 |
|
|
|
|
| 706 |
return out, report
|
| 707 |
|
| 708 |
|
| 709 |
+
def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
|
| 710 |
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 711 |
|
| 712 |
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
|
|
|
| 722 |
consolidated = team_id is None
|
| 723 |
# ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
|
| 724 |
# itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
|
| 725 |
+
wants = lambda keys: needed is None or bool(set(keys) & set(needed))
|
| 726 |
+
inventory_needed = wants(['on_hand', 'unit_cost', 'inv_value', 'qty_ltm', 'dos', 'stock_bucket',
|
| 727 |
+
'inventory_value_per_quantity', 'inventory_value_per_quantity_state', 'cover_gap_units',
|
| 728 |
+
'cover_gap_d', 'buy_now'])
|
| 729 |
+
inventory_snapshot = _inventory_by_code(t) if inventory_needed else {}
|
| 730 |
inventory_source_available = isinstance(inventory_snapshot, dict)
|
| 731 |
inv = inventory_snapshot if inventory_source_available else {}
|
| 732 |
+
bu_share = {} if consolidated or not inventory_needed else _bu_ltm_share(t, team_id)
|
| 733 |
sup = supplier_master()
|
| 734 |
# ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build:
|
| 735 |
# tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the
|
|
|
|
| 740 |
_prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both
|
| 741 |
except Exception: # noqa: BLE001
|
| 742 |
_prods = None
|
| 743 |
+
tiers, _tier_report = products.tier_prices_by_code(_prods) if wants(
|
| 744 |
+
[f'price_{n}' for n in range(1, 4)] + [f'unit_{n}' for n in range(1, 4)] +
|
| 745 |
+
[key for price, unit, _name, _n in products.PRICELIST_TIER_COLUMNS for key in (price, unit)]) else ({}, {})
|
| 746 |
+
pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) \
|
| 747 |
+
if wants(['pre_book_qty', 'open_backlog_qty']) else ({}, {}, {})
|
| 748 |
cat = _catalogue_by_code()
|
| 749 |
# ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
|
| 750 |
# same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
|
| 751 |
# will sell. Degrades to `{}` on a read failure, matching every other column source here — a
|
| 752 |
# blank cover gap is honest, a product grid that will not render is not.
|
| 753 |
try:
|
| 754 |
+
fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id) \
|
| 755 |
+
if wants(['demand_fwd', 'cover_gap_units', 'cover_gap_d', 'buy_now']) else ({}, {})
|
| 756 |
except Exception: # noqa: BLE001
|
| 757 |
fwd_demand, _fwd_report = {}, {"error": "forward demand unavailable"}
|
| 758 |
# The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
|
|
|
|
| 763 |
# because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds
|
| 764 |
# units, not brands, and pro-rating them onto a BU would print a number no Odoo query could
|
| 765 |
# reproduce. Blank, never zero, when the tenant has no FBA warehouse.
|
| 766 |
+
fba_by_code, _fba_report = _fba_by_code(cat) if inventory_needed or wants(['fba']) else ({}, {})
|
| 767 |
+
rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} if wants(
|
| 768 |
+
['rev_ytd', 'rev_ly', 'yoy_pct', 'qty_ytd', 'orders_ytd']) else {}
|
| 769 |
raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p
|
| 770 |
for p in (_prods or []) if p.get("default_code") or p.get("id")}
|
| 771 |
|
platform/modules/products.py
CHANGED
|
@@ -20,7 +20,7 @@ import datetime as dt
|
|
| 20 |
from pathlib import Path
|
| 21 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 22 |
from functools import lru_cache
|
| 23 |
-
|
| 24 |
import core.periods as P
|
| 25 |
import modules.sales as sales_mod
|
| 26 |
|
|
@@ -65,6 +65,10 @@ def _sku_rev(date_from, date_to, team_id=None, all_channels=False):
|
|
| 65 |
all_channels=all_channels),
|
| 66 |
['price_subtotal:sum', 'product_uom_qty:sum'],
|
| 67 |
['product_id', 'order_id'], lazy=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
codes = _code_map()
|
| 69 |
out = {}
|
| 70 |
orders_by_code = {}
|
|
@@ -246,6 +250,9 @@ def velocity_leaders(t=None, limit=25, team_id=None):
|
|
| 246 |
def _code_category():
|
| 247 |
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 248 |
(pid->category); the first record carrying a code sets that code's category."""
|
|
|
|
|
|
|
|
|
|
| 249 |
codes = _code_map()
|
| 250 |
cats = sales_mod._product_cat()
|
| 251 |
out = {}
|
|
@@ -630,6 +637,8 @@ def tier_cap_overflow(tiers):
|
|
| 630 |
fourth tier is dropped — and a dropped tier that nobody counts is the silent cap the rule
|
| 631 |
forbids. `validate_price_and_unit_cells` turns this into a named, reported figure.
|
| 632 |
"""
|
|
|
|
|
|
|
| 633 |
counts = {}
|
| 634 |
for tier in tiers or ():
|
| 635 |
counts[tier.get("pricelist")] = counts.get(tier.get("pricelist"), 0) + 1
|
|
@@ -685,6 +694,9 @@ def tier_prices_by_code(prods=None):
|
|
| 685 |
2,605 price rows against **0 customers and 0 orders**, so a SKU reading several pairs is
|
| 686 |
catalogue-true and commercially misleading. The entry keeps the list NAME for provenance.
|
| 687 |
"""
|
|
|
|
|
|
|
|
|
|
| 688 |
report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0,
|
| 689 |
"rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0,
|
| 690 |
"rules_with_packaging": 0}
|
|
|
|
| 20 |
from pathlib import Path
|
| 21 |
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 22 |
from functools import lru_cache
|
| 23 |
+
from harness.canonical_source import odoo as O
|
| 24 |
import core.periods as P
|
| 25 |
import modules.sales as sales_mod
|
| 26 |
|
|
|
|
| 65 |
all_channels=all_channels),
|
| 66 |
['price_subtotal:sum', 'product_uom_qty:sum'],
|
| 67 |
['product_id', 'order_id'], lazy=False)
|
| 68 |
+
from harness.canonical_source import active
|
| 69 |
+
source = active()
|
| 70 |
+
if source is not None:
|
| 71 |
+
return source.sku_revenue(g)
|
| 72 |
codes = _code_map()
|
| 73 |
out = {}
|
| 74 |
orders_by_code = {}
|
|
|
|
| 250 |
def _code_category():
|
| 251 |
"""code -> category name. Built from _code_map (pid->code) + the sales category map
|
| 252 |
(pid->category); the first record carrying a code sets that code's category."""
|
| 253 |
+
from harness.canonical_source import active
|
| 254 |
+
if active() is not None:
|
| 255 |
+
return active().categories()
|
| 256 |
codes = _code_map()
|
| 257 |
cats = sales_mod._product_cat()
|
| 258 |
out = {}
|
|
|
|
| 637 |
fourth tier is dropped — and a dropped tier that nobody counts is the silent cap the rule
|
| 638 |
forbids. `validate_price_and_unit_cells` turns this into a named, reported figure.
|
| 639 |
"""
|
| 640 |
+
if hasattr(tiers, 'overflow'):
|
| 641 |
+
return dict(tiers.overflow)
|
| 642 |
counts = {}
|
| 643 |
for tier in tiers or ():
|
| 644 |
counts[tier.get("pricelist")] = counts.get(tier.get("pricelist"), 0) + 1
|
|
|
|
| 694 |
2,605 price rows against **0 customers and 0 orders**, so a SKU reading several pairs is
|
| 695 |
catalogue-true and commercially misleading. The entry keeps the list NAME for provenance.
|
| 696 |
"""
|
| 697 |
+
from harness.canonical_source import active
|
| 698 |
+
if active() is not None:
|
| 699 |
+
return active().tiers(prods)
|
| 700 |
report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0,
|
| 701 |
"rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0,
|
| 702 |
"rules_with_packaging": 0}
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b22533130e1c89fb8fdbd8f6ae52395f79e0ba4df48f151b79047f6881d304aa
|
web/src/automation/AutomationBuilder.tsx
CHANGED
|
@@ -2360,7 +2360,8 @@ function TriggerProps({
|
|
| 2360 |
<option value="">Select a database…</option>
|
| 2361 |
{tables.map((t) => (
|
| 2362 |
<option key={t.key} value={t.key}>
|
| 2363 |
-
{t.label}
|
|
|
|
| 2364 |
</option>
|
| 2365 |
))}
|
| 2366 |
</select>
|
|
|
|
| 2360 |
<option value="">Select a database…</option>
|
| 2361 |
{tables.map((t) => (
|
| 2362 |
<option key={t.key} value={t.key}>
|
| 2363 |
+
{t.label}
|
| 2364 |
+
{t.rowCount != null && ` (${t.rowCount} ${t.rowCount === 1 ? "row" : "rows"})`}
|
| 2365 |
</option>
|
| 2366 |
))}
|
| 2367 |
</select>
|
web/src/automation/AutomationDetail.tsx
CHANGED
|
@@ -1597,7 +1597,8 @@ function AutomationDetail({
|
|
| 1597 |
) : null}
|
| 1598 |
{tables.map((t) => (
|
| 1599 |
<option key={t.key} value={t.key}>
|
| 1600 |
-
{t.label}
|
|
|
|
| 1601 |
</option>
|
| 1602 |
))}
|
| 1603 |
</select>
|
|
|
|
| 1597 |
) : null}
|
| 1598 |
{tables.map((t) => (
|
| 1599 |
<option key={t.key} value={t.key}>
|
| 1600 |
+
{t.label}
|
| 1601 |
+
{t.rowCount != null && ` (${t.rowCount} ${t.rowCount === 1 ? "row" : "rows"})`}
|
| 1602 |
</option>
|
| 1603 |
))}
|
| 1604 |
</select>
|
web/src/automation/automationApi.ts
CHANGED
|
@@ -921,7 +921,7 @@ export interface UserTable {
|
|
| 921 |
key: string;
|
| 922 |
label: string;
|
| 923 |
source: string;
|
| 924 |
-
rowCount: number;
|
| 925 |
/**
|
| 926 |
* `automation` is the machine-owner TAG (W23/R9's grey columns) — the id of the automation
|
| 927 |
* that owns the column, or `null`. Narrowed from `unknown` in wave 24 against the route that
|
|
|
|
| 921 |
key: string;
|
| 922 |
label: string;
|
| 923 |
source: string;
|
| 924 |
+
rowCount: number | null;
|
| 925 |
/**
|
| 926 |
* `automation` is the machine-owner TAG (W23/R9's grey columns) — the id of the automation
|
| 927 |
* that owns the column, or `null`. Narrowed from `unknown` in wave 24 against the route that
|
web/src/customer-grid/CustomerGrid.tsx
CHANGED
|
@@ -46,7 +46,7 @@ import { withoutAppliedLeaves } from "./drillLimits";
|
|
| 46 |
// ⭐⭐ W50-T03 / CONTRACT C1 — the windowed aggregate's envelope. Lane F serves the route
|
| 47 |
// (W50-T31); this side is written to C1's shape and degrades to the loaded window until it
|
| 48 |
// lands, which is the done-when's own observable rather than a workaround.
|
| 49 |
-
import type { AggregateResult } from "./apiBridge";
|
| 50 |
import {
|
| 51 |
EMPTY_WINDOW_PREDICATE, importantCountNote, limitSummary, lockedRecordsNote,
|
| 52 |
pagingSummary, serverViewCount, windowedCapabilityNote, windowedFoldNote, windowPredicateKey,
|
|
@@ -1361,7 +1361,9 @@ type ModeGroups = { groups: ModeGroup[]; totalGroups: number; totalRows: number;
|
|
| 1361 |
type ModeAgg = { colId: string; agg: string; p?: number };
|
| 1362 |
|
| 1363 |
async function mirrorModeGroups(table: string, req: RowWindowRequest, by: string,
|
| 1364 |
-
aggs: ModeAgg[], offset: number, size: number, abort: AbortSignal
|
|
|
|
|
|
|
| 1365 |
const response = await fetch(`${API_V1}/odoo-tables/${encodeURIComponent(table)}/aggregates`, {
|
| 1366 |
method: "POST", credentials: CREDENTIALS, signal: abort,
|
| 1367 |
headers: { "Content-Type": "application/json" },
|
|
@@ -1440,8 +1442,8 @@ function MirrorWindowPager({ offset, size, total, pending, label, onPage }: {
|
|
| 1440 |
}
|
| 1441 |
|
| 1442 |
function useMirrorGroups(table: string, request: RowWindowRequest, field: Field,
|
| 1443 |
-
aggs: ModeAgg[], offset: number, size: number, active = true) {
|
| 1444 |
-
const key = JSON.stringify([table, request, field.key, aggs, offset, size, active]);
|
| 1445 |
const [answer, setAnswer] = useState<{ key: string; data?: ModeGroups; error?: string }>({ key: "" });
|
| 1446 |
const [retry, setRetry] = useState(0);
|
| 1447 |
useEffect(() => {
|
|
@@ -1449,7 +1451,7 @@ function useMirrorGroups(table: string, request: RowWindowRequest, field: Field,
|
|
| 1449 |
let live = true;
|
| 1450 |
setAnswer({ key });
|
| 1451 |
if (!active) return () => { live = false; abort.abort(); };
|
| 1452 |
-
void mirrorModeGroups(table, request, field.key, aggs, offset, size, abort.signal).then(
|
| 1453 |
(data) => { if (live) setAnswer({ key, data }); },
|
| 1454 |
(error: unknown) => { if (live) setAnswer({ key, error: error instanceof Error
|
| 1455 |
? error.message : "Groups could not be read for this window." }); });
|
|
@@ -1462,21 +1464,23 @@ function useMirrorGroups(table: string, request: RowWindowRequest, field: Field,
|
|
| 1462 |
}
|
| 1463 |
|
| 1464 |
function MirrorRecordWindow({ table, request, field, group, titleKey, fieldByKey,
|
| 1465 |
-
cardKeys, size, calendar = false, onOpen }: {
|
| 1466 |
table: string; request: RowWindowRequest; field: Field; group: ModeGroup;
|
| 1467 |
titleKey: string; fieldByKey: Map<string, Field>; cardKeys: string[]; size: number;
|
| 1468 |
calendar?: boolean; hosted?: boolean; onOpen: (record: Row) => void;
|
|
|
|
| 1469 |
}) {
|
| 1470 |
const [offset, setOffset] = useState(0);
|
| 1471 |
const [retry, setRetry] = useState(0);
|
| 1472 |
-
const key = JSON.stringify([table, request, field.key, group.key, offset, size]);
|
| 1473 |
const [page, setPage] = useState<{ key: string; rows?: Row[]; total?: number; error?: string }>({ key: "" });
|
| 1474 |
const seq = useRef(0);
|
| 1475 |
useEffect(() => {
|
| 1476 |
const current = ++seq.current;
|
| 1477 |
let live = true;
|
| 1478 |
setPage({ key });
|
| 1479 |
-
|
|
|
|
| 1480 |
group: { by: field.key, key: group.key } }, (reason) => {
|
| 1481 |
if (live && seq.current === current) console.debug("[CustomerGrid]", "record window failure", reason);
|
| 1482 |
})
|
|
@@ -1549,15 +1553,16 @@ function MirrorRecordWindow({ table, request, field, group, titleKey, fieldByKey
|
|
| 1549 |
}
|
| 1550 |
|
| 1551 |
function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, size,
|
| 1552 |
-
clamped, aggs, summaryMap, hosted = false, onOpen, onAddOption }: {
|
| 1553 |
table: string; request: RowWindowRequest; field: Field; fieldByKey: Map<string, Field>;
|
| 1554 |
cardKeys: string[]; titleKey: string; size: number; clamped: boolean; aggs: ModeAgg[];
|
| 1555 |
summaryMap: SummaryMap; hosted?: boolean;
|
|
|
|
| 1556 |
onOpen: (record: Row) => void; onAddOption: ((anchor: AnchorRect) => void) | null;
|
| 1557 |
}) {
|
| 1558 |
const [offset, setOffset] = useState(0);
|
| 1559 |
const columnsPerPage = 12;
|
| 1560 |
-
const { data, error, retry } = useMirrorGroups(table, request, field, aggs, offset, columnsPerPage);
|
| 1561 |
useEffect(() => {
|
| 1562 |
if (data && offset >= data.totalGroups && offset > 0)
|
| 1563 |
setOffset(Math.max(0, Math.ceil(data.totalGroups / columnsPerPage) - 1) * columnsPerPage);
|
|
@@ -1583,6 +1588,7 @@ function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, s
|
|
| 1583 |
</div> : null;
|
| 1584 |
})}
|
| 1585 |
<MirrorRecordWindow table={table} request={request} field={field} group={group}
|
|
|
|
| 1586 |
fieldByKey={fieldByKey} cardKeys={cardKeys} titleKey={titleKey} size={size} hosted={hosted}
|
| 1587 |
onOpen={onOpen} />
|
| 1588 |
</div>;
|
|
@@ -1597,9 +1603,10 @@ function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, s
|
|
| 1597 |
}
|
| 1598 |
|
| 1599 |
function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, size,
|
| 1600 |
-
summary, metrics, hosted = false, onOpen }: {
|
| 1601 |
table: string; request: RowWindowRequest; field: Field; fieldByKey: Map<string, Field>;
|
| 1602 |
titleKey: string; today?: string; size: number; summary: boolean; hosted?: boolean;
|
|
|
|
| 1603 |
metrics?: { id: string; field: string; agg: string }[]; onOpen: (record: Row) => void;
|
| 1604 |
}) {
|
| 1605 |
const [pickedMonth, setMonth] = useState("");
|
|
@@ -1613,7 +1620,7 @@ function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, si
|
|
| 1613 |
] };
|
| 1614 |
const aggs = summary ? (metrics ?? []).map((metric) => ({ colId: metric.field,
|
| 1615 |
agg: metric.agg === "avg" ? "average" : metric.agg })) : [];
|
| 1616 |
-
const { data, error, retry } = useMirrorGroups(table, range, field, aggs, 0, 42, !!month);
|
| 1617 |
const shift = (delta: number) => setMonth(new Date(Date.UTC(year, m - 1 + delta, 1)).toISOString().slice(0, 7));
|
| 1618 |
const leading = month ? (new Date(Date.UTC(year, m - 1, 1)).getUTCDay() + 6) % 7 : 0;
|
| 1619 |
const cells = Array.from({ length: Math.ceil((leading + days) / 7) * 7 }, (_, i) =>
|
|
@@ -1648,6 +1655,7 @@ function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, si
|
|
| 1648 |
<span>{value == null ? "Unavailable" : formatServerAgg(f,
|
| 1649 |
{ agg: (metric.agg === "avg" ? "average" : metric.agg) as SummaryChoice["agg"] }, value)}</span></div> : null;
|
| 1650 |
}) : <MirrorRecordWindow key={iso} table={table} request={range} field={field} group={group}
|
|
|
|
| 1651 |
fieldByKey={fieldByKey} cardKeys={[]} titleKey={titleKey} size={size} calendar hosted={hosted}
|
| 1652 |
onOpen={onOpen} />}
|
| 1653 |
</>}
|
|
@@ -2050,6 +2058,14 @@ type RecordIdInitialPage = {
|
|
| 2050 |
payload: CustomersPayload;
|
| 2051 |
};
|
| 2052 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2053 |
interface CustomerGridProps {
|
| 2054 |
scope?: SurfaceScope;
|
| 2055 |
/** Shell-owned Data workspace location; grid resolves its view/field, never the global rail. */
|
|
@@ -2083,6 +2099,7 @@ interface CustomerGridProps {
|
|
| 2083 |
/** Exact linked pids to page on the server. */
|
| 2084 |
embeddedRecordIds?: readonly number[];
|
| 2085 |
embeddedInitialPage?: RecordIdInitialPage;
|
|
|
|
| 2086 |
/** A fresh drill is resolving ids; withhold the previous id page during that request. */
|
| 2087 |
embeddedIdsPending?: boolean;
|
| 2088 |
onEmbeddedTotalChange?: (total: number | null) => void;
|
|
@@ -2183,6 +2200,7 @@ interface LinkGridModalProps {
|
|
| 2183 |
table: SurfaceScope;
|
| 2184 |
recordIds: readonly number[];
|
| 2185 |
initialPage?: RecordIdInitialPage;
|
|
|
|
| 2186 |
idsPending?: boolean;
|
| 2187 |
editable?: boolean;
|
| 2188 |
single?: boolean;
|
|
@@ -2214,11 +2232,11 @@ interface LinkGridModalProps {
|
|
| 2214 |
// chrome slots and a `cellView` modifier while leaving existing callers and the focus trap intact.
|
| 2215 |
export function LinkGridModal({ label, table, recordIds, editable = false, single = false,
|
| 2216 |
excludeRecordId, selectionFilter, cellView = false, cellFilter, cellFields, onCellFilterChange, onCellDisplayChange,
|
| 2217 |
-
headerActions, footer, initialPage, idsPending = false,
|
| 2218 |
onOpenRecord, onSave, onClose }: LinkGridModalProps) {
|
| 2219 |
const panelRef = useRef<HTMLDivElement>(null);
|
| 2220 |
const [selectedIds, setSelectedIds] = useState<number[]>(() => [...recordIds]);
|
| 2221 |
-
const countKey = `${table}:${recordIdsKey(recordIds)}`;
|
| 2222 |
const [serverCount, setServerCount] = useState<{ key: string; total: number | null }>();
|
| 2223 |
const reportTotal = useCallback((total: number | null) => {
|
| 2224 |
setServerCount({ key: countKey, total });
|
|
@@ -2283,6 +2301,7 @@ export function LinkGridModal({ label, table, recordIds, editable = false, singl
|
|
| 2283 |
onCellDisplayChange={onCellDisplayChange}
|
| 2284 |
embeddedRecordIds={editable ? undefined : recordIds}
|
| 2285 |
embeddedInitialPage={editable ? undefined : initialPage}
|
|
|
|
| 2286 |
embeddedIdsPending={!editable && idsPending}
|
| 2287 |
onEmbeddedTotalChange={editable ? undefined : reportTotal}
|
| 2288 |
embeddedExcludeRecordId={editable ? excludeRecordId : undefined}
|
|
@@ -2489,9 +2508,9 @@ function linkSelectionPredicate(request: Pick<RowWindowRequest, "filters" | "fil
|
|
| 2489 |
/** Linked grids page their immutable restrictions through the caller-owned payload seam. */
|
| 2490 |
function useRecordIdWindow(scope: SurfaceScope, ids: readonly number[] | undefined,
|
| 2491 |
related: RelatedPin | null, initialPage?: RecordIdInitialPage, idsPending = false, cellView = false,
|
| 2492 |
-
selectionFilter?: FilterTree) {
|
| 2493 |
const candidates = !!selectionFilter?.nodes?.length;
|
| 2494 |
-
const baseKey = JSON.stringify([scope, ids, related, initialPage, idsPending, cellView]);
|
| 2495 |
const key = candidates ? baseKey + "::" + JSON.stringify(selectionFilter) : baseKey;
|
| 2496 |
const appliedCellFilters = cellView && ids !== undefined;
|
| 2497 |
const seed = !idsPending && ids !== undefined && !related && initialPage && initialPage.offset === 0
|
|
@@ -2528,9 +2547,10 @@ function useRecordIdWindow(scope: SurfaceScope, ids: readonly number[] | undefin
|
|
| 2528 |
return;
|
| 2529 |
}
|
| 2530 |
setPage((prev) => ({ key, payload: prev?.key === key ? prev.payload : null, pending: true }));
|
| 2531 |
-
|
|
|
|
| 2532 |
...(candidates ? linkSelectionPredicate(req, selectionFilter) : {}),
|
| 2533 |
-
...(ids !== undefined ? { ids } : {}), related }).then((payload) => {
|
| 2534 |
if (seq !== sequence.current || currentKey.current !== key) return;
|
| 2535 |
if (candidates && selectionFilter && payload && unresolvedConditions(selectionFilter.nodes, {
|
| 2536 |
fieldByKey: new Map(payload.fields.map((field) => [field.key, field])), today: payload.today,
|
|
@@ -2543,7 +2563,7 @@ function useRecordIdWindow(scope: SurfaceScope, ids: readonly number[] | undefin
|
|
| 2543 |
}
|
| 2544 |
setPage({ key, payload, pending: false });
|
| 2545 |
});
|
| 2546 |
-
}, [scope, ids, related, key, seed, initialPage, idsPending, appliedCellFilters, candidates, selectionFilter]);
|
| 2547 |
useEffect(() => {
|
| 2548 |
const refresh = () => { if (lastRequest.current) requestWindow(lastRequest.current, true); };
|
| 2549 |
// React can replay effect setup after cancelling its first request in StrictMode.
|
|
@@ -2580,6 +2600,7 @@ function CustomerGridSurface({
|
|
| 2580 |
embeddedPayload,
|
| 2581 |
embeddedRecordIds,
|
| 2582 |
embeddedInitialPage,
|
|
|
|
| 2583 |
embeddedIdsPending = false,
|
| 2584 |
onEmbeddedTotalChange,
|
| 2585 |
embeddedSelectable = false,
|
|
@@ -2643,7 +2664,7 @@ function CustomerGridSurface({
|
|
| 2643 |
() => embeddedIdsKey ? JSON.parse(embeddedIdsKey) as number[] : undefined, [embeddedIdsKey]);
|
| 2644 |
const selectionWindow = embedded && !!embeddedSelectionFilter?.nodes?.length;
|
| 2645 |
const idWindow = useRecordIdWindow(scope, embeddedIds, relatedPin, embeddedInitialPage, embeddedIdsPending, cellView,
|
| 2646 |
-
selectionWindow ? embeddedSelectionFilter : undefined);
|
| 2647 |
/**
|
| 2648 |
* ⭐⭐ W35-T23 (owner item 4 / R2) — **`previewReadOnly` IS GONE, AND THE SPLIT IS THE TICKET.**
|
| 2649 |
*
|
|
@@ -4459,9 +4480,9 @@ function CustomerGridSurface({
|
|
| 4459 |
const windowPredicate = useMemo(
|
| 4460 |
() => windowPredicateKey(config.filters, config.filterConj, config.sorts, search)
|
| 4461 |
+ (embeddedIdsKey || selectionWindow
|
| 4462 |
-
? JSON.stringify([scope, embeddedIdsKey, relatedPinKey, cellView, embeddedSelectionFilter]) : ""),
|
| 4463 |
[config.filters, config.filterConj, config.sorts, search, scope, embeddedIdsKey, relatedPinKey, cellView,
|
| 4464 |
-
selectionWindow, embeddedSelectionFilter]
|
| 4465 |
);
|
| 4466 |
/* HOTFIX 2026-09-24 — ONE PAGE SIZE FOR EVERY DATABASE (50 / 100 / 200, default 100). */
|
| 4467 |
const windowRequest = useCallback(
|
|
@@ -4983,7 +5004,8 @@ function CustomerGridSurface({
|
|
| 4983 |
return;
|
| 4984 |
}
|
| 4985 |
let live = true;
|
| 4986 |
-
|
|
|
|
| 4987 |
...(embeddedIds !== undefined ? { ids: embeddedIds } : {}),
|
| 4988 |
...linkSelectionPredicate({
|
| 4989 |
filters: cellView && embeddedIds !== undefined ? withoutAppliedLeaves(config.filters) : config.filters,
|
|
@@ -5020,7 +5042,7 @@ function CustomerGridSurface({
|
|
| 5020 |
// is part of the question, so moving between two tabs on one target table must re-ask. Its
|
| 5021 |
// identity is stable (see `relatedPinKey`), so this cannot loop.
|
| 5022 |
}, [serverWindowed, topic.key, summaryRequestKey, windowPredicate, config.groupBy, relatedPin, embeddedIds,
|
| 5023 |
-
serverAggRequestKey, cellView, embeddedSelectionFilter]);
|
| 5024 |
/**
|
| 5025 |
* What the bar actually paints: C1's numbers when the server answered, this browser's fold
|
| 5026 |
* over the loaded window otherwise — and the two are never mixed within one paint, because a
|
|
@@ -5179,7 +5201,8 @@ function CustomerGridSurface({
|
|
| 5179 |
error, which replaced the whole grid with "Cannot reach the server." for one group's page;
|
| 5180 |
the failure belongs to the group that asked, so its cause stays with that group. */
|
| 5181 |
let failure = "";
|
| 5182 |
-
|
|
|
|
| 5183 |
...windowRequest(offset),
|
| 5184 |
related: relatedPin,
|
| 5185 |
group: { by, key: groupKeyRaw.get(key) ?? null },
|
|
@@ -5199,7 +5222,7 @@ function CustomerGridSurface({
|
|
| 5199 |
return next;
|
| 5200 |
});
|
| 5201 |
});
|
| 5202 |
-
}, [config.groupBy, groupKeyRaw, scope, windowRequest, relatedPin]);
|
| 5203 |
useEffect(() => {
|
| 5204 |
if (!serverWindowed || !config.groupBy || groupSummaryOn) return;
|
| 5205 |
for (const key of expanded) {
|
|
@@ -12220,6 +12243,7 @@ function CustomerGridSurface({
|
|
| 12220 |
{displayMode === "calendar" &&
|
| 12221 |
(calendarField ? (
|
| 12222 |
serverWindowed ? <MirrorCalendar key={mirrorDetailKey + calendarField.key}
|
|
|
|
| 12223 |
table={topic.key} request={mirrorModeRequest} field={calendarField} fieldByKey={fieldByKey}
|
| 12224 |
titleKey={lockedKey} today={today} size={pageSize}
|
| 12225 |
summary={displaySpec?.calendarMode === "summary"} metrics={displaySpec?.calendarMetrics}
|
|
@@ -12272,6 +12296,7 @@ function CustomerGridSurface({
|
|
| 12272 |
{displayMode === "kanban" &&
|
| 12273 |
(kanbanField ? (
|
| 12274 |
serverWindowed ? <MirrorKanban key={mirrorDetailKey + kanbanField.key}
|
|
|
|
| 12275 |
table={topic.key} request={mirrorModeRequest} field={kanbanField} fieldByKey={fieldByKey}
|
| 12276 |
cardKeys={kanbanCardKeys} titleKey={lockedKey} size={pageSize}
|
| 12277 |
clamped={displaySpec?.kanbanClamp !== false} aggs={summaryRequest} summaryMap={summaryMap}
|
|
|
|
| 46 |
// ⭐⭐ W50-T03 / CONTRACT C1 — the windowed aggregate's envelope. Lane F serves the route
|
| 47 |
// (W50-T31); this side is written to C1's shape and degrades to the loaded window until it
|
| 48 |
// lands, which is the done-when's own observable rather than a workaround.
|
| 49 |
+
import type { AggregateRequest, AggregateResult } from "./apiBridge";
|
| 50 |
import {
|
| 51 |
EMPTY_WINDOW_PREDICATE, importantCountNote, limitSummary, lockedRecordsNote,
|
| 52 |
pagingSummary, serverViewCount, windowedCapabilityNote, windowedFoldNote, windowPredicateKey,
|
|
|
|
| 1361 |
type ModeAgg = { colId: string; agg: string; p?: number };
|
| 1362 |
|
| 1363 |
async function mirrorModeGroups(table: string, req: RowWindowRequest, by: string,
|
| 1364 |
+
aggs: ModeAgg[], offset: number, size: number, abort: AbortSignal,
|
| 1365 |
+
rowSource?: EmbeddedRowSource): Promise<ModeGroups> {
|
| 1366 |
+
if (rowSource) return rowSource.readGroups(req, by, aggs, offset, size);
|
| 1367 |
const response = await fetch(`${API_V1}/odoo-tables/${encodeURIComponent(table)}/aggregates`, {
|
| 1368 |
method: "POST", credentials: CREDENTIALS, signal: abort,
|
| 1369 |
headers: { "Content-Type": "application/json" },
|
|
|
|
| 1442 |
}
|
| 1443 |
|
| 1444 |
function useMirrorGroups(table: string, request: RowWindowRequest, field: Field,
|
| 1445 |
+
aggs: ModeAgg[], offset: number, size: number, active = true, rowSource?: EmbeddedRowSource) {
|
| 1446 |
+
const key = JSON.stringify([table, request, field.key, aggs, offset, size, active, rowSource?.key]);
|
| 1447 |
const [answer, setAnswer] = useState<{ key: string; data?: ModeGroups; error?: string }>({ key: "" });
|
| 1448 |
const [retry, setRetry] = useState(0);
|
| 1449 |
useEffect(() => {
|
|
|
|
| 1451 |
let live = true;
|
| 1452 |
setAnswer({ key });
|
| 1453 |
if (!active) return () => { live = false; abort.abort(); };
|
| 1454 |
+
void mirrorModeGroups(table, request, field.key, aggs, offset, size, abort.signal, rowSource).then(
|
| 1455 |
(data) => { if (live) setAnswer({ key, data }); },
|
| 1456 |
(error: unknown) => { if (live) setAnswer({ key, error: error instanceof Error
|
| 1457 |
? error.message : "Groups could not be read for this window." }); });
|
|
|
|
| 1464 |
}
|
| 1465 |
|
| 1466 |
function MirrorRecordWindow({ table, request, field, group, titleKey, fieldByKey,
|
| 1467 |
+
cardKeys, size, calendar = false, onOpen, rowSource }: {
|
| 1468 |
table: string; request: RowWindowRequest; field: Field; group: ModeGroup;
|
| 1469 |
titleKey: string; fieldByKey: Map<string, Field>; cardKeys: string[]; size: number;
|
| 1470 |
calendar?: boolean; hosted?: boolean; onOpen: (record: Row) => void;
|
| 1471 |
+
rowSource?: EmbeddedRowSource;
|
| 1472 |
}) {
|
| 1473 |
const [offset, setOffset] = useState(0);
|
| 1474 |
const [retry, setRetry] = useState(0);
|
| 1475 |
+
const key = JSON.stringify([table, request, field.key, group.key, offset, size, rowSource?.key]);
|
| 1476 |
const [page, setPage] = useState<{ key: string; rows?: Row[]; total?: number; error?: string }>({ key: "" });
|
| 1477 |
const seq = useRef(0);
|
| 1478 |
useEffect(() => {
|
| 1479 |
const current = ++seq.current;
|
| 1480 |
let live = true;
|
| 1481 |
setPage({ key });
|
| 1482 |
+
const readWindow = rowSource ? (_table: string, query: RowWindowRequest, _onFailure: (reason: string) => void) => rowSource.readWindow(query) : fetchTableWindow;
|
| 1483 |
+
void readWindow(table, { ...request, offset, limit: size,
|
| 1484 |
group: { by: field.key, key: group.key } }, (reason) => {
|
| 1485 |
if (live && seq.current === current) console.debug("[CustomerGrid]", "record window failure", reason);
|
| 1486 |
})
|
|
|
|
| 1553 |
}
|
| 1554 |
|
| 1555 |
function MirrorKanban({ table, request, field, fieldByKey, cardKeys, titleKey, size,
|
| 1556 |
+
clamped, aggs, summaryMap, hosted = false, onOpen, onAddOption, rowSource }: {
|
| 1557 |
table: string; request: RowWindowRequest; field: Field; fieldByKey: Map<string, Field>;
|
| 1558 |
cardKeys: string[]; titleKey: string; size: number; clamped: boolean; aggs: ModeAgg[];
|
| 1559 |
summaryMap: SummaryMap; hosted?: boolean;
|
| 1560 |
+
rowSource?: EmbeddedRowSource;
|
| 1561 |
onOpen: (record: Row) => void; onAddOption: ((anchor: AnchorRect) => void) | null;
|
| 1562 |
}) {
|
| 1563 |
const [offset, setOffset] = useState(0);
|
| 1564 |
const columnsPerPage = 12;
|
| 1565 |
+
const { data, error, retry } = useMirrorGroups(table, request, field, aggs, offset, columnsPerPage, true, rowSource);
|
| 1566 |
useEffect(() => {
|
| 1567 |
if (data && offset >= data.totalGroups && offset > 0)
|
| 1568 |
setOffset(Math.max(0, Math.ceil(data.totalGroups / columnsPerPage) - 1) * columnsPerPage);
|
|
|
|
| 1588 |
</div> : null;
|
| 1589 |
})}
|
| 1590 |
<MirrorRecordWindow table={table} request={request} field={field} group={group}
|
| 1591 |
+
rowSource={rowSource}
|
| 1592 |
fieldByKey={fieldByKey} cardKeys={cardKeys} titleKey={titleKey} size={size} hosted={hosted}
|
| 1593 |
onOpen={onOpen} />
|
| 1594 |
</div>;
|
|
|
|
| 1603 |
}
|
| 1604 |
|
| 1605 |
function MirrorCalendar({ table, request, field, fieldByKey, titleKey, today, size,
|
| 1606 |
+
summary, metrics, hosted = false, onOpen, rowSource }: {
|
| 1607 |
table: string; request: RowWindowRequest; field: Field; fieldByKey: Map<string, Field>;
|
| 1608 |
titleKey: string; today?: string; size: number; summary: boolean; hosted?: boolean;
|
| 1609 |
+
rowSource?: EmbeddedRowSource;
|
| 1610 |
metrics?: { id: string; field: string; agg: string }[]; onOpen: (record: Row) => void;
|
| 1611 |
}) {
|
| 1612 |
const [pickedMonth, setMonth] = useState("");
|
|
|
|
| 1620 |
] };
|
| 1621 |
const aggs = summary ? (metrics ?? []).map((metric) => ({ colId: metric.field,
|
| 1622 |
agg: metric.agg === "avg" ? "average" : metric.agg })) : [];
|
| 1623 |
+
const { data, error, retry } = useMirrorGroups(table, range, field, aggs, 0, 42, !!month, rowSource);
|
| 1624 |
const shift = (delta: number) => setMonth(new Date(Date.UTC(year, m - 1 + delta, 1)).toISOString().slice(0, 7));
|
| 1625 |
const leading = month ? (new Date(Date.UTC(year, m - 1, 1)).getUTCDay() + 6) % 7 : 0;
|
| 1626 |
const cells = Array.from({ length: Math.ceil((leading + days) / 7) * 7 }, (_, i) =>
|
|
|
|
| 1655 |
<span>{value == null ? "Unavailable" : formatServerAgg(f,
|
| 1656 |
{ agg: (metric.agg === "avg" ? "average" : metric.agg) as SummaryChoice["agg"] }, value)}</span></div> : null;
|
| 1657 |
}) : <MirrorRecordWindow key={iso} table={table} request={range} field={field} group={group}
|
| 1658 |
+
rowSource={rowSource}
|
| 1659 |
fieldByKey={fieldByKey} cardKeys={[]} titleKey={titleKey} size={size} calendar hosted={hosted}
|
| 1660 |
onOpen={onOpen} />}
|
| 1661 |
</>}
|
|
|
|
| 2058 |
payload: CustomersPayload;
|
| 2059 |
};
|
| 2060 |
|
| 2061 |
+
export interface EmbeddedRowSource {
|
| 2062 |
+
/** Reader and query identity. Page IDs never define this source's population. */
|
| 2063 |
+
key: string;
|
| 2064 |
+
readWindow: (request: RowWindowRequest) => Promise<CustomersPayload | null>;
|
| 2065 |
+
readAggregates: (request: AggregateRequest) => Promise<AggregateResult | null>;
|
| 2066 |
+
readGroups: (request: RowWindowRequest, by: string, aggs: ModeAgg[], offset: number, size: number) => Promise<ModeGroups>;
|
| 2067 |
+
}
|
| 2068 |
+
|
| 2069 |
interface CustomerGridProps {
|
| 2070 |
scope?: SurfaceScope;
|
| 2071 |
/** Shell-owned Data workspace location; grid resolves its view/field, never the global rail. */
|
|
|
|
| 2099 |
/** Exact linked pids to page on the server. */
|
| 2100 |
embeddedRecordIds?: readonly number[];
|
| 2101 |
embeddedInitialPage?: RecordIdInitialPage;
|
| 2102 |
+
embeddedRowSource?: EmbeddedRowSource;
|
| 2103 |
/** A fresh drill is resolving ids; withhold the previous id page during that request. */
|
| 2104 |
embeddedIdsPending?: boolean;
|
| 2105 |
onEmbeddedTotalChange?: (total: number | null) => void;
|
|
|
|
| 2200 |
table: SurfaceScope;
|
| 2201 |
recordIds: readonly number[];
|
| 2202 |
initialPage?: RecordIdInitialPage;
|
| 2203 |
+
rowSource?: EmbeddedRowSource;
|
| 2204 |
idsPending?: boolean;
|
| 2205 |
editable?: boolean;
|
| 2206 |
single?: boolean;
|
|
|
|
| 2232 |
// chrome slots and a `cellView` modifier while leaving existing callers and the focus trap intact.
|
| 2233 |
export function LinkGridModal({ label, table, recordIds, editable = false, single = false,
|
| 2234 |
excludeRecordId, selectionFilter, cellView = false, cellFilter, cellFields, onCellFilterChange, onCellDisplayChange,
|
| 2235 |
+
headerActions, footer, initialPage, rowSource, idsPending = false,
|
| 2236 |
onOpenRecord, onSave, onClose }: LinkGridModalProps) {
|
| 2237 |
const panelRef = useRef<HTMLDivElement>(null);
|
| 2238 |
const [selectedIds, setSelectedIds] = useState<number[]>(() => [...recordIds]);
|
| 2239 |
+
const countKey = `${table}:${rowSource?.key ?? recordIdsKey(recordIds)}`;
|
| 2240 |
const [serverCount, setServerCount] = useState<{ key: string; total: number | null }>();
|
| 2241 |
const reportTotal = useCallback((total: number | null) => {
|
| 2242 |
setServerCount({ key: countKey, total });
|
|
|
|
| 2301 |
onCellDisplayChange={onCellDisplayChange}
|
| 2302 |
embeddedRecordIds={editable ? undefined : recordIds}
|
| 2303 |
embeddedInitialPage={editable ? undefined : initialPage}
|
| 2304 |
+
embeddedRowSource={editable ? undefined : rowSource}
|
| 2305 |
embeddedIdsPending={!editable && idsPending}
|
| 2306 |
onEmbeddedTotalChange={editable ? undefined : reportTotal}
|
| 2307 |
embeddedExcludeRecordId={editable ? excludeRecordId : undefined}
|
|
|
|
| 2508 |
/** Linked grids page their immutable restrictions through the caller-owned payload seam. */
|
| 2509 |
function useRecordIdWindow(scope: SurfaceScope, ids: readonly number[] | undefined,
|
| 2510 |
related: RelatedPin | null, initialPage?: RecordIdInitialPage, idsPending = false, cellView = false,
|
| 2511 |
+
selectionFilter?: FilterTree, rowSource?: EmbeddedRowSource) {
|
| 2512 |
const candidates = !!selectionFilter?.nodes?.length;
|
| 2513 |
+
const baseKey = JSON.stringify([scope, rowSource?.key ?? ids, related, initialPage, idsPending, cellView]);
|
| 2514 |
const key = candidates ? baseKey + "::" + JSON.stringify(selectionFilter) : baseKey;
|
| 2515 |
const appliedCellFilters = cellView && ids !== undefined;
|
| 2516 |
const seed = !idsPending && ids !== undefined && !related && initialPage && initialPage.offset === 0
|
|
|
|
| 2547 |
return;
|
| 2548 |
}
|
| 2549 |
setPage((prev) => ({ key, payload: prev?.key === key ? prev.payload : null, pending: true }));
|
| 2550 |
+
const readWindow = rowSource ? rowSource.readWindow : (request: RowWindowRequest) => fetchTableWindow(scope, request);
|
| 2551 |
+
void readWindow({ ...req,
|
| 2552 |
...(candidates ? linkSelectionPredicate(req, selectionFilter) : {}),
|
| 2553 |
+
...(ids !== undefined && !rowSource ? { ids } : {}), related }).then((payload) => {
|
| 2554 |
if (seq !== sequence.current || currentKey.current !== key) return;
|
| 2555 |
if (candidates && selectionFilter && payload && unresolvedConditions(selectionFilter.nodes, {
|
| 2556 |
fieldByKey: new Map(payload.fields.map((field) => [field.key, field])), today: payload.today,
|
|
|
|
| 2563 |
}
|
| 2564 |
setPage({ key, payload, pending: false });
|
| 2565 |
});
|
| 2566 |
+
}, [scope, ids, related, key, seed, initialPage, idsPending, appliedCellFilters, candidates, selectionFilter, rowSource]);
|
| 2567 |
useEffect(() => {
|
| 2568 |
const refresh = () => { if (lastRequest.current) requestWindow(lastRequest.current, true); };
|
| 2569 |
// React can replay effect setup after cancelling its first request in StrictMode.
|
|
|
|
| 2600 |
embeddedPayload,
|
| 2601 |
embeddedRecordIds,
|
| 2602 |
embeddedInitialPage,
|
| 2603 |
+
embeddedRowSource,
|
| 2604 |
embeddedIdsPending = false,
|
| 2605 |
onEmbeddedTotalChange,
|
| 2606 |
embeddedSelectable = false,
|
|
|
|
| 2664 |
() => embeddedIdsKey ? JSON.parse(embeddedIdsKey) as number[] : undefined, [embeddedIdsKey]);
|
| 2665 |
const selectionWindow = embedded && !!embeddedSelectionFilter?.nodes?.length;
|
| 2666 |
const idWindow = useRecordIdWindow(scope, embeddedIds, relatedPin, embeddedInitialPage, embeddedIdsPending, cellView,
|
| 2667 |
+
selectionWindow ? embeddedSelectionFilter : undefined, embeddedRowSource);
|
| 2668 |
/**
|
| 2669 |
* ⭐⭐ W35-T23 (owner item 4 / R2) — **`previewReadOnly` IS GONE, AND THE SPLIT IS THE TICKET.**
|
| 2670 |
*
|
|
|
|
| 4480 |
const windowPredicate = useMemo(
|
| 4481 |
() => windowPredicateKey(config.filters, config.filterConj, config.sorts, search)
|
| 4482 |
+ (embeddedIdsKey || selectionWindow
|
| 4483 |
+
? JSON.stringify([scope, embeddedRowSource?.key ?? embeddedIdsKey, relatedPinKey, cellView, embeddedSelectionFilter]) : ""),
|
| 4484 |
[config.filters, config.filterConj, config.sorts, search, scope, embeddedIdsKey, relatedPinKey, cellView,
|
| 4485 |
+
selectionWindow, embeddedSelectionFilter, embeddedRowSource]
|
| 4486 |
);
|
| 4487 |
/* HOTFIX 2026-09-24 — ONE PAGE SIZE FOR EVERY DATABASE (50 / 100 / 200, default 100). */
|
| 4488 |
const windowRequest = useCallback(
|
|
|
|
| 5004 |
return;
|
| 5005 |
}
|
| 5006 |
let live = true;
|
| 5007 |
+
const readAggregates = embeddedRowSource ? (_table: string, request: AggregateRequest) => embeddedRowSource.readAggregates(request) : fetchAggregates;
|
| 5008 |
+
void readAggregates(topic.key, {
|
| 5009 |
...(embeddedIds !== undefined ? { ids: embeddedIds } : {}),
|
| 5010 |
...linkSelectionPredicate({
|
| 5011 |
filters: cellView && embeddedIds !== undefined ? withoutAppliedLeaves(config.filters) : config.filters,
|
|
|
|
| 5042 |
// is part of the question, so moving between two tabs on one target table must re-ask. Its
|
| 5043 |
// identity is stable (see `relatedPinKey`), so this cannot loop.
|
| 5044 |
}, [serverWindowed, topic.key, summaryRequestKey, windowPredicate, config.groupBy, relatedPin, embeddedIds,
|
| 5045 |
+
serverAggRequestKey, cellView, embeddedSelectionFilter, embeddedRowSource]);
|
| 5046 |
/**
|
| 5047 |
* What the bar actually paints: C1's numbers when the server answered, this browser's fold
|
| 5048 |
* over the loaded window otherwise — and the two are never mixed within one paint, because a
|
|
|
|
| 5201 |
error, which replaced the whole grid with "Cannot reach the server." for one group's page;
|
| 5202 |
the failure belongs to the group that asked, so its cause stays with that group. */
|
| 5203 |
let failure = "";
|
| 5204 |
+
const readGroupWindow = embeddedRowSource ? (_scope: string, request: RowWindowRequest, _onFailure: (reason: string) => void) => embeddedRowSource.readWindow(request) : fetchTableWindow;
|
| 5205 |
+
void readGroupWindow(scope, {
|
| 5206 |
...windowRequest(offset),
|
| 5207 |
related: relatedPin,
|
| 5208 |
group: { by, key: groupKeyRaw.get(key) ?? null },
|
|
|
|
| 5222 |
return next;
|
| 5223 |
});
|
| 5224 |
});
|
| 5225 |
+
}, [config.groupBy, groupKeyRaw, scope, windowRequest, relatedPin, embeddedRowSource]);
|
| 5226 |
useEffect(() => {
|
| 5227 |
if (!serverWindowed || !config.groupBy || groupSummaryOn) return;
|
| 5228 |
for (const key of expanded) {
|
|
|
|
| 12243 |
{displayMode === "calendar" &&
|
| 12244 |
(calendarField ? (
|
| 12245 |
serverWindowed ? <MirrorCalendar key={mirrorDetailKey + calendarField.key}
|
| 12246 |
+
rowSource={embeddedRowSource}
|
| 12247 |
table={topic.key} request={mirrorModeRequest} field={calendarField} fieldByKey={fieldByKey}
|
| 12248 |
titleKey={lockedKey} today={today} size={pageSize}
|
| 12249 |
summary={displaySpec?.calendarMode === "summary"} metrics={displaySpec?.calendarMetrics}
|
|
|
|
| 12296 |
{displayMode === "kanban" &&
|
| 12297 |
(kanbanField ? (
|
| 12298 |
serverWindowed ? <MirrorKanban key={mirrorDetailKey + kanbanField.key}
|
| 12299 |
+
rowSource={embeddedRowSource}
|
| 12300 |
table={topic.key} request={mirrorModeRequest} field={kanbanField} fieldByKey={fieldByKey}
|
| 12301 |
cardKeys={kanbanCardKeys} titleKey={lockedKey} size={pageSize}
|
| 12302 |
clamped={displaySpec?.kanbanClamp !== false} aggs={summaryRequest} summaryMap={summaryMap}
|
web/src/customer-grid/apiBridge.ts
CHANGED
|
@@ -1209,6 +1209,29 @@ export function aggregatePayload(body: unknown): AggregateResult | null {
|
|
| 1209 |
return out;
|
| 1210 |
}
|
| 1211 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1212 |
/**
|
| 1213 |
* `POST /api/v1/odoo-tables/<key>/aggregates` — C1, over the WHOLE predicate.
|
| 1214 |
*
|
|
@@ -1221,32 +1244,52 @@ export async function fetchAggregates(
|
|
| 1221 |
req: AggregateRequest
|
| 1222 |
): Promise<AggregateResult | null> {
|
| 1223 |
if (!req.aggs.length && !req.groupBy) return null;
|
| 1224 |
-
let
|
| 1225 |
-
|
| 1226 |
-
|
| 1227 |
-
|
| 1228 |
-
|
| 1229 |
-
|
| 1230 |
-
|
| 1231 |
-
|
| 1232 |
-
|
| 1233 |
-
|
| 1234 |
-
|
| 1235 |
-
|
| 1236 |
-
|
| 1237 |
-
|
| 1238 |
-
|
| 1239 |
-
|
| 1240 |
-
|
| 1241 |
-
|
| 1242 |
-
|
| 1243 |
-
|
| 1244 |
-
|
| 1245 |
-
|
| 1246 |
-
|
| 1247 |
-
|
| 1248 |
-
|
| 1249 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1250 |
}
|
| 1251 |
|
| 1252 |
// ═══════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
| 1209 |
return out;
|
| 1210 |
}
|
| 1211 |
|
| 1212 |
+
/** Refuse broken pagination metadata before another aggregate page is requested. */
|
| 1213 |
+
export function aggregatePageProgress(body: unknown, offset: number, size: number | undefined,
|
| 1214 |
+
expected?: number): { total?: number; done: boolean } | null {
|
| 1215 |
+
if (!body || typeof body !== "object" || size === undefined) return null;
|
| 1216 |
+
const meta = body as { total_groups?: number; group_offset?: number; group_limit?: number;
|
| 1217 |
+
has_more?: boolean };
|
| 1218 |
+
if (meta.total_groups === undefined) {
|
| 1219 |
+
return offset === 0 && expected === undefined
|
| 1220 |
+
&& (meta.has_more === undefined || meta.has_more === false)
|
| 1221 |
+
&& (meta.group_offset === undefined || meta.group_offset === 0)
|
| 1222 |
+
? { done: true } : null;
|
| 1223 |
+
}
|
| 1224 |
+
const total = meta.total_groups;
|
| 1225 |
+
if (!Number.isSafeInteger(total) || total < 0 || meta.group_offset !== offset
|
| 1226 |
+
|| !Number.isSafeInteger(meta.group_limit) || (meta.group_limit ?? 0) <= 0
|
| 1227 |
+
|| size > meta.group_limit!
|
| 1228 |
+
|| (expected !== undefined && expected !== total)) return null;
|
| 1229 |
+
const next = offset + size;
|
| 1230 |
+
if (next > total || (size === 0 && next < total)
|
| 1231 |
+
|| (meta.has_more !== undefined && meta.has_more !== (next < total))) return null;
|
| 1232 |
+
return { total, done: next === total };
|
| 1233 |
+
}
|
| 1234 |
+
|
| 1235 |
/**
|
| 1236 |
* `POST /api/v1/odoo-tables/<key>/aggregates` — C1, over the WHOLE predicate.
|
| 1237 |
*
|
|
|
|
| 1244 |
req: AggregateRequest
|
| 1245 |
): Promise<AggregateResult | null> {
|
| 1246 |
if (!req.aggs.length && !req.groupBy) return null;
|
| 1247 |
+
let offset = 0;
|
| 1248 |
+
let answer: AggregateResult | null = null;
|
| 1249 |
+
let expectedGroups: number | undefined;
|
| 1250 |
+
do {
|
| 1251 |
+
let res: Response;
|
| 1252 |
+
try {
|
| 1253 |
+
res = await fetch(
|
| 1254 |
+
`${API_V1}/odoo-tables/${encodeURIComponent(tableKey)}/aggregates`,
|
| 1255 |
+
{
|
| 1256 |
+
method: "POST",
|
| 1257 |
+
credentials: CREDENTIALS,
|
| 1258 |
+
headers: JSON_HEADERS,
|
| 1259 |
+
body: JSON.stringify({
|
| 1260 |
+
...(req.ids !== undefined ? { ids: req.ids } : {}),
|
| 1261 |
+
filters: req.filters ?? [],
|
| 1262 |
+
filterConj: req.filterConj === "or" ? "or" : "and",
|
| 1263 |
+
search: (req.search ?? "").trim(),
|
| 1264 |
+
...(req.groupBy ? { groupBy: req.groupBy } : {}),
|
| 1265 |
+
...(req.groupBy ? { groupOffset: offset, groupLimit: 1000 } : {}),
|
| 1266 |
+
/* ⛔ A BODY KEY HERE, A QUERY STRING ON `/rows` (`windowRowsPath`). Same object, two
|
| 1267 |
+
transports, because this door is a POST and that one is a GET. */
|
| 1268 |
+
...(req.related ? { related: req.related } : {}),
|
| 1269 |
+
aggs: req.aggs,
|
| 1270 |
+
}),
|
| 1271 |
+
}
|
| 1272 |
+
);
|
| 1273 |
+
} catch {
|
| 1274 |
+
return null;
|
| 1275 |
+
}
|
| 1276 |
+
if (handledSession(res) || !res.ok) return null;
|
| 1277 |
+
const body = await readJson(res);
|
| 1278 |
+
const page = aggregatePayload(body);
|
| 1279 |
+
if (!page) return null;
|
| 1280 |
+
if (!req.groupBy) return page;
|
| 1281 |
+
const progress = aggregatePageProgress(body, offset, page.groups?.length, expectedGroups);
|
| 1282 |
+
if (!progress) return null;
|
| 1283 |
+
expectedGroups = progress.total;
|
| 1284 |
+
if (!answer) answer = page;
|
| 1285 |
+
else {
|
| 1286 |
+
if (answer.totalRows !== page.totalRows
|
| 1287 |
+
|| JSON.stringify(answer.aggregates) !== JSON.stringify(page.aggregates)) return null;
|
| 1288 |
+
answer.groups = [...(answer.groups ?? []), ...(page.groups ?? [])];
|
| 1289 |
+
}
|
| 1290 |
+
offset += page.groups?.length ?? 0;
|
| 1291 |
+
if (progress.done) return answer;
|
| 1292 |
+
} while (true);
|
| 1293 |
}
|
| 1294 |
|
| 1295 |
// ═══════════════════════════════════════════════════════════════════════════════════════════
|
web/src/sheet/DrillModal.tsx
CHANGED
|
@@ -6,11 +6,9 @@
|
|
| 6 |
// popups that behave almost the same, and "almost" is where a focus trap or an escape key goes
|
| 7 |
// missing on one of them.
|
| 8 |
//
|
| 9 |
-
//
|
| 10 |
-
//
|
| 11 |
-
//
|
| 12 |
-
// component always calls `GET /sheets/{id}/cells/{addr}/drill` and never reads ids off the cell.
|
| 13 |
-
// Passing the cell's cached ids would be a stale count presented as a live one.
|
| 14 |
//
|
| 15 |
// ⭐ R13 ALSO FORBIDS DISCLOSING STALENESS HERE. A stale cell opens the same popup with the same
|
| 16 |
// chrome and NO comparison text: no "was 24,920", no badge, nothing. The staleness is a property of
|
|
@@ -55,10 +53,10 @@
|
|
| 55 |
|
| 56 |
import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
| 57 |
import { API_V1, CREDENTIALS } from "../apiContract";
|
| 58 |
-
import { fetchWorkspace } from "../customer-grid/apiBridge";
|
| 59 |
import { dropAppliedFilterLimits } from "../customer-grid/drillLimits";
|
| 60 |
import type { SurfaceScope } from "../customer-grid/apiBridge";
|
| 61 |
-
import type { CellDisplayChoice } from "../customer-grid/CustomerGrid";
|
| 62 |
import { dataPathHash } from "../customer-grid/dataPath";
|
| 63 |
import { AnchoredOverlay, BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface";
|
| 64 |
import type { CustomersPayload, Field, FilterTree, GridWorkspace, Row } from "../customer-grid/types";
|
|
@@ -239,12 +237,9 @@ function DrillFilterBand(
|
|
| 239 |
);
|
| 240 |
}
|
| 241 |
|
| 242 |
-
/**
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
*/
|
| 246 |
-
function drillCapText(drill: Pick<CellDrill, "record_ids" | "count"> | null): string {
|
| 247 |
-
if (!drill || drill.record_ids.length >= drill.count) return "";
|
| 248 |
return `Showing ${drill.record_ids.length.toLocaleString()} of ${drill.count.toLocaleString()} `
|
| 249 |
+ "matching records. Narrow the cell's filter to list the rest.";
|
| 250 |
}
|
|
@@ -739,13 +734,77 @@ function DrillModal(
|
|
| 739 |
today: drill.page.today,
|
| 740 |
recordsMutable: drill.page.recordsMutable,
|
| 741 |
limits: dropAppliedFilterLimits(drill.page.limits, source?.kind === "filter" ? source.tree : undefined),
|
| 742 |
-
counts: { shown: drill.page.rows.length, matched: drill.
|
| 743 |
-
total: drill.
|
| 744 |
};
|
| 745 |
return { offset: drill.page.offset, limit: drill.page.limit, token: drill.token, payload,
|
| 746 |
filter: source?.kind === "filter" ? source.tree : undefined };
|
| 747 |
}, [drill, fields]);
|
| 748 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
const identityField = fields.find((field) => field.pinned) ?? fields[0] ?? null;
|
| 750 |
const savedView = drill?.view_id
|
| 751 |
? workspace?.views?.find((view) => view.id === drill.view_id) ?? null
|
|
@@ -802,18 +861,33 @@ function DrillModal(
|
|
| 802 |
setRowsError("");
|
| 803 |
try {
|
| 804 |
const first = drill.page?.offset === 0 ? drill.page : undefined;
|
| 805 |
-
const recordRows: Row[] = [
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 809 |
if (!got.ok) throw new Error(got.message);
|
| 810 |
if (!got.value.page) throw new Error("The records for Export did not include a page. Try again.");
|
| 811 |
-
if (got.value.
|
| 812 |
-
|| JSON.stringify(got.value.record_ids) !== JSON.stringify(drill.record_ids))
|
| 813 |
throw new Error("These records changed while Export was preparing. Try again.");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 814 |
recordRows.push(...got.value.page.rows);
|
| 815 |
-
offset += got.value.page.
|
| 816 |
}
|
|
|
|
| 817 |
const exporter = await import("../customer-grid/export");
|
| 818 |
exporter.runExport(format, `${label} records`, first?.today, visibleFields, recordRows);
|
| 819 |
} catch (cause) {
|
|
@@ -914,6 +988,7 @@ function DrillModal(
|
|
| 914 |
label={label}
|
| 915 |
table={scope}
|
| 916 |
recordIds={drill.record_ids}
|
|
|
|
| 917 |
{...{ initialPage, idsPending }}
|
| 918 |
editable={false}
|
| 919 |
cellView
|
|
@@ -940,7 +1015,7 @@ function DrillModal(
|
|
| 940 |
className="sheet-cell-view-action"
|
| 941 |
aria-haspopup="menu"
|
| 942 |
aria-expanded={exportOpen}
|
| 943 |
-
disabled={!exportReady || exporting}
|
| 944 |
onClick={() => setExportOpen((open) => !open)}
|
| 945 |
>
|
| 946 |
<svg viewBox="0 0 16 16" aria-hidden="true">
|
|
@@ -950,9 +1025,9 @@ function DrillModal(
|
|
| 950 |
</button>
|
| 951 |
</div>
|
| 952 |
)}
|
| 953 |
-
footer={bandText || capText || rowsError || drill.
|
| 954 |
? <>
|
| 955 |
-
{drill.
|
| 956 |
<DrillFilterBand text="No records match this cell's filter." />
|
| 957 |
)}
|
| 958 |
{rowsError && <DrillFilterBand text={rowsError} />}
|
|
|
|
| 6 |
// popups that behave almost the same, and "almost" is where a focus trap or an escape key goes
|
| 7 |
// missing on one of them.
|
| 8 |
//
|
| 9 |
+
// R13 requires live rows and counts. The drill returns an exact count and page IDs.
|
| 10 |
+
// Its caller-owned row source replays structured queries under current permissions.
|
| 11 |
+
// Page IDs identify rows already read and never restrict the remaining population.
|
|
|
|
|
|
|
| 12 |
//
|
| 13 |
// ⭐ R13 ALSO FORBIDS DISCLOSING STALENESS HERE. A stale cell opens the same popup with the same
|
| 14 |
// chrome and NO comparison text: no "was 24,920", no badge, nothing. The staleness is a property of
|
|
|
|
| 53 |
|
| 54 |
import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
| 55 |
import { API_V1, CREDENTIALS } from "../apiContract";
|
| 56 |
+
import { aggregatePageProgress, aggregatePayload, fetchWorkspace } from "../customer-grid/apiBridge";
|
| 57 |
import { dropAppliedFilterLimits } from "../customer-grid/drillLimits";
|
| 58 |
import type { SurfaceScope } from "../customer-grid/apiBridge";
|
| 59 |
+
import type { CellDisplayChoice, EmbeddedRowSource } from "../customer-grid/CustomerGrid";
|
| 60 |
import { dataPathHash } from "../customer-grid/dataPath";
|
| 61 |
import { AnchoredOverlay, BodyPortal, OverlayProvider, useOverlayLayer } from "../customer-grid/OverlaySurface";
|
| 62 |
import type { CustomersPayload, Field, FilterTree, GridWorkspace, Row } from "../customer-grid/types";
|
|
|
|
| 237 |
);
|
| 238 |
}
|
| 239 |
|
| 240 |
+
/** Explain a legacy server's ID cap while page-based responses use their ordinary pager. */
|
| 241 |
+
function drillCapText(drill: Pick<CellDrill, "record_ids" | "count" | "record_ids_scope"> | null): string {
|
| 242 |
+
if (!drill || drill.record_ids_scope === "page" || drill.record_ids.length >= drill.count) return "";
|
|
|
|
|
|
|
|
|
|
| 243 |
return `Showing ${drill.record_ids.length.toLocaleString()} of ${drill.count.toLocaleString()} `
|
| 244 |
+ "matching records. Narrow the cell's filter to list the rest.";
|
| 245 |
}
|
|
|
|
| 734 |
today: drill.page.today,
|
| 735 |
recordsMutable: drill.page.recordsMutable,
|
| 736 |
limits: dropAppliedFilterLimits(drill.page.limits, source?.kind === "filter" ? source.tree : undefined),
|
| 737 |
+
counts: { shown: drill.page.rows.length, matched: drill.count,
|
| 738 |
+
total: drill.page.totalUnfiltered ?? drill.count, windowed: true },
|
| 739 |
};
|
| 740 |
return { offset: drill.page.offset, limit: drill.page.limit, token: drill.token, payload,
|
| 741 |
filter: source?.kind === "filter" ? source.tree : undefined };
|
| 742 |
}, [drill, fields]);
|
| 743 |
|
| 744 |
+
const rowSource = useMemo<EmbeddedRowSource | undefined>(() => {
|
| 745 |
+
if (!drill || drill.record_ids_scope !== "page") return undefined;
|
| 746 |
+
const source = scopeFromDrill(drill);
|
| 747 |
+
const filter = source?.kind === "filter" ? source.tree : undefined;
|
| 748 |
+
const read = async (query: import("./sheetApi").CellDrillQuery, offset = 0, limit = 100) => {
|
| 749 |
+
const got = await drillCell(workbookId, addr, tab, { query, offset, limit, filter });
|
| 750 |
+
if (!got.ok) { setRowsError(got.message); return null; }
|
| 751 |
+
return got.value;
|
| 752 |
+
};
|
| 753 |
+
return {
|
| 754 |
+
key: JSON.stringify([workbookId, addr, tab, drill.population, drill.filter, drill.view_id, reloadSeq]),
|
| 755 |
+
readGroups: async (request, by, aggs, offset, size) => {
|
| 756 |
+
const value = await read({ filters: request.filters, filterConj: request.filterConj,
|
| 757 |
+
search: request.search, groupBy: by, aggs, groupOffset: offset, groupLimit: size });
|
| 758 |
+
const summary = value?.summary;
|
| 759 |
+
if (!summary || !Array.isArray(summary.groups) || typeof summary.total_groups !== "number")
|
| 760 |
+
throw new Error("Groups could not be read for this window.");
|
| 761 |
+
if (summary.limits.length)
|
| 762 |
+
throw new Error(summary.limits.map(limit => limit.cause).filter(Boolean).join(". "));
|
| 763 |
+
return { groups: summary.groups as { key: string | number | null; label: string; count: number;
|
| 764 |
+
aggregates: Record<string, number | null> }[], totalGroups: summary.total_groups,
|
| 765 |
+
totalRows: summary.total_rows, aggregates: summary.aggregates };
|
| 766 |
+
},
|
| 767 |
+
readWindow: async (request) => {
|
| 768 |
+
const value = await read({ filters: request.filters, filterConj: request.filterConj,
|
| 769 |
+
sorts: request.sorts, search: request.search, group: request.group }, request.offset, request.limit);
|
| 770 |
+
if (!value?.page) return null;
|
| 771 |
+
const page = value.page;
|
| 772 |
+
return { ...initialPage?.payload, fields: page.fields?.length ? page.fields : fields,
|
| 773 |
+
rows: page.rows, today: page.today, recordsMutable: page.recordsMutable,
|
| 774 |
+
limits: dropAppliedFilterLimits(page.limits, filter),
|
| 775 |
+
counts: { shown: page.rows.length, matched: value.count,
|
| 776 |
+
total: page.totalUnfiltered ?? value.count, windowed: true } };
|
| 777 |
+
},
|
| 778 |
+
readAggregates: async (request) => {
|
| 779 |
+
let offset = 0;
|
| 780 |
+
let answer: ReturnType<typeof aggregatePayload> = null;
|
| 781 |
+
let total: number | undefined;
|
| 782 |
+
let population: string | null | undefined;
|
| 783 |
+
do {
|
| 784 |
+
const value = await read({ filters: request.filters, filterConj: request.filterConj,
|
| 785 |
+
search: request.search, aggs: request.aggs, groupBy: request.groupBy,
|
| 786 |
+
groupOffset: offset, groupLimit: 1000 });
|
| 787 |
+
const summary = value?.summary;
|
| 788 |
+
const page = summary ? aggregatePayload(summary) : null;
|
| 789 |
+
if (!page || !summary) return null;
|
| 790 |
+
if (!request.groupBy) return page;
|
| 791 |
+
const progress = aggregatePageProgress(summary, offset, page.groups?.length, total);
|
| 792 |
+
if (!progress) return null;
|
| 793 |
+
if (!progress.done && !value?.population) return null;
|
| 794 |
+
if (!answer) { answer = page; total = summary.total_groups; population = value?.population; }
|
| 795 |
+
else {
|
| 796 |
+
if (value?.population !== population || total !== summary.total_groups
|
| 797 |
+
|| page.totalRows !== answer.totalRows
|
| 798 |
+
|| JSON.stringify(page.aggregates) !== JSON.stringify(answer.aggregates)) return null;
|
| 799 |
+
answer.groups = [...(answer.groups ?? []), ...(page.groups ?? [])];
|
| 800 |
+
}
|
| 801 |
+
offset += page.groups?.length ?? 0;
|
| 802 |
+
if (progress.done) return answer;
|
| 803 |
+
} while (true);
|
| 804 |
+
},
|
| 805 |
+
};
|
| 806 |
+
}, [drill, workbookId, addr, tab, reloadSeq, initialPage, fields]);
|
| 807 |
+
|
| 808 |
const identityField = fields.find((field) => field.pinned) ?? fields[0] ?? null;
|
| 809 |
const savedView = drill?.view_id
|
| 810 |
? workspace?.views?.find((view) => view.id === drill.view_id) ?? null
|
|
|
|
| 861 |
setRowsError("");
|
| 862 |
try {
|
| 863 |
const first = drill.page?.offset === 0 ? drill.page : undefined;
|
| 864 |
+
const recordRows: Row[] = [];
|
| 865 |
+
const recordIds = new Set<number>();
|
| 866 |
+
let offset = 0;
|
| 867 |
+
if (!drill.population) throw new Error("These records have no freshness stamp. Reopen View data and try Export again.");
|
| 868 |
+
if (!Number.isSafeInteger(drill.count) || drill.count < 0)
|
| 869 |
+
throw new Error("These records have no complete count. Reopen View data and try Export again.");
|
| 870 |
+
while (offset < drill.count) {
|
| 871 |
+
const got = await drillCell(workbookId, addr, tab, { offset, limit: 1000,
|
| 872 |
+
filter: drill.view_id ? undefined : drill.filter, query: drill.query });
|
| 873 |
if (!got.ok) throw new Error(got.message);
|
| 874 |
if (!got.value.page) throw new Error("The records for Export did not include a page. Try again.");
|
| 875 |
+
if (got.value.population !== drill.population || got.value.count !== drill.count)
|
|
|
|
| 876 |
throw new Error("These records changed while Export was preparing. Try again.");
|
| 877 |
+
if (got.value.page.offset !== offset || got.value.page.total !== drill.count
|
| 878 |
+
|| got.value.page.rows.length > drill.count - offset)
|
| 879 |
+
throw new Error("Some records could not be read. Try Export again.");
|
| 880 |
+
if (!got.value.page.rows.length)
|
| 881 |
+
throw new Error("These records changed while Export was preparing. Try again.");
|
| 882 |
+
for (const row of got.value.page.rows) {
|
| 883 |
+
if (!Number.isSafeInteger(row.pid) || recordIds.has(row.pid))
|
| 884 |
+
throw new Error("Some records could not be read. Try Export again.");
|
| 885 |
+
recordIds.add(row.pid);
|
| 886 |
+
}
|
| 887 |
recordRows.push(...got.value.page.rows);
|
| 888 |
+
offset += got.value.page.rows.length;
|
| 889 |
}
|
| 890 |
+
if (recordRows.length !== drill.count) throw new Error("Some records could not be read. Try Export again.");
|
| 891 |
const exporter = await import("../customer-grid/export");
|
| 892 |
exporter.runExport(format, `${label} records`, first?.today, visibleFields, recordRows);
|
| 893 |
} catch (cause) {
|
|
|
|
| 988 |
label={label}
|
| 989 |
table={scope}
|
| 990 |
recordIds={drill.record_ids}
|
| 991 |
+
rowSource={rowSource}
|
| 992 |
{...{ initialPage, idsPending }}
|
| 993 |
editable={false}
|
| 994 |
cellView
|
|
|
|
| 1015 |
className="sheet-cell-view-action"
|
| 1016 |
aria-haspopup="menu"
|
| 1017 |
aria-expanded={exportOpen}
|
| 1018 |
+
disabled={!exportReady || exporting || idsPending}
|
| 1019 |
onClick={() => setExportOpen((open) => !open)}
|
| 1020 |
>
|
| 1021 |
<svg viewBox="0 0 16 16" aria-hidden="true">
|
|
|
|
| 1025 |
</button>
|
| 1026 |
</div>
|
| 1027 |
)}
|
| 1028 |
+
footer={bandText || capText || rowsError || drill.count === 0
|
| 1029 |
? <>
|
| 1030 |
+
{drill.count === 0 && (
|
| 1031 |
<DrillFilterBand text="No records match this cell's filter." />
|
| 1032 |
)}
|
| 1033 |
{rowsError && <DrillFilterBand text={rowsError} />}
|
web/src/sheet/sheetApi.ts
CHANGED
|
@@ -819,18 +819,34 @@ export interface CellDrillPage {
|
|
| 819 |
today?: string;
|
| 820 |
recordsMutable?: boolean;
|
| 821 |
limits?: import("../customer-grid/types").GridLimit[];
|
|
|
|
|
|
|
| 822 |
}
|
| 823 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 824 |
export interface CellDrill {
|
| 825 |
database: string;
|
| 826 |
view_id?: string;
|
| 827 |
filter?: unknown;
|
| 828 |
record_ids: number[];
|
|
|
|
|
|
|
| 829 |
count: number;
|
| 830 |
-
/** S66-T48: the cap the server applied to `record_ids`, or null when it listed every id. */
|
| 831 |
id_limit?: number | null;
|
| 832 |
page?: CellDrillPage;
|
| 833 |
token?: string | null;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 834 |
}
|
| 835 |
|
| 836 |
/**
|
|
@@ -838,7 +854,7 @@ export interface CellDrill {
|
|
| 838 |
* address against the first tab holding it, which drilled `Summary (All)` for a `Summary (SR)` cell.
|
| 839 |
*/
|
| 840 |
export const drillCell = (id: string, addr: string, tab?: string,
|
| 841 |
-
page?: { offset?: number; limit?: number; filter?: unknown }): Promise<Result<CellDrill>> => {
|
| 842 |
const query = new URLSearchParams();
|
| 843 |
if (tab) query.set("tab", tab);
|
| 844 |
if (page?.offset !== undefined) query.set("offset", String(page.offset));
|
|
@@ -846,6 +862,7 @@ export const drillCell = (id: string, addr: string, tab?: string,
|
|
| 846 |
// View data's UNSAVED filter: the server drills this tree in place of the stored one and
|
| 847 |
// writes nothing, so the grid gets the new ids before the cell write lands.
|
| 848 |
if (page?.filter !== undefined) query.set("filter", JSON.stringify(page.filter));
|
|
|
|
| 849 |
return call(
|
| 850 |
`/sheets/${encodeURIComponent(id)}/cells/${encodeURIComponent(addr)}/drill`
|
| 851 |
+ (query.size ? `?${query.toString()}` : ""),
|
|
|
|
| 819 |
today?: string;
|
| 820 |
recordsMutable?: boolean;
|
| 821 |
limits?: import("../customer-grid/types").GridLimit[];
|
| 822 |
+
total: number;
|
| 823 |
+
totalUnfiltered: number;
|
| 824 |
}
|
| 825 |
|
| 826 |
+
export type CellDrillQuery = Partial<Pick<import("../customer-grid/apiBridge").RowWindowRequest,
|
| 827 |
+
"filters" | "filterConj" | "sorts" | "search" | "group">> & {
|
| 828 |
+
aggs?: import("../customer-grid/apiBridge").AggregateRequest["aggs"];
|
| 829 |
+
groupBy?: string;
|
| 830 |
+
groupOffset?: number;
|
| 831 |
+
groupLimit?: number;
|
| 832 |
+
};
|
| 833 |
+
|
| 834 |
export interface CellDrill {
|
| 835 |
database: string;
|
| 836 |
view_id?: string;
|
| 837 |
filter?: unknown;
|
| 838 |
record_ids: number[];
|
| 839 |
+
/** Page identities only. The count and structured query describe the full population. */
|
| 840 |
+
record_ids_scope: "page";
|
| 841 |
count: number;
|
|
|
|
| 842 |
id_limit?: number | null;
|
| 843 |
page?: CellDrillPage;
|
| 844 |
token?: string | null;
|
| 845 |
+
population: string | null;
|
| 846 |
+
query: CellDrillQuery;
|
| 847 |
+
summary?: { ok: boolean; total_rows: number; aggregates: Record<string, number | null>;
|
| 848 |
+
groups?: unknown[]; total_groups?: number; group_offset?: number; group_limit?: number;
|
| 849 |
+
has_more?: boolean; limits: import("../customer-grid/types").GridLimit[] };
|
| 850 |
}
|
| 851 |
|
| 852 |
/**
|
|
|
|
| 854 |
* address against the first tab holding it, which drilled `Summary (All)` for a `Summary (SR)` cell.
|
| 855 |
*/
|
| 856 |
export const drillCell = (id: string, addr: string, tab?: string,
|
| 857 |
+
page?: { offset?: number; limit?: number; filter?: unknown; query?: CellDrillQuery }): Promise<Result<CellDrill>> => {
|
| 858 |
const query = new URLSearchParams();
|
| 859 |
if (tab) query.set("tab", tab);
|
| 860 |
if (page?.offset !== undefined) query.set("offset", String(page.offset));
|
|
|
|
| 862 |
// View data's UNSAVED filter: the server drills this tree in place of the stored one and
|
| 863 |
// writes nothing, so the grid gets the new ids before the cell write lands.
|
| 864 |
if (page?.filter !== undefined) query.set("filter", JSON.stringify(page.filter));
|
| 865 |
+
if (page?.query !== undefined) query.set("query", JSON.stringify(page.query));
|
| 866 |
return call(
|
| 867 |
`/sheets/${encodeURIComponent(id)}/cells/${encodeURIComponent(addr)}/drill`
|
| 868 |
+ (query.size ? `?${query.toString()}` : ""),
|