fsanyoto commited on
Commit
92127d3
·
verified ·
1 Parent(s): 7932416

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/amazon_relational.py +88 -19
  4. api/application_registry.py +6 -3
  5. api/assistant_stream.py +64 -5
  6. api/connectors_amazon.py +11 -5
  7. api/deps.py +61 -75
  8. api/main.py +54 -2
  9. api/odoo_relational.py +29 -11
  10. api/process_role.py +2 -4
  11. api/registry_source.py +768 -0
  12. api/request_class.py +36 -69
  13. api/request_timing.py +421 -0
  14. api/routes_admin.py +5 -2
  15. api/routes_connectors.py +12 -6
  16. api/routes_customers.py +55 -16
  17. api/routes_grid.py +94 -1
  18. api/routes_odoo_tables.py +786 -308
  19. api/routes_platform_admin.py +8 -8
  20. api/routes_products.py +27 -9
  21. api/routes_sheet.py +358 -113
  22. api/routes_tables.py +152 -58
  23. api/sheet_warm.py +147 -2
  24. platform/core/activity_log.py +111 -10
  25. platform/core/grid_events.py +11 -0
  26. platform/core/measure_resolve.py +4 -2
  27. platform/core/odoo.py +219 -6
  28. platform/core/refresh_proof.py +295 -0
  29. platform/core/request_timing.py +412 -0
  30. platform/core/store_pg.py +683 -59
  31. platform/core/user_tables.py +61 -12
  32. platform/harness/canonical_source.py +383 -0
  33. platform/harness/datastore.py +493 -12
  34. platform/harness/filter_sql.py +7 -3
  35. platform/harness/measure_filter.py +23 -11
  36. platform/harness/mirror_tokens.py +3 -15
  37. platform/harness/semantic.py +40 -14
  38. platform/modules/ar.py +19 -13
  39. platform/modules/backorders.py +8 -3
  40. platform/modules/customer_data.py +129 -48
  41. platform/modules/customers.py +25 -8
  42. platform/modules/inventory.py +9 -5
  43. platform/modules/map.py +6 -3
  44. platform/modules/product_data.py +56 -8
  45. platform/modules/products.py +13 -1
  46. platform/modules/sheet.py +8 -6
  47. requirements.txt +2 -2
  48. web/public/AIOS_DEPLOY_RECEIPT +1 -1
  49. web/src/automation/AutomationBuilder.tsx +2 -1
  50. web/src/automation/AutomationDetail.tsx +2 -1
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "5a226b6c1f70 (5a226b6c1f70)",
3
  "releases": [
4
  {
5
  "version": "v83",
 
1
  {
2
+ "current": "bb3a8714cd79044c8a7f99ddd29afbf5a891cdd3 (bb3a8714cd79)",
3
  "releases": [
4
  {
5
  "version": "v83",
VERSION CHANGED
@@ -1 +1 @@
1
- 5a226b6c1f70 (5a226b6c1f70)
 
1
+ bb3a8714cd79044c8a7f99ddd29afbf5a891cdd3 (bb3a8714cd79)
api/amazon_relational.py CHANGED
@@ -238,8 +238,14 @@ def _format(col, duck_type):
238
  return None
239
 
240
 
241
- def mirror_columns(cur, table):
242
  """[(name, type)] for a mirror table, or [] when it has never been synced."""
 
 
 
 
 
 
243
  try:
244
  return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
245
  except Exception: # noqa: BLE001
@@ -369,7 +375,7 @@ def _parent_lookup_sql(cur):
369
  return resolved
370
 
371
 
372
- def lookups(cur, key, names=None):
373
  """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
374
 
375
  `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
@@ -385,6 +391,23 @@ def lookups(cur, key, names=None):
385
  sku = sku if sku in names else None
386
  if not asin and not sku:
387
  return []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
  listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
389
  parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
390
  latest = _latest_listings_sql(listing_names)
@@ -529,14 +552,15 @@ def _quoted(name):
529
  AMAZON_STORED_BYTES = 4 * 1024 * 1024
530
 
531
 
532
- def _read_through(cur, key, count):
533
  """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
534
  ut = _odoo._ut()
535
  if key in READ_THROUGH or count > ut.MAX_ROWS:
536
  return True
537
  if not count:
538
  return False
539
- names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])]
 
540
  sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
541
  # Include the stored id, refreshed stamp and row-object punctuation, with the same
542
  # ASCII escaping as the document writer. The largest sampled row is conservative.
@@ -590,9 +614,9 @@ def _integer(value):
590
  return None if value in (None, "") else int(value)
591
 
592
 
593
- def grid_source(cur, key):
594
  """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
595
- columns = mirror_columns(cur, SOURCE[key])
596
  if not columns:
597
  return missing_source(key)
598
  names = {name for name, _dtype in columns}
@@ -610,11 +634,15 @@ def grid_source(cur, key):
610
 
611
  complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
612
  peer = '"_amazon_identity_peer"'
613
- equal = " AND ".join(f"{value(name, peer)} IS NOT DISTINCT FROM {value(name)}"
614
- for name in fallback if name in names) or "TRUE"
615
- # Equal incomplete keys retain source occurrence order, independent of mutable cells.
616
- occurrence = (f"(SELECT CAST(count(*) AS VARCHAR) FROM {source} AS {peer} "
617
- f"WHERE {equal} AND {peer}.rowid < {source}.rowid)")
 
 
 
 
618
  fallback_digest = digest(["'__amazon_missing_identity__'",
619
  *(value(name) for name in fallback), occurrence])
620
  digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
@@ -641,10 +669,26 @@ def grid_source(cur, key):
641
  cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
642
  # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
643
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
644
- for lookup in lookups(cur, key, names):
645
  cols[lookup["key"]] = (lookup["expr"], _text)
646
- return {"table": SOURCE[key], "where": "", "id": ident,
647
  "needs_excluded": False, "cols": cols}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
648
 
649
 
650
  def grid_sources(cur=None):
@@ -766,7 +810,7 @@ def wall_link_expression(source, predicate, identity):
766
  f"WHERE {predicate}) AS _wall_links)")
767
 
768
 
769
- def fill_link_cells(rows, fields, read_targets):
770
  """Populate derived Amazon links with one read per distinct target on this page.
771
 
772
  The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
@@ -775,17 +819,42 @@ def fill_link_cells(rows, fields, read_targets):
775
  Neither path stores history rows or caps the matching target set.
776
  """
777
  edges = link_edges(fields)
778
- requests = {}
779
  for _field, target, frm, on in edges:
780
  values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
781
  requests.setdefault(target, {}).setdefault(on, set()).update(values)
 
 
 
 
782
  indexes = {}
 
783
  for target, matches in requests.items():
784
  indexes[target] = {column: {} for column in matches}
785
- for row in read_targets(target, matches) if any(matches.values()) else ():
786
- for column in matches:
787
- if row.get(column) not in (None, ""):
788
- indexes[target][column].setdefault(str(row[column]), set()).add(int(row["pid"]))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
789
  for field, target, frm, on in edges:
790
  for row in rows:
791
  row[field["key"]] = ",".join(str(pid) for pid in sorted(
 
238
  return None
239
 
240
 
241
+ def mirror_columns(cur, table, *, strict=False):
242
  """[(name, type)] for a mirror table, or [] when it has never been synced."""
243
+ if strict:
244
+ # A successful empty metadata query proves absence on this cursor only.
245
+ return cur.execute(
246
+ "SELECT column_name, data_type FROM information_schema.columns "
247
+ "WHERE table_catalog = current_database() AND table_schema = current_schema() "
248
+ "AND table_name = ? ORDER BY ordinal_position", [table]).fetchall()
249
  try:
250
  return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
251
  except Exception: # noqa: BLE001
 
375
  return resolved
376
 
377
 
378
+ def lookups(cur, key, names=None, *, allowed=True):
379
  """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
380
 
381
  `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
 
391
  sku = sku if sku in names else None
392
  if not asin and not sku:
393
  return []
394
+ if not allowed:
395
+ # Keep declared cells blank without inspecting a denied lookup dependency.
396
+ keys = []
397
+ if asin:
398
+ if "parent_asin" not in names:
399
+ keys.append((LOOKUP_PARENT, asin, asin_via))
400
+ if not sku:
401
+ keys.append((LOOKUP_SKU, asin, asin_via))
402
+ elif sku:
403
+ if not names & _NATIVE_ASIN:
404
+ keys.append((LOOKUP_ASIN, sku, sku_via))
405
+ if "parent_asin" not in names:
406
+ keys.append((LOOKUP_PARENT, sku, sku_via))
407
+ return [{"key": name, "label": _LOOKUP_LABELS[name], "after": after,
408
+ "expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None,
409
+ "description": _LOOKUP_DESCRIPTIONS[name]}
410
+ for name, after, via in keys if name not in names]
411
  listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
412
  parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
413
  latest = _latest_listings_sql(listing_names)
 
552
  AMAZON_STORED_BYTES = 4 * 1024 * 1024
553
 
554
 
555
+ def _read_through(cur, key, count, *, columns=None):
556
  """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
557
  ut = _odoo._ut()
558
  if key in READ_THROUGH or count > ut.MAX_ROWS:
559
  return True
560
  if not count:
561
  return False
562
+ names = [name for name, _dtype in (columns if columns is not None
563
+ else mirror_columns(cur, SOURCE[key]))]
564
  sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
565
  # Include the stored id, refreshed stamp and row-object punctuation, with the same
566
  # ASCII escaping as the document writer. The largest sampled row is conservative.
 
614
  return None if value in (None, "") else int(value)
615
 
616
 
617
+ def grid_source(cur, key, *, columns=None, allow_lookups=True):
618
  """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
619
+ columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
620
  if not columns:
621
  return missing_source(key)
622
  names = {name for name, _dtype in columns}
 
634
 
635
  complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
636
  peer = '"_amazon_identity_peer"'
637
+ peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
638
+ partition = ", ".join(value(name, peer) for name in fallback)
639
+ # Rank only incomplete identities; earlier equal peers are exactly row_number minus one.
640
+ occurrence = (
641
+ '(SELECT CAST("_identity_occurrence" AS VARCHAR) FROM '
642
+ f'(SELECT {peer}.rowid AS "_identity_rowid", '
643
+ f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
644
+ f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
645
+ f'AS "_amazon_missing_identity" WHERE "_identity_rowid" = {source}.rowid)')
646
  fallback_digest = digest(["'__amazon_missing_identity__'",
647
  *(value(name) for name in fallback), occurrence])
648
  digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
 
669
  cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
670
  # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
671
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
672
+ for lookup in lookups(cur, key, names, allowed=allow_lookups):
673
  cols[lookup["key"]] = (lookup["expr"], _text)
674
+ spec = {"table": SOURCE[key], "where": "", "id": ident,
675
  "needs_excluded": False, "cols": cols}
676
+ if table not in amazon_store.DERIVED_TABLES:
677
+ spec["page_key"] = f"{source}.rowid"
678
+ return spec
679
+
680
+
681
+ def page_source(cur, key, definition=None, *, allow_lookups=True):
682
+ """Resolve one target's storage policy on the admitted cursor without provider discovery."""
683
+ columns = mirror_columns(cur, SOURCE[key], strict=True)
684
+ mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough"))
685
+ if not columns:
686
+ return missing_source(key) if mandatory else None
687
+ if not mandatory:
688
+ count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
689
+ if not _read_through(cur, key, count, columns=columns):
690
+ return None
691
+ return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
692
 
693
 
694
  def grid_sources(cur=None):
 
810
  f"WHERE {predicate}) AS _wall_links)")
811
 
812
 
813
+ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refusal=None):
814
  """Populate derived Amazon links with one read per distinct target on this page.
815
 
816
  The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
 
819
  Neither path stores history rows or caps the matching target set.
820
  """
821
  edges = link_edges(fields)
822
+ requests, weights = {}, {}
823
  for _field, target, frm, on in edges:
824
  values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
825
  requests.setdefault(target, {}).setdefault(on, set()).update(values)
826
+ for row in rows:
827
+ if row.get(frm) not in (None, ""):
828
+ slot = (target, on, str(row[frm]))
829
+ weights[slot] = weights.get(slot, 0) + 1
830
  indexes = {}
831
+ remaining = max_bytes
832
  for target, matches in requests.items():
833
  indexes[target] = {column: {} for column in matches}
834
+ iterator = iter(read_targets(target, matches) if any(matches.values()) else ())
835
+ try:
836
+ for row in iterator:
837
+ pid = int(row["pid"])
838
+ for column, wanted in matches.items():
839
+ value = str(row[column]) if row.get(column) not in (None, "") else None
840
+ if value not in wanted:
841
+ continue
842
+ bucket = indexes[target][column].get(value)
843
+ if bucket is not None and pid in bucket:
844
+ continue
845
+ if remaining is not None:
846
+ # Charge every output occurrence before allocating its distinct ID.
847
+ cost = (len(str(pid)) + (1 if bucket else 2)) * weights[target, column, value]
848
+ if cost > remaining:
849
+ raise capacity_refusal()
850
+ remaining -= cost
851
+ if bucket is None:
852
+ bucket = indexes[target][column][value] = set()
853
+ bucket.add(pid)
854
+ finally:
855
+ close = getattr(iterator, "close", None)
856
+ if close is not None:
857
+ close()
858
  for field, target, frm, on in edges:
859
  for row in rows:
860
  row[field["key"]] = ",".join(str(pid) for pid in sorted(
api/application_registry.py CHANGED
@@ -6,6 +6,7 @@ import copy
6
  import hashlib
7
  import json
8
  from core import shares, write_execution
 
9
  import app_rules
10
 
11
  from modules.finance_apps import (finance_application as _finance_application,
@@ -114,10 +115,11 @@ def _brand_analytics_refused(runtime):
114
  if path.resolve() == Path(datastore.DB_PATH).resolve():
115
  row = datastore.ro_cursor().execute(query, key).fetchone()
116
  else:
117
- with duckdb.connect(str(path), read_only=True) as cursor:
118
  row = cursor.execute(query, key).fetchone()
119
  return bool(row) and row[0] == "refused"
120
- except Exception: # noqa: BLE001
 
121
  return False
122
 
123
 
@@ -133,7 +135,8 @@ def brand_analytics_offered(session):
133
  and (enablement.get("databases") or {}).get(BRAND_ANALYTICS_DATABASE) is True)
134
  refused = _brand_analytics_refused(session.runtime) if connected and enabled else None
135
  return brand_analytics_holds(connected, enabled, refused)
136
- except Exception: # noqa: BLE001
 
137
  return False
138
 
139
 
 
6
  import hashlib
7
  import json
8
  from core import shares, write_execution
9
+ from core.request_timing import raise_if_resource_error
10
  import app_rules
11
 
12
  from modules.finance_apps import (finance_application as _finance_application,
 
115
  if path.resolve() == Path(datastore.DB_PATH).resolve():
116
  row = datastore.ro_cursor().execute(query, key).fetchone()
117
  else:
118
+ with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
119
  row = cursor.execute(query, key).fetchone()
120
  return bool(row) and row[0] == "refused"
121
+ except Exception as exc: # noqa: BLE001
122
+ raise_if_resource_error(exc)
123
  return False
124
 
125
 
 
135
  and (enablement.get("databases") or {}).get(BRAND_ANALYTICS_DATABASE) is True)
136
  refused = _brand_analytics_refused(session.runtime) if connected and enabled else None
137
  return brand_analytics_holds(connected, enabled, refused)
138
+ except Exception as exc: # noqa: BLE001
139
+ raise_if_resource_error(exc)
140
  return False
141
 
142
 
api/assistant_stream.py CHANGED
@@ -1,9 +1,13 @@
1
  """Expose one Query turn as server-sent events with same-turn error recovery."""
2
  import asyncio
3
  import contextvars
 
 
4
  import json
5
  import threading
6
 
 
 
7
  from fastapi import APIRouter, Body, Depends
8
  from starlette.responses import StreamingResponse
9
 
@@ -16,16 +20,65 @@ router = APIRouter(prefix="/api/v1", dependencies=[Depends(module_gate("analyst"
16
 
17
 
18
  class StreamGZipMiddleware:
19
- """Bypass compression for the streaming chat path only."""
20
 
21
  def __init__(self, app, **kwargs):
22
- from starlette.middleware.gzip import GZipMiddleware
23
  self.app = app
24
- self.compressed = GZipMiddleware(app, **kwargs)
 
25
 
26
  async def __call__(self, scope, receive, send):
27
- app = self.app if scope.get("path") == PATH else self.compressed
28
- await app(scope, receive, send)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
 
31
  def _event(name, data):
@@ -88,6 +141,12 @@ async def events(body, session, *, submit=None):
88
  put("text_delta", {"text": content[len(text):]})
89
  put("done", result)
90
  except Exception as exc:
 
 
 
 
 
 
91
  detail = getattr(exc, "detail", None)
92
  error = (detail or {}).get("error") if isinstance(detail, dict) else None
93
  put("error", {"code": (error or {}).get("code", "stream_failed"),
 
1
  """Expose one Query turn as server-sent events with same-turn error recovery."""
2
  import asyncio
3
  import contextvars
4
+ import gzip
5
+ import io
6
  import json
7
  import threading
8
 
9
+ import anyio
10
+
11
  from fastapi import APIRouter, Body, Depends
12
  from starlette.responses import StreamingResponse
13
 
 
20
 
21
 
22
  class StreamGZipMiddleware:
23
+ """Compress response chunks in the bounded anyio pool while passing SSE through."""
24
 
25
  def __init__(self, app, **kwargs):
 
26
  self.app = app
27
+ self.minimum_size = kwargs.get("minimum_size", 500)
28
+ self.compresslevel = kwargs.get("compresslevel", 9)
29
 
30
  async def __call__(self, scope, receive, send):
31
+ if scope.get("type") != "http" or scope.get("path") == PATH:
32
+ return await self.app(scope, receive, send)
33
+ from starlette.datastructures import Headers
34
+ from starlette.middleware.gzip import IdentityResponder
35
+ if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""):
36
+ responder = _OffLoopGZipResponder(self.app, self.minimum_size, self.compresslevel)
37
+ else:
38
+ responder = IdentityResponder(self.app, self.minimum_size)
39
+ await responder(scope, receive, send)
40
+
41
+
42
+ from starlette.middleware.gzip import GZipResponder, IdentityResponder
43
+
44
+
45
+ class _OffLoopGZipResponder(GZipResponder):
46
+ """Retain Starlette's header and chunk behavior with compression outside the event loop."""
47
+
48
+ def __init__(self, app, minimum_size, compresslevel):
49
+ IdentityResponder.__init__(self, app, minimum_size)
50
+ self.compresslevel = compresslevel
51
+ self.gzip_file = None
52
+
53
+ async def __call__(self, scope, receive, send):
54
+ try:
55
+ await IdentityResponder.__call__(self, scope, receive, send)
56
+ finally:
57
+ if self.gzip_file is not None:
58
+ def close():
59
+ self.gzip_file.close()
60
+ self.gzip_buffer.close()
61
+ with anyio.CancelScope(shield=True):
62
+ await anyio.to_thread.run_sync(close)
63
+
64
+ def _compress(self, body, more):
65
+ if self.gzip_file is None:
66
+ self.gzip_buffer = io.BytesIO()
67
+ self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer,
68
+ compresslevel=self.compresslevel)
69
+ return GZipResponder.apply_compression(self, body, more_body=more)
70
+
71
+ async def send_with_compression(self, message):
72
+ if (message["type"] == "http.response.body"
73
+ and not self.content_encoding_set and not self.content_type_is_excluded):
74
+ body, more = message.get("body", b""), message.get("more_body", False)
75
+ if self.started or more or len(body) >= self.minimum_size:
76
+ self._compressed_body = await anyio.to_thread.run_sync(
77
+ lambda: self._compress(body, more))
78
+ await super().send_with_compression(message)
79
+
80
+ def apply_compression(self, body, *, more_body):
81
+ return self._compressed_body
82
 
83
 
84
  def _event(name, data):
 
141
  put("text_delta", {"text": content[len(text):]})
142
  put("done", result)
143
  except Exception as exc:
144
+ from request_timing import resource_cause
145
+ cause = resource_cause(exc)
146
+ if cause is not None:
147
+ put("error", {"code": "server_busy", "cause": cause,
148
+ "message": "The server is busy. Try again shortly.", "retry": False})
149
+ return
150
  detail = getattr(exc, "detail", None)
151
  error = (detail or {}).get("error") if isinstance(detail, dict) else None
152
  put("error", {"code": (error or {}).get("code", "stream_failed"),
api/connectors_amazon.py CHANGED
@@ -29,6 +29,8 @@ _PLATFORM = Path(__file__).resolve().parents[2] / "platform"
29
  if str(_PLATFORM) not in sys.path:
30
  sys.path.insert(0, str(_PLATFORM))
31
 
 
 
32
  # ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
33
  # map and the transport; this module adds the probe and the registration and nothing else.
34
  from harness.amazon_client import ( # noqa: F401,E402
@@ -189,7 +191,8 @@ def _mirror_counts(cur, az):
189
  try:
190
  counts[table] = int(cur.execute(
191
  f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
192
- except Exception: # noqa: BLE001
 
193
  continue
194
  return counts
195
 
@@ -337,7 +340,8 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
337
  try:
338
  purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
339
  purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
340
- except Exception:
 
341
  purge_plan, purge_message, purge_unknown = None, "", True
342
 
343
  def finish(status):
@@ -407,10 +411,11 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
407
  entities = _sync_entities(_cur, cadence)
408
  mirrored = _mirror_counts(_cur, az)
409
  else:
410
- with duckdb.connect(str(path), read_only=True) as cur:
411
  entities = _sync_entities(cur, cadence)
412
  mirrored = _mirror_counts(cur, az)
413
- except Exception: # noqa: BLE001
 
414
  # No exception text: paths, provider payloads and credentials never become status copy.
415
  return finish(unavailable)
416
  if entities is None:
@@ -642,7 +647,8 @@ def read_database_enablement(rt):
642
  keys = database_keys()
643
  try:
644
  current = rt.get(ENABLEMENT_KEY) if rt is not None else None
645
- except Exception: # noqa: BLE001
 
646
  return {"available": False, "databases": {key: False for key in keys}}
647
  # Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
648
  # the migration default rather than malformed state.
 
29
  if str(_PLATFORM) not in sys.path:
30
  sys.path.insert(0, str(_PLATFORM))
31
 
32
+ from core.request_timing import raise_if_resource_error
33
+
34
  # ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
35
  # map and the transport; this module adds the probe and the registration and nothing else.
36
  from harness.amazon_client import ( # noqa: F401,E402
 
191
  try:
192
  counts[table] = int(cur.execute(
193
  f"SELECT count(*) FROM {az.table_name(table)}").fetchone()[0])
194
+ except Exception as exc: # noqa: BLE001
195
+ raise_if_resource_error(exc)
196
  continue
197
  return counts
198
 
 
340
  try:
341
  purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
342
  purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
343
+ except Exception as exc:
344
+ raise_if_resource_error(exc)
345
  purge_plan, purge_message, purge_unknown = None, "", True
346
 
347
  def finish(status):
 
411
  entities = _sync_entities(_cur, cadence)
412
  mirrored = _mirror_counts(_cur, az)
413
  else:
414
+ with _ds.bounded_connection(duckdb.connect(str(path), read_only=True)) as cur:
415
  entities = _sync_entities(cur, cadence)
416
  mirrored = _mirror_counts(cur, az)
417
+ except Exception as exc: # noqa: BLE001
418
+ raise_if_resource_error(exc)
419
  # No exception text: paths, provider payloads and credentials never become status copy.
420
  return finish(unavailable)
421
  if entities is None:
 
647
  keys = database_keys()
648
  try:
649
  current = rt.get(ENABLEMENT_KEY) if rt is not None else None
650
+ except Exception as exc: # noqa: BLE001
651
+ raise_if_resource_error(exc)
652
  return {"available": False, "databases": {key: False for key in keys}}
653
  # Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
654
  # the migration default rather than malformed state.
api/deps.py CHANGED
@@ -16,8 +16,10 @@ import difflib
16
  import os
17
  import sys
18
  import time
 
19
  from collections import OrderedDict
20
  from dataclasses import dataclass
 
21
  from pathlib import Path
22
 
23
  from fastapi import Depends, HTTPException, Request, Response
@@ -1429,47 +1431,10 @@ def assistant_source_status(session: Session, databases=None):
1429
  return out
1430
 
1431
 
1432
- #: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
1433
- #: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
1434
- #: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
1435
- #: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
1436
- #: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
1437
- #: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
1438
- #: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
1439
- #: answers 503 when the store is unreadable); a cached registry is never served in its place.
1440
- #: The key carries the identity of the reader and the store it reads, so a swapped store or a
1441
- #: patched `users.registry` / `store.get` misses instead of replaying another registry.
1442
- _SESSION_REGISTRY = {"key": None, "reg": None}
1443
- _SESSION_REGISTRY_LOCK = __import__("threading").Lock()
1444
-
1445
-
1446
  def _session_registry():
1447
- """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed.
1448
-
1449
- ⛔ POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`),
1450
- and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
1451
- fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
1452
- memo on HF), and its `get()` already holds the bucket for the life of the process, so the
1453
- re-read this memo removes does not exist there: it reads straight through.
1454
- """
1455
- key = None
1456
- try:
1457
- backend = store.backend()
1458
- token = (store.revision("users") or {}).get("token") if backend == "pg" else None
1459
- if token not in (None, ""):
1460
- where = os.environ.get("AIOS_TENANT") or "royal-imports"
1461
- key = (backend, where, id(users.registry), id(store.get), str(token))
1462
- except Exception: # noqa: BLE001 -- fail-closed below
1463
- key = None
1464
- if key is not None:
1465
- with _SESSION_REGISTRY_LOCK:
1466
- if _SESSION_REGISTRY["key"] == key:
1467
- return _SESSION_REGISTRY["reg"]
1468
- reg = users.registry() or {}
1469
- if key is not None:
1470
- with _SESSION_REGISTRY_LOCK:
1471
- _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
1472
- return reg
1473
 
1474
 
1475
  def _user_for(claims):
@@ -1531,26 +1496,16 @@ def _user_for(claims):
1531
  return None
1532
 
1533
 
1534
- # --- wave 19 (R4): the LAST-ACTIVE stamp, throttled in this process ------------------------------
1535
- # "Is anyone actually using this account?" is one of the two questions the Loopable admin plane
1536
- # answers, and the only place that can honestly answer it is the session resolver — every
1537
- # authenticated request passes through here.
1538
- #
1539
- # ⛔ THEREFORE IT MUST COST ALMOST NOTHING. A store write per request would put the HF store's
1540
- # 256-commits/hour budget on the critical path of the entire API, and a store READ per request to
1541
- # decide whether to write would be little better. So the staleness decision is made from a
1542
- # PROCESS-LOCAL map: no I/O at all on the 99.97% of requests inside the window, and the write
1543
- # itself (once an hour per account) happens on `core.users`' own background thread, so no request
1544
- # ever waits for a hub round-trip. `core.users` explains why that write is SYNC and not the
1545
- # coalesced async path — `users.json` is shared with the Streamlit host and an async flush rebases
1546
- # on this process's cache, which would let a stamp revert somebody else's deactivation.
1547
- #
1548
- # HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
1549
- # write once per window (still bounded); a restart re-arms every user's first request; and the
1550
- # resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
1551
- # renders it as. Bounded independently so it cannot grow into a leak on a deployment
1552
- # with many accounts.
1553
  _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
 
 
1554
  _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
1555
  _ACTIVE_MAX_TRACKED = 4096
1556
 
@@ -1572,30 +1527,59 @@ def _principal(tenant, username):
1572
 
1573
 
1574
  def note_active(uname, when=None):
1575
- """Record that `uname` was just seen — WITHOUT writing. The login path calls this because it
1576
- has already stamped `last_active` itself; without it the very next request would find no entry
1577
- and stamp again immediately, making the "at most once an hour" rule false by one write per
1578
- sign-in."""
1579
  key = (uname or "").strip().lower()
1580
  if not key:
1581
  return
1582
- _ACTIVE_SEEN[key] = when if when is not None else time.time()
1583
- _ACTIVE_SEEN.move_to_end(key)
1584
- while len(_ACTIVE_SEEN) > _ACTIVE_MAX_TRACKED:
1585
- _ACTIVE_SEEN.popitem(last=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1586
 
1587
 
1588
  def _touch_active(uname):
1589
- """Stamp `last_active` at most once per window per account, per process. Never raises."""
1590
  try:
1591
  key = (uname or "").strip().lower()
1592
  if not key:
1593
  return
1594
  now = time.time()
1595
- last = _ACTIVE_SEEN.get(key)
1596
- if last is not None and (now - last) < _ACTIVE_EVERY:
1597
- return
1598
- note_active(key, now)
 
1599
  users.touch_active(key)
1600
  except Exception: # noqa: BLE001 — telemetry may never break a request
1601
  pass
@@ -1640,13 +1624,15 @@ def require_session(request: Request, response: Response) -> Session:
1640
  # `session.tenant` read back from something the request could influence.
1641
  response.headers[_TENANT_HEADER] = str(claims["t"])
1642
  response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
1643
- # R4: last AFTER the session is fully resolved — a stamp is only true of a request that was
1644
- # actually admitted, and putting it here means no failed-auth path can ever write one.
1645
  # qa-runner has no global account record. Never let the normal activity telemetry create or
1646
  # touch one through the default tenant-#0 users store.
1647
  if (not user.get("qa_runner")
1648
  and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
1649
- _touch_active(user.get("username"))
 
 
1650
  return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
1651
 
1652
 
 
16
  import os
17
  import sys
18
  import time
19
+ import threading
20
  from collections import OrderedDict
21
  from dataclasses import dataclass
22
+ from datetime import datetime, timezone
23
  from pathlib import Path
24
 
25
  from fastapi import Depends, HTTPException, Request, Response
 
1431
  return out
1432
 
1433
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1434
  def _session_registry():
1435
+ """Read the checked request registry through the store's revision-and-timestamp text cache."""
1436
+ # The same request snapshot serves roster hashing and people helpers through users.registry.
1437
+ return users.registry() or {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1438
 
1439
 
1440
  def _user_for(claims):
 
1496
  return None
1497
 
1498
 
1499
+ # --- R4 activity: read observations and persisted-write throttling are separate ----------------
1500
+ # GET/HEAD/OPTIONS only observe admitted activity in bounded process memory. They never start a
1501
+ # store stamp, even after restart or window expiry. Admin projections may show these observations;
1502
+ # they disappear on restart and are not flushed. Passive notification polling remains excluded.
1503
+ # Login already persists both stamps. Other admitted mutating requests may launch core.users'
1504
+ # fresh-read background merge once per account/window/process. Reads cannot prime that throttle.
1505
+ # Both maps evict oldest entries at the bound; eviction or restart re-arms the next mutation.
 
 
 
 
 
 
 
 
 
 
 
 
1506
  _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
1507
+ _ACTIVE_OBSERVED: "OrderedDict[tuple[str, str], float]" = OrderedDict()
1508
+ _ACTIVE_LOCK = threading.Lock()
1509
  _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
1510
  _ACTIVE_MAX_TRACKED = 4096
1511
 
 
1527
 
1528
 
1529
  def note_active(uname, when=None):
1530
+ """Prime the persisted-write throttle after login's stamp; read observations never call this."""
 
 
 
1531
  key = (uname or "").strip().lower()
1532
  if not key:
1533
  return
1534
+ with _ACTIVE_LOCK:
1535
+ _remember_active(_ACTIVE_SEEN, key, when if when is not None else time.time())
1536
+
1537
+
1538
+ def _remember_active(target, key, when):
1539
+ """Bound an activity map while its caller holds _ACTIVE_LOCK."""
1540
+ target[key] = when
1541
+ target.move_to_end(key)
1542
+ while len(target) > _ACTIVE_MAX_TRACKED:
1543
+ target.popitem(last=False)
1544
+
1545
+
1546
+ def _observe_active(tenant, uname):
1547
+ """Observe admitted activity without I/O or changing the persisted-write throttle."""
1548
+ key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower())
1549
+ if all(key):
1550
+ with _ACTIVE_LOCK:
1551
+ _remember_active(_ACTIVE_OBSERVED, key, time.time())
1552
+
1553
+
1554
+ def observed_last_active(tenant, uname, persisted=None):
1555
+ """Project the newer persisted or process-local observation for this exact tenant/account."""
1556
+ key = (str(tenant or "").strip().lower(), str(uname or "").strip().lower())
1557
+ with _ACTIVE_LOCK:
1558
+ observed = _ACTIVE_OBSERVED.get(key)
1559
+ if observed is None:
1560
+ return persisted
1561
+ if persisted:
1562
+ try:
1563
+ saved = datetime.fromisoformat(str(persisted).replace("Z", "+00:00"))
1564
+ if saved.tzinfo is not None and saved.timestamp() >= observed:
1565
+ return persisted
1566
+ except (ValueError, OverflowError):
1567
+ pass
1568
+ return datetime.fromtimestamp(observed, timezone.utc).isoformat(timespec="seconds")
1569
 
1570
 
1571
  def _touch_active(uname):
1572
+ """Throttle an admitted mutation's persisted stamp per account/process. Never raises."""
1573
  try:
1574
  key = (uname or "").strip().lower()
1575
  if not key:
1576
  return
1577
  now = time.time()
1578
+ with _ACTIVE_LOCK:
1579
+ last = _ACTIVE_SEEN.get(key)
1580
+ if last is not None and (now - last) < _ACTIVE_EVERY:
1581
+ return
1582
+ _remember_active(_ACTIVE_SEEN, key, now)
1583
  users.touch_active(key)
1584
  except Exception: # noqa: BLE001 — telemetry may never break a request
1585
  pass
 
1624
  # `session.tenant` read back from something the request could influence.
1625
  response.headers[_TENANT_HEADER] = str(claims["t"])
1626
  response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
1627
+ # Observe only AFTER authentication admission. Read methods stay store-read-only; only an
1628
+ # admitted mutation can launch the persisted stamp, without moving it outside request timing.
1629
  # qa-runner has no global account record. Never let the normal activity telemetry create or
1630
  # touch one through the default tenant-#0 users store.
1631
  if (not user.get("qa_runner")
1632
  and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
1633
+ _observe_active(claims["t"], user.get("username"))
1634
+ if request.method not in {"GET", "HEAD", "OPTIONS"}:
1635
+ _touch_active(user.get("username"))
1636
  return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
1637
 
1638
 
api/main.py CHANGED
@@ -140,7 +140,16 @@ _PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
140
  if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
141
  "date")}
142
 
143
- app = FastAPI(title="AIOS web API")
 
 
 
 
 
 
 
 
 
144
 
145
  # The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with
146
  # the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
@@ -159,6 +168,10 @@ def _error_shape(request: Request, exc: StarletteHTTPException):
159
  422 from a malformed path param) is wrapped here so a client never has to branch on two
160
  different error bodies.
161
  """
 
 
 
 
162
  detail = exc.detail
163
  if isinstance(detail, dict) and "error" in detail:
164
  body = detail
@@ -183,12 +196,51 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
183
  adapter, which has no durable session dict to degrade into. A 200 over a write that
184
  evaporated is the failure the whole rule exists to prevent.
185
  """
 
 
 
 
186
  return JSONResponse(
187
  {"error": {"code": "store_unavailable",
188
  "message": "the tenant store is unavailable. No change was saved"}},
189
  status_code=503)
190
 
191
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
  @app.exception_handler(freeze_notice.DataInputFrozen)
193
  def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
194
  """Use the existing readable error envelope for every maintenance refusal."""
@@ -306,7 +358,7 @@ def public_freeze_notice():
306
 
307
 
308
  @app.get("/api/health")
309
- def health(challenge: str = ""):
310
  """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
311
  reveal anything about the deployment beyond "the process is up". No version, no tenant list,
312
  no config: a health endpoint is the one URL every scanner finds first.
 
140
  if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
141
  "date")}
142
 
143
+ from request_timing import RequestTiming # noqa: E402
144
+
145
+
146
+ class _TimedFastAPI(FastAPI):
147
+ def build_middleware_stack(self):
148
+ # Include error responses, freeze refusals, admission and compressed body sends.
149
+ return RequestTiming(super().build_middleware_stack())
150
+
151
+
152
+ app = _TimedFastAPI(title="AIOS web API")
153
 
154
  # The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with
155
  # the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
 
168
  422 from a malformed path param) is wrapped here so a client never has to branch on two
169
  different error bodies.
170
  """
171
+ from request_timing import resource_response
172
+ busy = resource_response(exc)
173
+ if busy is not None:
174
+ return busy
175
  detail = exc.detail
176
  if isinstance(detail, dict) and "error" in detail:
177
  body = detail
 
196
  adapter, which has no durable session dict to degrade into. A 200 over a write that
197
  evaporated is the failure the whole rule exists to prevent.
198
  """
199
+ from request_timing import resource_response
200
+ busy = resource_response(exc)
201
+ if busy is not None:
202
+ return busy
203
  return JSONResponse(
204
  {"error": {"code": "store_unavailable",
205
  "message": "the tenant store is unavailable. No change was saved"}},
206
  status_code=503)
207
 
208
 
209
+ async def _resource_unavailable(request: Request, exc: Exception):
210
+ from request_timing import resource_response
211
+ return resource_response(exc)
212
+
213
+
214
+ @app.exception_handler(Exception)
215
+ async def _unexpected_error(request: Request, exc: Exception):
216
+ from request_timing import resource_response
217
+ busy = resource_response(exc)
218
+ if busy is not None:
219
+ return busy
220
+ return JSONResponse({"error": {"code": "server_error", "message": "The request could not finish."}},
221
+ status_code=500)
222
+
223
+
224
+ from core.odoo import OdooPoolTimeout
225
+ from core.store_pg import SharedReadTimeout
226
+ from core.request_timing import RequestDeadlineExceeded
227
+ from request_timing import DeadlineCapacityExceeded
228
+ from duckdb import OutOfMemoryException
229
+ for _resource_error in (OdooPoolTimeout, SharedReadTimeout, RequestDeadlineExceeded,
230
+ DeadlineCapacityExceeded, OutOfMemoryException):
231
+ app.add_exception_handler(_resource_error, _resource_unavailable)
232
+ try:
233
+ from psycopg_pool import PoolTimeout, TooManyRequests
234
+ from psycopg import OperationalError, InterfaceError
235
+ from psycopg.errors import QueryCanceled, LockNotAvailable, IdleInTransactionSessionTimeout
236
+ except ImportError:
237
+ pass
238
+ else:
239
+ for _resource_error in (PoolTimeout, TooManyRequests, QueryCanceled, LockNotAvailable,
240
+ OperationalError, InterfaceError, IdleInTransactionSessionTimeout):
241
+ app.add_exception_handler(_resource_error, _resource_unavailable)
242
+
243
+
244
  @app.exception_handler(freeze_notice.DataInputFrozen)
245
  def _data_input_frozen(request: Request, exc: freeze_notice.DataInputFrozen):
246
  """Use the existing readable error envelope for every maintenance refusal."""
 
358
 
359
 
360
  @app.get("/api/health")
361
+ async def health(challenge: str = ""):
362
  """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
363
  reveal anything about the deployment beyond "the process is up". No version, no tenant list,
364
  no config: a health endpoint is the one URL every scanner finds first.
api/odoo_relational.py CHANGED
@@ -3534,17 +3534,35 @@ def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False
3534
  f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}'
3535
  for k in keys)
3536
  cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})')
3537
- values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
3538
- for r in (rows or []) if isinstance(r, dict)]
3539
- if values:
3540
- if bulk:
3541
- # Column arrays avoid 40,000 Python/DuckDB execute round trips for stored shipments.
3542
- selects = ", ".join(
3543
- f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
3544
- cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
3545
- else:
3546
- placeholders = ", ".join(["?"] * len(keys))
3547
- cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3548
  return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
3549
 
3550
 
 
3534
  f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}'
3535
  for k in keys)
3536
  cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})')
3537
+ import itertools
3538
+ source = iter(rows if rows is not None else ())
3539
+ try:
3540
+ while True:
3541
+ batch = list(itertools.islice(source, 2048))
3542
+ if not batch:
3543
+ break
3544
+ values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
3545
+ for r in batch if isinstance(r, dict)]
3546
+ # Serialized values no longer need the dictionaries held by this batch.
3547
+ batch.clear()
3548
+ del batch
3549
+ if not values:
3550
+ del values
3551
+ continue
3552
+ if bulk:
3553
+ # Every insertion holds at most 2,048 rows, including column-array conversion.
3554
+ selects = ", ".join(
3555
+ f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
3556
+ cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
3557
+ else:
3558
+ placeholders = ", ".join(["?"] * len(keys))
3559
+ cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
3560
+ values.clear()
3561
+ del values
3562
+ finally:
3563
+ close = getattr(source, "close", None)
3564
+ if callable(close):
3565
+ close()
3566
  return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
3567
 
3568
 
api/process_role.py CHANGED
@@ -46,10 +46,8 @@ os.environ["AIOS_PROCESS_ROLE"] = ROLE
46
  if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
47
  print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
48
  if ROLE == "web":
49
- # The web process keeps a smaller DuckDB budget. apply_limits reads this at connect time.
50
- os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "3GB")
51
- if os.environ.get("AIOS_WEB_DUCKDB_THREADS"):
52
- os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ["AIOS_WEB_DUCKDB_THREADS"])
53
 
54
  _LOCK = threading.Lock()
55
  _STARTED = {}
 
46
  if ROLE_SOURCE != "AIOS_PROCESS_ROLE" and ROLE == "web":
47
  print(f"[process-role] {ROLE_SOURCE}: role=web. Heavy jobs run only in the sync worker.", flush=True)
48
  if ROLE == "web":
49
+ os.environ.setdefault("AIOS_DUCKDB_MEMORY", os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB")
50
+ os.environ.setdefault("AIOS_DUCKDB_THREADS", os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2")
 
 
51
 
52
  _LOCK = threading.Lock()
53
  _STARTED = {}
api/registry_source.py ADDED
@@ -0,0 +1,768 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bounded stored-source reads before canonical grid projection."""
2
+ import copy
3
+ from contextlib import closing
4
+ from dataclasses import is_dataclass, replace
5
+
6
+ from deps import err
7
+
8
+ BATCH_ROWS = 1024
9
+
10
+
11
+ def with_runtime(session, runtime):
12
+ """Retain ordinary lightweight host sessions beside the API dataclass."""
13
+ if is_dataclass(session):
14
+ return replace(session, runtime=runtime)
15
+ actor = copy.copy(session)
16
+ actor.runtime = runtime
17
+ return actor
18
+
19
+
20
+ def backend(runtime):
21
+ """Resolve the admitted runtime's physical store through its tenant guard."""
22
+ while hasattr(runtime, '_runtime'):
23
+ runtime = runtime._runtime
24
+ document_store = getattr(runtime, '_document_store', None)
25
+ if callable(document_store):
26
+ return document_store()
27
+ reader = runtime._store() if callable(getattr(runtime, '_store', None)) else runtime
28
+ from core import store
29
+ return store._d() if reader is store else reader
30
+
31
+
32
+ class MetadataStore:
33
+ """Keep per-record enrichment states outside the definition vocabulary."""
34
+ schema_only = True
35
+
36
+ def __init__(self, runtime):
37
+ self._runtime = runtime
38
+
39
+ def __getattr__(self, name):
40
+ return getattr(self._runtime, name)
41
+
42
+ def get_projection(self, name, drop=()):
43
+ if name == 'user_tables':
44
+ drop = tuple(dict.fromkeys([*drop, 'aiEnrich']))
45
+ return self._runtime.get_projection(name, drop=drop)
46
+
47
+
48
+ def paths(runtime, name, selected):
49
+ """Read explicit values through the paged store door, or a borrowed offline document."""
50
+ if not selected:
51
+ return []
52
+ reader = backend(runtime)
53
+ method = getattr(reader, 'get_json_paths', None)
54
+ if callable(method):
55
+ return method(getattr(runtime, 'store_key', str)(name), selected)
56
+ doc = runtime.get(name) or {}
57
+ out = []
58
+ for path in selected:
59
+ value = doc
60
+ for part in path:
61
+ value = value.get(str(part)) if isinstance(value, dict) else None
62
+ out.append(copy.deepcopy(value))
63
+ return out
64
+
65
+
66
+ def items(runtime, name, path=(), *, selected=None):
67
+ """Yield bounded batches through a streaming store door or the legacy offline reader."""
68
+ reader = backend(runtime)
69
+ if selected is not None:
70
+ selected = iter(selected)
71
+ import itertools
72
+ while True:
73
+ keys = list(itertools.islice(selected, BATCH_ROWS))
74
+ if not keys:
75
+ return
76
+ values = paths(runtime, name, [[*path, str(key)] for key in keys])
77
+ yield [(str(key), value) for key, value in zip(keys, values) if value is not None]
78
+ else:
79
+ stream = getattr(reader, 'iter_json_items', None)
80
+ if callable(stream):
81
+ try:
82
+ yield from stream(getattr(runtime, 'store_key', str)(name), path, batch_size=BATCH_ROWS)
83
+ except RuntimeError as exc:
84
+ from core.request_timing import raise_if_resource_error
85
+ raise_if_resource_error(exc)
86
+ code = str(exc).split(':', 1)[0]
87
+ if code in {'canonical_source_unavailable', 'canonical_source_invalid'}:
88
+ raise err(503, code, str(exc).partition(':')[2].strip()) from exc
89
+ raise
90
+ return
91
+ else:
92
+ # Offline compatibility retains its existing whole-document read.
93
+ import itertools
94
+ doc = runtime.get(name) or {}
95
+ for part in path:
96
+ doc = doc.get(str(part)) if isinstance(doc, dict) else None
97
+ if doc is None:
98
+ return
99
+ if not isinstance(doc, dict):
100
+ raise err(409, 'canonical_source_invalid', f'{name} does not contain a row object.')
101
+ source = iter(doc.items())
102
+ while True:
103
+ batch = list(itertools.islice(source, BATCH_ROWS))
104
+ if not batch:
105
+ return
106
+ yield batch
107
+
108
+
109
+ def array_values(runtime, name, path=(), *, drop=()):
110
+ """Stream array entries through the bound store with an ordinary offline fallback."""
111
+ reader = backend(runtime)
112
+ method = getattr(reader, 'iter_json_values', None)
113
+ if callable(method):
114
+ yield from method(getattr(runtime, 'store_key', str)(name), path, drop=drop, batch_size=BATCH_ROWS)
115
+ return
116
+ value = runtime.get(name) or {}
117
+ for part in path:
118
+ value = value[int(part)] if isinstance(value, list) else value.get(str(part), {})
119
+ import itertools
120
+ source = enumerate(value if isinstance(value, list) else [])
121
+ while True:
122
+ batch = list(itertools.islice(source, BATCH_ROWS))
123
+ if not batch:
124
+ return
125
+ yield [(str(index), {k: v for k, v in entry.items() if k not in drop}
126
+ if isinstance(entry, dict) else entry) for index, entry in batch]
127
+
128
+
129
+ def object_values(runtime, name, path=(), *, drop=(), omit=()):
130
+ """Read nested definitions with record operands excluded at the backing store."""
131
+ reader = backend(runtime)
132
+ method = getattr(reader, 'get_json_object', None)
133
+ if callable(method):
134
+ return method(getattr(runtime, 'store_key', str)(name), path, drop=drop, omit=omit)
135
+ value = runtime.get(name) or {}
136
+ for part in path:
137
+ value = value.get(str(part), {}) if isinstance(value, dict) else {}
138
+ return {key: ({k: v for k, v in entry.items() if k not in drop} if isinstance(entry, dict) else entry)
139
+ for key, entry in value.items() if key not in omit} if isinstance(value, dict) else {}
140
+
141
+
142
+ def members(runtime, name, path, ids, *, id_key=None, source_limit=None):
143
+ """Read complete selected membership through the bound store's SQL predicate."""
144
+ reader = backend(runtime)
145
+ method = getattr(reader, 'get_json_members', None)
146
+ if callable(method):
147
+ return method(getattr(runtime, 'store_key', str)(name), path, ids,
148
+ id_key=id_key, source_limit=source_limit)
149
+ value = runtime.get(name) or {}
150
+ for part in path:
151
+ value = value[int(part)] if isinstance(value, list) else value.get(str(part), {})
152
+ selected, seen, out = {int(pid) for pid in ids}, set(), []
153
+ for index, item in enumerate(value if isinstance(value, list) else []):
154
+ if source_limit is not None and index >= source_limit:
155
+ break
156
+ raw = item.get(id_key) if id_key is not None and isinstance(item, dict) else item
157
+ try:
158
+ pid = int(raw)
159
+ except (TypeError, ValueError):
160
+ continue
161
+ if pid in selected and pid not in seen:
162
+ out.append(item)
163
+ seen.add(pid)
164
+ return out
165
+
166
+
167
+ def generation(session, table_key):
168
+ """Check population and projection buckets without reading their record values."""
169
+ from core import schema_overlay, shares, store_pg, user_tables
170
+ runtime, reader = session.runtime, backend(session.runtime)
171
+ names = [user_tables.STORE_KEY, table_key + '_table_workspace', table_key + '__shared',
172
+ table_key + '_docs', user_tables.ZONE_STORE_KEY, schema_overlay.STORE_KEY,
173
+ shares.SHARES_KEY, table_key + '_cohorts']
174
+ if table_key in {'customer_data', 'product_data'}:
175
+ workspace = 'customer_table_workspace' if table_key == 'customer_data' else 'product_table_workspace'
176
+ names += [workspace, workspace + '__shared', 'customer_ledger', 'ar_days_to_pay',
177
+ 'geocoords', 'loopable_route_system', 'customer_docs', 'product_docs',
178
+ 'customer_cohorts', 'product_cohorts']
179
+ if isinstance(reader, store_pg.PgStore):
180
+ key = getattr(runtime, 'store_key', str)
181
+ # A post-read check must cross the backing validation boundary, including updated_at.
182
+ tokens = store_pg._read_revisions([key(name) for name in names], reader.tenant_slug)
183
+ else:
184
+ revision = getattr(runtime, 'revision', None)
185
+ if not callable(revision):
186
+ return None # Ordinary offline readers retain their original snapshot semantics.
187
+ tokens = {name: revision(name) for name in names}
188
+ return copy.deepcopy((session.user, tokens))
189
+
190
+
191
+ def require_generation(session, table_key, before):
192
+ if before is not None and generation(session, table_key) != before:
193
+ raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
194
+
195
+
196
+ class BatchStore:
197
+ """Lend canonical readers one row batch and its overlays, with every write refused."""
198
+
199
+ def __init__(self, runtime, table_key, definition, rows, *, metadata=None, uname=None, needed=None):
200
+ self._runtime, self.table_key = runtime, table_key
201
+ self.definition = dict(definition, rows=dict(rows))
202
+ self.row_ids = [str(key) for key, _ in rows]
203
+ self.docs = {}
204
+ self.metadata = metadata if metadata is not None else {}
205
+ self.uname = uname
206
+ self._canonical_needed = needed
207
+ self.measure_memo, self.mset_memo, self.pool_cache = {}, {}, {}
208
+ from core import user_tables
209
+ for field in definition.get('fields') or []:
210
+ if field.get('type') != 'ai_enrich':
211
+ continue
212
+ col = str(field['key'])
213
+ if needed is not None and col not in needed:
214
+ continue
215
+ values = paths(runtime, user_tables.STORE_KEY,
216
+ [[table_key, user_tables.AI_ENRICH_MARK_KEY, col, pid] for pid in self.row_ids])
217
+ self.definition.setdefault(user_tables.AI_ENRICH_MARK_KEY, {})[col] = {
218
+ pid: mark for pid, mark in zip(self.row_ids, values) if isinstance(mark, dict)}
219
+
220
+ def __getattr__(self, name):
221
+ return getattr(self._runtime, name)
222
+
223
+ def selected_values(self, name, path, ids):
224
+ ids = [str(pid) for pid in ids]
225
+ return {pid: value for pid, value in zip(ids, paths(self._runtime, name,
226
+ [[*path, pid] for pid in ids])) if value is not None}
227
+
228
+ def update(self, *args, **kwargs):
229
+ raise err(503, 'canonical_source_read_only', 'Refresh the database definition before reading its rows.')
230
+
231
+ put = upload_bytes = delete_path = update
232
+
233
+ def _workspace(self, name):
234
+ reader = backend(self._runtime)
235
+ project = getattr(reader, 'get_view_workspace', None) or getattr(reader, '_get_view_workspace', None)
236
+ if name not in self.metadata:
237
+ self.metadata[name] = (project(getattr(self._runtime, 'store_key', str)(name))
238
+ if callable(project) else self._runtime.get_projection(name, drop=('overlays',))) or {}
239
+ meta = copy.deepcopy(self.metadata[name])
240
+ reader = backend(self._runtime)
241
+ stream = getattr(reader, 'iter_view_members', None) or getattr(reader, '_iter_view_members', None)
242
+ selected = set(map(int, self.row_ids))
243
+ def invalid_membership():
244
+ return err(503, 'canonical_view_membership_invalid',
245
+ 'A saved view contains membership entries but no valid record IDs. '
246
+ 'Repair its membership before reading this database.')
247
+ for owner, ws in meta.items():
248
+ if not isinstance(ws, dict) or owner.startswith('_'):
249
+ continue
250
+ for vid, view in (ws.get('views') or {}).items():
251
+ cfg = view.get('config') if isinstance(view, dict) else None
252
+ if not isinstance(cfg, dict):
253
+ continue
254
+ if 'memberPids' in cfg:
255
+ authored = cfg.get('memberPids') or ()
256
+ if authored:
257
+ for raw in authored:
258
+ try:
259
+ int(raw)
260
+ except (TypeError, ValueError):
261
+ continue
262
+ break
263
+ else:
264
+ raise invalid_membership()
265
+ continue
266
+ if not callable(stream):
267
+ if callable(project):
268
+ raise err(503, 'canonical_source_unavailable', 'Saved view membership cannot be read safely.')
269
+ continue # An ordinary offline document did not strip memberships.
270
+ first = [None]
271
+ saw_any = [False]
272
+ def values():
273
+ parts = stream(getattr(self._runtime, 'store_key', str)(name),
274
+ [owner, 'views', vid, 'config', 'memberPids'],
275
+ getattr(self.metadata[name], 'source_revision', None), batch_size=BATCH_ROWS)
276
+ try:
277
+ with closing(parts):
278
+ for batch in parts:
279
+ for _, raw in batch:
280
+ saw_any[0] = True
281
+ try:
282
+ pid = int(raw)
283
+ except (TypeError, ValueError):
284
+ continue
285
+ if first[0] is None:
286
+ first[0] = pid
287
+ yield pid
288
+ except RuntimeError as exc:
289
+ from core.request_timing import raise_if_resource_error
290
+ raise_if_resource_error(exc)
291
+ raise err(503, 'canonical_source_unavailable', 'Saved view membership changed or is unavailable.') from exc
292
+ cur = getattr(self, '_canonical_source_cursor', None)
293
+ if cur is not None and selected:
294
+ from routes_odoo_tables import _membership_relation
295
+ cache_key = ('view_members', name, owner, vid)
296
+ if cache_key not in self.metadata:
297
+ operand = _membership_relation(cur, values())
298
+ self.metadata[cache_key] = (operand, first[0], saw_any[0])
299
+ operand, first[0], saw_any[0] = self.metadata[cache_key]
300
+ if saw_any[0] and first[0] is None:
301
+ raise invalid_membership()
302
+ kept = [int(row[0]) for row in cur.execute(
303
+ f'SELECT DISTINCT * FROM ({operand.sql}) members(pid) '
304
+ f'WHERE pid IN ({",".join("?" for _ in selected)}) ORDER BY pid',
305
+ [*operand.params, *selected]).fetchall()]
306
+ else:
307
+ # Metadata consumes no row memberships; retain only the curated witness.
308
+ with closing(values()) as source:
309
+ next(source, None)
310
+ kept = []
311
+ if saw_any[0] and first[0] is None:
312
+ # Config sanitization drops invalid entries; [] would mean all rows.
313
+ # Refuse instead of inventing an ID to stand for empty curated intent.
314
+ raise invalid_membership()
315
+ # Retain an actual authored witness when this batch has no member. An empty
316
+ # list would turn a curated view into all rows. Full SQL view resolution uses
317
+ # the original revision-bound backend, never this batch-local projection.
318
+ cfg['memberPids'] = kept if kept or first[0] is None else [first[0]]
319
+ for owner, ws in meta.items():
320
+ if not isinstance(ws, dict) or owner.startswith('_') or owner != self.uname:
321
+ continue
322
+ values = paths(self._runtime, name, [[owner, 'overlays', pid] for pid in self.row_ids])
323
+ # Only this owner's selected cells have been read, so admit a new bounded document.
324
+ meta[owner] = dict(ws, overlays={pid: value for pid, value in zip(self.row_ids, values)
325
+ if isinstance(value, dict)})
326
+ return meta
327
+
328
+ def _definitions(self):
329
+ from core import user_tables
330
+ if user_tables.STORE_KEY not in self.metadata:
331
+ self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection(
332
+ user_tables.STORE_KEY, drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {}
333
+ return {**self.metadata[user_tables.STORE_KEY], self.table_key: self.definition}
334
+
335
+ def get(self, name, fresh=False):
336
+ from core import user_tables
337
+ if name == user_tables.STORE_KEY:
338
+ return self._definitions()
339
+ if name not in self.docs or fresh:
340
+ if name.endswith('_table_workspace'):
341
+ value = self._workspace(name)
342
+ elif name.endswith('__shared'):
343
+ selected = paths(self._runtime, name, [['fields'],
344
+ *[['cells', pid] for pid in self.row_ids]])
345
+ fields, values = selected[0] or {}, selected[1:]
346
+ value = {'fields': fields, 'cells': {pid: cell for pid, cell in zip(self.row_ids, values)
347
+ if isinstance(cell, dict)}}
348
+ elif name in {self.table_key + '_docs', 'customer_docs', 'product_docs'}:
349
+ values = (paths(self._runtime, name, [[pid] for pid in self.row_ids])
350
+ if self._canonical_needed is None else [None] * len(self.row_ids))
351
+ value = {pid: docs for pid, docs in zip(self.row_ids, values) if isinstance(docs, list)}
352
+ else:
353
+ if name not in self.metadata or fresh:
354
+ self.metadata[name] = self._runtime.get(name, fresh=fresh) if fresh else self._runtime.get(name)
355
+ value = self.metadata[name]
356
+ self.docs[name] = value
357
+ return copy.deepcopy(self.docs[name])
358
+
359
+ def _read_strict(self, name):
360
+ return self.get(name)
361
+
362
+ def get_entry(self, name, entry):
363
+ from core import user_tables
364
+ if name == user_tables.STORE_KEY:
365
+ return self.definition if str(entry) == self.table_key else self._runtime.get_entry(name, entry)
366
+ return (self.get(name) or {}).get(str(entry))
367
+
368
+ def get_projection(self, name, drop=()):
369
+ if name == 'user_tables':
370
+ from core import store
371
+ return store._project(self._definitions(), frozenset(drop))
372
+ return self._runtime.get_projection(name, drop=drop)
373
+
374
+
375
+ class CanonicalStore(BatchStore):
376
+ """Lend the existing customer/product assemblies one canonical source batch."""
377
+ schema_only = False
378
+ _canonical_read_only = True
379
+
380
+ def __init__(self, session, table_key, rows, *, cur, metadata, needed):
381
+ super().__init__(session.runtime, table_key, {'fields': []},
382
+ [(str(row['pid']), {}) for row in rows], metadata=metadata,
383
+ uname=session.uname, needed=needed)
384
+ self._canonical_source_cursor = cur
385
+ self._canonical_rows = rows if table_key == 'customer_data' else None
386
+ self._canonical_product_rows = rows if table_key == 'product_data' else None
387
+ from core import perm_scope
388
+ team, agent = perm_scope.derive_pool_scope(session.user, table_key)
389
+ import time
390
+ key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
391
+ self.pool_cache[key] = (time.time(), rows)
392
+
393
+ def get_projection(self, name, drop=()):
394
+ if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
395
+ return self.get(name)
396
+ if name.endswith('_table_workspace') or name.endswith('__shared'):
397
+ from core import store
398
+ return store._project(self.get(name), frozenset(drop))
399
+ return super().get_projection(name, drop=drop)
400
+
401
+ def get(self, name, fresh=False):
402
+ if name in self.docs and not fresh:
403
+ return copy.deepcopy(self.docs[name])
404
+ if name.endswith('_cohorts'):
405
+ from modules import cohort
406
+ definitions = object_values(self._runtime, name, [self.uname], drop=('members',))
407
+ value = {self.uname: {key: dict(meta, members=members(self._runtime, name,
408
+ [self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
409
+ for key, meta in definitions.items() if isinstance(meta, dict)}}
410
+ elif name == 'loopable_route_system':
411
+ value = object_values(self._runtime, name, omit=('routes', 'zones'))
412
+ value['routes'], value['zones'] = [], []
413
+ for batch in array_values(self._runtime, name, ['routes'], drop=('stops',)):
414
+ for index, route in batch:
415
+ if isinstance(route, dict):
416
+ route = dict(route, stops=members(self._runtime, name, ['routes', index, 'stops'],
417
+ self.row_ids, id_key='customerPid'))
418
+ value['routes'].append(route)
419
+ for batch in array_values(self._runtime, name, ['zones'], drop=(
420
+ 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
421
+ value['zones'].extend(meta for _, meta in batch)
422
+ elif name == 'table_zones':
423
+ value = {self.table_key: []}
424
+ for batch in array_values(self._runtime, name, [self.table_key], drop=(
425
+ 'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')):
426
+ value[self.table_key].extend(meta for _, meta in batch)
427
+ else:
428
+ return super().get(name, fresh=fresh)
429
+ self.docs[name] = value
430
+ return copy.deepcopy(value)
431
+
432
+ def _definitions(self):
433
+ from core import user_tables
434
+ if user_tables.STORE_KEY not in self.metadata:
435
+ self.metadata[user_tables.STORE_KEY] = self._runtime.get_projection(user_tables.STORE_KEY,
436
+ drop=(*user_tables.DEF_ONLY_DROPS, 'aiEnrich')) or {}
437
+ return self.metadata[user_tables.STORE_KEY]
438
+
439
+ def get_entry(self, name, entry):
440
+ if name == 'user_tables':
441
+ return self._definitions().get(str(entry))
442
+ return super().get_entry(name, entry)
443
+
444
+
445
+ def canonical_metadata(session, table_key):
446
+ """Read the canonical permitted definitions without constructing source rows."""
447
+ import routes_customers as customers
448
+ import routes_products as products
449
+ session.require(table_key)
450
+ lent = CanonicalStore(session, table_key, [], cur=None, metadata={}, needed=frozenset())
451
+ actor = with_runtime(session, customers._AssemblyStore(lent, schema_only=True))
452
+ build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
453
+ return build(actor, scope='product' if table_key == 'product_data' else 'customer',
454
+ consume_corrections=False, with_rows=False, needed=frozenset())
455
+
456
+
457
+ def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
458
+ """Refuse a failed compiled source by name without publishing an empty population."""
459
+ from fastapi import HTTPException
460
+ from core.request_timing import raise_if_resource_error
461
+ try:
462
+ with closing(_canonical_batches(session, cur, table_key, ids=ids, needed=needed,
463
+ metric_window=metric_window, related=related)) as batches:
464
+ yield from batches
465
+ except HTTPException:
466
+ raise
467
+ except Exception as exc:
468
+ raise_if_resource_error(exc)
469
+ raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc
470
+
471
+
472
+ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
473
+ """Run canonical family, permission, overlay and measure projections over bounded identities."""
474
+ import aios_grid
475
+ import routes_customers as customers
476
+ import routes_products as products
477
+ import routes_sheet as sheet
478
+ from core import perm_scope, user_tables, route_system
479
+ from modules import customer_data, product_data
480
+ if getattr(session.runtime, 'key', None) != 'royal-imports':
481
+ raise err(503, 'canonical_source_unavailable', f'{table_key} has no compiled source for this tenant.')
482
+ session.require(table_key)
483
+ team, agent = perm_scope.derive_pool_scope(session.user, table_key)
484
+ before = generation(session, table_key)
485
+ metadata = {}
486
+ definition = canonical_metadata(session, table_key)
487
+ # Every permanent wall operand is populated before apply_row_scope judges the batch.
488
+ tree = perm_scope.entry(session.user, table_key) or {}
489
+ wall = tree.get('filter') or {}
490
+ def keys(node):
491
+ if isinstance(node, dict):
492
+ if node.get('colId'):
493
+ yield str(node['colId'])
494
+ for group in ('nodes', 'children', 'filters'):
495
+ for child in node.get(group) or []:
496
+ yield from keys(child)
497
+ required = None if needed is None else set(needed) | set(keys(wall))
498
+ if required is not None:
499
+ from core.if_field import references
500
+ vocabulary = {f['key']: f for f in [*definition.get('fields', []),
501
+ *(definition.get('_permission_scope') or {}).get('fields', [])] if isinstance(f, dict) and f.get('key')}
502
+ changed = True
503
+ while changed:
504
+ previous = set(required)
505
+ for key in list(required):
506
+ field = vocabulary.get(key) or {}
507
+ required.update(match.strip() for match in aios_grid._FORMULA_REF.findall(str(field.get('formula') or '')))
508
+ if field.get('type') == 'if':
509
+ required.update(references(field.get('if') or {}))
510
+ if field.get('type') == 'created_time':
511
+ required.add('_created')
512
+ changed = previous != required
513
+ # Zone and Route projections consume location and identity operands before display projection.
514
+ if required is not None and {'zone', 'routes'} & required:
515
+ required.update({'lat', 'lon'})
516
+ runtime = CanonicalStore(session, table_key, [], cur=cur, metadata=metadata, needed=required)
517
+ all_keys = {f['key'] for f in definition['fields']} if required is None else required
518
+ raw = user_tables.enabled_odoo_schema_fields(runtime, model='product.product') if table_key == 'product_data' else []
519
+ if required is not None:
520
+ raw = [field for field in raw if field.get('key') in required]
521
+ import routes_keychain
522
+ import time
523
+ paused = routes_keychain.odoo_paused(session.runtime)
524
+ pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
525
+ cached = session.runtime.pool_cache.get(pool_key)
526
+ if cached and (paused or time.time() - cached[0] < 900):
527
+ import itertools
528
+ def borrowed():
529
+ source = (r for r in cached[1] if ids is None or r['pid'] in ids)
530
+ while True:
531
+ batch = list(itertools.islice(source, BATCH_ROWS))
532
+ if not batch:
533
+ return
534
+ yield batch
535
+ batches = borrowed()
536
+ elif paused:
537
+ import itertools
538
+ key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
539
+ hit = session.runtime.pool_cache.get(key)
540
+ if hit:
541
+ def frozen():
542
+ source = (r for r in hit[1] if ids is None or r['pid'] in ids)
543
+ while True:
544
+ batch = list(itertools.islice(source, BATCH_ROWS))
545
+ if not batch:
546
+ return
547
+ yield batch
548
+ batches = frozen()
549
+ elif table_key == 'customer_data':
550
+ path = ['odoo_pool', routes_keychain._snap_scope_key(team, agent)]
551
+ stamp = paths(session.runtime, routes_keychain.SNAPSHOT_KEY, [[*path, 'ts']])[0]
552
+ if stamp is None:
553
+ raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
554
+ def frozen():
555
+ for batch in array_values(session.runtime, routes_keychain.SNAPSHOT_KEY, [*path, 'rows']):
556
+ yield [r for _, r in batch if isinstance(r, dict) and (ids is None or r.get('pid') in ids)]
557
+ batches = frozen()
558
+ else:
559
+ raise err(503, 'connector_paused', 'This source is paused and its scoped snapshot is unavailable.')
560
+ elif table_key == 'product_data':
561
+ batches = product_data.source_batches(cur, st=runtime, team_id=team, ids=ids, needed=required,
562
+ raw_fields=[field['rawName'] for field in raw])
563
+ else:
564
+ batches = customer_data.source_batches(cur, st=runtime, agent_name=agent, team_id=team,
565
+ ids=ids, needed=required, ledger_batches=items(session.runtime, customer_data.LEDGER_KEY,
566
+ selected=ids) if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY) else ())
567
+ with closing(batches):
568
+ for rows in batches:
569
+ runtime = CanonicalStore(session, table_key, rows, cur=cur, metadata=metadata, needed=required)
570
+ actor = with_runtime(session, runtime)
571
+ build = products._compute_product_assembly if table_key == 'product_data' else customers._compute_grid_assembly
572
+ g = build(actor, scope='product' if table_key == 'product_data' else 'customer',
573
+ consume_corrections=False, with_rows=True, needed=all_keys)
574
+ kept = g['fields'] if needed is None else [f for f in g['fields'] if f['key'] in needed]
575
+ derived, bounded, limits = sheet._metric_derived(actor, table_key,
576
+ dict(g, fields=kept), metric_window)
577
+ if table_key == 'product_data':
578
+ projected = products.product_rows(dict(g, fields=kept, derived=derived)) if kept else [
579
+ {'pid': r['pid']} for r in g['rows_src']]
580
+ projected = user_tables.zone_cells(table_key, projected, allowed_row_ids=g['pids'], st=runtime)
581
+ else:
582
+ projected = aios_grid.rows_from_pool(g['rows_src'], kept, g['ws'].get('overlays'), derived=derived) \
583
+ if kept else [{'pid': r['pid']} for r in g['rows_src']]
584
+ projected = route_system.apply_customer_projections(projected, g['routes'], g['zones'])
585
+ if related:
586
+ from routes_odoo_tables import narrow_related_rows
587
+ projected = narrow_related_rows(actor, table_key, projected, related)
588
+ from routes_tables import scope_linked_metrics
589
+ projected, _fields, _refusals = scope_linked_metrics(actor, table_key, projected,
590
+ kept, g['measures'], related)
591
+ from core import grid_events
592
+ docs = grid_events.docs_for(g['pids'], scope_key='customer', uname=session.uname,
593
+ admin=session.admin, st=runtime) if table_key == 'customer_data' and ids is not None and needed is None else {}
594
+ yield {'rows': projected, 'fields': g['fields'], 'today': g['today'], 'docs': docs,
595
+ 'bounded': bounded, 'unbounded': limits,
596
+ 'limits': list(g['workspace'].get('limits') or [])}
597
+ projected.clear()
598
+ runtime.pool_cache.clear()
599
+ runtime.measure_memo.clear()
600
+ require_generation(session, table_key, before)
601
+
602
+
603
+ def canonical_grid(session, cur, table_key, *, needed=None, metric_window=None):
604
+ """Materialize only required canonical source and wall columns into the isolated cursor."""
605
+ import odoo_relational as rel
606
+ import routes_odoo_tables as rot
607
+ from modules import sheet
608
+ g = canonical_metadata(session, table_key)
609
+ fields = g['fields']
610
+ agent_field = {'key': 'agent_id', 'type': 'int', 'label': 'Current agent ID'}
611
+ join_agent = False
612
+ if table_key == 'customer_data' and (needed is None or 'agent_id' in needed):
613
+ from core import perm_scope
614
+ hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime)
615
+ if not {'agent', 'agent_id'} & set(hidden):
616
+ if 'agent_id' in rel.columns(cur, 'res_partner'):
617
+ fields = [f for f in fields if f['key'] != 'agent_id'] + [agent_field]
618
+ join_agent = True
619
+ elif needed is not None and 'agent_id' in needed:
620
+ raise err(409, 'related_invalid', 'customer_data.agent_id is unavailable in the partner mirror.')
621
+ kept = fields if needed is None else [f for f in fields if f['key'] in needed]
622
+ unbounded, bounded = {}, set()
623
+ def rows():
624
+ seen = False
625
+ empty_limits = {}
626
+ with closing(canonical_batches(session, cur, table_key, needed=needed, metric_window=metric_window)) as batches:
627
+ for batch in batches:
628
+ if batch['rows']:
629
+ seen = True
630
+ unbounded.update(batch['unbounded'])
631
+ bounded.update(batch['bounded'])
632
+ else:
633
+ empty_limits.update(batch['unbounded'])
634
+ yield from batch['rows']
635
+ batch['rows'].clear()
636
+ if not seen:
637
+ unbounded.update(empty_limits)
638
+ typed = sheet.mirror_fields(kept)
639
+ import hashlib
640
+ import json
641
+ slot = hashlib.sha1(json.dumps([metric_window],
642
+ sort_keys=True, default=str).encode()).hexdigest()[:12]
643
+ spec = rel.registry_grid_spec(cur, f'{table_key}__at_{slot}', typed, rows(), bulk=True)
644
+ if join_agent:
645
+ cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id=partner.agent_id FROM res_partner partner WHERE grid.pid=partner.id')
646
+ scope_fields = (g.get('_permission_scope') or {}).get('fields', fields)
647
+ return cur, dict(spec, fields=fields, scope_applied=True,
648
+ scope_fields=tuple(f['key'] for f in scope_fields),
649
+ bounded=tuple(bounded), unbounded=unbounded), rot._filter_columns(spec, typed)
650
+
651
+
652
+ def canonical_page(session, cur, table_key, ids, *, related=None):
653
+ """Hydrate a selected page through canonical projections without constructing a full payload."""
654
+ ids = list(dict.fromkeys(int(pid) for pid in ids))
655
+ found, docs, payload = {}, {}, None
656
+ with closing(canonical_batches(session, cur, table_key, ids=ids, related=related)) as batches:
657
+ for batch in batches:
658
+ payload = {key: value for key, value in batch.items()
659
+ if key not in {'rows', 'docs', 'bounded', 'unbounded'}}
660
+ found.update((int(row['pid']), row) for row in batch['rows'])
661
+ docs.update(batch.get('docs') or {})
662
+ if payload is None:
663
+ g = canonical_metadata(session, table_key)
664
+ payload = {'fields': g['fields'], 'today': g['today'], 'limits': list(g['workspace'].get('limits') or [])}
665
+ return dict(payload, rows=[found[pid] for pid in ids if pid in found], docs=docs)
666
+
667
+
668
+ def stored_batches(session, table_key, *, ids=None, related=None, needed=None, cur=None):
669
+ """Project bounded stored batches through the table's existing permission and overlay reader."""
670
+ import routes_tables as tables
671
+ from core import user_tables
672
+ metadata_actor = with_runtime(session, MetadataStore(session.runtime))
673
+ definition = tables._defn_or_refuse(metadata_actor, table_key, defs_only=True, scope_applied=True)
674
+ if definition.get('readThrough'):
675
+ raise err(409, 'canonical_source_unavailable', f'{table_key} requires its declared SQL source.')
676
+ metadata = {}
677
+ with closing(items(session.runtime, user_tables.STORE_KEY, [table_key, 'rows'], selected=ids)) as batches:
678
+ for rows in batches:
679
+ runtime = BatchStore(session.runtime, table_key, definition, rows,
680
+ metadata=metadata, uname=session.uname, needed=needed)
681
+ runtime._canonical_source_cursor = cur
682
+ actor = with_runtime(session, runtime)
683
+ # table_rows remains the canonical evaluator. Its document contains this batch only.
684
+ payload = tables.table_rows(table_key, session=actor, related=related)
685
+ if needed is not None:
686
+ payload = dict(payload, rows=[{k: v for k, v in row.items() if k == 'pid' or k in needed}
687
+ for row in payload['rows']])
688
+ yield payload
689
+
690
+
691
+ def metadata_limits(session, table_key, metadata):
692
+ """Retain the editable capacity receipt without asking for a population document."""
693
+ from core import user_tables
694
+ limits = list((metadata.get('workspace') or {}).get('limits') or [])
695
+ report = user_tables.limit_report(table_key, st=user_tables.lend(
696
+ session.runtime, user_tables={table_key: metadata['defn']}))
697
+ if report and report not in limits:
698
+ limits.append(report)
699
+ return limits
700
+
701
+
702
+ def stored_grid(session, cur, table_key, *, needed=None):
703
+ """Build an exact SQL population through bounded canonical table projections."""
704
+ import odoo_relational as rel
705
+ import routes_odoo_tables as rot
706
+ import routes_tables as tables
707
+ from modules import sheet
708
+
709
+ before = generation(session, table_key)
710
+ metadata_actor = with_runtime(session, MetadataStore(session.runtime))
711
+ metadata = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True,
712
+ consume_corrections=False)
713
+ fields = metadata['fields']
714
+ kept = fields if needed is None else [f for f in fields if f['key'] in needed]
715
+
716
+ def rows():
717
+ with closing(stored_batches(session, table_key, needed=needed)) as batches:
718
+ for payload in batches:
719
+ yield from payload['rows']
720
+ payload['rows'].clear()
721
+
722
+ spec_fields = sheet.mirror_fields(kept)
723
+ spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows(), bulk=True)
724
+ require_generation(session, table_key, before)
725
+ spec = dict(spec, fields=fields, scope_applied=True,
726
+ limits=metadata_limits(session, table_key, metadata))
727
+ return cur, spec, rot._filter_columns(spec, spec_fields)
728
+
729
+
730
+ def stored_page(session, table_key, ids, *, related=None):
731
+ """Hydrate only selected identities through the canonical stored-row projection."""
732
+ before = generation(session, table_key)
733
+ ids = list(dict.fromkeys(int(pid) for pid in ids))
734
+ payload, found, docs, zones = None, {}, {}, {}
735
+ with closing(stored_batches(session, table_key, ids=ids, related=related)) as batches:
736
+ for batch in batches:
737
+ payload = {k: v for k, v in batch.items() if k != 'rows'}
738
+ found.update((int(row['pid']), row) for row in batch['rows'])
739
+ docs.update(batch.get('docs') or {})
740
+ for zone in batch.get('zones') or []:
741
+ key = str(zone.get('id') or zone.get('name'))
742
+ if key not in zones:
743
+ zones[key] = dict(zone, memberships=[])
744
+ zones[key]['memberships'].extend(zone.get('memberships') or [])
745
+ if payload is None:
746
+ import routes_tables as tables
747
+ metadata_actor = with_runtime(session, MetadataStore(session.runtime))
748
+ g = tables.ut_assembly(metadata_actor, table_key, with_rows=False, fields_only=True,
749
+ consume_corrections=False)
750
+ payload = {'fields': g['fields'], 'today': g['today'],
751
+ 'limits': metadata_limits(session, table_key, g)}
752
+ rows = [found[pid] for pid in ids if pid in found]
753
+ require_generation(session, table_key, before)
754
+ return dict(payload, rows=rows, docs=docs, zones=list(zones.values()),
755
+ scope={**payload.get('scope', {}), 'rowCount': len(rows)})
756
+
757
+
758
+ def page_cells(runtime, table_key, pids):
759
+ """Read a page's shared cells through the canonical parser and coherent row-bounded snapshots."""
760
+ from core import shared_overlay
761
+ import itertools
762
+ source, out = iter(pids), {}
763
+ while True:
764
+ batch = list(itertools.islice(source, BATCH_ROWS))
765
+ if not batch:
766
+ return out
767
+ reader = BatchStore(runtime, table_key, {'fields': []}, [(str(pid), {}) for pid in batch])
768
+ out.update(shared_overlay.cells(table_key, batch, st=reader))
api/request_class.py CHANGED
@@ -10,7 +10,9 @@ import asyncio
10
  import json
11
  import os
12
  import re
 
13
  import threading
 
14
  from urllib.parse import parse_qs
15
 
16
  BUSY_CODE = "server_busy"
@@ -120,65 +122,6 @@ class _Counters:
120
  _COUNTERS = _Counters()
121
 
122
 
123
- class _StackSampler:
124
- """Opt-in (AIOS_HEAVY_PROFILE=1): samples app frames while heavy requests run, logs the hot ones."""
125
-
126
- INTERVAL = 0.2
127
-
128
- def __init__(self):
129
- self.lock = threading.Lock()
130
- self.active = 0
131
- self.counts = {}
132
- self.thread = None
133
-
134
- def begin(self):
135
- with self.lock:
136
- self.active += 1
137
- if self.active == 1:
138
- self.counts = {}
139
- if self.thread is None or not self.thread.is_alive():
140
- import process_role
141
- self.thread = process_role.thread(target=self._run, name="heavy-profile", kind="light")
142
- self.thread.start()
143
-
144
- def end(self, label, seconds):
145
- with self.lock:
146
- self.active -= 1
147
- counts = dict(self.counts)
148
- if seconds < 3 or not counts:
149
- return
150
- top = sorted(counts.items(), key=lambda kv: -kv[1])[:18]
151
- print(f"[heavy-profile] {label} {seconds:.1f}s " + " | ".join(
152
- f"{name} {n * self.INTERVAL:.1f}s" for name, n in top), flush=True)
153
-
154
- def _run(self):
155
- import sys
156
- import time
157
- me = threading.get_ident()
158
- while True:
159
- time.sleep(self.INTERVAL)
160
- with self.lock:
161
- if self.active <= 0:
162
- self.thread = None
163
- return
164
- for tid, frame in sys._current_frames().items():
165
- if tid == me:
166
- continue
167
- seen = set()
168
- while frame is not None:
169
- code = frame.f_code
170
- path = code.co_filename.replace("\\", "/")
171
- if ("/api/" in path or "/platform/" in path) and "site-packages" not in path:
172
- name = f"{code.co_name}@{path.rsplit('/', 1)[-1]}"
173
- if name not in seen:
174
- seen.add(name)
175
- self.counts[name] = self.counts.get(name, 0) + 1
176
- frame = frame.f_back
177
-
178
-
179
- _SAMPLER = _StackSampler() if os.environ.get("AIOS_HEAVY_PROFILE") == "1" else None
180
-
181
-
182
  def status():
183
  """Snapshot of heavy request admission for a settings or health surface."""
184
  c = _COUNTERS
@@ -204,8 +147,8 @@ def _env_float(name, default):
204
  return default
205
 
206
 
207
- async def _send_busy(send):
208
- body = json.dumps({"error": {"code": BUSY_CODE, "message": BUSY_MESSAGE}}).encode()
209
  await send({"type": "http.response.start", "status": 503,
210
  "headers": [(b"content-type", b"application/json"),
211
  (b"content-length", str(len(body)).encode()),
@@ -253,22 +196,53 @@ class HeavyGate:
253
  await self._run_heavy(scope, body_msgs, receive, send)
254
 
255
  async def _run_heavy(self, scope, body_msgs, receive, send):
 
256
  sem = self._semaphore()
257
  c = self.counters
258
  c.add("queued", 1)
 
 
259
  acquire = asyncio.ensure_future(sem.acquire())
260
  gone = asyncio.ensure_future(_wait_disconnect(receive))
261
  try:
262
- done, _ = await asyncio.wait({acquire, gone}, timeout=self.queue_seconds,
 
263
  return_when=asyncio.FIRST_COMPLETED)
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  finally:
 
 
 
 
265
  c.add("queued", -1)
266
  if not acquire.done():
267
  acquire.cancel()
268
  if not gone.done():
269
  gone.cancel()
 
270
  # A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
271
  got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
 
 
 
 
 
 
 
 
 
272
  if acquire not in done or gone in done:
273
  if got:
274
  sem.release()
@@ -292,17 +266,10 @@ class HeavyGate:
292
 
293
  c.add("in_flight", 1)
294
  c.add("admitted", 1)
295
- started = asyncio.get_running_loop().time()
296
- if _SAMPLER is not None:
297
- _SAMPLER.begin()
298
  try:
299
  await self.app(scope, _replay(body_msgs, receive), send_wrapped)
300
  finally:
301
  release()
302
- if _SAMPLER is not None:
303
- q = scope.get("query_string", b"").decode("latin-1")[:60]
304
- _SAMPLER.end(f"{scope.get('method')} {scope.get('path')}?{q}",
305
- asyncio.get_running_loop().time() - started)
306
 
307
 
308
  async def _read_body(receive, limit):
 
10
  import json
11
  import os
12
  import re
13
+ import sys
14
  import threading
15
+ import time
16
  from urllib.parse import parse_qs
17
 
18
  BUSY_CODE = "server_busy"
 
122
  _COUNTERS = _Counters()
123
 
124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  def status():
126
  """Snapshot of heavy request admission for a settings or health surface."""
127
  c = _COUNTERS
 
147
  return default
148
 
149
 
150
+ async def _send_busy(send, cause="heavy_queue_timeout"):
151
+ body = json.dumps({"error": {"code": BUSY_CODE, "cause": cause, "message": BUSY_MESSAGE}}).encode()
152
  await send({"type": "http.response.start", "status": 503,
153
  "headers": [(b"content-type", b"application/json"),
154
  (b"content-length", str(len(body)).encode()),
 
196
  await self._run_heavy(scope, body_msgs, receive, send)
197
 
198
  async def _run_heavy(self, scope, body_msgs, receive, send):
199
+ queued_at = time.perf_counter()
200
  sem = self._semaphore()
201
  c = self.counters
202
  c.add("queued", 1)
203
+ timing = sys.modules.get("core.request_timing")
204
+ budget_error = getattr(timing, "RequestDeadlineExceeded", TimeoutError)
205
  acquire = asyncio.ensure_future(sem.acquire())
206
  gone = asyncio.ensure_future(_wait_disconnect(receive))
207
  try:
208
+ timeout = timing.remaining_budget(self.queue_seconds) if timing else self.queue_seconds
209
+ done, _ = await asyncio.wait({acquire, gone}, timeout=timeout,
210
  return_when=asyncio.FIRST_COMPLETED)
211
+ if timing:
212
+ timing.remaining_budget()
213
+ except budget_error:
214
+ if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
215
+ sem.release()
216
+ c.add("busy_refusals", 1)
217
+ await _send_busy(send, "request_deadline")
218
+ return
219
+ except BaseException:
220
+ # Cancellation can race a successful acquire before the handler owns the permit.
221
+ if acquire.done() and not acquire.cancelled() and acquire.exception() is None:
222
+ sem.release()
223
+ raise
224
  finally:
225
+ # The classifier can also run alone, without the app or an active timing scope.
226
+ timing = sys.modules.get("core.request_timing")
227
+ if timing is not None:
228
+ timing.record_queue((time.perf_counter() - queued_at) * 1000)
229
  c.add("queued", -1)
230
  if not acquire.done():
231
  acquire.cancel()
232
  if not gone.done():
233
  gone.cancel()
234
+ await asyncio.gather(acquire, gone, return_exceptions=True)
235
  # A permit can land in the same tick as a timeout or disconnect. Hold it only if used.
236
  got = acquire.done() and not acquire.cancelled() and acquire.exception() is None
237
+ try:
238
+ if timing:
239
+ timing.remaining_budget()
240
+ except budget_error:
241
+ if got:
242
+ sem.release()
243
+ c.add("busy_refusals", 1)
244
+ await _send_busy(send, "request_deadline")
245
+ return
246
  if acquire not in done or gone in done:
247
  if got:
248
  sem.release()
 
266
 
267
  c.add("in_flight", 1)
268
  c.add("admitted", 1)
 
 
 
269
  try:
270
  await self.app(scope, _replay(body_msgs, receive), send_wrapped)
271
  finally:
272
  release()
 
 
 
 
273
 
274
 
275
  async def _read_body(receive, limit):
api/request_timing.py ADDED
@@ -0,0 +1,421 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Outermost ASGI timing, bounded route statistics and request-associated stack sampling."""
2
+ from __future__ import annotations
3
+
4
+ import asyncio
5
+ import contextvars
6
+ import json
7
+ import logging
8
+ import math
9
+ import os
10
+ import sys
11
+ import threading
12
+ import time
13
+ from collections import OrderedDict, deque
14
+
15
+ from core import request_timing as timing
16
+
17
+ LOG = logging.getLogger("aios.request_timing")
18
+ _STATS_LOCK = threading.Lock()
19
+ _STATS = OrderedDict()
20
+ _ROUTE_LIMIT = 512
21
+
22
+
23
+ def request_timeout_s():
24
+ try:
25
+ value = float(os.environ.get("AIOS_REQUEST_TIMEOUT_S", "45"))
26
+ return max(.01, value) if math.isfinite(value) else 45.0
27
+ except ValueError:
28
+ return 45.0
29
+
30
+
31
+ _BUSY_CAUSES = {
32
+ "PoolTimeout": "postgres_pool_timeout", "TooManyRequests": "postgres_pool_full",
33
+ "QueryCanceled": "postgres_statement_timeout", "LockNotAvailable": "postgres_lock_timeout",
34
+ "SharedReadTimeout": "postgres_shared_read_timeout", "OdooPoolTimeout": "odoo_pool_timeout",
35
+ "RequestDeadlineExceeded": "request_deadline", "DeadlineCapacityExceeded": "deadline_capacity",
36
+ "OperationalError": "postgres_connection_unavailable", "InterfaceError": "postgres_connection_unavailable",
37
+ "ConnectionTimeout": "postgres_connection_timeout",
38
+ "IdleInTransactionSessionTimeout": "postgres_idle_transaction_timeout",
39
+ "OutOfMemoryException": "duckdb_memory_limit",
40
+ "PostgresOperationTimeout": "postgres_operation_timeout",
41
+ "OdooRequestTimeout": "odoo_rpc_timeout",
42
+ }
43
+
44
+
45
+ def resource_cause(error):
46
+ """Find a resource refusal through route wrappers without exposing database messages."""
47
+ seen = set()
48
+ while error is not None and id(error) not in seen:
49
+ seen.add(id(error))
50
+ cause = _BUSY_CAUSES.get(type(error).__name__)
51
+ if cause:
52
+ return cause
53
+ error = error.__cause__ or error.__context__
54
+ return None
55
+
56
+
57
+ def resource_response(error):
58
+ from starlette.responses import JSONResponse
59
+ cause = resource_cause(error)
60
+ if cause is None:
61
+ return None
62
+ return JSONResponse({"error": {"code": "server_busy", "cause": cause,
63
+ "message": "The server is busy. Try again shortly."}},
64
+ status_code=503, headers={"Retry-After": "1"})
65
+
66
+
67
+ class DeadlineCapacityExceeded(RuntimeError):
68
+ """The bounded statement watchdog has no free registration slot."""
69
+
70
+
71
+ class _StatementAlarm:
72
+ def __init__(self, owner, connection, acc):
73
+ self.owner, self.connection, self.acc = owner, connection, acc
74
+ self.lock = threading.Lock()
75
+ self.active, self.expired = True, False
76
+
77
+ def fire(self):
78
+ with self.lock:
79
+ if self.active and (self.acc.cancelled or time.monotonic() >= self.acc.deadline):
80
+ self.expired = True
81
+ self.active = False
82
+ self.connection.interrupt()
83
+
84
+ def close(self):
85
+ # Serialize disarm with interrupt before the cursor can execute another statement.
86
+ with self.lock:
87
+ self.active = False
88
+ expired = self.expired
89
+ with self.owner.lock:
90
+ self.owner.active.discard(self)
91
+ return expired
92
+
93
+
94
+ class _DeadlineWatchdog:
95
+ INTERVAL = .01
96
+ CAPACITY = 128
97
+
98
+ def __init__(self):
99
+ self.lock = threading.Lock()
100
+ self.active = set()
101
+ self.thread = None
102
+
103
+ def arm(self, connection, acc):
104
+ import process_role
105
+ with self.lock:
106
+ if len(self.active) >= self.CAPACITY:
107
+ raise DeadlineCapacityExceeded("request statement watchdog is full")
108
+ alarm = _StatementAlarm(self, connection, acc)
109
+ self.active.add(alarm)
110
+ if self.thread is None:
111
+ job = process_role.thread(target=self._run, name="request-deadline", kind="light")
112
+ if not process_role.admitted(job):
113
+ self.active.remove(alarm)
114
+ raise DeadlineCapacityExceeded("request statement watchdog is unavailable")
115
+ self.thread = job
116
+ try:
117
+ job.start()
118
+ except BaseException:
119
+ self.thread = None
120
+ self.active.remove(alarm)
121
+ raise
122
+ return alarm
123
+
124
+ def _run(self):
125
+ while True:
126
+ time.sleep(self.INTERVAL)
127
+ with self.lock:
128
+ if not self.active:
129
+ self.thread = None
130
+ return
131
+ active = tuple(self.active)
132
+ for alarm in active:
133
+ try:
134
+ alarm.fire()
135
+ except Exception:
136
+ LOG.warning("request query interrupt failed", exc_info=False)
137
+
138
+
139
+ _DEADLINES = _DeadlineWatchdog()
140
+ timing.configure_deadlines(_DEADLINES.arm)
141
+
142
+ _LOCK_PROBE_SLOTS = threading.BoundedSemaphore(8)
143
+
144
+
145
+ class _LockProbe:
146
+ def __init__(self, pid, diagnose):
147
+ self.pid, self.diagnose = pid, diagnose
148
+ self.lock = threading.Lock()
149
+ self.active, self.fired = True, False
150
+ self.job = None
151
+
152
+ def run(self):
153
+ with self.lock:
154
+ if not self.active:
155
+ return
156
+ self.fired = True
157
+ try:
158
+ self.diagnose(self.pid)
159
+ finally:
160
+ _LOCK_PROBE_SLOTS.release()
161
+
162
+ def close(self):
163
+ with self.lock:
164
+ if not self.active:
165
+ return
166
+ self.active = False
167
+ self.job.cancel()
168
+ if not self.fired:
169
+ _LOCK_PROBE_SLOTS.release()
170
+
171
+
172
+ def _arm_lock_probe(pid, wait_s, diagnose):
173
+ import process_role
174
+ if not _LOCK_PROBE_SLOTS.acquire(blocking=False):
175
+ return None
176
+ probe = _LockProbe(pid, diagnose)
177
+ try:
178
+ probe.job = process_role.thread(target=probe.run, name="store-lock-diagnosis",
179
+ kind="light", delay=max(.01, wait_s * .75))
180
+ probe.job.start()
181
+ except Exception:
182
+ _LOCK_PROBE_SLOTS.release()
183
+ LOG.warning("store lock diagnosis could not start", exc_info=False)
184
+ return None
185
+ return probe
186
+
187
+
188
+ timing.configure_lock_probes(_arm_lock_probe)
189
+
190
+
191
+ def slow_threshold_ms():
192
+ try:
193
+ value = float(os.environ.get("AIOS_SLOW_REQUEST_MS", "1000"))
194
+ return max(0.0, value) if math.isfinite(value) else 1000.0
195
+ except ValueError:
196
+ return 1000.0
197
+
198
+
199
+ def route_template(scope):
200
+ route = scope.get("route")
201
+ return getattr(route, "path", None) or scope.get("_timing_route", "<unmatched>")
202
+
203
+
204
+ def _resolve_route(scope):
205
+ # Refusals can finish before routing. Match registered templates without invoking a handler.
206
+ from starlette.routing import Match
207
+ for route in getattr(scope.get("app"), "routes", ()):
208
+ match, child = route.matches(scope)
209
+ if match == Match.FULL:
210
+ scope["_timing_route"] = getattr(route, "path", None) or "<unmatched>"
211
+ scope["_timing_endpoint"] = child.get("endpoint")
212
+ return
213
+
214
+
215
+ def _save_stats(scope, status, milliseconds):
216
+ key = (scope.get("method", ""), route_template(scope))
217
+ with _STATS_LOCK:
218
+ values = _STATS.setdefault(key, deque(maxlen=500))
219
+ _STATS.move_to_end(key)
220
+ values.append((milliseconds, int(status is not None and status >= 500)))
221
+ while len(_STATS) > _ROUTE_LIMIT:
222
+ _STATS.popitem(last=False)
223
+
224
+
225
+ def route_stats():
226
+ """Return process-wide route timings without tenant, user or request identifiers."""
227
+ with _STATS_LOCK:
228
+ snapshot = [(key, list(values)) for key, values in _STATS.items()]
229
+ rows = []
230
+ for (method, route), values in snapshot:
231
+ ordered = sorted(value[0] for value in values)
232
+ rows.append({"method": method, "route": route, "count": len(values),
233
+ "5xx": sum(value[1] for value in values),
234
+ "p50": round(ordered[math.ceil(len(ordered) * .50) - 1], 2),
235
+ "p95": round(ordered[math.ceil(len(ordered) * .95) - 1], 2),
236
+ "max": round(ordered[-1], 2)})
237
+ return {"window": 500, "unit": "ms", "routes": rows}
238
+
239
+
240
+ def _app_frames(frame):
241
+ names = set()
242
+ while frame is not None:
243
+ code = frame.f_code
244
+ path = code.co_filename.replace("\\", "/")
245
+ if ("/api/" in path or "/platform/" in path) and "site-packages" not in path:
246
+ names.add(f"{code.co_name}@{path.rsplit('/', 1)[-1]}")
247
+ frame = frame.f_back
248
+ return names
249
+
250
+
251
+ def _coroutine_frames(task):
252
+ frames, seen = [], set()
253
+ coro = task.get_coro()
254
+ while coro is not None and id(coro) not in seen:
255
+ seen.add(id(coro))
256
+ frame = getattr(coro, "cr_frame", None) or getattr(coro, "gi_frame", None)
257
+ if frame is not None:
258
+ frames.append(frame)
259
+ coro = getattr(coro, "cr_await", None) or getattr(coro, "gi_yieldfrom", None)
260
+ return frames
261
+
262
+
263
+ class _StackSampler:
264
+ """Sample only matched worker contexts and coroutine scopes, including light requests."""
265
+
266
+ INTERVAL = .2
267
+
268
+ def __init__(self):
269
+ self.lock = threading.Lock()
270
+ self.active = {}
271
+ self.thread = None
272
+
273
+ def begin(self, acc, scope):
274
+ import process_role
275
+ with self.lock:
276
+ self.active[acc] = (scope, asyncio.get_running_loop())
277
+ if self.thread is None:
278
+ job = process_role.thread(target=self._run, name="request-profile", kind="light")
279
+ if process_role.admitted(job):
280
+ self.thread = job
281
+ job.start()
282
+
283
+ def end(self, acc):
284
+ with self.lock:
285
+ self.active.pop(acc, None)
286
+
287
+ def sample(self):
288
+ with self.lock:
289
+ active = dict(self.active)
290
+ matched = {acc: set() for acc in active}
291
+ scopes = {id(scope): acc for acc, (scope, _loop) in active.items()}
292
+ threads = timing.associated_threads()
293
+ for tid, frame in sys._current_frames().items():
294
+ acc = threads.get(tid)
295
+ probe = frame
296
+ while probe is not None and acc not in active:
297
+ candidate = probe.f_locals.get("scope")
298
+ owner = scopes.get(id(candidate))
299
+ if owner is not None and active[owner][0] is candidate:
300
+ acc = owner
301
+ path = probe.f_code.co_filename.replace("\\", "/")
302
+ if "/anyio/" in path and probe.f_code.co_name == "run":
303
+ context = probe.f_locals.get("context")
304
+ if isinstance(context, contextvars.Context):
305
+ acc = context.get(timing.CURRENT)
306
+ probe = probe.f_back
307
+ if acc in active:
308
+ matched[acc].update(_app_frames(frame))
309
+ for loop in {loop for _scope, loop in active.values()}:
310
+ try:
311
+ tasks = asyncio.all_tasks(loop)
312
+ except RuntimeError:
313
+ continue
314
+ for task in tasks:
315
+ frames = _coroutine_frames(task)
316
+ owners = set()
317
+ for frame in frames:
318
+ candidate = frame.f_locals.get("scope")
319
+ owner = scopes.get(id(candidate))
320
+ if owner is not None and active[owner][0] is candidate:
321
+ owners.add(owner)
322
+ for acc in owners:
323
+ for frame in frames:
324
+ matched[acc].update(_app_frames(frame))
325
+ for acc, names in matched.items():
326
+ with acc.lock:
327
+ if not acc.closed:
328
+ acc.samples.update(names)
329
+
330
+ def _run(self):
331
+ while True:
332
+ time.sleep(self.INTERVAL)
333
+ with self.lock:
334
+ if not self.active:
335
+ self.thread = None
336
+ return
337
+ try:
338
+ self.sample()
339
+ except Exception:
340
+ LOG.debug("request sampler skipped a changing stack", exc_info=True)
341
+
342
+
343
+ _SAMPLER = _StackSampler()
344
+
345
+
346
+ class RequestTiming:
347
+ """Measure final body sends without buffering or consuming the receive channel."""
348
+
349
+ def __init__(self, app):
350
+ self.app = app
351
+
352
+ async def __call__(self, scope, receive, send):
353
+ if scope.get("type") != "http":
354
+ return await self.app(scope, receive, send)
355
+ with timing.request_scope() as acc:
356
+ acc.deadline = time.monotonic() + request_timeout_s()
357
+ status, failed, disconnected, complete = None, False, False, False
358
+ threshold = slow_threshold_ms()
359
+ _resolve_route(scope)
360
+ _SAMPLER.begin(acc, scope)
361
+
362
+ async def receive_timed():
363
+ nonlocal disconnected
364
+ # Response streaming remains open after headers, including SSE disconnect waits.
365
+ if status is None:
366
+ remaining = timing.remaining_budget()
367
+ try:
368
+ message = await asyncio.wait_for(receive(), remaining)
369
+ except asyncio.TimeoutError as exc:
370
+ raise timing.RequestDeadlineExceeded("request body deadline exceeded") from exc
371
+ timing.remaining_budget()
372
+ else:
373
+ message = await receive()
374
+ if message["type"] == "http.disconnect":
375
+ disconnected = True
376
+ acc.cancelled = True
377
+ return message
378
+
379
+ async def send_timed(message):
380
+ nonlocal status, complete
381
+ if message["type"] == "http.response.start":
382
+ values = acc.snapshot()
383
+ elapsed = (time.perf_counter() - acc.started) * 1000
384
+ app_ms = max(0., elapsed - values["queue_ms"] - values["store_ms"] - values["duckdb_ms"])
385
+ # Headers describe time until headers. Final streaming time is logged after completion.
386
+ header = (f'queue;dur={values["queue_ms"]:.2f}, store;dur={values["store_ms"]:.2f}, '
387
+ f'duckdb;dur={values["duckdb_ms"]:.2f}, app;dur={app_ms:.2f}')
388
+ message = dict(message)
389
+ message["headers"] = [(key, value) for key, value in message.get("headers", [])
390
+ if key.lower() != b"server-timing"] + [(b"server-timing", header.encode("ascii"))]
391
+ await send(message)
392
+ if message["type"] == "http.response.start":
393
+ status = message["status"]
394
+ elif message["type"] == "http.response.body":
395
+ with acc.lock:
396
+ acc.response_bytes += len(message.get("body", b""))
397
+ complete = not message.get("more_body", False)
398
+
399
+ try:
400
+ await self.app(scope, receive_timed, send_timed)
401
+ except BaseException:
402
+ failed = True
403
+ raise
404
+ finally:
405
+ elapsed = (time.perf_counter() - acc.started) * 1000
406
+ _SAMPLER.end(acc)
407
+ with acc.lock:
408
+ acc.closed = True
409
+ acc.cancelled = True
410
+ samples = sorted(acc.samples.items(), key=lambda item: -item[1])[:18]
411
+ _save_stats(scope, status, elapsed)
412
+ if elapsed >= threshold:
413
+ endpoint = scope.get("endpoint") or scope.get("_timing_endpoint")
414
+ values = acc.snapshot()
415
+ values["app_ms"] = round(max(0., elapsed - values["queue_ms"] - values["store_ms"]
416
+ - values["duckdb_ms"]), 2)
417
+ values.update(method=scope.get("method", ""), route=route_template(scope), status=status,
418
+ total_ms=round(elapsed, 2), failed=failed, disconnected=disconnected,
419
+ complete=complete, endpoint=getattr(endpoint, "__name__", "<unresolved>"),
420
+ samples={name: round(count * _SAMPLER.INTERVAL * 1000, 2) for name, count in samples})
421
+ LOG.warning("[slow-request] %s", json.dumps(values, separators=(",", ":"), ensure_ascii=True))
api/routes_admin.py CHANGED
@@ -40,7 +40,7 @@ import core.registry as registry
40
  import core.schema_overlay as schema_overlay # W46-T36 / C1 - the overlay merge + R12 wall
41
  import core.store as store
42
 
43
- from deps import Session, err, perms, require_session, users
44
  from routes_auth import _public_user
45
 
46
  router = APIRouter(prefix="/api/v1")
@@ -112,6 +112,7 @@ def _registry():
112
  def _view(uname, rec, governed=None, st=None, surface_keys=None):
113
  """What an administrator may see about an account. NEVER `salt` or `hash` — this function is
114
  the only projection these routes use, so there is one place that can leak and it does not.
 
115
 
116
  ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both,
117
  resolved once for the whole request; the single-record routes pass neither, because their
@@ -127,7 +128,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
127
  "email": rec.get("email") or None,
128
  "tenant": _tenant_of(rec),
129
  "active": bool(rec.get("active", True)),
130
- "last_active": rec.get("last_active"),
131
  # The session-revocation handle. An admin needs it: it is the only visible evidence
132
  # that "sign them out everywhere" actually happened.
133
  "epoch": int(rec.get("epoch") or 0),
@@ -2805,6 +2806,7 @@ def settings(session: Session = Depends(require_session)):
2805
  modules = perms.allowed_modules(session.user)
2806
  from core import freeze_notice
2807
  import process_role
 
2808
  roles = process_role.status()
2809
  worker = roles["worker"]
2810
  if not perms.is_admin(session.user):
@@ -2813,6 +2815,7 @@ def settings(session: Session = Depends(require_session)):
2813
  return {
2814
  # Which process role answered, and whether its sync worker is alive and beating.
2815
  "processRole": roles["role"],
 
2816
  "worker": worker,
2817
  "freezeNotice": freeze_notice.notice(),
2818
  # The build this Space is running. Behind the session on purpose — see `main.py::health`,
 
40
  import core.schema_overlay as schema_overlay # W46-T36 / C1 - the overlay merge + R12 wall
41
  import core.store as store
42
 
43
+ from deps import Session, err, observed_last_active, perms, require_session, users
44
  from routes_auth import _public_user
45
 
46
  router = APIRouter(prefix="/api/v1")
 
112
  def _view(uname, rec, governed=None, st=None, surface_keys=None):
113
  """What an administrator may see about an account. NEVER `salt` or `hash` — this function is
114
  the only projection these routes use, so there is one place that can leak and it does not.
115
+ `last_active` includes this tenant/account's temporary read observations without saving them.
116
 
117
  ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both,
118
  resolved once for the whole request; the single-record routes pass neither, because their
 
128
  "email": rec.get("email") or None,
129
  "tenant": _tenant_of(rec),
130
  "active": bool(rec.get("active", True)),
131
+ "last_active": observed_last_active(_tenant_of(rec), uname, rec.get("last_active")),
132
  # The session-revocation handle. An admin needs it: it is the only visible evidence
133
  # that "sign them out everywhere" actually happened.
134
  "epoch": int(rec.get("epoch") or 0),
 
2806
  modules = perms.allowed_modules(session.user)
2807
  from core import freeze_notice
2808
  import process_role
2809
+ from request_timing import route_stats
2810
  roles = process_role.status()
2811
  worker = roles["worker"]
2812
  if not perms.is_admin(session.user):
 
2815
  return {
2816
  # Which process role answered, and whether its sync worker is alive and beating.
2817
  "processRole": roles["role"],
2818
+ "requestTiming": route_stats() if perms.is_admin(session.user) else None,
2819
  "worker": worker,
2820
  "freezeNotice": freeze_notice.notice(),
2821
  # The build this Space is running. Behind the session on purpose — see `main.py::health`,
api/routes_connectors.py CHANGED
@@ -51,6 +51,7 @@ from deps import Session, err, require_session, running_main
51
  #: connector is in.
52
  from core import connector_directory
53
  from core import connector_sync
 
54
  from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
55
  from harness import amazon_client
56
  #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
@@ -995,6 +996,7 @@ def _entry_types(rt, uname=""):
995
  scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
996
  return types, scopes, None
997
  except Exception as e: # noqa: BLE001
 
998
  return set(), {}, f"{type(e).__name__}: {e}"
999
 
1000
 
@@ -2902,9 +2904,10 @@ def _amazon_completed_sync(rt):
2902
  return None, False
2903
  if path.resolve() == Path(datastore.DB_PATH).resolve():
2904
  return read(datastore.ro_cursor())
2905
- with duckdb.connect(str(path), read_only=True) as cursor:
2906
  return read(cursor)
2907
- except Exception:
 
2908
  return None, False
2909
 
2910
 
@@ -2931,9 +2934,10 @@ def _daily_completed_sync(rt, connector_key):
2931
  return None, False
2932
  if path.resolve() == Path(datastore.DB_PATH).resolve():
2933
  return read(datastore.ro_cursor())
2934
- with duckdb.connect(str(path), read_only=True) as cursor:
2935
  return read(cursor)
2936
- except Exception:
 
2937
  return None, False
2938
  return None, False
2939
 
@@ -4642,12 +4646,13 @@ def _amazon_databases(rt, is_admin, spec=None):
4642
  # trims an over-long cause itself; a credential shape still refuses, and the refusal carries
4643
  # the guarded rows that passed, so only the refused tables read "Sync status is unavailable
4644
  # for this database." (a missing status is exactly what `_amazon_database_sync_health`
4645
- # degrades). Any other failure reading status degrades every row, never the whole page.
4646
  try:
4647
  states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
4648
  except cx.AmazonStatusSafetyError as refused:
4649
  states = dict(getattr(refused, "safe_status", None) or {})
4650
- except Exception: # noqa: BLE001
 
4651
  states = {}
4652
 
4653
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
@@ -5401,6 +5406,7 @@ def _plain_read_failure(exc, code, sentence):
5401
  `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
5402
  and no exception text (which can echo a payload) reaches the wire.
5403
  """
 
5404
  import logging # noqa: PLC0415
5405
  logging.getLogger(__name__).exception("connector read failed: %s", code)
5406
  name = type(exc).__name__
 
51
  #: connector is in.
52
  from core import connector_directory
53
  from core import connector_sync
54
+ from core.request_timing import raise_if_resource_error
55
  from core.keychain import AMAZON_ROTATION_DEADLINE_FIELD
56
  from harness import amazon_client
57
  #: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default,
 
996
  scopes[t] = str(e.get("scope") or rk.DEFAULT_SCOPE)
997
  return types, scopes, None
998
  except Exception as e: # noqa: BLE001
999
+ raise_if_resource_error(e)
1000
  return set(), {}, f"{type(e).__name__}: {e}"
1001
 
1002
 
 
2904
  return None, False
2905
  if path.resolve() == Path(datastore.DB_PATH).resolve():
2906
  return read(datastore.ro_cursor())
2907
+ with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
2908
  return read(cursor)
2909
+ except Exception as exc:
2910
+ raise_if_resource_error(exc)
2911
  return None, False
2912
 
2913
 
 
2934
  return None, False
2935
  if path.resolve() == Path(datastore.DB_PATH).resolve():
2936
  return read(datastore.ro_cursor())
2937
+ with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
2938
  return read(cursor)
2939
+ except Exception as exc:
2940
+ raise_if_resource_error(exc)
2941
  return None, False
2942
  return None, False
2943
 
 
4646
  # trims an over-long cause itself; a credential shape still refuses, and the refusal carries
4647
  # the guarded rows that passed, so only the refused tables read "Sync status is unavailable
4648
  # for this database." (a missing status is exactly what `_amazon_database_sync_health`
4649
+ # degrades). Resource failures propagate; other unreadable status degrades every row.
4650
  try:
4651
  states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
4652
  except cx.AmazonStatusSafetyError as refused:
4653
  states = dict(getattr(refused, "safe_status", None) or {})
4654
+ except Exception as exc: # noqa: BLE001
4655
+ raise_if_resource_error(exc)
4656
  states = {}
4657
 
4658
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
 
5406
  `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
5407
  and no exception text (which can echo a payload) reaches the wire.
5408
  """
5409
+ raise_if_resource_error(exc)
5410
  import logging # noqa: PLC0415
5411
  logging.getLogger(__name__).exception("connector read failed: %s", code)
5412
  name = type(exc).__name__
api/routes_customers.py CHANGED
@@ -121,15 +121,7 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r
121
  found = {}
122
  if requested:
123
  try:
124
- with store_pg._pool().connection() as con:
125
- rows = con.execute(store_pg._q(
126
- "SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])",
127
- "store_kv", handle.tenant_slug), (sorted(requested),)).fetchall()
128
- for name, raw_rev, stamp in rows:
129
- rev = int(raw_rev) if raw_rev is not None else 0
130
- found[name] = {"rev": rev,
131
- "updated_at": stamp.timestamp() if hasattr(stamp, "timestamp") else None,
132
- "token": f"pg:{rev}"}
133
  except Exception as exc:
134
  if type(exc).__name__ in {'PoolTimeout', 'OperationalError', 'InterfaceError', 'ConnectionTimeout'}:
135
  from core.grid_events import StoreUnavailable
@@ -189,6 +181,9 @@ def _pool_for(rt, team_id, agent):
189
  # capability result; empty is safe, another tenant's customer book is not.
190
  if rt.key != "royal-imports":
191
  return []
 
 
 
192
 
193
  import modules.customer_data as cl
194
  import routes_keychain
@@ -460,18 +455,44 @@ class _AssemblyStore:
460
  self.tokens[name] = (self._validated_tokens[name] if name in self._validated_tokens
461
  else _assembly_revision(self._runtime, name))
462
  except AttributeError as exc:
 
 
 
463
  self.cacheable = False
464
  if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
465
  print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
 
 
 
 
 
 
 
466
  key = (name, shape)
467
  if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
468
- self.docs[key] = read()
 
 
 
 
 
 
 
 
469
  # TableStore readers can mutate nested projections; lend an independent read each time.
470
  return deepcopy(self.docs[key])
471
 
472
  def get(self, name, fresh=False):
 
 
 
473
  if self.schema_only and fresh:
474
  self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
 
 
 
 
 
475
  if self.schema_only and name.endswith("_table_workspace"):
476
  return self._read(name, ("projection", ("overlays",)),
477
  lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh)
@@ -482,10 +503,14 @@ class _AssemblyStore:
482
  self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
483
 
484
  def get_projection(self, name, drop=()):
 
 
485
  return self._read(name, ("projection", tuple(drop)),
486
  lambda: self._runtime.get_projection(name, drop=drop))
487
 
488
  def get_entry(self, name, entry):
 
 
489
  if self.schema_only:
490
  backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime
491
  if not callable(getattr(backend, "get_entry", None)):
@@ -504,6 +529,9 @@ class _AssemblyStore:
504
  return self._read(name, "exists", lambda: self._runtime.exists(name))
505
 
506
  def revision(self, name):
 
 
 
507
  token = _assembly_revision(self._runtime, name)
508
  self.tokens.setdefault(name, token)
509
  return token
@@ -600,20 +628,30 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
600
  result = ((*prefix, json.dumps(None)), {})
601
  return result
602
  names = {name for name, _token in revisions}
 
 
 
 
 
 
 
 
603
  with _grid_assembly_lock:
604
  for memo_key in list(rt.pool_cache):
605
  if memo_key[:-2] == prefix:
606
  names.update(name for name, _token in memo_key[-1])
607
- tokens, users_stamp = _assembly_revisions(rt, names, uses_roster,
608
  getattr(session, "tenant", None), roster_memo)
609
  result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
610
  return result
611
 
612
  def own_copy(result):
613
- # The pool is already shared by scope_cache and rows_from_pool never mutates it.
614
- # Copy the rest together to preserve internal aliases without lending mutable state.
615
- out = deepcopy({k: v for k, v in result.items() if k != "rows_src"})
616
- out["rows_src"] = result["rows_src"]
 
 
617
  out["workspace"]["storageKey"] = storage_key
618
  return out
619
 
@@ -653,7 +691,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
653
 
654
  def valid(revisions, tokens):
655
  if any(name not in tokens for name, _token in revisions):
656
- tokens, _users = _assembly_revisions(rt, [name for name, _token in revisions])
 
657
  result = all(json.dumps(tokens[name], sort_keys=True, default=str) == token
658
  for name, token in revisions)
659
  return result
 
121
  found = {}
122
  if requested:
123
  try:
124
+ found = store_pg._revision_many(sorted(requested), tenant_slug=handle.tenant_slug)
 
 
 
 
 
 
 
 
125
  except Exception as exc:
126
  if type(exc).__name__ in {'PoolTimeout', 'OperationalError', 'InterfaceError', 'ConnectionTimeout'}:
127
  from core.grid_events import StoreUnavailable
 
181
  # capability result; empty is safe, another tenant's customer book is not.
182
  if rt.key != "royal-imports":
183
  return []
184
+ bounded = getattr(rt, '_canonical_rows', None)
185
+ if bounded is not None:
186
+ return bounded
187
 
188
  import modules.customer_data as cl
189
  import routes_keychain
 
455
  self.tokens[name] = (self._validated_tokens[name] if name in self._validated_tokens
456
  else _assembly_revision(self._runtime, name))
457
  except AttributeError as exc:
458
+ if self.schema_only and callable(getattr(self._runtime, 'revision', None)):
459
+ from core.shared_overlay import SchemaUnavailable
460
+ raise SchemaUnavailable('table definition revisions are unavailable') from exc
461
  self.cacheable = False
462
  if os.environ.get("AIOS_HEAVY_PROFILE") == "1":
463
  print(f"[assembly-miss] revision unreadable for {name}: {exc}", flush=True)
464
+ except Exception as exc:
465
+ if self.schema_only:
466
+ from core.request_timing import raise_if_resource_error
467
+ from core.shared_overlay import SchemaUnavailable
468
+ raise_if_resource_error(exc)
469
+ raise SchemaUnavailable('table definition revisions are unavailable') from exc
470
+ raise
471
  key = (name, shape)
472
  if fresh or (not self.cacheable and not self.schema_only) or key not in self.docs:
473
+ try:
474
+ self.docs[key] = read()
475
+ except Exception as exc:
476
+ if self.schema_only:
477
+ from core.request_timing import raise_if_resource_error
478
+ from core.shared_overlay import SchemaUnavailable
479
+ raise_if_resource_error(exc)
480
+ raise SchemaUnavailable('table definitions are unavailable') from exc
481
+ raise
482
  # TableStore readers can mutate nested projections; lend an independent read each time.
483
  return deepcopy(self.docs[key])
484
 
485
  def get(self, name, fresh=False):
486
+ if fresh:
487
+ from core.request_timing import clear_read_memo
488
+ clear_read_memo()
489
  if self.schema_only and fresh:
490
  self.docs = {key: value for key, value in self.docs.items() if key[0] != name}
491
+ if self.schema_only and name == "user_tables":
492
+ from core.user_tables import DEF_ONLY_DROPS
493
+ drops = (*DEF_ONLY_DROPS, 'aiEnrich')
494
+ return self._read(name, ("projection", drops),
495
+ lambda: self._runtime.get_projection(name, drop=drops), fresh=fresh)
496
  if self.schema_only and name.endswith("_table_workspace"):
497
  return self._read(name, ("projection", ("overlays",)),
498
  lambda: self._runtime.get_projection(name, drop=("overlays",)), fresh=fresh)
 
503
  self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)), fresh=fresh)
504
 
505
  def get_projection(self, name, drop=()):
506
+ if self.schema_only and name == 'user_tables':
507
+ drop = tuple(dict.fromkeys([*drop, 'aiEnrich']))
508
  return self._read(name, ("projection", tuple(drop)),
509
  lambda: self._runtime.get_projection(name, drop=drop))
510
 
511
  def get_entry(self, name, entry):
512
+ if self.schema_only and name == 'user_tables':
513
+ return (self.get(name) or {}).get(str(entry))
514
  if self.schema_only:
515
  backend = self._runtime._store() if callable(getattr(self._runtime, "_store", None)) else self._runtime
516
  if not callable(getattr(backend, "get_entry", None)):
 
529
  return self._read(name, "exists", lambda: self._runtime.exists(name))
530
 
531
  def revision(self, name):
532
+ # Revisionless schema reads cannot supply a token for a permission cache.
533
+ if self.schema_only and not callable(getattr(self._runtime, "revision", None)):
534
+ return None
535
  token = _assembly_revision(self._runtime, name)
536
  self.tokens.setdefault(name, token)
537
  return token
 
628
  result = ((*prefix, json.dumps(None)), {})
629
  return result
630
  names = {name for name, _token in revisions}
631
+ if (not with_rows and (scope.startswith('ut_') or scope in {'customer', 'product'})
632
+ and _assembly_pg_store(assembly_rt if assembly_rt is not None else rt) is not None):
633
+ # Check known metadata together before any borrowed read asks for its own token.
634
+ names.update({'user_tables', 'object_shares', 'automations', 'loopable_route_system',
635
+ 'table_zones', 'ut_schema_overlay', 'database_setup', scope + '_cohorts',
636
+ scope + '_table_workspace', scope + '_table_workspace__shared'})
637
+ if scope.startswith('ut_'):
638
+ names.add(scope + '__shared')
639
  with _grid_assembly_lock:
640
  for memo_key in list(rt.pool_cache):
641
  if memo_key[:-2] == prefix:
642
  names.update(name for name, _token in memo_key[-1])
643
+ tokens, users_stamp = _assembly_revisions(assembly_rt if assembly_rt is not None else rt, names, uses_roster,
644
  getattr(session, "tenant", None), roster_memo)
645
  result = ((*prefix, json.dumps(users_stamp, sort_keys=True, default=str)), tokens)
646
  return result
647
 
648
  def own_copy(result):
649
+ # Both aliases lend immutable source rows. Copy the request-shaped metadata together.
650
+ borrowed = {id(result['rows_src']): result['rows_src']}
651
+ wall_rows = (result.get('_permission_scope') or {}).get('rows')
652
+ if wall_rows is not None:
653
+ borrowed[id(wall_rows)] = wall_rows
654
+ out = deepcopy(result, borrowed)
655
  out["workspace"]["storageKey"] = storage_key
656
  return out
657
 
 
691
 
692
  def valid(revisions, tokens):
693
  if any(name not in tokens for name, _token in revisions):
694
+ tokens, _users = _assembly_revisions(assembly_rt if assembly_rt is not None else rt,
695
+ [name for name, _token in revisions])
696
  result = all(json.dumps(tokens[name], sort_keys=True, default=str) == token
697
  for name, token in revisions)
698
  return result
api/routes_grid.py CHANGED
@@ -51,8 +51,9 @@ def _ctx(session: Session, fields, pids, *, permission_fields=None, **kw):
51
  if scope_key == "product":
52
  import modules.product_data as pd
53
  from routes_products import MODULE as _PMOD, pd_fields
 
54
 
55
- module, canonical = _PMOD, pd_fields(consolidated=True, st=session.runtime)
56
  kw.setdefault("table", pd.table_ops(session.runtime))
57
  hidden = perm_scope.hidden_keys(session.user, module,
58
  canonical if permission_fields is None else permission_fields,
@@ -1224,6 +1225,98 @@ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "custome
1224
  return out
1225
 
1226
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1227
  def _builtin_history_context(session, scope, *, restoring=False):
1228
  """Resolve history through the same assembly and write context as grid events."""
1229
  from routes_customers import grid_assembly
 
51
  if scope_key == "product":
52
  import modules.product_data as pd
53
  from routes_products import MODULE as _PMOD, pd_fields
54
+ from registry_source import MetadataStore
55
 
56
+ module, canonical = _PMOD, pd_fields(consolidated=True, st=MetadataStore(session.runtime))
57
  kw.setdefault("table", pd.table_ops(session.runtime))
58
  hidden = perm_scope.hidden_keys(session.user, module,
59
  canonical if permission_fields is None else permission_fields,
 
1225
  return out
1226
 
1227
 
1228
+ def _builtin_activity_metadata(session, scope, *, st):
1229
+ """Return detached visible fields, complete permission fields, hidden keys and no row identities for one Activity page."""
1230
+ from copy import copy, deepcopy
1231
+ import aios_grid
1232
+ import odoo_relational
1233
+ from core import perm_scope, route_system, schema_overlay
1234
+ from modules import cohort
1235
+ from routes_customers import _AssemblyStore, _merge_shared_fields, shared_fields
1236
+
1237
+ if scope not in ("customer", "product"):
1238
+ raise err(400, "bad_scope", "Activity requires a customer or product database.")
1239
+ reader = st if isinstance(st, _AssemblyStore) and st.schema_only else _AssemblyStore(st, schema_only=True)
1240
+ borrowed = copy(session)
1241
+ borrowed.runtime = reader
1242
+ module = "product_data" if scope == "product" else MODULE
1243
+ borrowed.require(module)
1244
+ if scope == "product":
1245
+ from routes_products import scoped_pool
1246
+ import modules.product_data as pd
1247
+ _, _, _, contract = scoped_pool(borrowed, with_rows=False)
1248
+ definitions = pd.shared_fields(st=reader)
1249
+ table_key = pd.TABLE_KEY
1250
+ else:
1251
+ contract = list(aios_grid.FIELDS) + odoo_relational.module_link_fields(MODULE)
1252
+ definitions = shared_fields(st=reader)
1253
+ table_key = "customer_table_workspace"
1254
+
1255
+ if not reader.available():
1256
+ from core.grid_events import StoreUnavailable
1257
+ raise StoreUnavailable("Activity source fields require an available workspace.")
1258
+ document = reader.get(table_key) or {}
1259
+ personal = (document.get(session.uname) or {}).get("fields") or {}
1260
+ has_cohorts = bool(cohort.scoped(scope, st=reader).all_for(session.uname))
1261
+ fields = aios_grid.fields_from_workspace({"fields": personal}, cohorts=has_cohorts,
1262
+ scope_key=scope, fields_base=contract)
1263
+ if scope == "customer":
1264
+ # Shared definitions supersede personal collisions before the dependent-field wall.
1265
+ fields = _merge_shared_fields(fields, definitions, session=borrowed)
1266
+ else:
1267
+ fields = aios_grid.apply_choice_overrides(fields, definitions)
1268
+ from core import shares
1269
+ for field in fields:
1270
+ saved = definitions.get(field.get("key"))
1271
+ if isinstance(saved, dict) and saved.get("granted") is True:
1272
+ field["granted"] = True
1273
+ if isinstance(saved.get("createdBy"), str):
1274
+ field["createdBy"] = saved["createdBy"]
1275
+ have = {field["key"] for field in fields}
1276
+ fields += [dict(field, source="overlay", shared=True)
1277
+ for key, field in definitions.items() if key not in have]
1278
+ for field in fields:
1279
+ if field.get("granted") is True:
1280
+ role = shares.role_for("field", shares.field_oid(module, field["key"]),
1281
+ session.uname, is_admin=session.admin, st=reader)
1282
+ if role:
1283
+ field["sharedRole"] = role
1284
+
1285
+ # Keep projection field identities without loading Route/Zone objects or membership cells.
1286
+ projection = route_system.customer_projection_fields([], [])
1287
+ if scope == "product":
1288
+ projection = [field for field in projection if field["key"] == route_system.ZONE_FIELD_KEY]
1289
+ projected = {field["key"] for field in projection}
1290
+ fields = [field for field in fields if field["key"] not in projected] + projection
1291
+ corpus = deepcopy(fields)
1292
+ hidden = perm_scope.hidden_keys(session.user, module, corpus, st=reader)
1293
+ fields = [field for field in fields if field["key"] not in hidden]
1294
+ wall_contract = contract if scope == "product" else aios_grid.FIELDS
1295
+ fields, excluded = schema_overlay.for_grid(module, fields, st=reader, contract=wall_contract)
1296
+ return {"fields": deepcopy(fields), "pids": frozenset(),
1297
+ "defn": {"fields": corpus}, "hidden": hidden | excluded,
1298
+ "_permission_scope": {"row_fields": deepcopy(wall_contract)}}
1299
+
1300
+
1301
+ def _builtin_activity_pids(session, database, wanted, metadata, *, st):
1302
+ """Authorize only requested builtin identities through the ordinary pool and permanent row walls."""
1303
+ from core import perm_scope
1304
+ team_id, agent = perm_scope.derive_pool_scope(session.user, database)
1305
+ if database == "customer_data":
1306
+ from routes_customers import _pool_for
1307
+ rows = _pool_for(st, team_id, agent)
1308
+ elif database == "product_data":
1309
+ from routes_products import _pool_for
1310
+ rows = _pool_for(st, team_id)
1311
+ else:
1312
+ raise err(400, "bad_scope", "Activity requires a customer or product database.")
1313
+ rows = [row for row in rows if str(row.get("pid")) in wanted]
1314
+ # The row wall needs real shared cells, never a schema projection with missing values.
1315
+ rows = perm_scope.apply_row_scope(rows, session.user, database,
1316
+ metadata["_permission_scope"]["row_fields"], st=session.runtime)
1317
+ return frozenset(row["pid"] for row in rows)
1318
+
1319
+
1320
  def _builtin_history_context(session, scope, *, restoring=False):
1321
  """Resolve history through the same assembly and write context as grid events."""
1322
  from routes_customers import grid_assembly
api/routes_odoo_tables.py CHANGED
@@ -289,8 +289,7 @@ def odoo_tables_status(brief: bool = Query(False),
289
  from harness import datastore as _ds
290
  frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
291
  else {"table": spec["table"]})
292
- n = _ds.window(select="1", where=spec.get("where") or "",
293
- order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
294
  else:
295
  n = -1 # unknown, and never reported as zero
296
  mirror_refreshed = mirror_stamp(
@@ -652,7 +651,9 @@ def _sync_entities(cur, cadence_seconds):
652
  rows = cur.execute(
653
  "SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
654
  "FROM _sync_state ORDER BY entity").fetchall()
655
- except Exception: # noqa: BLE001
 
 
656
  return None
657
  out = {}
658
  for entity, phase, last_id, cursor_wd, nrows, updated_at in rows:
@@ -777,25 +778,19 @@ GRID_SOURCES = {}
777
  _SOURCE_PROVIDERS = []
778
  _SOURCE_ANALYSES = {}
779
 
780
- #: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
781
- #: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
782
- #: keys to the shared dict as it went, and returned the dict to any caller that found it
783
- #: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
784
- #: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
785
- #: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
786
- #: this lock and publishes it in one `dict.update`; a reader waits on the lock until
787
- #: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
788
- #: Amazon fallback) happens only after a build has been published, so it can never stand in
789
- #: for one either.
790
  _SOURCES_LOCK = threading.RLock()
791
  _SOURCES_BUILT = False
792
- _SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
 
 
793
 
794
 
795
  def invalidate_sources():
796
  """Forget measured bindings after a connector publishes a new mirror schema."""
797
- global _SOURCES_BUILT
798
  with _SOURCES_LOCK:
 
799
  _SOURCES_BUILT = False
800
  GRID_SOURCES.clear()
801
 
@@ -847,54 +842,54 @@ def _sources():
847
  importing that module at file-import time would drag the Odoo layer into every process that
848
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
849
 
850
- ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
851
- lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
852
- and so does one that holds only admissions made since the last invalidation.
853
  """
854
- global _SOURCES_BUILT, _SOURCES_BUILDER
855
  if _SOURCES_BUILT and GRID_SOURCES:
856
  return GRID_SOURCES
857
  with _SOURCES_LOCK:
858
- if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
859
  return GRID_SOURCES
860
- _SOURCES_BUILDER = threading.get_ident()
861
- try:
862
- built = {}
863
- for build in list(_SOURCE_PROVIDERS):
864
- try:
865
- extra = build() or {}
866
- except Exception: # noqa: BLE001
867
- # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
868
- # grids must not go dark because a newer connector's module is unhappy, and
869
- # vice versa.
870
- continue
871
- if extra:
872
- built.update(extra)
873
- _ut().register_connected(*extra)
874
- # A key admitted since the last invalidation was measured at least as recently as
875
- # this build, so it stands; everything else comes from the build. ONE update, so a
876
- # lock-free reader of the dict sees the old entry or the new one, never a gap.
877
  GRID_SOURCES.update({**built, **GRID_SOURCES})
878
  _SOURCES_BUILT = True
879
- finally:
880
- _SOURCES_BUILDER = None
 
 
881
  return GRID_SOURCES
882
 
883
 
884
- def _learn_amazon_sources(cur=None):
885
- """Publish the complete current Amazon binding set, including restored missing entries."""
886
- _sources()
 
 
 
887
  if cur is None:
888
  extra = _amazon_sources()
889
  else:
890
  import amazon_relational
891
  extra = amazon_relational.grid_sources(cur)
892
- if not extra:
893
- return
894
  with _SOURCES_LOCK:
895
- # One known key cannot certify every history table. Replace old missing bindings as
896
- # one publication, so the first rows AND aggregates request sees the restored schema.
897
  GRID_SOURCES.update(extra)
 
898
  _ut().register_connected(*extra)
899
 
900
 
@@ -1297,13 +1292,7 @@ def _amazon_sources():
1297
 
1298
 
1299
  def require_amazon_mirror(table_key, session, definition=None, st=None):
1300
- """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
1301
-
1302
- The provider contributes no keys on an Odoo-only mirror. A persisted read-through
1303
- definition still must refuse there instead of falling through to its empty stored rows.
1304
- S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
1305
- window route stops paying a second definitions projection for this one check.
1306
- """
1307
  if not str(table_key).startswith("ut_amzn_"):
1308
  return
1309
  import amazon_relational as amazon
@@ -1321,26 +1310,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
1321
  if mismatch:
1322
  raise err(409, "cross_tenant_store", mismatch)
1323
  try:
1324
- spec = _sources().get(table_key)
1325
- if (spec is not None or definition.get("readThrough")
1326
- or table_key in amazon.READ_THROUGH):
1327
- # Check current table presence even if a binding was cached before a restart.
1328
- current = amazon.grid_source(cur, table_key)
1329
- if current.get("missing"):
1330
- raise err(503, "amazon_mirror_not_loaded", current["missing"])
1331
- # A dynamically oversized snapshot may have a durable stub before this process
1332
- # has measured it. Admit its binding without changing another source's registry.
1333
- # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
1334
- # key. Admitted into an empty or Amazon-less registry, this one key used to make
1335
- # `_sources()` skip its build, so the grid resolved while its link targets
1336
- # (orders, read-through) were dispatched as stored tables: a first-page 500.
1337
- if spec is None or spec.get("missing"):
1338
- _learn_amazon_sources(cur)
1339
- with _SOURCES_LOCK:
1340
- _sources()
1341
- held = GRID_SOURCES.get(table_key)
1342
- if held is None or held.get("missing"):
1343
- GRID_SOURCES[table_key] = current
1344
  finally:
1345
  cur.close()
1346
 
@@ -1512,8 +1482,7 @@ def _unmaterialisable(cur=None):
1512
  try:
1513
  frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
1514
  else {"table": spec["table"]})
1515
- total = datastore.window(select="1", where=spec.get("where") or "",
1516
- order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
1517
  except Exception: # noqa: BLE001
1518
  total = None
1519
  if total is not None and total > max_rows:
@@ -1557,7 +1526,27 @@ def _degrade(expr, have_by_alias, default_alias=""):
1557
  return expr if bare.lower() in have else "NULL"
1558
 
1559
 
1560
- def _source_for(cur, table_key):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1561
  """The resolved spec for one connected grid, or None when this grid is not read-through yet.
1562
 
1563
  ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
@@ -1584,28 +1573,15 @@ def _source_for(cur, table_key):
1584
  # table and turns Products ordered into the wrong 404.
1585
  if module_cache_source(key) is not None:
1586
  return None
 
 
 
 
1587
  spec = _sources().get(key)
1588
- if (not spec or spec.get("missing")) and key.startswith("ut_amzn_"):
1589
- # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
1590
- # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
1591
- # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
1592
- # read-through Amazon grid answered "no read-through binding". S70-T12 re-asks the
1593
- # complete Amazon provider on this restored cursor, including cached missing entries.
1594
- _learn_amazon_sources(cur)
1595
- spec = GRID_SOURCES.get(key)
1596
  if not spec:
1597
  return None
1598
  if spec.get("missing"):
1599
  raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1600
- if key.startswith("ut_amzn_"):
1601
- import amazon_relational as amazon
1602
- if key in amazon.SOURCE:
1603
- # The registry survives syncs. Resolved parent data embedded in its old
1604
- # expressions must follow this cursor's revision rather than the boot revision.
1605
- derived = {lookup["key"]: (lookup["expr"], amazon._text)
1606
- for lookup in amazon.lookups(cur, key)}
1607
- spec = {**spec, "cols": {**{k: v for k, v in spec["cols"].items()
1608
- if k not in amazon.LOOKUP_KEYS}, **derived}}
1609
  from harness import datastore
1610
  if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
1611
  try:
@@ -1658,9 +1634,21 @@ def _source_for(cur, table_key):
1658
  for key, (expr, cast) in spec["cols"].items():
1659
  cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
1660
  have_by_alias), cast)
 
 
1661
  return {**spec, "cols": cols}
1662
 
1663
 
 
 
 
 
 
 
 
 
 
 
1664
  #: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
1665
  #: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of
1666
  #: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of
@@ -1874,7 +1862,8 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
1874
  if table_key in trail:
1875
  raise ValueError("a relation has a circular access rule")
1876
  trail = (*trail, table_key)
1877
- handle = _ut().lend_defs(st)
 
1878
  declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
1879
  columns = _filter_columns(spec, fields)
1880
  guards, resolved, targets = [], {}, {}
@@ -1895,13 +1884,14 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
1895
  targets[key] = None
1896
  return None
1897
  definition = _ut().get(key, st=handle)
1898
- target_spec = _source_for(cur, key)
1899
  if target_spec is None and str(key).startswith("ut_amzn_"):
1900
  # Small Amazon targets (notably listings) are stored snapshots, but
1901
  # their raw mirror still answers the same automatic link in SQL.
1902
  import amazon_relational
1903
  if key in amazon_relational.SOURCE:
1904
- target_spec = amazon_relational.grid_source(cur, key)
 
1905
  if not definition or target_spec is None or target_spec.get("missing"):
1906
  refuse(field)
1907
  target_fields = definition.get("fields") or []
@@ -2375,7 +2365,12 @@ def _related_source_value(session, entry, record):
2375
  if not source:
2376
  return None
2377
  try:
2378
- rows = perm_scope.scoped_table(session.user, source, st=session.runtime) or []
 
 
 
 
 
2379
  except Exception: # noqa: BLE001
2380
  return None
2381
  want = str(record)
@@ -2386,21 +2381,25 @@ def _related_source_value(session, entry, record):
2386
  return None
2387
 
2388
 
2389
- def _materialised_table(table_key):
2390
- """Storage dispatch, independent of the browser's fallible status roster.
2391
-
2392
- ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
2393
- (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
2394
- the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
2395
- """
2396
  if module_cache_source(table_key) is not None:
2397
  return True
2398
- if table_key in _sources():
2399
- return False
2400
  if str(table_key).startswith("ut_amzn_"):
2401
- _learn_amazon_sources()
2402
- return table_key not in GRID_SOURCES
2403
- return True
 
 
 
 
 
 
 
 
 
 
 
2404
 
2405
 
2406
  #: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
@@ -2425,11 +2424,17 @@ _UNREAD = ...
2425
 
2426
 
2427
  def _user_tables_token(rt):
2428
- """The durable change token of this tenant's `user_tables` bucket, or None when unreadable."""
2429
- try:
2430
- return rt.revision(_ut().STORE_KEY)["token"]
2431
- except Exception: # noqa: BLE001
2432
  return None
 
 
 
 
 
 
 
2433
 
2434
 
2435
  def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
@@ -2463,17 +2468,10 @@ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
2463
 
2464
 
2465
  def _page_defs(session):
2466
- """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's.
2467
-
2468
- Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked
2469
- against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link
2470
- target does not pay a second revision round trip.
2471
- The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
2472
- `user_tables` revision instead of once per question. Kept only when every definition really is
2473
- a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
2474
- 28 MB document on a failed read and that must never be held, let alone copied per request.
2475
- ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
2476
- lends lazily live for this one request and are never carried to the next.
2477
  """
2478
  import copy
2479
  import core.store as store
@@ -2483,15 +2481,23 @@ def _page_defs(session):
2483
  token = _user_tables_token(rt)
2484
 
2485
  def keep(v):
2486
- tables = [d for d in (v or {}).values() if isinstance(d, dict)]
 
2487
  return bool(tables) and all(store.is_projected(d) for d in tables)
2488
 
2489
- defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt),
2490
- keep=keep, clone=copy.deepcopy, token=token)
2491
- return ut.lend(rt, **{ut.STORE_KEY: defs}), token
 
 
 
 
 
 
 
2492
 
2493
 
2494
- def _stored_link_rows(session, table, token=_UNREAD):
2495
  """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
2496
 
2497
  ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
@@ -2505,7 +2511,8 @@ def _stored_link_rows(session, table, token=_UNREAD):
2505
 
2506
  rt = session.runtime
2507
  if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
2508
- or module_cache_source(table) is not None or not _materialised_table(table)):
 
2509
  return perm_scope._read(table, session.user, rt)
2510
  return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
2511
  keep=lambda v: bool(v[1]),
@@ -2528,8 +2535,17 @@ def _traversal_defn_or_refuse(session, table_key, st=None):
2528
  return defn
2529
 
2530
 
 
 
 
 
 
 
 
 
2531
  def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
2532
- traversal_of=None, st=None, rev=..., normalize_match=False): # `...` IS `_UNREAD`
 
2533
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
2534
 
2535
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
@@ -2547,9 +2563,13 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2547
  import core.relations as relations
2548
 
2549
  columns = set(columns) | {relations.IDENTITY_COL}
2550
- if not _materialised_table(table):
2551
- read = (_compiled_read(table, session, traversal_intermediate=True, st=st)
2552
- if traversal_of is not None else _compiled_read(table, session, st=st))
 
 
 
 
2553
  spec = read["spec"]
2554
  bindings = dict(spec["cols"], pid=(spec["id"], int))
2555
  if columns - set(bindings):
@@ -2560,10 +2580,17 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2560
  if values is None:
2561
  continue
2562
  expr, cast = bindings[key]
2563
- values = list(values)
2564
  if normalize_match and key != "pid":
2565
  expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
2566
  cast = lambda value: str(value).strip().lower()
 
 
 
 
 
 
 
 
2567
  parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
2568
  params.extend(cast(value) for value in values)
2569
  if matches is not None:
@@ -2571,6 +2598,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2571
  for key, values in matches.items():
2572
  expr, cast = bindings[key]
2573
  values = sorted(values)
 
 
 
2574
  if values:
2575
  alternatives.append(f"{expr} IN ({','.join('?' for _ in values)})")
2576
  params.extend(cast(value) for value in values)
@@ -2579,14 +2609,30 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2579
  projection = ", ".join(bindings[key][0] for key in names)
2580
  source = spec.get("from_sql") or spec["table"]
2581
  predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
2582
- return [dict(zip(names, row)) for row in read["cur"].execute(
2583
- f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2584
 
2585
  # This is an internal join, never a cell response. Apply the same permanent wall on the
2586
  # complete contract before taking any join value, including a join on a hidden column.
2587
- if traversal_of is None and not perm_scope.may_read(
2588
- session.user, table, st=st if st is not None else session.runtime):
2589
- raise err(403, "forbidden_table", "This account cannot open the linked database.")
2590
  if module_cache_source(table) is not None:
2591
  session.require(table)
2592
  # The revision memo serves a caller that handed this request's token (the page's link fills,
@@ -2597,7 +2643,7 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2597
  fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
2598
  if definition is not None else perm_scope._read(table, session.user, session.runtime))
2599
  else:
2600
- fields, rows = _stored_link_rows(session, table, token=rev)
2601
  if ids is not None:
2602
  wanted = {str(value) for value in ids}
2603
  rows = [row for row in rows if str(row.get("pid")) in wanted]
@@ -2615,8 +2661,9 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2615
  wanted = {normalize(value) for value in values}
2616
  rows = [row for row in rows if normalize(row.get(key)) in wanted]
2617
  if matches is not None:
2618
- wanted = {key: {str(value) for value in values} for key, values in matches.items()}
2619
- rows = [row for row in rows if any(str(row.get(key)) in values
 
2620
  for key, values in wanted.items())]
2621
  return [{key: row.get(key) for key in columns} for row in rows]
2622
 
@@ -2627,7 +2674,48 @@ def _relation_cell_ids(value):
2627
  return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
2628
 
2629
 
2630
- def _relation_leg_ids(session, source, source_ids, entry):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2631
  """Follow one declared leg, including manual, inverse and through links."""
2632
  import core.relations as relations
2633
 
@@ -2645,6 +2733,9 @@ def _relation_leg_ids(session, source, source_ids, entry):
2645
  source_col, target_col = ("pid", field) if reverse else (field, "pid")
2646
  elif inverse:
2647
  source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
 
 
 
2648
  source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
2649
  values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
2650
  # A column-keyed edge cannot fall back to the wire record id. In particular,
@@ -2669,11 +2760,11 @@ def _relation_leg_ids(session, source, source_ids, entry):
2669
  traversal_of=str(entry.get("key") or via.get("key") or ""))
2670
  values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
2671
  if (manual and reverse) or (inverse and not reverse):
2672
- rows = _relation_rows(session, target, [target_col])
2673
- rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values]
2674
  else:
2675
  rows = _relation_rows(session, target, [target_col], match=(target_col, values),
2676
- where=entry.get("where"))
2677
  return {int(row["pid"]) for row in rows}
2678
 
2679
 
@@ -2686,7 +2777,7 @@ def _relation_label(key):
2686
  if table == key and label), str(key))
2687
 
2688
 
2689
- def _relation_target_ids(session, target, related):
2690
  """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
2691
  import core.relations as relations
2692
  from routes_grid import _may_open, related_path_entries
@@ -2721,11 +2812,13 @@ def _relation_target_ids(session, target, related):
2721
  if not _may_open(session, leg["target"]):
2722
  raise err(403, "forbidden_table",
2723
  f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
2724
- if not ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
 
 
2725
  # An earlier, authorised Amazon hop matched nothing. There is no next
2726
  # source record to read, and that is an empty relation, not a missing key.
2727
  continue
2728
- ids = _relation_leg_ids(session, table, ids, leg)
2729
  return ids
2730
 
2731
 
@@ -2961,7 +3054,9 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
2961
  related = _json_arg(related, "related")
2962
  if related in (None, "", {}):
2963
  return None, [], []
2964
- ids = _relation_target_ids(session, table_key, related)
 
 
2965
  sql, params = relations.pin_sql(spec["id"], sorted(ids))
2966
  notes = []
2967
  if len(ids) > _PIN_REPORT_AT:
@@ -3020,6 +3115,119 @@ def _active_cohort_leaves(nodes):
3020
  return out
3021
 
3022
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3023
  def _view_store(session, table_key):
3024
  """The workspace document a saved view of THIS database lives in.
3025
 
@@ -3028,7 +3236,43 @@ def _view_store(session, table_key):
3028
  to fail it: the client reads the views `/workspace` served out of this same document.
3029
  """
3030
  import core.table_store as table_store
3031
- return table_store.make(f"{table_key}_table_workspace", st=session.runtime)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3032
 
3033
 
3034
  def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits):
@@ -3052,8 +3296,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3052
  * no filters and a curated `memberPids` is the client's `else if (curated)` branch: the
3053
  membership IS the view;
3054
  * no filters and no members means the view holds every row in scope;
3055
- * the ids are materialised in ONE `SELECT DISTINCT` under `base_where`, uncapped, the same
3056
- way `_related_pin` does it, so the wall rides along and standing rule 1 is kept.
3057
 
3058
  THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY:
3059
  resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets`
@@ -3082,6 +3325,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3082
 
3083
  store = _view_store(session, table_key)
3084
  sets, memo, unresolved = {}, {}, []
 
3085
  counter = [0]
3086
 
3087
  def key_for():
@@ -3092,10 +3336,11 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3092
  return "__view_%d__" % counter[0]
3093
 
3094
  def resolve(vid, seen):
3095
- """`('ids', [...])` / `('all', None)` / `(None, reason)` for one view id."""
3096
  if vid in memo:
3097
  return memo[vid]
3098
  if vid in seen:
 
3099
  # `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left
3100
  # ABSENT, so a stored loop degrades to an explained empty table, not a hung request.
3101
  return None, "it refers to itself through another saved view"
@@ -3115,12 +3360,24 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3115
  "own records rather than from this database's live connection")
3116
  memo[vid] = got
3117
  return got
3118
- members = []
3119
- for raw in (cfg.get("memberPids") or ()):
 
 
 
 
 
3120
  try:
3121
- members.append(int(raw))
3122
- except (TypeError, ValueError): # noqa: PERF203
3123
- continue
 
 
 
 
 
 
 
3124
  raw_nodes = cfg.get("filters") or []
3125
  if not isinstance(raw_nodes, list):
3126
  raw_nodes = []
@@ -3129,10 +3386,14 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3129
  # "anything active": with no nodes at all the curated set IS the view, and with nodes
3130
  # that all happen to be half-built the view holds everything. Two different answers
3131
  # from one list, so the emptiness is checked before the compiler sees it.
3132
- got = ("ids", members) if members else ("all", None)
3133
  memo[vid] = got
3134
  return got
3135
  nodes = rewrite(raw_nodes, seen + (vid,))
 
 
 
 
3136
  unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols))
3137
  if unknown:
3138
  # ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it
@@ -3146,7 +3407,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3146
  try:
3147
  pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or"
3148
  else "and"),
3149
- columns=cols, member_ids=(members or None),
3150
  id_sql=spec["id"], today=_today(), cohort_sets=sets)
3151
  except ValueError as exc:
3152
  got = (None, "it uses a condition this database's live connection cannot evaluate ("
@@ -3167,9 +3428,9 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3167
  return got
3168
  frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
3169
  where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql
3170
- ids = [int(r[0]) for r in cur.execute(
3171
  f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}',
3172
- list(wall_params) + list(pred.params)).fetchall() if r and r[0] is not None]
3173
  got = ("ids", ids)
3174
  memo[vid] = got
3175
  return got
@@ -3200,7 +3461,7 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3200
  if kind == "ids":
3201
  sets[key] = got
3202
  op = "noneOf" if not_in else "anyOf"
3203
- if len(got) >= _PIN_REPORT_AT:
3204
  limits.append({
3205
  "subject": "view", "effect": "slow",
3206
  "cause": f"the saved view this condition names holds {len(got):,} rows, "
@@ -3220,7 +3481,12 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
3220
  out.append({"colId": "__cohort__", "op": op, "value": key})
3221
  return out
3222
 
3223
- tree = rewrite(tree, ())
 
 
 
 
 
3224
  if unresolved:
3225
  limits.append({
3226
  "subject": ", ".join(vid for vid, _why in unresolved),
@@ -3240,9 +3506,9 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
3240
  `__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
3241
 
3242
  A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
3243
- here the way the client answers it: the residue (every non-rank condition) under the wall is
3244
- the domain, and `filter_eval.resolve_rank_sets` (the Python port of the client's set pass,
3245
- ties by ascending pid) picks the members. Only the id and the ranked columns are read, uncapped.
3246
  """
3247
  fe = _fe()
3248
  leaves = fe._rank_leaves(tree or [], cols)
@@ -3270,30 +3536,8 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
3270
  params.extend(pred.params)
3271
  frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
3272
  answers = {}
3273
- python_leaves = []
3274
  for n in leaves:
3275
- got = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
3276
- if got is None:
3277
- python_leaves.append(n)
3278
- else:
3279
- answers[id(n)] = got
3280
- if python_leaves:
3281
- # A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and
3282
- # the ranked columns once, uncapped, and let `resolve_rank_sets` answer it.
3283
- ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols})
3284
- select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked])
3285
- sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "")
3286
- rows = []
3287
- for r in cur.execute(sql, params).fetchall():
3288
- if not r or r[0] is None:
3289
- continue
3290
- row = {"pid": int(r[0])}
3291
- for i, c in enumerate(ranked, start=1):
3292
- row[c] = r[i]
3293
- rows.append(row)
3294
- resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols)
3295
- for n in python_leaves:
3296
- answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ())
3297
  sets = dict(cohort_sets or {})
3298
  counter = [0]
3299
  wanted = {id(n) for n in leaves}
@@ -3322,66 +3566,101 @@ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, coho
3322
  out.append(n)
3323
  return out
3324
 
3325
- return rewrite(tree), sets
 
 
 
3326
 
3327
 
3328
  def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
3329
- """One GLOBAL rank leaf as SQL: a `SqlMembership` selecting its ids, or None for the resolver.
3330
-
3331
- Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a
3332
- paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds,
3333
- and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line
3334
- for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values
3335
- are ABSENT, ties break by ascending id in both directions, a bound outside the builder's
3336
- whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than
3337
- buckets is empty. The domain's count and mean are read once here and inlined as literals.
3338
- """
3339
  fe = _fe()
3340
  Sql = _fs().SqlMembership
3341
  empty = Sql("SELECT NULL WHERE FALSE", [])
3342
- scope, _field = fe._rank_scope(leaf, None)
3343
- if scope != "global":
3344
- return None
3345
  col = cols.get(leaf.get("colId")) or {}
3346
  if col.get("type") not in fe.NUMERIC_TYPES:
3347
  return empty
3348
  v = f"TRY_CAST({col['sql']} AS DOUBLE)"
3349
- dom = (f"SELECT {id_sql} AS pid, {v} AS v FROM {frm_body} WHERE "
3350
- + (f"({where}) AND " if where else "") + f"{v} IS NOT NULL AND isfinite({v})")
 
 
3351
  params = list(params)
3352
  op = leaf.get("op")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3353
  if op in ("topN", "bottomN"):
3354
  bound = fe._rank_bound(leaf.get("value"), 1, 10000)
3355
  if bound is None:
3356
  return empty
3357
- direction = "DESC" if op == "topN" else "ASC"
3358
- return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {bound}",
3359
- params)
3360
- count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone()
3361
- count = int(count or 0)
3362
- if not count:
3363
- return empty
3364
- if op in ("inTopPct", "inBottomPct"):
3365
  pct = fe._rank_bound(leaf.get("value"), 1, 100)
3366
  if pct is None:
3367
  return empty
3368
- take = min(count, max(1, math.ceil(count * pct / 100)))
3369
- direction = "DESC" if op == "inTopPct" else "ASC"
3370
- return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}",
3371
- params)
3372
- if op in ("aboveAvg", "belowAvg"):
3373
- mean = float(total) / count
3374
- cmp = ">" if op == "aboveAvg" else "<"
3375
- return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params)
3376
- if op in ("inQuartile", "inDecile"):
3377
  buckets = 4 if op == "inQuartile" else 10
3378
  wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
3379
- if wanted is None or count < buckets:
3380
  return empty
3381
- return Sql(f"SELECT pid FROM (SELECT pid, row_number() OVER (ORDER BY v ASC, pid ASC) - 1"
3382
- f" AS idx FROM ({dom}) d) r WHERE least({buckets}, (idx * {buckets}) // {count}"
3383
- f" + 1) = {int(wanted)}", params)
3384
- return None
 
 
 
3385
 
3386
  def _may_see_view(session, vid, store):
3387
  """May this caller open that saved view? The same predicate the view rail asks.
@@ -3404,7 +3683,12 @@ def resolved_fields(table_key, defn):
3404
  fields = rel.namespaced_fields(mk_fields(), connection)
3405
  break
3406
  if fields is None:
3407
- if table_key not in _sources():
 
 
 
 
 
3408
  raise err(404, "not_connected", "that database is not a connected grid")
3409
  fields = (defn or {}).get("fields") or []
3410
  if not fields:
@@ -3509,7 +3793,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
3509
  if mismatch:
3510
  raise err(409, "cross_tenant_store", mismatch)
3511
 
3512
- spec = _source_for(cur, table_key)
3513
  if spec is None:
3514
  from core import perm_scope
3515
  rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
@@ -3771,36 +4055,96 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
3771
 
3772
 
3773
  @contextmanager
3774
- def _materialised_read(table_key, session, query, related, ids=None):
3775
  """One compiler for native rows and their aggregate sibling, over the walled payload.
3776
 
3777
  The temporary connection contains only this request's already stripped, pinned rows and is
3778
  closed before returning. It never registers data on the tenant's shared mirror connection.
3779
  """
3780
  import duckdb
 
 
3781
 
3782
- if table_key == "customer_data":
3783
- from routes_customers import _payload
 
 
 
 
 
 
 
 
3784
  session.require(table_key)
3785
- payload = _payload(session, related=related)
3786
- elif table_key == "product_data":
3787
- from routes_products import products
3788
- session.require(table_key)
3789
- payload = products(session=session, related=related)
 
 
 
 
 
 
3790
  else:
3791
  from routes_tables import table_rows
3792
  payload = table_rows(table_key, session=session, related=related)
3793
- if ids is not None:
3794
- wanted = set(ids)
3795
- payload = {**payload, "rows": [row for row in payload["rows"]
3796
- if int(row["pid"]) in wanted]}
3797
  fields = payload["fields"]
3798
- with duckdb.connect(":memory:") as cur:
3799
- cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
3800
- [json.dumps(payload["rows"], default=str)])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3801
  bindings = {}
3802
  for field in fields:
3803
  key = str(field["key"])
 
 
3804
  pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
3805
  expr = f"json_extract_string(payload, '{pointer}')"
3806
  numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
@@ -3833,7 +4177,10 @@ def _materialised_read(table_key, session, query, related, ids=None):
3833
  where = f"({where}) AND {search.sql}" if where else search.sql
3834
  params.extend(search.params)
3835
  yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
3836
- "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
 
 
 
3837
 
3838
 
3839
  def _group_pin(cols, group_by, group_key):
@@ -3901,7 +4248,30 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
3901
 
3902
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
3903
  "sorts": _json_arg(sorts, "sorts"), "search": search})
3904
- with _materialised_read(table_key, session, query, related, ids=ids) as read:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3905
  spec = read["spec"]
3906
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
3907
  where, params = read["where"], list(read["params"])
@@ -3909,19 +4279,73 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
3909
  if gsql:
3910
  where = f"({where}) AND {gsql}" if where else gsql
3911
  params.extend(gparams)
3912
- win = datastore.window(select=f'{spec["id"]} AS pid', where=where,
3913
  params=tuple(params), order_by=order or spec["id"],
3914
- offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
3915
  payload = read["payload"]
3916
- by_id = {int(row["pid"]): row for row in payload["rows"]}
3917
- rows = [by_id[int(row[0])] for row in win["rows"]]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3918
  if win["clamped"]:
3919
  read["limits"].append({"subject": "limit", "effect": "window_clamped",
3920
  "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
3921
  "recommendation": "Continue paging with offset. Every matching row remains available."})
 
 
3922
  return {**payload, "ok": True, "rows": rows, "total": win["total"],
3923
- "totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"],
3924
- "limits": read["limits"], "scope": {"table": table_key, "readThrough": True}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3925
 
3926
 
3927
  @router.get("/odoo-tables/{table_key}/rows")
@@ -3950,7 +4374,7 @@ def odoo_table_rows(table_key: str,
3950
  that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in
3951
  `limits` with its cause and a recommendation — R6's second sentence — and never silently
3952
  dropped, because an ignored condition widens.
3953
-
3954
  ⭐⭐ `related` (W51-T05 / C6) IS A **JSON QUERY STRING**, copying the `filters` precedent one
3955
  argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
3956
  window to the rows linked to that ONE record, ahead of `filters`, and `total` /
@@ -3976,13 +4400,10 @@ def odoo_table_rows(table_key: str,
3976
  "limit, filters, sorts and search in the query string.")
3977
  ids = _record_ids(body)
3978
 
3979
- # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
3980
- # asks it instead of reading its own definitions projection (one store round trip fewer on
3981
- # every Amazon page). Not reordered past it: the check may register this grid's binding,
3982
- # which is what `_materialised_table` below then reads.
3983
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
3984
- require_amazon_mirror(table_key, session, st=page[0] if page else None)
3985
- if _materialised_table(table_key):
3986
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
3987
  sorts, search, related, groupBy, groupKey, ids=ids)
3988
 
@@ -4016,7 +4437,8 @@ def odoo_table_rows(table_key: str,
4016
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
4017
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
4018
  where=where, params=tuple(params), order_by=order,
4019
- offset=offset, limit=(limit or None), cur=cur, page_first=True, **frm)
 
4020
  if win["clamped"]:
4021
  limits.append({
4022
  "subject": "limit", "effect": "window_clamped",
@@ -4058,12 +4480,15 @@ def odoo_table_rows(table_key: str,
4058
  if not _may_open(session, target, st=page_st):
4059
  return []
4060
  return _relation_rows(session, target, list(matches), matches=matches,
4061
- st=page_st, rev=page_rev)
 
 
4062
 
4063
  # Fields are the current reader's contract; target rows cross the same wall
4064
  # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
4065
- amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets)
4066
- overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
 
4067
 
4068
  # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
4069
  #
@@ -4078,20 +4503,16 @@ def odoo_table_rows(table_key: str,
4078
  # the wall and the predicate, so the argument it demands is the list we just fetched.
4079
  # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
4080
  # answered "may this session open this surface".
4081
- # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request
4082
- # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
4083
- # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
4084
  shared_defs = _so().fields(table_key, st=page_st)
4085
  if shared_defs:
4086
  declared = {field["key"]: field for field in fields}
4087
  declared.update({key: dict(field, source="overlay") for key, field in shared_defs.items()})
4088
  fields = list(declared.values())
4089
- for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
4090
- st=page_st).items():
4091
  overlays.setdefault(pid, {}).update(cells)
4092
  if_keys = {field["key"] for field in fields if field.get("type") == "if"}
4093
- for row in rows_src:
4094
- overlays[str(row["pid"])].update({key: row.get(key, "") for key in if_keys})
4095
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
4096
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
4097
  # in an overlay column that reads a hidden mirror column was outside its reach - and a formula
@@ -4108,11 +4529,16 @@ def odoo_table_rows(table_key: str,
4108
  rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
4109
  overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
4110
  for pid, cells in overlays.items()}
4111
- rows = aios_grid.rows_from_pool(rows_src, fields, overlays)
 
 
 
 
 
 
4112
 
4113
- unfiltered = win["total"] if where == base_where else datastore.window(
4114
- select="1", where=base_where, params=tuple(wall_params), order_by=spec["id"],
4115
- limit=1, cur=cur, **frm)["total"]
4116
 
4117
  from core import connector_spec
4118
  rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
@@ -4135,7 +4561,7 @@ def odoo_table_rows(table_key: str,
4135
  # ══════════════════════════════════════════════════════════
4136
 
4137
  #: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers
4138
- #: that omit groupLimit receive the complete grouping; card modes page columns explicitly.
4139
  MAX_GROUPS = 1_000
4140
 
4141
  #: One request asks for at most this many summaries. Each is one more aggregate expression in a
@@ -4143,10 +4569,52 @@ MAX_GROUPS = 1_000
4143
  MAX_AGGREGATES = 200
4144
 
4145
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4146
  def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
4147
- """Resolve each page's Odoo link cells through the scoped relationship reader."""
4148
  from routes_grid import _may_open
4149
  declared = {field["key"] for field in fields}
 
4150
  for field in fields:
4151
  bag = field.get("link") or {}
4152
  if field.get("type") != "link" or not bag.get("on") or not bag.get("from"):
@@ -4154,19 +4622,36 @@ def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
4154
  target, on, source = bag["table"], bag["on"], bag["from"]
4155
  if _rel().base_table_key(target) not in _rel().READ_THROUGH_KEYS:
4156
  continue
4157
- matches = list({row.get(source) for row in rows if row.get(source) not in (None, "")})
4158
- targets = []
4159
- if source in declared and matches and _may_open(session, target, st=st):
4160
- targets = _relation_rows(session, target, [on], match=(on, matches), st=st, rev=rev,
4161
- normalize_match=True)
4162
- index = {}
4163
- for row in targets:
4164
- index.setdefault(str(row.get(on) or "").strip().lower(), []).append(str(row["pid"]))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4165
  for row in rows:
4166
- ids = sorted(set(index.get(str(row.get(source) or "").strip().lower(), [])), key=int)
4167
- if bag.get("single"):
4168
- ids = ids[:1]
4169
- row[field["key"]] = ",".join(ids[:_ut().LINK_MAX_IDS])
 
 
4170
 
4171
 
4172
  def _agg_number(value):
@@ -4358,15 +4843,10 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
4358
  f"THEN 1 ELSE 0 END"),
4359
  where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
4360
  projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
4361
- if group_limit is not None:
4362
- rows = datastore.window(
4363
- select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
4364
- group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"]
4365
- else:
4366
- source = frm.get("from_sql") or frm["table"]
4367
- tail = f" WHERE {where}" if where else ""
4368
- rows = cur.execute(f"SELECT {projection} FROM {source}{tail} GROUP BY {gexpr} "
4369
- "ORDER BY 2 DESC, 1 ASC", tuple(params)).fetchall()
4370
  groups = []
4371
  for r in rows:
4372
  vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
@@ -4375,7 +4855,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
4375
  "count": int(r[1] or 0), "aggregates": vals})
4376
  return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
4377
  "total_groups": distinct, "group_offset": group_offset,
4378
- "group_limit": group_limit, "limits": limits}
 
4379
 
4380
 
4381
  @router.post("/odoo-tables/{table_key}/aggregates")
@@ -4424,13 +4905,11 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
4424
  raise err(400, "bad_selection", "groupOffset must be a nonnegative integer")
4425
  if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
4426
  raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
4427
- if group_offset and group_limit is None:
4428
- raise err(400, "bad_selection", "Send groupLimit when paging groups")
4429
 
4430
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4431
- require_amazon_mirror(table_key, session, st=page[0] if page else None)
4432
 
4433
- if _materialised_table(table_key):
4434
  with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
4435
  import odoo_default_scope
4436
  if (odoo_default_scope.applies(table_key)
@@ -4481,7 +4960,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
4481
  "limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
4482
  if out["groups"] is not None:
4483
  body_out["groups"] = out["groups"]
4484
- body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit")})
4485
  return body_out
4486
 
4487
 
@@ -4611,8 +5090,7 @@ def population(table_key, cur=None, rt=None):
4611
  if spec is None:
4612
  return None
4613
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
4614
- return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"],
4615
- limit=1, cur=cur, **frm)["total"]
4616
 
4617
 
4618
  def whole_pool(table_key, cur=None, rt=None):
 
289
  from harness import datastore as _ds
290
  frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
291
  else {"table": spec["table"]})
292
+ n = _ds.count_rows(where=spec.get("where") or "", cur=cur, **frm)
 
293
  else:
294
  n = -1 # unknown, and never reported as zero
295
  mirror_refreshed = mirror_stamp(
 
651
  rows = cur.execute(
652
  "SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
653
  "FROM _sync_state ORDER BY entity").fetchall()
654
+ except Exception as exc: # noqa: BLE001
655
+ from core.request_timing import raise_if_resource_error
656
+ raise_if_resource_error(exc)
657
  return None
658
  out = {}
659
  for entity, phase, last_id, cursor_wd, nrows, updated_at in rows:
 
778
  _SOURCE_PROVIDERS = []
779
  _SOURCE_ANALYSES = {}
780
 
781
+ # Provider SQL runs outside the publication lock. Invalidation rejects an older build.
 
 
 
 
 
 
 
 
 
782
  _SOURCES_LOCK = threading.RLock()
783
  _SOURCES_BUILT = False
784
+ _SOURCES_BUILDER = None # compatibility for callers that reset legacy build state
785
+ _SOURCES_BUILDING = threading.local()
786
+ _SOURCES_EPOCH = 0
787
 
788
 
789
  def invalidate_sources():
790
  """Forget measured bindings after a connector publishes a new mirror schema."""
791
+ global _SOURCES_BUILT, _SOURCES_EPOCH
792
  with _SOURCES_LOCK:
793
+ _SOURCES_EPOCH += 1
794
  _SOURCES_BUILT = False
795
  GRID_SOURCES.clear()
796
 
 
842
  importing that module at file-import time would drag the Odoo layer into every process that
843
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
844
 
845
+ Providers build a local dictionary outside the publication lock.
 
 
846
  """
847
+ global _SOURCES_BUILT
848
  if _SOURCES_BUILT and GRID_SOURCES:
849
  return GRID_SOURCES
850
  with _SOURCES_LOCK:
851
+ if (_SOURCES_BUILT and GRID_SOURCES) or getattr(_SOURCES_BUILDING, "active", False):
852
  return GRID_SOURCES
853
+ epoch = _SOURCES_EPOCH
854
+ from harness import datastore
855
+ generation = datastore.mirror_generation()
856
+ _SOURCES_BUILDING.active = True
857
+ try:
858
+ built = {}
859
+ for build in list(_SOURCE_PROVIDERS):
860
+ try:
861
+ extra = build() or {}
862
+ except Exception: # noqa: BLE001
863
+ continue
864
+ built.update(extra)
865
+ with _SOURCES_LOCK:
866
+ if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
867
+ return GRID_SOURCES
 
 
868
  GRID_SOURCES.update({**built, **GRID_SOURCES})
869
  _SOURCES_BUILT = True
870
+ if built:
871
+ _ut().register_connected(*built)
872
+ finally:
873
+ _SOURCES_BUILDING.active = False
874
  return GRID_SOURCES
875
 
876
 
877
+ def _learn_amazon_sources(cur=None, *, tenant=None):
878
+ """Refresh discovery for non-page callers without retaining negative results."""
879
+ with _SOURCES_LOCK:
880
+ epoch = _SOURCES_EPOCH
881
+ from harness import datastore
882
+ generation = datastore.mirror_generation()
883
  if cur is None:
884
  extra = _amazon_sources()
885
  else:
886
  import amazon_relational
887
  extra = amazon_relational.grid_sources(cur)
 
 
888
  with _SOURCES_LOCK:
889
+ if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
890
+ return
891
  GRID_SOURCES.update(extra)
892
+ if extra:
893
  _ut().register_connected(*extra)
894
 
895
 
 
1292
 
1293
 
1294
  def require_amazon_mirror(table_key, session, definition=None, st=None):
1295
+ """Admit one Amazon grid and return its binding, or None for a stored snapshot."""
 
 
 
 
 
 
1296
  if not str(table_key).startswith("ut_amzn_"):
1297
  return
1298
  import amazon_relational as amazon
 
1310
  if mismatch:
1311
  raise err(409, "cross_tenant_store", mismatch)
1312
  try:
1313
+ return _amazon_page_source(cur, table_key, definition, allow_lookups=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1314
  finally:
1315
  cur.close()
1316
 
 
1482
  try:
1483
  frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql")
1484
  else {"table": spec["table"]})
1485
+ total = datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm)
 
1486
  except Exception: # noqa: BLE001
1487
  total = None
1488
  if total is not None and total > max_rows:
 
1526
  return expr if bare.lower() in have else "NULL"
1527
 
1528
 
1529
+ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_lookups=True):
1530
+ """Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
1531
+ import amazon_relational as amazon
1532
+
1533
+ try:
1534
+ if binding:
1535
+ columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
1536
+ spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
1537
+ else:
1538
+ spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
1539
+ except Exception as exc: # noqa: BLE001
1540
+ from core.request_timing import raise_if_resource_error
1541
+ raise_if_resource_error(exc)
1542
+ raise err(503, "amazon_source_unavailable",
1543
+ "This Amazon database could not be resolved. Retry the request.") from exc
1544
+ if spec is not None and spec.get("missing"):
1545
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1546
+ return spec
1547
+
1548
+
1549
+ def _source_for(cur, table_key, *, allow_lookups=True):
1550
  """The resolved spec for one connected grid, or None when this grid is not read-through yet.
1551
 
1552
  ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
 
1573
  # table and turns Products ordered into the wrong 404.
1574
  if module_cache_source(key) is not None:
1575
  return None
1576
+ if key.startswith("ut_amzn_"):
1577
+ import amazon_relational as amazon
1578
+ if key in amazon.SOURCE:
1579
+ return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
1580
  spec = _sources().get(key)
 
 
 
 
 
 
 
 
1581
  if not spec:
1582
  return None
1583
  if spec.get("missing"):
1584
  raise err(503, "amazon_mirror_not_loaded", spec["missing"])
 
 
 
 
 
 
 
 
 
1585
  from harness import datastore
1586
  if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
1587
  try:
 
1634
  for key, (expr, cast) in spec["cols"].items():
1635
  cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
1636
  have_by_alias), cast)
1637
+ if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
1638
+ spec = {**spec, "page_key": spec["id"]}
1639
  return {**spec, "cols": cols}
1640
 
1641
 
1642
+ def _page_source_for(cur, table_key, st=None, *, session=None):
1643
+ """Plan admitted lookups afresh because an autocommit cursor can change snapshots."""
1644
+ if str(table_key).startswith("ut_amzn_") and session is not None:
1645
+ import amazon_relational as amazon
1646
+ from routes_grid import _may_open
1647
+ allowed = _may_open(session, amazon.LISTINGS, st=st)
1648
+ return _source_for(cur, table_key, allow_lookups=allowed)
1649
+ return _source_for(cur, table_key)
1650
+
1651
+
1652
  #: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
1653
  #: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of
1654
  #: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of
 
1862
  if table_key in trail:
1863
  raise ValueError("a relation has a circular access rule")
1864
  trail = (*trail, table_key)
1865
+ handle = (st if isinstance(st, _ut()._Lent) and getattr(st, "schema_only", False)
1866
+ else _ut().lend_defs(st, schema_only=True))
1867
  declared = {f["key"]: f for f in fields if isinstance(f, dict) and f.get("key")}
1868
  columns = _filter_columns(spec, fields)
1869
  guards, resolved, targets = [], {}, {}
 
1884
  targets[key] = None
1885
  return None
1886
  definition = _ut().get(key, st=handle)
1887
+ target_spec = _page_source_for(cur, key, handle, session=session)
1888
  if target_spec is None and str(key).startswith("ut_amzn_"):
1889
  # Small Amazon targets (notably listings) are stored snapshots, but
1890
  # their raw mirror still answers the same automatic link in SQL.
1891
  import amazon_relational
1892
  if key in amazon_relational.SOURCE:
1893
+ target_spec = amazon_relational.grid_source(cur, key,
1894
+ allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle))
1895
  if not definition or target_spec is None or target_spec.get("missing"):
1896
  refuse(field)
1897
  target_fields = definition.get("fields") or []
 
2365
  if not source:
2366
  return None
2367
  try:
2368
+ cursor = getattr(session.runtime, '_canonical_source_cursor', None)
2369
+ if cursor is not None and source in {'customer_data', 'product_data'}:
2370
+ import registry_source
2371
+ rows = registry_source.canonical_page(session, cursor, source, [record])['rows']
2372
+ else:
2373
+ rows = perm_scope.scoped_table(session.user, source, st=session.runtime) or []
2374
  except Exception: # noqa: BLE001
2375
  return None
2376
  want = str(record)
 
2381
  return None
2382
 
2383
 
2384
+ def _materialised_table(table_key, *, cur=None, st=None):
2385
+ """Dispatch one target against its cursor without treating registry misses as evidence."""
 
 
 
 
 
2386
  if module_cache_source(table_key) is not None:
2387
  return True
 
 
2388
  if str(table_key).startswith("ut_amzn_"):
2389
+ from harness import datastore
2390
+ import amazon_relational as amazon
2391
+ if table_key not in amazon.SOURCE:
2392
+ return True
2393
+ own_cursor = cur is None
2394
+ cur = datastore.ro_cursor() if own_cursor else cur
2395
+ try:
2396
+ definition = _ut().get(table_key, st=st) if st is not None else None
2397
+ spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
2398
+ return spec is None
2399
+ finally:
2400
+ if own_cursor:
2401
+ cur.close()
2402
+ return table_key not in _sources()
2403
 
2404
 
2405
  #: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
 
2424
 
2425
 
2426
  def _user_tables_token(rt):
2427
+ """Return the durable revision and timestamp, or decline a revisionless reader."""
2428
+ reader = getattr(rt, "revision", None)
2429
+ if not callable(reader):
 
2430
  return None
2431
+ try:
2432
+ revision = reader(_ut().STORE_KEY)
2433
+ return (revision["token"], revision.get("updated_at")) if revision else None
2434
+ except Exception as exc: # noqa: BLE001
2435
+ from core.request_timing import raise_if_resource_error
2436
+ raise_if_resource_error(exc)
2437
+ raise err(503, "schema_unavailable", "table definition revisions are unavailable") from exc
2438
 
2439
 
2440
  def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
 
2468
 
2469
 
2470
  def _page_defs(session):
2471
+ """Return one schema-only `(snapshot, revision)` for a page and its scoped targets.
2472
+
2473
+ Shared metadata and source plans belong to the fresh lend, never to the revision memo.
2474
+ Projection and revision failures refuse instead of falling back to a whole document.
 
 
 
 
 
 
 
2475
  """
2476
  import copy
2477
  import core.store as store
 
2481
  token = _user_tables_token(rt)
2482
 
2483
  def keep(v):
2484
+ # Activity is a generated descriptor, not a stored row-bearing definition.
2485
+ tables = [d for key, d in (v or {}).items() if key != "ut_activity" and isinstance(d, dict)]
2486
  return bool(tables) and all(store.is_projected(d) for d in tables)
2487
 
2488
+ try:
2489
+ defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt, schema_only=True),
2490
+ keep=keep, clone=copy.deepcopy, token=token)
2491
+ except Exception as exc: # noqa: BLE001
2492
+ from core.request_timing import raise_if_resource_error
2493
+ raise_if_resource_error(exc)
2494
+ raise err(503, "schema_unavailable", "table definitions are unavailable") from exc
2495
+ lent = ut.lend(rt, **{ut.STORE_KEY: defs})
2496
+ lent.schema_only = True
2497
+ return lent, token
2498
 
2499
 
2500
+ def _stored_link_rows(session, table, token=_UNREAD, *, materialised=None):
2501
  """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
2502
 
2503
  ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
 
2511
 
2512
  rt = session.runtime
2513
  if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
2514
+ or module_cache_source(table) is not None
2515
+ or not (materialised if materialised is not None else _materialised_table(table))):
2516
  return perm_scope._read(table, session.user, rt)
2517
  return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
2518
  keep=lambda v: bool(v[1]),
 
2535
  return defn
2536
 
2537
 
2538
+ class _RelationRows:
2539
+ def __init__(self, read):
2540
+ self.read = read
2541
+
2542
+ def __iter__(self):
2543
+ return self.read()
2544
+
2545
+
2546
  def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
2547
+ traversal_of=None, st=None, rev=..., normalize_match=False,
2548
+ compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD`
2549
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
2550
 
2551
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
 
2563
  import core.relations as relations
2564
 
2565
  columns = set(columns) | {relations.IDENTITY_COL}
2566
+ if traversal_of is None and not perm_scope.may_read(
2567
+ session.user, table, st=st if st is not None else session.runtime):
2568
+ raise err(403, "forbidden_table", "This account cannot open the linked database.")
2569
+ if not (materialised if materialised is not None else _materialised_table(table)):
2570
+ read = compiled if compiled is not None else (
2571
+ _compiled_read(table, session, traversal_intermediate=True, st=st)
2572
+ if traversal_of is not None else _compiled_read(table, session, st=st))
2573
  spec = read["spec"]
2574
  bindings = dict(spec["cols"], pid=(spec["id"], int))
2575
  if columns - set(bindings):
 
2580
  if values is None:
2581
  continue
2582
  expr, cast = bindings[key]
 
2583
  if normalize_match and key != "pid":
2584
  expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
2585
  cast = lambda value: str(value).strip().lower()
2586
+ if isinstance(values, _fs().SqlMembership):
2587
+ operand = _membership_relation(read['cur'], _membership_values(values),
2588
+ normalize=cast, sql_type='HUGEINT' if cast is int else 'DOUBLE' if cast is float else 'VARCHAR',
2589
+ skip_invalid=False)
2590
+ parts.append(f'{expr} IN ({operand.sql})')
2591
+ params.extend(operand.params)
2592
+ continue
2593
+ values = list(values)
2594
  parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
2595
  params.extend(cast(value) for value in values)
2596
  if matches is not None:
 
2598
  for key, values in matches.items():
2599
  expr, cast = bindings[key]
2600
  values = sorted(values)
2601
+ if normalize_match:
2602
+ expr = f"LOWER(TRIM(CAST({expr} AS VARCHAR)))"
2603
+ cast = lambda value: str(value).strip().lower()
2604
  if values:
2605
  alternatives.append(f"{expr} IN ({','.join('?' for _ in values)})")
2606
  params.extend(cast(value) for value in values)
 
2609
  projection = ", ".join(bindings[key][0] for key in names)
2610
  source = spec.get("from_sql") or spec["table"]
2611
  predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
2612
+ if spec.get("page_key") and not spec.get("from_sql"):
2613
+ key = spec["page_key"]
2614
+ sql = (f'WITH _relation_keys AS MATERIALIZED (SELECT {key} AS "_target_key" '
2615
+ f'FROM {source} WHERE {predicate}) SELECT {projection} FROM {source} '
2616
+ f'SEMI JOIN _relation_keys ON {key} = _relation_keys."_target_key"')
2617
+ else:
2618
+ sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
2619
+ def batches():
2620
+ try:
2621
+ cursor = read["cur"].execute(sql, params)
2622
+ while True:
2623
+ batch = cursor.fetchmany(2048)
2624
+ if not batch:
2625
+ return
2626
+ for row in batch:
2627
+ yield dict(zip(names, row))
2628
+ batch.clear()
2629
+ finally:
2630
+ if stream and compiled is None:
2631
+ read["cur"].close()
2632
+ return _RelationRows(batches) if stream else list(batches())
2633
 
2634
  # This is an internal join, never a cell response. Apply the same permanent wall on the
2635
  # complete contract before taking any join value, including a join on a hidden column.
 
 
 
2636
  if module_cache_source(table) is not None:
2637
  session.require(table)
2638
  # The revision memo serves a caller that handed this request's token (the page's link fills,
 
2643
  fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
2644
  if definition is not None else perm_scope._read(table, session.user, session.runtime))
2645
  else:
2646
+ fields, rows = _stored_link_rows(session, table, token=rev, materialised=True)
2647
  if ids is not None:
2648
  wanted = {str(value) for value in ids}
2649
  rows = [row for row in rows if str(row.get("pid")) in wanted]
 
2661
  wanted = {normalize(value) for value in values}
2662
  rows = [row for row in rows if normalize(row.get(key)) in wanted]
2663
  if matches is not None:
2664
+ normalize = (lambda value: str(value).strip().lower()) if normalize_match else str
2665
+ wanted = {key: {normalize(value) for value in values} for key, values in matches.items()}
2666
+ rows = [row for row in rows if any(normalize(row.get(key)) in values
2667
  for key, values in wanted.items())]
2668
  return [{key: row.get(key) for key in columns} for row in rows]
2669
 
 
2674
  return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
2675
 
2676
 
2677
+ def _relation_leg_membership(session, source, source_ids, entry, cur, *, source_col,
2678
+ target_col, manual, inverse, reverse):
2679
+ """Follow a scoped leg with SQL operands at every population-sized boundary."""
2680
+ import core.relations as relations
2681
+ target, link, via = str(entry['target']), entry.get('link') or {}, entry.get('via') or {}
2682
+ observed = {'rows': 0, 'complete': True, 'available': False}
2683
+ def source_values():
2684
+ for row in _relation_rows(session, source, [source_col], ids=source_ids, stream=True):
2685
+ observed['rows'] += 1
2686
+ observed['complete'] &= source_col in row
2687
+ if row.get(source_col) not in (None, ''):
2688
+ observed['available'] = True
2689
+ if (manual and not reverse) or (inverse and reverse):
2690
+ yield from _relation_cell_ids(row[source_col])
2691
+ else:
2692
+ yield str(row[source_col])
2693
+ values = _membership_relation(cur, source_values(), normalize=str, sql_type='VARCHAR')
2694
+ if source_col != relations.IDENTITY_COL and not observed['available']:
2695
+ if (source.startswith('ut_amzn_') and target.startswith('ut_amzn_')
2696
+ and observed['rows'] and observed['complete']):
2697
+ return values
2698
+ raise err(400, 'unknown_relation', relations.unresolvable_source_refusal(source, entry))
2699
+ if (manual and not reverse) or (inverse and reverse):
2700
+ target_col = 'pid'
2701
+ through = relations.declared_through(link) if not manual else None
2702
+ if through:
2703
+ left = str(link['throughTo'] if reverse else link['throughOn'])
2704
+ right = str(link['throughOn'] if reverse else link['throughTo'])
2705
+ middle = _relation_rows(session, str(through), [left, right], match=(left, values),
2706
+ traversal_of=str(entry.get('key') or via.get('key') or ''), stream=True)
2707
+ values = _membership_relation(cur,
2708
+ (str(row[right]) for row in middle if row.get(right) not in (None, '')),
2709
+ normalize=str, sql_type='VARCHAR')
2710
+ if (manual and reverse) or (inverse and not reverse):
2711
+ rows = _relation_rows(session, target, [target_col], stream=True)
2712
+ return _relation_cell_membership(cur, rows, target_col, values)
2713
+ rows = _relation_rows(session, target, [target_col], match=(target_col, values),
2714
+ where=entry.get('where'), stream=True)
2715
+ return _membership_relation(cur, (row['pid'] for row in rows))
2716
+
2717
+
2718
+ def _relation_leg_ids(session, source, source_ids, entry, membership_cur=None):
2719
  """Follow one declared leg, including manual, inverse and through links."""
2720
  import core.relations as relations
2721
 
 
2733
  source_col, target_col = ("pid", field) if reverse else (field, "pid")
2734
  elif inverse:
2735
  source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
2736
+ if membership_cur is not None:
2737
+ return _relation_leg_membership(session, source, source_ids, entry, membership_cur,
2738
+ source_col=source_col, target_col=target_col, manual=manual, inverse=inverse, reverse=reverse)
2739
  source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
2740
  values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
2741
  # A column-keyed edge cannot fall back to the wire record id. In particular,
 
2760
  traversal_of=str(entry.get("key") or via.get("key") or ""))
2761
  values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
2762
  if (manual and reverse) or (inverse and not reverse):
2763
+ rows = _relation_rows(session, target, [target_col], stream=True)
2764
+ rows = (row for row in rows if _relation_cell_ids(row.get(target_col)) & values)
2765
  else:
2766
  rows = _relation_rows(session, target, [target_col], match=(target_col, values),
2767
+ where=entry.get("where"), stream=True)
2768
  return {int(row["pid"]) for row in rows}
2769
 
2770
 
 
2777
  if table == key and label), str(key))
2778
 
2779
 
2780
+ def _relation_target_ids(session, target, related, membership_cur=None):
2781
  """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
2782
  import core.relations as relations
2783
  from routes_grid import _may_open, related_path_entries
 
2812
  if not _may_open(session, leg["target"]):
2813
  raise err(403, "forbidden_table",
2814
  f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
2815
+ empty_ids = (not ids.params if membership_cur is not None
2816
+ and isinstance(ids, _fs().SqlMembership) else not ids)
2817
+ if empty_ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
2818
  # An earlier, authorised Amazon hop matched nothing. There is no next
2819
  # source record to read, and that is an empty relation, not a missing key.
2820
  continue
2821
+ ids = _relation_leg_ids(session, table, ids, leg, membership_cur)
2822
  return ids
2823
 
2824
 
 
3054
  related = _json_arg(related, "related")
3055
  if related in (None, "", {}):
3056
  return None, [], []
3057
+ ids = _relation_target_ids(session, table_key, related, membership_cur=cur)
3058
+ if isinstance(ids, _fs().SqlMembership):
3059
+ return f'COALESCE({spec["id"]} IN ({ids.sql}), FALSE)', list(ids.params), []
3060
  sql, params = relations.pin_sql(spec["id"], sorted(ids))
3061
  notes = []
3062
  if len(ids) > _PIN_REPORT_AT:
 
3115
  return out
3116
 
3117
 
3118
+ class _RelationLease(int):
3119
+ """A bound scalar keeps its cursor-local relations alive through copied query params.
3120
+
3121
+ Every leaf gets its own lease. Releasing one query cannot invalidate another leaf or query
3122
+ on a retained cursor. No process/session map holds leases after their parameters disappear.
3123
+ """
3124
+ def __new__(cls, cur):
3125
+ value = super().__new__(cls, 1)
3126
+ value.cur, value.names = cur, []
3127
+ return value
3128
+
3129
+ def table(self, suffix=""):
3130
+ import uuid
3131
+ name = '_request_members_' + uuid.uuid4().hex + suffix
3132
+ self.names.append(name)
3133
+ return name
3134
+
3135
+ def __reduce_ex__(self, protocol):
3136
+ raise TypeError('Request-owned membership cannot be persisted or cached')
3137
+
3138
+ def close(self):
3139
+ # Cleanup remains possible after the request's query deadline has fired.
3140
+ cur = getattr(self.cur, '_cursor', self.cur)
3141
+ while self.names:
3142
+ name = self.names[-1]
3143
+ cur.execute(f'DROP TABLE IF EXISTS "{name}"')
3144
+ self.names.pop()
3145
+
3146
+ def __del__(self):
3147
+ try:
3148
+ self.close()
3149
+ except Exception:
3150
+ pass # A closed cursor has already destroyed its TEMP relations.
3151
+
3152
+
3153
+ def _relation_cell_membership(cur, rows, column, values):
3154
+ """Match complete manual/inverse cells in SQL without retaining the target population."""
3155
+ import csv
3156
+ import tempfile
3157
+ from pathlib import Path
3158
+ from core.request_timing import remaining_budget
3159
+ lease = _RelationLease(cur)
3160
+ name = lease.table('_cells')
3161
+ count = 0
3162
+ with tempfile.TemporaryDirectory(prefix='aios-related-') as directory:
3163
+ path = Path(directory) / 'cells.csv'
3164
+ with path.open('w', encoding='utf-8', newline='') as output:
3165
+ writer = csv.writer(output)
3166
+ for index, row in enumerate(rows):
3167
+ if index % 2048 == 0:
3168
+ remaining_budget()
3169
+ for token in _relation_cell_ids(row.get(column)):
3170
+ writer.writerow([int(row['pid']), token])
3171
+ count += 1
3172
+ if not count:
3173
+ return _fs().SqlMembership('SELECT NULL WHERE FALSE', [])
3174
+ cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT * FROM read_csv(?, header=false, '
3175
+ "columns={'pid':'BIGINT','token':'VARCHAR'})", [str(path)])
3176
+ return _fs().SqlMembership(f'SELECT pid FROM "{name}" WHERE token IN ({values.sql}) AND ?=1',
3177
+ [*values.params, lease])
3178
+
3179
+
3180
+ def _membership_values(membership):
3181
+ """Replay SQL membership in bounded batches on its owning cursor."""
3182
+ if not membership.params:
3183
+ return
3184
+ cur = membership.params[-1].cur
3185
+ cur.execute(membership.sql, membership.params)
3186
+ while batch := cur.fetchmany(2048):
3187
+ for row in batch:
3188
+ yield row[0]
3189
+ batch.clear()
3190
+
3191
+
3192
+ def _membership_relation(cur, values, *, normalize=int, sql_type='HUGEINT', skip_invalid=True):
3193
+ """Normalize a complete operand with bounded Python memory, then own its SQL lifetime."""
3194
+ import tempfile
3195
+ lease = _RelationLease(cur)
3196
+ name = lease.table()
3197
+ count = 0
3198
+ # Spooling also permits the producer to read this same cursor without an INSERT replacing
3199
+ # its active result. No population-sized list or Python set is constructed.
3200
+ import csv
3201
+ from pathlib import Path
3202
+ from core.request_timing import remaining_budget
3203
+ with tempfile.TemporaryDirectory(prefix='aios-members-') as directory:
3204
+ path = Path(directory) / 'members.csv'
3205
+ source = iter(values)
3206
+ try:
3207
+ with path.open('w', encoding='utf-8', newline='') as output:
3208
+ writer = csv.writer(output)
3209
+ for index, raw in enumerate(source):
3210
+ if index % 2048 == 0:
3211
+ remaining_budget()
3212
+ try:
3213
+ value = normalize(raw)
3214
+ except (TypeError, ValueError):
3215
+ if not skip_invalid or normalize is not int:
3216
+ raise
3217
+ continue
3218
+ writer.writerow([str(value)])
3219
+ count += 1
3220
+ finally:
3221
+ close = getattr(source, 'close', None)
3222
+ if close:
3223
+ close()
3224
+ if count:
3225
+ cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT pid FROM read_csv(?, '
3226
+ "header=false, columns={'pid':'VARCHAR'}, force_not_null=['pid'])", [str(path)])
3227
+ return _fs().SqlMembership(f'SELECT TRY_CAST(pid AS {sql_type}) FROM "{name}" WHERE ?=1',
3228
+ [lease]) if count else _fs().SqlMembership('SELECT NULL WHERE FALSE', [])
3229
+
3230
+
3231
  def _view_store(session, table_key):
3232
  """The workspace document a saved view of THIS database lives in.
3233
 
 
3236
  to fail it: the client reads the views `/workspace` served out of this same document.
3237
  """
3238
  import core.table_store as table_store
3239
+ from routes_customers import _AssemblyStore
3240
+ runtime = session.runtime
3241
+ from registry_source import backend as document_backend
3242
+ backend = document_backend(runtime)
3243
+ project = getattr(backend, 'get_view_workspace', None) or getattr(backend, '_get_view_workspace', None)
3244
+ stream_values = getattr(backend, 'iter_json_values', None) or getattr(backend, '_iter_json_values', None)
3245
+ class ViewStore(_AssemblyStore):
3246
+ def get(self, name, fresh=False):
3247
+ if callable(project) and name == f'{table_key}_table_workspace':
3248
+ key = getattr(runtime, 'store_key', str)(name)
3249
+ return self._read(name, 'view_operands', lambda: project(key), fresh=fresh)
3250
+ return super().get(name, fresh=fresh)
3251
+
3252
+ def _read_strict(self, name):
3253
+ return self.get(name)
3254
+ result = table_store.make(f"{table_key}_table_workspace", st=ViewStore(runtime, schema_only=True))
3255
+ if callable(project):
3256
+ def members(owner, vid):
3257
+ name = f'{table_key}_table_workspace'
3258
+ key = getattr(runtime, 'store_key', str)(name)
3259
+ path = [owner, 'views', vid, 'config', 'memberPids']
3260
+ document = result.st.get(name)
3261
+ stable = getattr(backend, 'iter_view_members', None) or getattr(backend, '_iter_view_members', None)
3262
+ parts = (stable(key, path, getattr(document, 'source_revision', None), batch_size=2048)
3263
+ if callable(stable) else stream_values(key, path, batch_size=2048))
3264
+ from contextlib import closing
3265
+ try:
3266
+ with closing(parts):
3267
+ for batch in parts:
3268
+ for _, value in batch:
3269
+ yield value
3270
+ except Exception as exc:
3271
+ from core.request_timing import raise_if_resource_error
3272
+ raise_if_resource_error(exc)
3273
+ raise err(503, 'schema_unavailable', 'Saved view membership is unavailable. Try again.') from exc
3274
+ result.member_values = members
3275
+ return result
3276
 
3277
 
3278
  def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits):
 
3296
  * no filters and a curated `memberPids` is the client's `else if (curated)` branch: the
3297
  membership IS the view;
3298
  * no filters and no members means the view holds every row in scope;
3299
+ * membership stays a `SELECT DISTINCT` subquery under the wall, without a Python ID list.
 
3300
 
3301
  THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY:
3302
  resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets`
 
3325
 
3326
  store = _view_store(session, table_key)
3327
  sets, memo, unresolved = {}, {}, []
3328
+ cyclic = set()
3329
  counter = [0]
3330
 
3331
  def key_for():
 
3336
  return "__view_%d__" % counter[0]
3337
 
3338
  def resolve(vid, seen):
3339
+ """Return SQL membership, authored IDs, all rows, or an unresolved reason."""
3340
  if vid in memo:
3341
  return memo[vid]
3342
  if vid in seen:
3343
+ cyclic.update(seen[seen.index(vid):])
3344
  # `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left
3345
  # ABSENT, so a stored loop degrades to an explained empty table, not a hung request.
3346
  return None, "it refers to itself through another saved view"
 
3360
  "own records rather than from this database's live connection")
3361
  memo[vid] = got
3362
  return got
3363
+ source = (store.member_values(owner, vid) if 'memberPids' not in cfg
3364
+ and callable(getattr(store, 'member_values', None))
3365
+ else (cfg.get('memberPids') or ()))
3366
+ has_members = False
3367
+ def authored_values():
3368
+ nonlocal has_members
3369
+ values = iter(source)
3370
  try:
3371
+ for raw in values:
3372
+ # A nonempty authored operand stays curated even if normalization
3373
+ # rejects every value. An actually empty operand means all rows.
3374
+ has_members = True
3375
+ yield raw
3376
+ finally:
3377
+ close = getattr(values, 'close', None)
3378
+ if close:
3379
+ close()
3380
+ members = _membership_relation(cur, authored_values())
3381
  raw_nodes = cfg.get("filters") or []
3382
  if not isinstance(raw_nodes, list):
3383
  raw_nodes = []
 
3386
  # "anything active": with no nodes at all the curated set IS the view, and with nodes
3387
  # that all happen to be half-built the view holds everything. Two different answers
3388
  # from one list, so the emptiness is checked before the compiler sees it.
3389
+ got = ("ids", members) if has_members else ("all", None)
3390
  memo[vid] = got
3391
  return got
3392
  nodes = rewrite(raw_nodes, seen + (vid,))
3393
+ if vid in cyclic:
3394
+ got = (None, "it refers to itself through another saved view")
3395
+ memo[vid] = got
3396
+ return got
3397
  unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols))
3398
  if unknown:
3399
  # ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it
 
3407
  try:
3408
  pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or"
3409
  else "and"),
3410
+ columns=cols, member_ids=(members if has_members else None),
3411
  id_sql=spec["id"], today=_today(), cohort_sets=sets)
3412
  except ValueError as exc:
3413
  got = (None, "it uses a condition this database's live connection cannot evaluate ("
 
3428
  return got
3429
  frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
3430
  where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql
3431
+ ids = _fs().SqlMembership(
3432
  f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}',
3433
+ list(wall_params) + list(pred.params))
3434
  got = ("ids", ids)
3435
  memo[vid] = got
3436
  return got
 
3461
  if kind == "ids":
3462
  sets[key] = got
3463
  op = "noneOf" if not_in else "anyOf"
3464
+ if not isinstance(got, _fs().SqlMembership) and len(got) >= _PIN_REPORT_AT:
3465
  limits.append({
3466
  "subject": "view", "effect": "slow",
3467
  "cause": f"the saved view this condition names holds {len(got):,} rows, "
 
3481
  out.append({"colId": "__cohort__", "op": op, "value": key})
3482
  return out
3483
 
3484
+ try:
3485
+ tree = rewrite(tree, ())
3486
+ finally:
3487
+ # Recursive closures must not postpone relation release until cyclic GC.
3488
+ memo.clear()
3489
+ resolve = rewrite = None
3490
  if unresolved:
3491
  limits.append({
3492
  "subject": ", ".join(vid for vid, _why in unresolved),
 
3506
  `__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
3507
 
3508
  A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
3509
+ here as SQL over the non-rank residue under the wall. Finite values, group scopes and
3510
+ ascending-ID ties follow `filter_eval.resolve_rank_sets`. Averages stream sorted values
3511
+ through runtime Python sum with bounded batches. Other ranks remain entirely in SQL.
3512
  """
3513
  fe = _fe()
3514
  leaves = fe._rank_leaves(tree or [], cols)
 
3536
  params.extend(pred.params)
3537
  frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
3538
  answers = {}
 
3539
  for n in leaves:
3540
+ answers[id(n)] = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3541
  sets = dict(cohort_sets or {})
3542
  counter = [0]
3543
  wanted = {id(n) for n in leaves}
 
3566
  out.append(n)
3567
  return out
3568
 
3569
+ try:
3570
+ return rewrite(tree), sets
3571
+ finally:
3572
+ rewrite = None
3573
 
3574
 
3575
  def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
3576
+ """Resolve global and explicit group ranks as SQL membership over the walled residue."""
 
 
 
 
 
 
 
 
 
3577
  fe = _fe()
3578
  Sql = _fs().SqlMembership
3579
  empty = Sql("SELECT NULL WHERE FALSE", [])
3580
+ scope, field = fe._rank_scope(leaf, None)
3581
+ if scope == "group" and field not in cols:
3582
+ return empty
3583
  col = cols.get(leaf.get("colId")) or {}
3584
  if col.get("type") not in fe.NUMERIC_TYPES:
3585
  return empty
3586
  v = f"TRY_CAST({col['sql']} AS DOUBLE)"
3587
+ partition = cols[field]["sql"] if scope == "group" else "NULL"
3588
+ dom = (f"SELECT {id_sql} AS pid, {v} AS v, {partition} AS bucket FROM {frm_body} WHERE "
3589
+ + (f"({where}) AND " if where else "")
3590
+ + f"{id_sql} IS NOT NULL AND {v} IS NOT NULL AND isfinite({v})")
3591
  params = list(params)
3592
  op = leaf.get("op")
3593
+ if op in ("aboveAvg", "belowAvg"):
3594
+ import itertools
3595
+ import tempfile
3596
+ from pathlib import Path
3597
+
3598
+ # Python 3.12 sum uses compensation. SQL sum/fsum cannot replace its sorted oracle.
3599
+ domain = f"SELECT *, dense_rank() OVER (ORDER BY bucket) AS bucket_id FROM ({dom}) d"
3600
+ lease = _RelationLease(cur)
3601
+ name = lease.table('_mean')
3602
+ frozen = lease.table('_domain')
3603
+ # The mean and selected IDs must share one source snapshot across both statements.
3604
+ cur.execute(f'CREATE OR REPLACE TEMP TABLE "{frozen}" AS {domain}', params)
3605
+ cur.execute(f'SELECT bucket_id, v FROM "{frozen}" ORDER BY bucket_id, v, pid')
3606
+
3607
+ def values():
3608
+ while True:
3609
+ batch = cur.fetchmany(4096)
3610
+ if not batch:
3611
+ return
3612
+ yield from batch
3613
+
3614
+ # Spill scalar group means, never a Python map of groups or a list of population rows.
3615
+ with tempfile.TemporaryDirectory(prefix="aios-rank-") as directory:
3616
+ path = Path(directory) / "means.csv"
3617
+ groups = 0
3618
+ with path.open("w", encoding="ascii") as output:
3619
+ for bucket, rows in itertools.groupby(values(), key=lambda row: row[0]):
3620
+ count = 0
3621
+
3622
+ def numbers():
3623
+ nonlocal count
3624
+ for _, number in rows:
3625
+ count += 1
3626
+ yield number
3627
+
3628
+ mean = sum(numbers()) / count
3629
+ output.write(f"{bucket},{mean!r}\n")
3630
+ groups += 1
3631
+ cur.execute(f'CREATE OR REPLACE TEMP TABLE "{name}" (bucket_id BIGINT, mean DOUBLE)')
3632
+ if groups:
3633
+ cur.execute(f'INSERT INTO "{name}" SELECT * FROM read_csv(?, header=false, '
3634
+ "columns={'bucket_id':'BIGINT', 'mean':'DOUBLE'})", [str(path)])
3635
+ cmp = ">" if op == "aboveAvg" else "<"
3636
+ return Sql(f'SELECT d.pid FROM "{frozen}" d JOIN "{name}" m USING (bucket_id) '
3637
+ f"WHERE NOT isnan(m.mean) AND d.v {cmp} m.mean AND ?=1", [lease])
3638
+ direction = "DESC" if op in ("topN", "inTopPct") else "ASC"
3639
+ ranked = (f"SELECT pid, v, row_number() OVER (PARTITION BY bucket ORDER BY v {direction}, "
3640
+ "pid ASC) AS rn, count(*) OVER (PARTITION BY bucket) AS n "
3641
+ f"FROM ({dom}) d")
3642
  if op in ("topN", "bottomN"):
3643
  bound = fe._rank_bound(leaf.get("value"), 1, 10000)
3644
  if bound is None:
3645
  return empty
3646
+ predicate = f"rn <= {bound}"
3647
+ elif op in ("inTopPct", "inBottomPct"):
 
 
 
 
 
 
3648
  pct = fe._rank_bound(leaf.get("value"), 1, 100)
3649
  if pct is None:
3650
  return empty
3651
+ predicate = f"rn <= greatest(1, ceil(n * {pct} / 100.0))"
3652
+ elif op in ("inQuartile", "inDecile"):
 
 
 
 
 
 
 
3653
  buckets = 4 if op == "inQuartile" else 10
3654
  wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
3655
+ if wanted is None:
3656
  return empty
3657
+ # The Python resolver clears every group if any finite-value group is too small.
3658
+ ranked = f"SELECT *, min(n) OVER () AS smallest_group FROM ({ranked}) ranked_groups"
3659
+ predicate = (f"smallest_group >= {buckets} AND "
3660
+ f"least({buckets}, ((rn - 1) * {buckets}) // n + 1) = {wanted}")
3661
+ else:
3662
+ return empty
3663
+ return Sql(f"SELECT pid FROM ({ranked}) r WHERE {predicate}", params)
3664
 
3665
  def _may_see_view(session, vid, store):
3666
  """May this caller open that saved view? The same predicate the view rail asks.
 
3683
  fields = rel.namespaced_fields(mk_fields(), connection)
3684
  break
3685
  if fields is None:
3686
+ if str(table_key).startswith("ut_amzn_"):
3687
+ import amazon_relational as amazon
3688
+ connected = table_key in amazon.SOURCE
3689
+ else:
3690
+ connected = table_key in _sources()
3691
+ if not connected:
3692
  raise err(404, "not_connected", "that database is not a connected grid")
3693
  fields = (defn or {}).get("fields") or []
3694
  if not fields:
 
3793
  if mismatch:
3794
  raise err(409, "cross_tenant_store", mismatch)
3795
 
3796
+ spec = _page_source_for(cur, table_key, st, session=session)
3797
  if spec is None:
3798
  from core import perm_scope
3799
  rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
 
4055
 
4056
 
4057
  @contextmanager
4058
+ def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None):
4059
  """One compiler for native rows and their aggregate sibling, over the walled payload.
4060
 
4061
  The temporary connection contains only this request's already stripped, pinned rows and is
4062
  closed before returning. It never registers data on the tenant's shared mirror connection.
4063
  """
4064
  import duckdb
4065
+ import itertools
4066
+ from harness import datastore
4067
 
4068
+ compiled = payload is None and table_key in {'customer_data', 'product_data'}
4069
+ streamed = payload is None and table_key != 'ut_activity'
4070
+ source_generation = None
4071
+ if streamed:
4072
+ import registry_source
4073
+ source_generation = registry_source.generation(session, table_key)
4074
+ if payload is not None:
4075
+ pass
4076
+ elif compiled:
4077
+ import registry_source
4078
  session.require(table_key)
4079
+ metadata = registry_source.canonical_metadata(session, table_key)
4080
+ payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [],
4081
+ 'limits': list(metadata['workspace'].get('limits') or [])}
4082
+ elif streamed:
4083
+ import routes_tables
4084
+ from registry_source import MetadataStore, with_runtime, metadata_limits
4085
+ metadata_actor = with_runtime(session, MetadataStore(session.runtime))
4086
+ metadata = routes_tables.ut_assembly(metadata_actor, table_key, with_rows=False,
4087
+ fields_only=True, consume_corrections=False)
4088
+ payload = {'fields': metadata['fields'], 'today': metadata['today'], 'rows': [],
4089
+ 'limits': metadata_limits(session, table_key, metadata)}
4090
  else:
4091
  from routes_tables import table_rows
4092
  payload = table_rows(table_key, session=session, related=related)
 
 
 
 
4093
  fields = payload["fields"]
4094
+ if compiled:
4095
+ source_cur, mismatch = _mirror_cur(session)
4096
+ if mismatch:
4097
+ raise err(409, 'cross_tenant_store', mismatch)
4098
+ if source_cur is None:
4099
+ raise err(503, 'canonical_source_unavailable', f'{table_key} requires its tenant mirror.')
4100
+ connection = source_cur.cursor()
4101
+ else:
4102
+ connection = datastore.bounded_connection(duckdb.connect(":memory:"))
4103
+ with connection as cur:
4104
+ cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
4105
+ wanted = set(ids) if ids is not None else None
4106
+ if streamed:
4107
+ import registry_source
4108
+ def source_rows():
4109
+ from contextlib import closing
4110
+ factory = registry_source.canonical_batches if compiled else registry_source.stored_batches
4111
+ options = {'ids': ids, 'related': related, 'needed': needed}
4112
+ if compiled:
4113
+ parts = factory(session, cur, table_key, **options)
4114
+ else:
4115
+ parts = factory(session, table_key, **options)
4116
+ with closing(parts) as parts:
4117
+ for part in parts:
4118
+ if compiled:
4119
+ payload.setdefault('docs', {}).update(part.get('docs') or {})
4120
+ yield from part['rows']
4121
+ part['rows'].clear()
4122
+ rows = source_rows()
4123
+ else:
4124
+ rows = payload['rows']
4125
+ source = ((index, row) for index, row in enumerate(rows)
4126
+ if wanted is None or int(row["pid"]) in wanted)
4127
+ source_count = 0
4128
+ try:
4129
+ while True:
4130
+ batch = list(itertools.islice(source, 2048))
4131
+ if not batch:
4132
+ break
4133
+ cur.execute("INSERT INTO pinned_rows SELECT unnest(?::BIGINT[]), unnest(?::JSON[])",
4134
+ [[index for index, _ in batch],
4135
+ [json.dumps(row, default=str) for _, row in batch]])
4136
+ source_count += len(batch)
4137
+ batch.clear()
4138
+ finally:
4139
+ source.close()
4140
+ close = getattr(rows, "close", None)
4141
+ if callable(close):
4142
+ close()
4143
  bindings = {}
4144
  for field in fields:
4145
  key = str(field["key"])
4146
+ if streamed and needed is not None and key not in needed:
4147
+ continue
4148
  pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
4149
  expr = f"json_extract_string(payload, '{pointer}')"
4150
  numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
 
4177
  where = f"({where}) AND {search.sql}" if where else search.sql
4178
  params.extend(search.params)
4179
  yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
4180
+ "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload,
4181
+ "source_count": source_count, 'streamed': streamed}
4182
+ if streamed:
4183
+ registry_source.require_generation(session, table_key, source_generation)
4184
 
4185
 
4186
  def _group_pin(cols, group_by, group_key):
 
4248
 
4249
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
4250
  "sorts": _json_arg(sorts, "sorts"), "search": search})
4251
+ payload = None
4252
+ window_offset = offset
4253
+ if table_key == 'ut_activity':
4254
+ payload, offset, limit = _activity_window_payload(session, offset, limit, related)
4255
+ if ids is not None:
4256
+ wanted = set(ids)
4257
+ payload = dict(payload, rows=[row for row in payload['rows'] if int(row['pid']) in wanted])
4258
+ if not (query['filters'] or query['sorts'] or query['search']
4259
+ or (isinstance(groupBy, str) and groupBy.strip())):
4260
+ return {**payload, 'ok': True, 'total': len(payload['rows']),
4261
+ 'totalUnfiltered': len(payload['rows']), 'offset': offset, 'limit': limit,
4262
+ 'scope': {'table': table_key, 'readThrough': True}}
4263
+ window_offset = 0
4264
+ needed = _leaf_cols(query['filters']) - _PSEUDO_COLS
4265
+ needed.update(sort['colId'] for sort in query['sorts'])
4266
+ if isinstance(groupBy, str) and groupBy.strip():
4267
+ needed.add(groupBy.strip())
4268
+ for leaf in _fe()._rank_leaves(query['filters'], {key: {} for key in needed}):
4269
+ _mode, group_field = _fe()._rank_scope(leaf, None)
4270
+ if group_field:
4271
+ needed.add(group_field)
4272
+ if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
4273
+ needed = None
4274
+ with _materialised_read(table_key, session, query, related, ids=ids, payload=payload, needed=needed) as read:
4275
  spec = read["spec"]
4276
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
4277
  where, params = read["where"], list(read["params"])
 
4279
  if gsql:
4280
  where = f"({where}) AND {gsql}" if where else gsql
4281
  params.extend(gparams)
4282
+ win = datastore.window(select='_row_index', where=where,
4283
  params=tuple(params), order_by=order or spec["id"],
4284
+ offset=window_offset, limit=limit or None, cur=read["cur"], **read["frm"])
4285
  payload = read["payload"]
4286
+ indexes = [int(row[0]) for row in win['rows']]
4287
+ value_sql = read['spec']['id'] if read['streamed'] else 'payload'
4288
+ values = (read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows '
4289
+ 'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
4290
+ if indexes else [])
4291
+ if read['streamed']:
4292
+ import registry_source
4293
+ by_index = {int(index): int(value) for index, value in values}
4294
+ page_ids = [by_index[index] for index in indexes]
4295
+ if table_key in {'customer_data', 'product_data'}:
4296
+ payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
4297
+ else:
4298
+ payload = registry_source.stored_page(session, table_key, page_ids, related=related)
4299
+ rows = payload['rows']
4300
+ else:
4301
+ by_index = {int(index): json.loads(value) for index, value in values}
4302
+ rows = [by_index[index] for index in indexes]
4303
  if win["clamped"]:
4304
  read["limits"].append({"subject": "limit", "effect": "window_clamped",
4305
  "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
4306
  "recommendation": "Continue paging with offset. Every matching row remains available."})
4307
+ limits = list(payload.get('limits') or [])
4308
+ limits.extend(report for report in read['limits'] if report not in limits)
4309
  return {**payload, "ok": True, "rows": rows, "total": win["total"],
4310
+ "totalUnfiltered": read["source_count"],
4311
+ "offset": offset if table_key == 'ut_activity' else win["offset"], "limit": win["limit"],
4312
+ "limits": limits, "scope": {"table": table_key, "readThrough": True}}
4313
+
4314
+
4315
+ def _activity_window_payload(session, offset, limit, related):
4316
+ """Walk native cursors to the requested offset without retaining skipped event values."""
4317
+ from core import activity_log
4318
+ from harness import datastore
4319
+ from routes_tables import table_rows
4320
+
4321
+ offset = max(0, int(offset or 0))
4322
+ requested = int(limit or datastore.WINDOW_DEFAULT)
4323
+ limit = min(max(requested, 1), datastore.WINDOW_MAX, activity_log.MAX_PAGE)
4324
+ remaining, cursor = offset, None
4325
+ for _ in range(16):
4326
+ size = min(remaining, activity_log.MAX_PAGE) if remaining else limit
4327
+ payload = table_rows('ut_activity', session=session, related=related, cursor=cursor, limit=size)
4328
+ next_cursor = payload.get('next_cursor')
4329
+ if next_cursor is not None and next_cursor == cursor:
4330
+ raise err(503, 'activity_cursor_stalled', 'Reload Activity to continue reading events.')
4331
+ if not remaining:
4332
+ break
4333
+ remaining -= len(payload['rows'])
4334
+ if next_cursor is None:
4335
+ payload = dict(payload, rows=[])
4336
+ break
4337
+ cursor = next_cursor
4338
+ else:
4339
+ raise HTTPException(status_code=409, detail={'error': {
4340
+ 'code': 'activity_cursor_required',
4341
+ 'message': 'Continue this deep Activity window through the native cursor route.'},
4342
+ 'next_cursor': cursor})
4343
+ if requested != limit:
4344
+ payload = dict(payload, limits=[*(payload.get('limits') or []), {
4345
+ 'subject': 'limit', 'effect': 'window_clamped',
4346
+ 'cause': f'One response carries at most {limit} Activity rows.',
4347
+ 'recommendation': 'Continue with the returned Activity cursor or the next offset.'}])
4348
+ return payload, offset, limit
4349
 
4350
 
4351
  @router.get("/odoo-tables/{table_key}/rows")
 
4374
  that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in
4375
  `limits` with its cause and a recommendation — R6's second sentence — and never silently
4376
  dropped, because an ignored condition widens.
4377
+
4378
  ⭐⭐ `related` (W51-T05 / C6) IS A **JSON QUERY STRING**, copying the `filters` precedent one
4379
  argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
4380
  window to the rows linked to that ONE record, ahead of `filters`, and `total` /
 
4400
  "limit, filters, sorts and search in the query string.")
4401
  ids = _record_ids(body)
4402
 
4403
+ # Admission resolves this Amazon database without publishing a process-wide binding.
 
 
 
4404
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4405
+ admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4406
+ if (admitted is None if page is not None else _materialised_table(table_key)):
4407
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
4408
  sorts, search, related, groupBy, groupKey, ids=ids)
4409
 
 
4437
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
4438
  win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
4439
  where=where, params=tuple(params), order_by=order,
4440
+ offset=offset, limit=(limit or None), cur=cur, page_first=True,
4441
+ page_key=spec.get("page_key"), **frm)
4442
  if win["clamped"]:
4443
  limits.append({
4444
  "subject": "limit", "effect": "window_clamped",
 
4480
  if not _may_open(session, target, st=page_st):
4481
  return []
4482
  return _relation_rows(session, target, list(matches), matches=matches,
4483
+ st=page_st, rev=page_rev,
4484
+ stream=True,
4485
+ materialised=_materialised_table(target, cur=cur, st=page_st))
4486
 
4487
  # Fields are the current reader's contract; target rows cross the same wall
4488
  # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
4489
+ amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets,
4490
+ max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal)
4491
+ overlays = {}
4492
 
4493
  # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
4494
  #
 
4503
  # the wall and the predicate, so the argument it demands is the list we just fetched.
4504
  # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
4505
  # answered "may this session open this surface".
4506
+ # Definitions use the projected reader. Values use coherent page-bound snapshots.
 
 
4507
  shared_defs = _so().fields(table_key, st=page_st)
4508
  if shared_defs:
4509
  declared = {field["key"]: field for field in fields}
4510
  declared.update({key: dict(field, source="overlay") for key, field in shared_defs.items()})
4511
  fields = list(declared.values())
4512
+ from registry_source import page_cells
4513
+ for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
4514
  overlays.setdefault(pid, {}).update(cells)
4515
  if_keys = {field["key"] for field in fields if field.get("type") == "if"}
 
 
4516
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
4517
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
4518
  # in an overlay column that reads a hidden mirror column was outside its reach - and a formula
 
4529
  rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
4530
  overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
4531
  for pid, cells in overlays.items()}
4532
+ rows = []
4533
+ for row in rows_src:
4534
+ # Physical payloads stay separate even when business identities share a pid.
4535
+ cells = {key: value for key, value in row.items() if key != "pid"}
4536
+ cells.update(overlays.get(str(row["pid"]), {}))
4537
+ cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden})
4538
+ rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells}))
4539
 
4540
+ unfiltered = win["total"] if where == base_where else datastore.count_rows(
4541
+ where=base_where, params=tuple(wall_params), cur=cur, **frm)
 
4542
 
4543
  from core import connector_spec
4544
  rows = connector_spec.decode_scoped_rows(table_key, rows, user=session.user, st=session.runtime)
 
4561
  # ══════════════════════════════════════════════════════════
4562
 
4563
  #: Maximum requested group PAGE size, never a ceiling on reachable groups. Legacy callers
4564
+ #: that omit groupLimit receive one page and exact traversal metadata.
4565
  MAX_GROUPS = 1_000
4566
 
4567
  #: One request asks for at most this many summaries. Each is one more aggregate expression in a
 
4569
  MAX_AGGREGATES = 200
4570
 
4571
 
4572
+ LINK_VALUE_MAX_BYTES = 1_048_576
4573
+
4574
+
4575
+ def _link_capacity_refusal():
4576
+ """Name the complete-value capacity and the action that reduces it."""
4577
+ return err(503, 'relation_value_capacity',
4578
+ f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
4579
+ 'Reduce the page size or open the related database with filters.')
4580
+
4581
+
4582
+ def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
4583
+ """Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
4584
+ read = read if read is not None else _compiled_read(target, session, st=st)
4585
+ spec, cur = read['spec'], read['cur']
4586
+ bindings = dict(spec['cols'], pid=(spec['id'], int))
4587
+ if on not in bindings:
4588
+ raise err(409, 'relation_target_unavailable', 'A required relation column is unavailable.')
4589
+ expr = f'LOWER(TRIM(CAST({bindings[on][0]} AS VARCHAR)))'
4590
+ wanted = _membership_relation(cur, values, normalize=str, sql_type='VARCHAR')
4591
+ lease = _RelationLease(cur)
4592
+ name = lease.table('_link_values')
4593
+ source = spec.get('from_sql') or spec['table']
4594
+ where = read['where'] or '1=1'
4595
+ try:
4596
+ cur.execute(f'CREATE TEMP TABLE "{name}" AS SELECT DISTINCT {expr} AS k, '
4597
+ f'{spec["id"]} AS pid FROM {source} WHERE ({where}) AND {expr} IN ({wanted.sql})',
4598
+ [*read['params'], *wanted.params])
4599
+ population = f'(SELECT k, min(pid) AS pid FROM "{name}" GROUP BY k)' if single else f'"{name}"'
4600
+ required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
4601
+ f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
4602
+ if required > remaining:
4603
+ raise _link_capacity_refusal()
4604
+ # The byte check bounds both the aggregate and its Python result. IDs are never a
4605
+ # Python population; DISTINCT and numeric ordering preserve the existing wire value.
4606
+ cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
4607
+ f'FROM {population} GROUP BY k').fetchall())
4608
+ return cells, required
4609
+ finally:
4610
+ lease.close()
4611
+
4612
+
4613
  def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
4614
+ """Resolve page link cells through scoped SQL and a complete-value byte budget."""
4615
  from routes_grid import _may_open
4616
  declared = {field["key"] for field in fields}
4617
+ edges, requested = [], {}
4618
  for field in fields:
4619
  bag = field.get("link") or {}
4620
  if field.get("type") != "link" or not bag.get("on") or not bag.get("from"):
 
4622
  target, on, source = bag["table"], bag["on"], bag["from"]
4623
  if _rel().base_table_key(target) not in _rel().READ_THROUGH_KEYS:
4624
  continue
4625
+ edges.append((field, target, on, source, bag))
4626
+ if source in declared:
4627
+ values = {str(row[source]).strip().lower() for row in rows if row.get(source) not in (None, "")}
4628
+ if values:
4629
+ requested.setdefault(target, {}).setdefault(on, set()).update(values)
4630
+ remaining = LINK_VALUE_MAX_BYTES
4631
+ target_reads, resolved = {}, {}
4632
+ for field, target, on, source, bag in edges:
4633
+ if source not in declared:
4634
+ for row in rows:
4635
+ row[field["key"]] = ""
4636
+ continue
4637
+ matches = requested.get(target, {}).get(on, ())
4638
+ if not matches or not _may_open(session, target, st=st):
4639
+ cells = {}
4640
+ else:
4641
+ if target not in target_reads:
4642
+ target_reads[target] = _compiled_read(target, session, st=st)
4643
+ cache_key = (target, on, bool(bag.get('single')))
4644
+ if cache_key not in resolved:
4645
+ resolved[cache_key], _required = _sql_link_cells(session, target, on, matches, st=st,
4646
+ single=cache_key[2], remaining=remaining, read=target_reads[target])
4647
+ cells = resolved[cache_key]
4648
  for row in rows:
4649
+ value = cells.get(str(row.get(source) or '').strip().lower(), '')
4650
+ # Count repeated cells as well: several page rows may share one target key.
4651
+ remaining -= len(value) + (2 if value else 0)
4652
+ if remaining < 0:
4653
+ raise _link_capacity_refusal()
4654
+ row[field['key']] = value
4655
 
4656
 
4657
  def _agg_number(value):
 
4843
  f"THEN 1 ELSE 0 END"),
4844
  where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
4845
  projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
4846
+ group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
4847
+ rows = datastore.window(
4848
+ select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
4849
+ group_by=gexpr, offset=group_offset, limit=group_limit, cur=cur, **frm)["rows"]
 
 
 
 
 
4850
  groups = []
4851
  for r in rows:
4852
  vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
 
4855
  "count": int(r[1] or 0), "aggregates": vals})
4856
  return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
4857
  "total_groups": distinct, "group_offset": group_offset,
4858
+ "group_limit": group_limit, "has_more": group_offset + len(groups) < distinct,
4859
+ "limits": limits}
4860
 
4861
 
4862
  @router.post("/odoo-tables/{table_key}/aggregates")
 
4905
  raise err(400, "bad_selection", "groupOffset must be a nonnegative integer")
4906
  if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
4907
  raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
 
 
4908
 
4909
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4910
+ admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4911
 
4912
+ if (admitted is None if page is not None else _materialised_table(table_key)):
4913
  with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
4914
  import odoo_default_scope
4915
  if (odoo_default_scope.applies(table_key)
 
4960
  "limits": out["limits"], "scope": {"table": table_key, "readThrough": True}}
4961
  if out["groups"] is not None:
4962
  body_out["groups"] = out["groups"]
4963
+ body_out.update({k: out[k] for k in ("total_groups", "group_offset", "group_limit", "has_more")})
4964
  return body_out
4965
 
4966
 
 
5090
  if spec is None:
5091
  return None
5092
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
5093
+ return datastore.count_rows(where=spec.get("where") or "", cur=cur, **frm)
 
5094
 
5095
 
5096
  def whole_pool(table_key, cur=None, rt=None):
api/routes_platform_admin.py CHANGED
@@ -47,7 +47,7 @@ import core.do_usage as do_usage
47
  import core.store as store
48
  import operator_control
49
  import voice_control_plane as voice
50
- from deps import Session, err, require_session, users
51
 
52
  @asynccontextmanager
53
  async def _neon_lifespan(app):
@@ -329,7 +329,7 @@ def _user_rows(tenant=None):
329
  `routes_admin._view`'s discipline: one function that can leak, and it does not.
330
 
331
  R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
332
- rendered as "never" rather than as a fabricated date.
333
  """
334
  try:
335
  reg = users.registry() or {}
@@ -351,7 +351,7 @@ def _user_rows(tenant=None):
351
  "role": rec.get("role", "user"),
352
  "active": bool(rec.get("active", True)),
353
  "lastLogin": rec.get("last_login") or "",
354
- "lastActive": rec.get("last_active") or "",
355
  "platformAdmin": rec.get("platform_admin") is True,
356
  })
357
  return rows
@@ -699,14 +699,14 @@ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)):
699
  """Every account on the platform, or one tenant's — the drill behind the Users count.
700
 
701
  R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
702
- `routes_auth.login`, `lastActive` by `deps.require_session` (throttled to once an hour per
703
- account per process). Absent means never seen, and the pane renders it as "never".
704
  """
705
  rows = _user_rows(tenant)
706
  return {"users": rows, "count": len(rows),
707
- "stampsNote": ("Login and activity stamps started with this release; accounts that "
708
- "have not signed in since show no date rather than an invented one. "
709
- "Activity is recorded at most once an hour per account.")}
710
 
711
 
712
  @router.get("/databases")
 
47
  import core.store as store
48
  import operator_control
49
  import voice_control_plane as voice
50
+ from deps import Session, err, observed_last_active, require_session, users
51
 
52
  @asynccontextmanager
53
  async def _neon_lifespan(app):
 
329
  `routes_admin._view`'s discipline: one function that can leak, and it does not.
330
 
331
  R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
332
+ rendered as "never" when neither a persisted stamp nor a process-local observation exists.
333
  """
334
  try:
335
  reg = users.registry() or {}
 
351
  "role": rec.get("role", "user"),
352
  "active": bool(rec.get("active", True)),
353
  "lastLogin": rec.get("last_login") or "",
354
+ "lastActive": observed_last_active(slug, uname, rec.get("last_active")) or "",
355
  "platformAdmin": rec.get("platform_admin") is True,
356
  })
357
  return rows
 
699
  """Every account on the platform, or one tenant's — the drill behind the Users count.
700
 
701
  R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
702
+ `routes_auth.login`. `lastActive` includes bounded process-local read observations; persisted
703
+ activity comes from login and throttled admitted mutations. Absent means no recorded activity.
704
  """
705
  rows = _user_rows(tenant)
706
  return {"users": rows, "count": len(rows),
707
+ "stampsNote": ("Activity dates include recent visits observed by this server. "
708
+ "Visits are tracked temporarily. Saved dates come from sign-ins "
709
+ "and changes, and can be older after a server restart.")}
710
 
711
 
712
  @router.get("/databases")
api/routes_products.py CHANGED
@@ -53,6 +53,9 @@ def _pool_for(rt, team_id):
53
  # fail-closed seam rather than widening it by accident.
54
  if rt.key != "royal-imports":
55
  return []
 
 
 
56
 
57
  import modules.product_data as pd
58
  from core import user_tables
@@ -128,13 +131,14 @@ def scoped_pool(session: Session, *, with_rows: bool = True):
128
  except Exception as e:
129
  raise err(503, "pool_unavailable",
130
  f"the product catalogue could not be built; {str(e)[:160]}")
131
- # Image is a tenant-wide created field. Seed/repair its definition before
132
- # the field wall runs so every reader receives the same metadata and the
133
- # Leadership account owns the permission editor from the first request.
134
  import modules.product_data as pd
 
 
135
  if with_rows:
136
- pd.image_field_definition(st=session.runtime)
137
- fields_base = pd_fields(consolidated=team_id is None, st=session.runtime)
138
  # The SAME wall the customer assembly applies, in the same order: rows first (before pids
139
  # are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
140
  # then the values stripped from the rows as well as the field list.
@@ -217,7 +221,7 @@ def product_measures(session):
217
  return []
218
 
219
 
220
- def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
221
  """`{pid: {field key: value}}` for every measure column on this grid.
222
 
223
  ⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's
@@ -236,6 +240,11 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
236
  Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries.
237
  """
238
  import aios_grid
 
 
 
 
 
239
 
240
  mfields = aios_grid.measure_fields_of(fields)
241
  if not mfields or not offer:
@@ -263,7 +272,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
263
  out = {}
264
  for (dfrom, dto), group in by_window.items():
265
  keys = tuple(sorted({f["measure"]["key"] for f in group}))
266
- mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys)
 
267
  if mkey not in memo:
268
  try:
269
  memo[mkey] = sem.entity_measure_values(
@@ -272,7 +282,8 @@ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer):
272
  # here. The service filter exists to stop Delivery Charges polluting a SKU
273
  # RANKING; on a per-row column the row IS the service product, and excluding
274
  # it would print a FALSE 0 — which under C1's rule asserts "sold nothing".
275
- exclude_services=False, offer=offer)
 
276
  except Exception as e: # noqa: BLE001
277
  _measure_err("product-measure-column", e)
278
  # Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure
@@ -441,8 +452,9 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
441
  import aios_grid
442
  import core.perm_scope as perm_scope
443
  import modules.product_data as pd
 
444
  pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows)
445
- shared_defs = pd.shared_fields(st=session.runtime)
446
 
447
  ctx = grid_events.EventCtx(
448
  uname=session.uname, allowed_pids=pids, fields=[],
@@ -841,6 +853,12 @@ def pd_fields(consolidated=True, st=None):
841
  # (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
842
  if st is not None:
843
  image = pd.image_field_definition(st=st)
 
 
 
 
 
 
844
  else:
845
  # Contract checks and offline tooling have no tenant runtime. Keep a
846
  # truthful local projection with the same created-field semantics.
 
53
  # fail-closed seam rather than widening it by accident.
54
  if rt.key != "royal-imports":
55
  return []
56
+ bounded = getattr(rt, '_canonical_product_rows', None)
57
+ if bounded is not None:
58
+ return bounded
59
 
60
  import modules.product_data as pd
61
  from core import user_tables
 
131
  except Exception as e:
132
  raise err(503, "pool_unavailable",
133
  f"the product catalogue could not be built; {str(e)[:160]}")
134
+ # Project detached definitions without seeding fields or replacing caller grants.
135
+ # Keep the original runtime for row/value hydration and authoritative mutations.
 
136
  import modules.product_data as pd
137
+ from registry_source import MetadataStore
138
+ metadata = MetadataStore(session.runtime)
139
  if with_rows:
140
+ pd.image_field_definition(st=metadata)
141
+ fields_base = pd_fields(consolidated=team_id is None, st=metadata)
142
  # The SAME wall the customer assembly applies, in the same order: rows first (before pids
143
  # are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
144
  # then the values stripped from the rows as well as the field list.
 
221
  return []
222
 
223
 
224
+ def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer, *, cur=None):
225
  """`{pid: {field key: value}}` for every measure column on this grid.
226
 
227
  ⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's
 
240
  Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries.
241
  """
242
  import aios_grid
243
+ if cur is None:
244
+ from harness.canonical_source import active
245
+ source = active()
246
+ if source is not None:
247
+ cur = source.cur
248
 
249
  mfields = aios_grid.measure_fields_of(fields)
250
  if not mfields or not offer:
 
272
  out = {}
273
  for (dfrom, dto), group in by_window.items():
274
  keys = tuple(sorted({f["measure"]["key"] for f in group}))
275
+ mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys,
276
+ tuple(sorted(codes.values(), key=str)))
277
  if mkey not in memo:
278
  try:
279
  memo[mkey] = sem.entity_measure_values(
 
282
  # here. The service filter exists to stop Delivery Charges polluting a SKU
283
  # RANKING; on a per-row column the row IS the service product, and excluding
284
  # it would print a FALSE 0 — which under C1's rule asserts "sold nothing".
285
+ exclude_services=False, offer=offer, group_keys=list(codes.values()),
286
+ **({'cur': cur} if cur is not None else {}))
287
  except Exception as e: # noqa: BLE001
288
  _measure_err("product-measure-column", e)
289
  # Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure
 
452
  import aios_grid
453
  import core.perm_scope as perm_scope
454
  import modules.product_data as pd
455
+ from registry_source import MetadataStore
456
  pids, team_id, rows_src, fields_base = scoped_pool(session, with_rows=with_rows)
457
+ shared_defs = pd.shared_fields(st=MetadataStore(session.runtime))
458
 
459
  ctx = grid_events.EventCtx(
460
  uname=session.uname, allowed_pids=pids, fields=[],
 
853
  # (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
854
  if st is not None:
855
  image = pd.image_field_definition(st=st)
856
+ # An explicit grant still governs Image when its stored definition is missing
857
+ # or predates the grant marker. Let the existing permission closure decide access.
858
+ from core import shares
859
+ record = shares.grants("field", shares.field_oid(MODULE, PRODUCT_IMAGE_KEY), st=st)
860
+ if record.get("owner") or record.get("entries"):
861
+ image = dict(image, granted=True)
862
  else:
863
  # Contract checks and offline tooling have no tenant runtime. Keep a
864
  # truthful local projection with the same created-field semantics.
api/routes_sheet.py CHANGED
@@ -259,7 +259,7 @@ def _mirror(session: Session):
259
  if connector_defn is None:
260
  native = _native_table_definition(session, table_key)
261
  if native is not None:
262
- cur, spec, cols = _native_table_mirror(session, table_key)
263
  return cur, {**spec, "declared_fields": native.get("fields") or []}, cols
264
  raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
265
  fields = list(connector_defn.get("fields") or [])
@@ -301,12 +301,12 @@ def _mirror(session: Session):
301
  stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
302
  # The store keeps records as {row id: row}. Iterating that dict as a list yields only
303
  # the ids, which materialises an EMPTY table and answers every cell blank.
304
- rows = [row for row in (stored.values() if isinstance(stored, dict) else stored)
305
- if isinstance(row, dict)]
306
  fields = rel.shipment_fields()
307
  hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
308
  fields = [field for field in fields if field["key"] not in hidden]
309
- if not rows:
310
  # S75-T59: a full pass stores a rowless definition that reads through the mirror.
311
  spec = rot._source_for(cur, table_key)
312
  if spec is None:
@@ -419,23 +419,40 @@ def _native_table_definition(session, table_key):
419
  return defn
420
 
421
 
422
- def _native_table_mirror(session, table_key):
423
  """Reuse Data's scoped grid/spec reader, with a cursor owned by this Sheet request.
424
 
425
  The payload has already applied permanent predicates, allowed IDs and field/BU walls.
426
  Authored Sheet filters still compile below, against only the visible column vocabulary.
427
  """
428
- import routes_odoo_tables as rot
 
 
429
  memo = getattr(session, "_sheet_read_memo", None)
430
  if not isinstance(memo, dict) or "sources" not in memo:
431
  raise RuntimeError("A native Sheet source needs a request scope.")
432
  slot = ("native_source", table_key)
433
- if slot not in memo:
434
- read = memo["sources"].enter_context(rot._materialised_read(
435
- table_key, session, {"filters": [], "filterConj": "and", "search": ""}, None))
436
- spec = {**read["spec"], "fields": read["payload"]["fields"],
437
- "scope_applied": True, "limits": list(read["limits"])}
438
- memo[slot] = (read["cur"], spec, read["cols"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
439
  return memo[slot]
440
 
441
 
@@ -535,10 +552,18 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr
535
  with_rows=False, fields_only=not with_rows, needed=needed)
536
  return memo[slot]
537
  if table_key == "product_data":
 
 
 
 
538
  from routes_products import product_assembly
539
  memo[slot] = product_assembly(session, scope="product",
540
  consume_corrections=False, with_rows=with_rows, needed=needed)
541
  return memo[slot]
 
 
 
 
542
  from routes_customers import grid_assembly
543
  memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False,
544
  with_rows=with_rows, needed=needed)
@@ -565,6 +590,9 @@ def _registry_assembly(session: Session, table_key: str, *, with_rows: bool = Tr
565
  def _sheet_permission_reader(session):
566
  """Lend a captured schema/grant snapshot only inside the current Sheet request."""
567
  memo = getattr(session, "_sheet_read_memo", None)
 
 
 
568
  return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime
569
 
570
 
@@ -798,6 +826,22 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits):
798
  spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0])
799
  total_rows = int(cur.execute(count_sql, params).fetchone()[0])
800
  tail = count_sql[len("SELECT COUNT(*)"):]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
801
  sql = "SELECT " + ", ".join([*groups, *expressions]) + tail
802
  if groups:
803
  positions = ", ".join(str(index + 1) for index in range(len(groups)))
@@ -816,14 +860,6 @@ def _pivot_result(cur, spec, columns, pivot, count_sql, params, limits):
816
  row_keys.setdefault(row_key, len(row_keys))
817
  column_keys.setdefault(column_key, len(column_keys))
818
  values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]]
819
- # The matrix below is dense, so its size is rows x columns x values whatever the data holds.
820
- # Standing rule 1: a bound that cannot be lifted is reported with its cause and its fix.
821
- cells_needed = len(row_keys) * len(column_keys) * len(pivot["values"])
822
- if cells_needed > _PIVOT_MAX_CELLS:
823
- raise err(409, "pivot_too_large",
824
- f"This pivot needs {len(row_keys):,} rows by {len(column_keys):,} columns, "
825
- f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. "
826
- "Move a field to Filters, or use a field with fewer distinct values.")
827
  # SQL orders the entire tuple, but a column tuple may first occur in a later row group.
828
  # Sort each axis independently using its JSON spelling for deterministic mixed/null keys.
829
  ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
@@ -1014,6 +1050,32 @@ def _linked_filter_fields(database, session):
1014
  return {"fields": offered, "databases": databases, "unavailable": unavailable}
1015
 
1016
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1017
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
1018
  """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
1019
 
@@ -1046,7 +1108,9 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
1046
  column just because a filter named it would invent an empty column for a name this database
1047
  does not have, and the wall would then compile against nothing and quietly stop restricting.
1048
  """
1049
- import aios_grid
 
 
1050
  import odoo_relational as rel
1051
  import routes_odoo_tables as rot
1052
  from modules import sheet as sheet_mod
@@ -1058,8 +1122,8 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
1058
  derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window)
1059
  # Authorization has already scoped rows using the full wall vocabulary. Restrict only
1060
  # cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default.
1061
- rows = (aios_grid.rows_from_pool(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"),
1062
- derived=derived) if kept else [{"pid": row["pid"]} for row in g["rows_src"]])
1063
  if table_key == "customer_data" and (needed is None or "agent_id" in needed):
1064
  from core import perm_scope
1065
  agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
@@ -1068,8 +1132,7 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
1068
  # Join only to already-scoped customer rows. pid is the actual res.partner id.
1069
  have = rel.columns(cur, "res_partner")
1070
  if "agent_id" in have:
1071
- agents = dict(cur.execute("SELECT id, agent_id FROM res_partner").fetchall())
1072
- rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows]
1073
  fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
1074
  elif needed is not None and "agent_id" in needed:
1075
  raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
@@ -1086,35 +1149,14 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
1086
  spec_fields = sheet_mod.mirror_fields(kept)
1087
  # The related customer side can contain thousands of permitted rows. Insert its column
1088
  # arrays in one call; per-row execute round trips delayed the first drill page by seconds.
1089
- spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows, bulk=True)
1090
- # ⛔⛔ THE MATERIALISED TABLE IS GIVEN THE WINDOW'S OWN NAME, AND WITHOUT THIS THE MEMO ABOVE
1091
- # SERVES ONE WINDOW'S ROWS UNDER ANOTHER WINDOW'S ANSWER (found by W53-T14, live).
1092
- # `odoo_relational.registry_grid_spec` names its temp table `_registry_grid_<table_key>` and
1093
- # `CREATE OR REPLACE`s it, so the name carries the DATABASE and not the PERIOD, while
1094
- # `_mirror`'s memo is keyed by `(table_key, window)`. Resolve one customer_data cell at no
1095
- # period and a second at `{month, 2026-08}` inside ONE request and the second build replaces
1096
- # the first one's table underneath the spec the memo is still holding; a third cell reading
1097
- # the first slot then runs its SQL against August's rows.
1098
- # ⚠ MEASURED, and it is silent: `avg` over `measure_sales_the_last_12_months_6hlkf` answered
1099
- # **138.3942281513749** in a batch that had already resolved that month (508,322.0 / 3,673)
1100
- # and **1710.679825755513** on its own, which is the all-time number the preview door and the
1101
- # grid both give. No refusal, no limits row, a plausible number under the wrong label.
1102
- # ⛔ IT IS FIXED HERE RATHER THAN IN `registry_grid_spec` BECAUSE THAT FILE BELONGS TO ANOTHER
1103
- # LANE THIS WAVE. The copy is one `CREATE TABLE AS SELECT` over rows already in memory, next
1104
- # to an assembly that costs 20 to 130 seconds, so it is not a cost worth conditioning on.
1105
- # ⚠ ALWAYS, INCLUDING THE `None` WINDOW: the unqualified name is REPLACED by every later
1106
- # build, so leaving the no-period case pointing at it is exactly the case measured above.
1107
- # ⛔⛔ W54-T36 — THE COLUMN SET IS PART OF THAT NAME FOR THE SAME REASON THE WINDOW IS. Two
1108
- # cells on one database at one window may now be materialised over DIFFERENT columns, and a
1109
- # narrower later build would otherwise replace a wider earlier one under a spec the memo is
1110
- # still holding, so a query would address a column its own table no longer has.
1111
  slot = hashlib.sha1(
1112
  json.dumps([metric_window, None if needed is None else sorted(needed)],
1113
  sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12]
1114
- at_window = f'{spec["table"]}__at_{slot}'
1115
- cur.execute(f'CREATE OR REPLACE TEMP TABLE "{at_window}" AS '
1116
- f'SELECT * FROM "{spec["table"]}"')
1117
- spec = {**spec, "table": at_window}
 
1118
  # ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal`
1119
  # has to name WHY a column cannot be totalled, and the normalised copy has already thrown that
1120
  # away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds
@@ -1123,6 +1165,22 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
1123
  return cur, spec, rot._filter_columns(spec, spec_fields)
1124
 
1125
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1126
  def _metric_derived(session: Session, table_key: str, g, metric_window):
1127
  """`(derived, bounded_keys, unbounded)` for one assembly at the cell's period.
1128
 
@@ -1156,10 +1214,7 @@ def _metric_derived(session: Session, table_key: str, g, metric_window):
1156
  seen.update(cells or ())
1157
  live = [f for f in mfields if f.get("key") in seen]
1158
 
1159
- out = {pid: dict(cells or {}) for pid, cells in derived.items()}
1160
- for f in mfields: # every Metric column starts blank at a new window
1161
- for cells in out.values():
1162
- cells.pop(f.get("key"), None)
1163
 
1164
  bounded, unbounded = [], {}
1165
  for f in live:
@@ -1170,12 +1225,9 @@ def _metric_derived(session: Session, table_key: str, g, metric_window):
1170
  cells = _measure_cells_at(session, table_key, g, rewritten)
1171
  except Exception: # noqa: BLE001
1172
  cells = {}
1173
- got = False
1174
- for pid, values in (cells or {}).items():
1175
- if key in values:
1176
- out.setdefault(pid, {})[key] = values[key]
1177
- got = True
1178
  if got:
 
1179
  bounded.append(key)
1180
  else:
1181
  unbounded[key] = sheet_mod.metric_window_limit(
@@ -1209,10 +1261,12 @@ def _measure_cells_at(session: Session, table_key: str, g, field):
1209
  import routes_products as rp
1210
  return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"),
1211
  g.get("today"), "sheet-period", {},
1212
- rp.product_measures(session))
 
1213
  from core import measure_resolve
1214
  return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(),
1215
- g.get("today"), "sheet-period", {})
 
1216
 
1217
 
1218
  def _offered_view(session: Session, database: str, view_id: str):
@@ -1570,8 +1624,21 @@ def activity_read_policy(session: Session, object):
1570
  source_identity = identity[:3] + (databases,)
1571
  sources = memo.setdefault("activity_sources", {}) if isinstance(memo, dict) else {}
1572
  admitted = sources.setdefault(source_identity, {})
 
 
1573
 
1574
  def cell_allowed(cell):
 
 
 
 
 
 
 
 
 
 
 
1575
  if cell is None:
1576
  return True
1577
  if not isinstance(cell, dict):
@@ -4785,7 +4852,7 @@ def _metric_column_aggregate(session, database, aggregate, window):
4785
  # assembly for another database: that would create a field on the wrong database.
4786
  if database not in ("customer_data", "product_data"):
4787
  refuse("measure_not_offered")
4788
- g = _registry_assembly(session, database)
4789
  # Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue
4790
  # with the served offer too: product has a narrower entity catalogue, and revoked Metric
4791
  # capability makes the account's offer empty. A browser key can never widen either offer.
@@ -4845,7 +4912,7 @@ def _metric_column_aggregate(session, database, aggregate, window):
4845
  if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys
4846
  or raw["key"] in {f.get("key") for f in g["fields"]}):
4847
  _forget_sheet_database(session, database)
4848
- found = existing(_registry_assembly(session, database).get("fields"))
4849
  if found is not None:
4850
  return column(found)
4851
  refuse("metric_column_conflict", label)
@@ -4897,6 +4964,7 @@ def _sheet_request_scope(session):
4897
  previous = getattr(session, "_sheet_read_memo", None)
4898
  previous_assemblies = getattr(session, "_sheet_assemblies", None)
4899
  previous_context = getattr(session, "_sheet_scope_context", None)
 
4900
  sources = contextlib.ExitStack()
4901
  memo = {"today": _sheet_today(session), "sources": sources,
4902
  "stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
@@ -4916,6 +4984,7 @@ def _sheet_request_scope(session):
4916
  session._sheet_read_memo = previous
4917
  session._sheet_assemblies = previous_assemblies
4918
  session._sheet_scope_context = previous_context
 
4919
 
4920
 
4921
  def _request_scoped(route):
@@ -5681,6 +5750,16 @@ def _binding_related_entry(session, database, name):
5681
  return entry
5682
 
5683
 
 
 
 
 
 
 
 
 
 
 
5684
  def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None):
5685
  """Materialise the far side on the near cursor, then hand metadata to the compiler."""
5686
  from core import perm_scope, relations
@@ -5727,8 +5806,11 @@ def _binding_related_query(session, database, entry, cur, spec, columns, *, fiel
5727
  raise err(409, "relation_cursor_mismatch",
5728
  f"Relationship '{name}' cannot be queried on the same live connection as "
5729
  f"'{database}'. Retry after both databases share a connection.")
 
5730
  hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
5731
  st=session.runtime)
 
 
5732
  rcols = {key: value for key, value in rcols.items() if key not in hidden}
5733
  declared = {field["key"]: field for field in rspec.get("fields") or []}
5734
 
@@ -5750,7 +5832,7 @@ def _binding_related_query(session, database, entry, cur, spec, columns, *, fiel
5750
  raise unknown_field(key)
5751
  return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
5752
  "spec": rspec, "columns": rcols, "where": entry.get("where"),
5753
- "wall_nodes": wall_nodes, "wall_conj": wall_conj,
5754
  "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]}
5755
 
5756
 
@@ -5869,11 +5951,12 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
5869
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
5870
  cols = dict(cols)
5871
  if spec.get("scope_applied"):
5872
- wall_nodes, wall_conj = [], "and"
5873
  unreadable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
5874
  - sheet_mod.related_filter_ids(scope_nodes))
5875
  field = binding["aggregate"].get("field")
5876
- if field and field not in cols:
 
5877
  unreadable.add(field)
5878
  if unreadable:
5879
  declared = {field.get("key") for field in spec.get("declared_fields", [])}
@@ -6307,7 +6390,7 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
6307
  raise _refuse(exc)
6308
  _require_reader(session, book)
6309
  database = _require_database(session, binding["database"])
6310
- fields = list(_registry_assembly(session, database).get("fields") or [])
6311
  hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
6312
  visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
6313
  requested = binding["fields"]
@@ -6399,15 +6482,36 @@ _DRILL_MIRROR_LOCK = threading.RLock()
6399
  _DRILL_MIRROR_MAX = 6
6400
 
6401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6402
  def _drill_tokens(session, databases):
6403
  """Unknown freshness disables reuse. No workspace documents are retained here."""
6404
  from core import sheet_binding
 
6405
  tokens = {}
6406
  for db in sorted(databases):
6407
  try:
6408
  import harness.mirror_tokens # noqa: F401
6409
  tokens[db] = sheet_binding.database_change_token(_store(session), db)
6410
  except Exception as exc:
 
6411
  _token_unread(session, db, exc)
6412
  tokens[db] = None
6413
  return tokens
@@ -6417,6 +6521,7 @@ def _drill_tokens(session, databases):
6417
  def _drill_mirror(session, database, window, needed, databases):
6418
  """A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction."""
6419
  from core import sheet_binding, perm_scope
 
6420
  import datetime
6421
 
6422
  if _native_table_definition(session, database) is not None:
@@ -6433,7 +6538,8 @@ def _drill_mirror(session, database, window, needed, databases):
6433
  if callable(stamp):
6434
  try:
6435
  wall = stamp(session.tenant, session.uname, database, user=session.user)
6436
- except Exception:
 
6437
  pass
6438
  if not wall:
6439
  token = None
@@ -6446,7 +6552,7 @@ def _drill_mirror(session, database, window, needed, databases):
6446
  "_sheet_drill_entries")
6447
  absent = object()
6448
  previous = {name: getattr(session, name, absent) for name in attrs}
6449
- with _DRILL_MIRROR_LOCK:
6450
  hit = _DRILL_MIRRORS.get(key) if token else None
6451
  session._sheet_drill_isolated = True
6452
  session._sheet_drill_cursor = hit["owned"] if hit else None
@@ -6492,16 +6598,23 @@ def _drill_mirror(session, database, window, needed, databases):
6492
  setattr(session, name, value)
6493
 
6494
 
6495
- def _drill_page(session, database, record_ids, offset, limit):
6496
  """Use fetchTableWindow's server door, including its row and field walls and wire shape."""
6497
  import routes_odoo_tables as rot
6498
  from harness import datastore
6499
 
6500
  size = min(limit, datastore.WINDOW_MAX)
6501
- ids = record_ids[offset:offset + size]
6502
- payload = rot.odoo_table_rows(database, offset=0, limit=size, filters=None,
6503
- filterConj="and", sorts=None, search=None, related=None, groupBy=None, groupKey=None,
6504
- session=session, body={"ids": ids}) if ids else {"rows": [], "fields": []}
 
 
 
 
 
 
 
6505
  by_id = {int(row["pid"]): row for row in payload["rows"]}
6506
  limits = list(payload.get("limits") or [])
6507
  if size != limit:
@@ -6509,10 +6622,117 @@ def _drill_page(session, database, record_ids, offset, limit):
6509
  "cause": f"One response carries at most {datastore.WINDOW_MAX} rows.",
6510
  "recommendation": "Continue paging with offset. Every matching row remains available."})
6511
  return {**payload, "offset": offset, "limit": size, "limits": limits,
6512
- "total": len(record_ids), "totalUnfiltered": len(record_ids),
6513
  "rows": [by_id[pid] for pid in ids if pid in by_id]}
6514
 
6515
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6516
  def _drill_draft_binding(cell, binding, draft_filter):
6517
  """The stored binding with View data's UNSAVED filter in place of its own (S68-T23 finding).
6518
 
@@ -6546,35 +6766,20 @@ def _drill_draft_binding(cell, binding, draft_filter):
6546
  def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
6547
  session: Session = Depends(require_session),
6548
  offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1),
6549
- draft_filter: str | None = Query(default=None, alias="filter")):
6550
- """`{database, view_id?, filter?, record_ids, count, id_limit}` for `LinkGridModal` (R6, R13, R17).
6551
-
6552
- ⭐⭐ S66-T48 — `tab` (an id or a name, as `GET /sheets/{id}/tabs/{tab}` accepts) NAMES THE TAB
6553
- THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a
6554
- Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old
6555
- first-tab rule still answers, so an older client keeps working.
6556
-
6557
- ⭐⭐ S66-T48 — NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching
6558
- id; the popup pages them and sends the set in a POST body. Only a database whose records a
6559
- person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so
6560
- the popup says "showing X of N" rather than presenting X as the whole.
6561
-
6562
- ⛔ A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
6563
- would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
6564
- clicked on.
6565
-
6566
- ⭐ THE IDS ARE READ LIVE, NOT TAKEN FROM THE CELL (R13). A stale cell opens the same popup with
6567
- no comparison text, and the count beside those ids is the LIVE count, because `LinkGridModal`
6568
- prints the length of what it is handed and runs no query of its own.
6569
- """
6570
- from core import sheets
6571
  from modules import sheet as sheet_mod
6572
  import core.perm_scope as perm_scope
 
 
6573
 
6574
  # Python callers omit FastAPI's Query sentinels. Offset has no maximum; only the page does.
6575
  offset = offset if isinstance(offset, int) else 0
6576
  limit = limit if isinstance(limit, int) else 100
6577
  draft_filter = draft_filter if isinstance(draft_filter, str) else None
 
6578
  if offset < 0 or limit < 1:
6579
  raise err(400, "bad_drill_page", "Choose a nonnegative record offset and a positive page size.")
6580
 
@@ -6645,15 +6850,31 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
6645
  raise err(400, exc.code, exc.message)
6646
  needed = (sheet_mod.filter_columns_named(scope_nodes)
6647
  | sheet_mod.filter_columns_named(wall_nodes))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6648
  related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6649
  databases = {database}
6650
  databases.update(_binding_related_entry(session, database, via)["target"]
6651
  for via in sheet_mod.related_vias(scope_nodes))
 
6652
  with _drill_mirror(session, database, window, needed, databases) as (made, token):
6653
  cur, spec, cols = made
6654
  cols = dict(cols)
6655
  # Apply the aggregate's field wall before compiling either the count or the IDs.
6656
  hidden = perm_scope.hidden_keys(session.user, database, spec.get("fields") or [], st=session.runtime)
 
6657
  if spec.get("scope_applied"):
6658
  wall_nodes, wall_conj = [], "and"
6659
  unavailable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
@@ -6672,9 +6893,6 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
6672
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
6673
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6674
  scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
6675
- ids_sql, ids_params, _ = sheet_mod.build_ids_query(
6676
- spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
6677
- wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
6678
  except sheet_mod.SheetRefusal as exc:
6679
  raise err(400, exc.code, exc.message)
6680
  except ValueError:
@@ -6683,25 +6901,52 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
6683
  "connection. Edit the scope or ask an administrator to update the restriction.")
6684
  if spec.get("scope_applied") and limits:
6685
  raise err(409, "scope_not_applicable", "This cell's filter cannot be applied to this database.")
6686
- total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
6687
-
6688
- # S66-T48: preserve the existing mutable-table cap and every legacy answer key.
6689
- from core import user_tables
6690
- id_limit = None
6691
- if (int(total or 0) > user_tables.MAX_ROWS
6692
- and user_tables.records_mutable(database, st=session.runtime)):
6693
- id_limit = user_tables.MAX_ROWS
6694
- ids_sql = f"{ids_sql} LIMIT {int(id_limit)}"
6695
- record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6696
 
6697
  limits = list(limits) + list(spec.get("limits") or [])
6698
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
6699
- "id_limit": id_limit, "limits": limits}
 
6700
  if scope.get("view_id"):
6701
  out["view_id"] = scope["view_id"]
6702
  else:
6703
  out["filter"] = scope.get("filter")
6704
- out["page"] = _drill_page(session, database, record_ids, offset, limit)
 
 
 
 
 
 
6705
  out["token"] = token
6706
  return out
6707
 
 
259
  if connector_defn is None:
260
  native = _native_table_definition(session, table_key)
261
  if native is not None:
262
+ cur, spec, cols = _native_table_mirror(session, table_key, needed=needed)
263
  return cur, {**spec, "declared_fields": native.get("fields") or []}, cols
264
  raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
265
  fields = list(connector_defn.get("fields") or [])
 
301
  stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
302
  # The store keeps records as {row id: row}. Iterating that dict as a list yields only
303
  # the ids, which materialises an EMPTY table and answers every cell blank.
304
+ rows = (row for row in (stored.values() if isinstance(stored, dict) else stored)
305
+ if isinstance(row, dict))
306
  fields = rel.shipment_fields()
307
  hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
308
  fields = [field for field in fields if field["key"] not in hidden]
309
+ if not stored:
310
  # S75-T59: a full pass stores a rowless definition that reads through the mirror.
311
  spec = rot._source_for(cur, table_key)
312
  if spec is None:
 
419
  return defn
420
 
421
 
422
+ def _native_table_mirror(session, table_key, *, needed=None):
423
  """Reuse Data's scoped grid/spec reader, with a cursor owned by this Sheet request.
424
 
425
  The payload has already applied permanent predicates, allowed IDs and field/BU walls.
426
  Authored Sheet filters still compile below, against only the visible column vocabulary.
427
  """
428
+ import duckdb
429
+ from harness import datastore
430
+ import registry_source
431
  memo = getattr(session, "_sheet_read_memo", None)
432
  if not isinstance(memo, dict) or "sources" not in memo:
433
  raise RuntimeError("A native Sheet source needs a request scope.")
434
  slot = ("native_source", table_key)
435
+ if table_key == 'ut_activity':
436
+ if slot not in memo:
437
+ import routes_odoo_tables as rot
438
+ read = memo['sources'].enter_context(rot._materialised_read(table_key, session,
439
+ {'filters': [], 'filterConj': 'and', 'search': ''}, None))
440
+ spec = dict(read['spec'], fields=read['payload']['fields'],
441
+ scope_applied=True, limits=list(read['limits']))
442
+ memo[slot] = (read['cur'], spec, read['cols'])
443
+ return memo[slot]
444
+ shape = ('native_columns', table_key)
445
+ built = memo.get(shape)
446
+ if slot not in memo or (built is not None and (needed is None or not set(needed) <= built)):
447
+ wanted = None if needed is None else set(needed) | (built or set())
448
+ owned = getattr(session, '_sheet_drill_cursor', None)
449
+ if owned is None:
450
+ cur = memo['sources'].enter_context(datastore.bounded_connection(duckdb.connect(':memory:')))
451
+ session._sheet_drill_cursor = (cur, None)
452
+ else:
453
+ cur = owned[0]
454
+ memo[slot] = registry_source.stored_grid(session, cur, table_key, needed=wanted)
455
+ memo[shape] = wanted
456
  return memo[slot]
457
 
458
 
 
552
  with_rows=False, fields_only=not with_rows, needed=needed)
553
  return memo[slot]
554
  if table_key == "product_data":
555
+ if not with_rows:
556
+ from registry_source import canonical_metadata
557
+ memo[slot] = canonical_metadata(session, table_key)
558
+ return memo[slot]
559
  from routes_products import product_assembly
560
  memo[slot] = product_assembly(session, scope="product",
561
  consume_corrections=False, with_rows=with_rows, needed=needed)
562
  return memo[slot]
563
+ if not with_rows:
564
+ from registry_source import canonical_metadata
565
+ memo[slot] = canonical_metadata(session, table_key)
566
+ return memo[slot]
567
  from routes_customers import grid_assembly
568
  memo[slot] = grid_assembly(session, scope="customer", consume_corrections=False,
569
  with_rows=with_rows, needed=needed)
 
590
  def _sheet_permission_reader(session):
591
  """Lend a captured schema/grant snapshot only inside the current Sheet request."""
592
  memo = getattr(session, "_sheet_read_memo", None)
593
+ if (isinstance(memo, dict) and 'permission_reader' not in memo
594
+ and callable(getattr(session.runtime, 'get_projection', None))):
595
+ _capture_sheet_permissions(session)
596
  return memo.get("permission_reader", session.runtime) if isinstance(memo, dict) else session.runtime
597
 
598
 
 
826
  spec, {"fn": value["agg"], "field": field}, {**columns, field: numeric})[0])
827
  total_rows = int(cur.execute(count_sql, params).fetchone()[0])
828
  tail = count_sql[len("SELECT COUNT(*)"):]
829
+ # Count both dense axes before fetching any grouped records or allocating the matrix.
830
+ def axis_count(expressions):
831
+ if not expressions:
832
+ return 1
833
+ axis = "SELECT " + ", ".join(expressions) + tail
834
+ axis += " GROUP BY " + ", ".join(str(i + 1) for i in range(len(expressions)))
835
+ return int(cur.execute(f"SELECT count(*) FROM ({axis}) axis", params).fetchone()[0])
836
+
837
+ row_count = axis_count(groups[:len(row_fields)])
838
+ column_count = axis_count(groups[len(row_fields):])
839
+ cells_needed = row_count * column_count * len(pivot["values"])
840
+ if cells_needed > _PIVOT_MAX_CELLS:
841
+ raise err(409, "pivot_too_large",
842
+ f"This pivot needs {row_count:,} rows by {column_count:,} columns, "
843
+ f"{cells_needed:,} cells, and one pivot holds at most {_PIVOT_MAX_CELLS:,}. "
844
+ "Move a field to Filters, or use a field with fewer distinct values.")
845
  sql = "SELECT " + ", ".join([*groups, *expressions]) + tail
846
  if groups:
847
  positions = ", ".join(str(index + 1) for index in range(len(groups)))
 
860
  row_keys.setdefault(row_key, len(row_keys))
861
  column_keys.setdefault(column_key, len(column_keys))
862
  values[row_key, column_key] = [rot._agg_number(value) for value in record[len(groups):]]
 
 
 
 
 
 
 
 
863
  # SQL orders the entire tuple, but a column tuple may first occur in a later row group.
864
  # Sort each axis independently using its JSON spelling for deterministic mixed/null keys.
865
  ordered_rows = sorted(row_keys, key=lambda key: json.dumps(key, ensure_ascii=False))
 
1050
  return {"fields": offered, "databases": databases, "unavailable": unavailable}
1051
 
1052
 
1053
+ def _registry_rows(source, fields, overlays, derived):
1054
+ """Project borrowed scoped inputs through the grid's converter in bounded batches."""
1055
+ import itertools
1056
+ import aios_grid
1057
+ source = iter(source)
1058
+ while True:
1059
+ batch = list(itertools.islice(source, 2048))
1060
+ if not batch:
1061
+ return
1062
+ try:
1063
+ if fields:
1064
+ projected = aios_grid.rows_from_pool(batch, fields, overlays, derived=derived)
1065
+ # Release source rows before yielding their independent projections.
1066
+ batch.clear()
1067
+ try:
1068
+ yield from projected
1069
+ finally:
1070
+ projected.clear()
1071
+ del projected
1072
+ else:
1073
+ yield from ({"pid": row["pid"]} for row in batch)
1074
+ finally:
1075
+ batch.clear()
1076
+ del batch
1077
+
1078
+
1079
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
1080
  """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
1081
 
 
1108
  column just because a filter named it would invent an empty column for a name this database
1109
  does not have, and the wall would then compile against nothing and quietly stop restricting.
1110
  """
1111
+ if table_key in {'customer_data', 'product_data'}:
1112
+ from registry_source import canonical_grid
1113
+ return canonical_grid(session, cur, table_key, needed=needed, metric_window=metric_window)
1114
  import odoo_relational as rel
1115
  import routes_odoo_tables as rot
1116
  from modules import sheet as sheet_mod
 
1122
  derived, bounded, limits = _metric_derived(session, table_key, metric_input, metric_window)
1123
  # Authorization has already scoped rows using the full wall vocabulary. Restrict only
1124
  # cell materialization; an empty selection is ids-only, not rows_from_pool's all-fields default.
1125
+ rows = _registry_rows(g["rows_src"], kept, (g.get("ws") or {}).get("overlays"), derived)
1126
+ join_agent = False
1127
  if table_key == "customer_data" and (needed is None or "agent_id" in needed):
1128
  from core import perm_scope
1129
  agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
 
1132
  # Join only to already-scoped customer rows. pid is the actual res.partner id.
1133
  have = rel.columns(cur, "res_partner")
1134
  if "agent_id" in have:
1135
+ join_agent = True
 
1136
  fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
1137
  elif needed is not None and "agent_id" in needed:
1138
  raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
 
1149
  spec_fields = sheet_mod.mirror_fields(kept)
1150
  # The related customer side can contain thousands of permitted rows. Insert its column
1151
  # arrays in one call; per-row execute round trips delayed the first drill page by seconds.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1152
  slot = hashlib.sha1(
1153
  json.dumps([metric_window, None if needed is None else sorted(needed)],
1154
  sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12]
1155
+ spec = rel.registry_grid_spec(cur, f"{table_key}__at_{slot}", spec_fields, rows, bulk=True)
1156
+ if join_agent:
1157
+ cur.execute(f'UPDATE "{spec["table"]}" AS grid SET agent_id = partner.agent_id '
1158
+ 'FROM res_partner AS partner WHERE grid.pid = partner.id')
1159
+ # Window and column identities name the table before insertion, preserving every memo slot.
1160
  # ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal`
1161
  # has to name WHY a column cannot be totalled, and the normalised copy has already thrown that
1162
  # away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds
 
1165
  return cur, spec, rot._filter_columns(spec, spec_fields)
1166
 
1167
 
1168
+ class _PeriodDerived:
1169
+ """Read borrowed metric maps per row without copying their full population."""
1170
+
1171
+ def __init__(self, base, cleared):
1172
+ self.base, self.cleared, self.columns = base, cleared, []
1173
+
1174
+ def get(self, pid, default=None):
1175
+ cells = {key: value for key, value in (self.base.get(pid) or {}).items()
1176
+ if key not in self.cleared}
1177
+ for key, column in self.columns:
1178
+ values = column.get(pid) or {}
1179
+ if key in values:
1180
+ cells[key] = values[key]
1181
+ return cells or default
1182
+
1183
+
1184
  def _metric_derived(session: Session, table_key: str, g, metric_window):
1185
  """`(derived, bounded_keys, unbounded)` for one assembly at the cell's period.
1186
 
 
1214
  seen.update(cells or ())
1215
  live = [f for f in mfields if f.get("key") in seen]
1216
 
1217
+ out = _PeriodDerived(derived, {f.get("key") for f in mfields})
 
 
 
1218
 
1219
  bounded, unbounded = [], {}
1220
  for f in live:
 
1225
  cells = _measure_cells_at(session, table_key, g, rewritten)
1226
  except Exception: # noqa: BLE001
1227
  cells = {}
1228
+ got = any(key in values for values in (cells or {}).values())
 
 
 
 
1229
  if got:
1230
+ out.columns.append((key, cells))
1231
  bounded.append(key)
1232
  else:
1233
  unbounded[key] = sheet_mod.metric_window_limit(
 
1261
  import routes_products as rp
1262
  return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"),
1263
  g.get("today"), "sheet-period", {},
1264
+ rp.product_measures(session),
1265
+ cur=getattr(session.runtime, '_canonical_source_cursor', None))
1266
  from core import measure_resolve
1267
  return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(),
1268
+ g.get("today"), "sheet-period", {},
1269
+ cur=getattr(session.runtime, '_canonical_source_cursor', None))
1270
 
1271
 
1272
  def _offered_view(session: Session, database: str, view_id: str):
 
1624
  source_identity = identity[:3] + (databases,)
1625
  sources = memo.setdefault("activity_sources", {}) if isinstance(memo, dict) else {}
1626
  admitted = sources.setdefault(source_identity, {})
1627
+ policies = memo.setdefault("activity_bindings", {}) if isinstance(memo, dict) else {}
1628
+ bindings = policies.setdefault(source_identity, {})
1629
 
1630
  def cell_allowed(cell):
1631
+ if not isinstance(cell, dict) or cell.get("kind") != "bound":
1632
+ return resolve_cell(cell)
1633
+ signature = json.dumps(cell, sort_keys=True, separators=(",", ":"), allow_nan=False)
1634
+ if signature not in bindings:
1635
+ result = resolve_cell(cell)
1636
+ if len(bindings) >= 256:
1637
+ bindings.clear()
1638
+ bindings[signature] = result
1639
+ return bindings[signature]
1640
+
1641
+ def resolve_cell(cell):
1642
  if cell is None:
1643
  return True
1644
  if not isinstance(cell, dict):
 
4852
  # assembly for another database: that would create a field on the wrong database.
4853
  if database not in ("customer_data", "product_data"):
4854
  refuse("measure_not_offered")
4855
+ g = _registry_assembly(session, database, with_rows=False)
4856
  # Customer assembly gates measure_fields(team_id) on may_metrics. Intersect the catalogue
4857
  # with the served offer too: product has a narrower entity catalogue, and revoked Metric
4858
  # capability makes the account's offer empty. A browser key can never widen either offer.
 
4912
  if (raw["key"] in stored or raw["key"] in personal or raw["key"] in ctx.hidden_keys
4913
  or raw["key"] in {f.get("key") for f in g["fields"]}):
4914
  _forget_sheet_database(session, database)
4915
+ found = existing(_registry_assembly(session, database, with_rows=False).get("fields"))
4916
  if found is not None:
4917
  return column(found)
4918
  refuse("metric_column_conflict", label)
 
4964
  previous = getattr(session, "_sheet_read_memo", None)
4965
  previous_assemblies = getattr(session, "_sheet_assemblies", None)
4966
  previous_context = getattr(session, "_sheet_scope_context", None)
4967
+ previous_cursor = getattr(session, '_sheet_drill_cursor', None)
4968
  sources = contextlib.ExitStack()
4969
  memo = {"today": _sheet_today(session), "sources": sources,
4970
  "stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
 
4984
  session._sheet_read_memo = previous
4985
  session._sheet_assemblies = previous_assemblies
4986
  session._sheet_scope_context = previous_context
4987
+ session._sheet_drill_cursor = previous_cursor
4988
 
4989
 
4990
  def _request_scoped(route):
 
5750
  return entry
5751
 
5752
 
5753
+ def _remaining_row_wall(spec, nodes, conj):
5754
+ """Trust a scoped source only when its declared wall vocabulary covers every operand."""
5755
+ if not spec.get("scope_applied"):
5756
+ return nodes, conj
5757
+ from modules import sheet as sheet_mod
5758
+ if "scope_fields" in spec and sheet_mod.filter_columns_named(nodes) - set(spec["scope_fields"]):
5759
+ raise err(409, "scope_not_applicable", "Your row restriction cannot be applied to this database. Ask an administrator to review it.")
5760
+ return [], "and"
5761
+
5762
+
5763
  def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None):
5764
  """Materialise the far side on the near cursor, then hand metadata to the compiler."""
5765
  from core import perm_scope, relations
 
5806
  raise err(409, "relation_cursor_mismatch",
5807
  f"Relationship '{name}' cannot be queried on the same live connection as "
5808
  f"'{database}'. Retry after both databases share a connection.")
5809
+ wall_nodes, wall_conj = _remaining_row_wall(rspec, wall_nodes, wall_conj)
5810
  hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
5811
  st=session.runtime)
5812
+ # The far-side wall compiles against the full map: a wall column is often hidden from its reader.
5813
+ wall_cols = dict(rcols)
5814
  rcols = {key: value for key, value in rcols.items() if key not in hidden}
5815
  declared = {field["key"]: field for field in rspec.get("fields") or []}
5816
 
 
5832
  raise unknown_field(key)
5833
  return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
5834
  "spec": rspec, "columns": rcols, "where": entry.get("where"),
5835
+ "wall_nodes": wall_nodes, "wall_conj": wall_conj, "wall_columns": wall_cols,
5836
  "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]}
5837
 
5838
 
 
5951
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
5952
  cols = dict(cols)
5953
  if spec.get("scope_applied"):
5954
+ wall_nodes, wall_conj = _remaining_row_wall(spec, wall_nodes, wall_conj)
5955
  unreadable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
5956
  - sheet_mod.related_filter_ids(scope_nodes))
5957
  field = binding["aggregate"].get("field")
5958
+ known_fields = {f.get("key") for f in spec.get("fields", [])}
5959
+ if field and field not in cols and ("declared_fields" in spec or field in known_fields):
5960
  unreadable.add(field)
5961
  if unreadable:
5962
  declared = {field.get("key") for field in spec.get("declared_fields", [])}
 
6390
  raise _refuse(exc)
6391
  _require_reader(session, book)
6392
  database = _require_database(session, binding["database"])
6393
+ fields = list(_registry_assembly(session, database, with_rows=False).get("fields") or [])
6394
  hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
6395
  visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
6396
  requested = binding["fields"]
 
6482
  _DRILL_MIRROR_MAX = 6
6483
 
6484
 
6485
+ @contextlib.contextmanager
6486
+ def _drill_mirror_lock():
6487
+ """Include shared-cursor contention in the caller's absolute request budget."""
6488
+ from core import request_timing
6489
+ timeout = request_timing.remaining_budget()
6490
+ acquired = (_DRILL_MIRROR_LOCK.acquire() if timeout is None
6491
+ else _DRILL_MIRROR_LOCK.acquire(timeout=timeout))
6492
+ if not acquired:
6493
+ request_timing.remaining_budget()
6494
+ budget = request_timing.current_budget()
6495
+ error = budget[1] if budget is not None else request_timing.RequestDeadlineExceeded
6496
+ raise error('drill mirror lock deadline exceeded')
6497
+ try:
6498
+ request_timing.remaining_budget()
6499
+ yield
6500
+ finally:
6501
+ _DRILL_MIRROR_LOCK.release()
6502
+
6503
+
6504
  def _drill_tokens(session, databases):
6505
  """Unknown freshness disables reuse. No workspace documents are retained here."""
6506
  from core import sheet_binding
6507
+ from core.request_timing import raise_if_resource_error
6508
  tokens = {}
6509
  for db in sorted(databases):
6510
  try:
6511
  import harness.mirror_tokens # noqa: F401
6512
  tokens[db] = sheet_binding.database_change_token(_store(session), db)
6513
  except Exception as exc:
6514
+ raise_if_resource_error(exc)
6515
  _token_unread(session, db, exc)
6516
  tokens[db] = None
6517
  return tokens
 
6521
  def _drill_mirror(session, database, window, needed, databases):
6522
  """A small reader-scoped LRU of query mirrors, with serialized cursor use and eviction."""
6523
  from core import sheet_binding, perm_scope
6524
+ from core.request_timing import raise_if_resource_error
6525
  import datetime
6526
 
6527
  if _native_table_definition(session, database) is not None:
 
6538
  if callable(stamp):
6539
  try:
6540
  wall = stamp(session.tenant, session.uname, database, user=session.user)
6541
+ except Exception as exc:
6542
+ raise_if_resource_error(exc)
6543
  pass
6544
  if not wall:
6545
  token = None
 
6552
  "_sheet_drill_entries")
6553
  absent = object()
6554
  previous = {name: getattr(session, name, absent) for name in attrs}
6555
+ with _drill_mirror_lock():
6556
  hit = _DRILL_MIRRORS.get(key) if token else None
6557
  session._sheet_drill_isolated = True
6558
  session._sheet_drill_cursor = hit["owned"] if hit else None
 
6598
  setattr(session, name, value)
6599
 
6600
 
6601
+ def _drill_page(session, database, record_ids, offset, limit, total):
6602
  """Use fetchTableWindow's server door, including its row and field walls and wire shape."""
6603
  import routes_odoo_tables as rot
6604
  from harness import datastore
6605
 
6606
  size = min(limit, datastore.WINDOW_MAX)
6607
+ ids = record_ids
6608
+ import odoo_relational as rel
6609
+ stored = (database.startswith('ut_') and database not in {'ut_activity', rel.SHIPMENTS_KEY}
6610
+ and not database.startswith('ut_amzn_') and rot._materialised_table(database))
6611
+ if ids and stored:
6612
+ import registry_source
6613
+ payload = registry_source.stored_page(session, database, ids)
6614
+ else:
6615
+ payload = rot.odoo_table_rows(database, offset=0, limit=size, filters=None,
6616
+ filterConj="and", sorts=None, search=None, related=None, groupBy=None, groupKey=None,
6617
+ session=session, body={"ids": ids}) if ids else {"rows": [], "fields": []}
6618
  by_id = {int(row["pid"]): row for row in payload["rows"]}
6619
  limits = list(payload.get("limits") or [])
6620
  if size != limit:
 
6622
  "cause": f"One response carries at most {datastore.WINDOW_MAX} rows.",
6623
  "recommendation": "Continue paging with offset. Every matching row remains available."})
6624
  return {**payload, "offset": offset, "limit": size, "limits": limits,
6625
+ "total": total, "totalUnfiltered": total,
6626
  "rows": [by_id[pid] for pid in ids if pid in by_id]}
6627
 
6628
 
6629
+ def _drill_query(raw):
6630
+ """Accept structured narrowing and presentation options, never caller SQL."""
6631
+ import routes_odoo_tables as rot
6632
+ import aios_grid
6633
+ try:
6634
+ value = json.loads(raw) if isinstance(raw, str) else {}
6635
+ except ValueError:
6636
+ raise err(400, "bad_drill_query", "The record query could not be read. Reopen View data.")
6637
+ allowed = {"filters", "filterConj", "sorts", "search", "ids", "group", "aggs",
6638
+ "groupBy", "groupOffset", "groupLimit"}
6639
+ if not isinstance(value, dict) or set(value) - allowed:
6640
+ raise err(400, "bad_drill_query", "Use record filters and paging options for this query.")
6641
+ query = {**value, **rot._selection_query(value)}
6642
+ budget = [0]
6643
+
6644
+ def check_nodes(nodes, depth=1):
6645
+ # Calendar range queries add one group and one date rule to an authored filter tree.
6646
+ if (depth > aios_grid.MAX_FILTER_DEPTH + 1 or not isinstance(nodes, list)
6647
+ or len(nodes) > aios_grid.MAX_FILTER_SIBLINGS):
6648
+ raise err(400, "bad_drill_query", "Use a valid record filter tree.")
6649
+ for node in nodes:
6650
+ budget[0] += 1
6651
+ if budget[0] > aios_grid.MAX_FILTER_NODES + 2 or not isinstance(node, dict):
6652
+ raise err(400, "bad_drill_query", "Use a valid record filter tree.")
6653
+ if "children" in node:
6654
+ if node.get("conj", "and") not in ("and", "or"):
6655
+ raise err(400, "bad_drill_query", "Choose all or any for this filter group.")
6656
+ check_nodes(node["children"], depth + 1)
6657
+ elif (not isinstance(node.get("colId"), str)
6658
+ or node.get("op") not in aios_grid.FILTER_OPS | aios_grid.COHORT_OPS | aios_grid.VIEW_OPS):
6659
+ raise err(400, "bad_drill_query", "Choose a field and condition for this filter.")
6660
+
6661
+ check_nodes(query["filters"])
6662
+ if "ids" in value:
6663
+ query["ids"] = rot._record_ids(value)
6664
+ if "aggs" in value and (not isinstance(value["aggs"], list)
6665
+ or len(value["aggs"]) > rot.MAX_AGGREGATES
6666
+ or any(not isinstance(agg, dict) or not isinstance(agg.get("colId"), str)
6667
+ or not isinstance(agg.get("agg"), str) for agg in value["aggs"])):
6668
+ raise err(400, "bad_drill_query", "Choose a list of field summaries.")
6669
+ if query.get("groupBy") is not None and not isinstance(query["groupBy"], str):
6670
+ raise err(400, "bad_drill_query", "Choose a grouping field.")
6671
+ for key, low, high in (("groupOffset", 0, None), ("groupLimit", 1, rot.MAX_GROUPS)):
6672
+ if key in query and (type(query[key]) is not int or query[key] < low
6673
+ or (high is not None and query[key] > high)):
6674
+ raise err(400, "bad_drill_query", "Choose a valid group offset and page size.")
6675
+ if query.get("group") is not None and (not isinstance(query["group"], dict)
6676
+ or set(query["group"]) != {"by", "key"}
6677
+ or not isinstance(query["group"]["by"], str)):
6678
+ raise err(400, "bad_drill_query", "Choose a grouping field and value.")
6679
+ if any(not isinstance(sort, dict) or not isinstance(sort.get("colId"), str)
6680
+ or sort.get("dir") not in ("asc", "desc") for sort in query["sorts"]):
6681
+ raise err(400, "bad_drill_query", "Choose a field and direction for each sort.")
6682
+ return query
6683
+
6684
+
6685
+ def _drill_read_sql(session, database, cur, spec, cols, count_sql, params, query, limits):
6686
+ """Intersect grid options with the cell population before counting or paging."""
6687
+ import routes_odoo_tables as rot
6688
+ source = spec.get("from_sql") or spec["table"]
6689
+ prefix = f"SELECT COUNT(*) FROM {source}"
6690
+ if not count_sql.startswith(prefix):
6691
+ raise err(409, "scope_not_applicable", "This cell's record scope cannot be paged.")
6692
+ where = count_sql[len(prefix):].strip()
6693
+ where = where[len("WHERE "):] if where else "TRUE"
6694
+ where = f"({where})"
6695
+ params = list(params)
6696
+ tree = query["filters"]
6697
+ named = rot._leaf_cols(tree) - rot._PSEUDO_COLS
6698
+ named.update(sort["colId"] for sort in query["sorts"])
6699
+ if query.get("group"):
6700
+ named.add(query["group"]["by"])
6701
+ if named - set(cols):
6702
+ raise err(403, "field_unreadable", "This record query names an unavailable field.")
6703
+ try:
6704
+ had_cohort = bool(rot._active_cohort_leaves(tree))
6705
+ tree, sets = rot._views_as_cohorts(tree, database, session, spec, cols,
6706
+ where, params, cur, limits)
6707
+ tree, sets = rot._ranks_as_cohorts(tree, query["filterConj"], spec, cols,
6708
+ where, params, cur, sets, had_cohort, rot._today(), limits)
6709
+ pred = rot._fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols,
6710
+ id_sql=spec["id"], today=rot._today(), cohort_sets=sets)
6711
+ if pred is not None:
6712
+ if pred.uses_aggregate:
6713
+ raise ValueError("A record filter cannot use an aggregate.")
6714
+ where += f" AND ({pred.sql})"
6715
+ params.extend(pred.params)
6716
+ if query["search"].strip():
6717
+ search = rot._fs().compile_search(query["search"].strip(), cols)
6718
+ if search is not None:
6719
+ where += f" AND ({search.sql})"
6720
+ params.extend(search.params)
6721
+ if query.get("group"):
6722
+ group = query["group"]
6723
+ sql, values = rot._group_pin(cols, group["by"], json.dumps(group["key"]))
6724
+ where += f" AND ({sql})"
6725
+ params.extend(values)
6726
+ if query.get("ids") is not None:
6727
+ ids = query["ids"]
6728
+ where += f" AND {spec['id']} IN (SELECT unnest(?::BIGINT[]))"
6729
+ params.append(ids)
6730
+ order = rot._fs().compile_order_by(query["sorts"], cols, tiebreak_sql=spec["id"])
6731
+ except ValueError as exc:
6732
+ raise err(400, "filter_unsupported", str(exc))
6733
+ return source, f"({where})", params, order or spec["id"]
6734
+
6735
+
6736
  def _drill_draft_binding(cell, binding, draft_filter):
6737
  """The stored binding with View data's UNSAVED filter in place of its own (S68-T23 finding).
6738
 
 
6766
  def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
6767
  session: Session = Depends(require_session),
6768
  offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=1),
6769
+ draft_filter: str | None = Query(default=None, alias="filter"),
6770
+ query: str | None = Query(default=None)):
6771
+ """Return exact counts and one SQL page, with page IDs explicitly marked as incomplete."""
6772
+ from core import sheets, sheet_binding
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6773
  from modules import sheet as sheet_mod
6774
  import core.perm_scope as perm_scope
6775
+ import routes_odoo_tables as rot
6776
+ from harness import datastore
6777
 
6778
  # Python callers omit FastAPI's Query sentinels. Offset has no maximum; only the page does.
6779
  offset = offset if isinstance(offset, int) else 0
6780
  limit = limit if isinstance(limit, int) else 100
6781
  draft_filter = draft_filter if isinstance(draft_filter, str) else None
6782
+ query = _drill_query(query)
6783
  if offset < 0 or limit < 1:
6784
  raise err(400, "bad_drill_page", "Choose a nonnegative record offset and a positive page size.")
6785
 
 
6850
  raise err(400, exc.code, exc.message)
6851
  needed = (sheet_mod.filter_columns_named(scope_nodes)
6852
  | sheet_mod.filter_columns_named(wall_nodes))
6853
+ needed.update(rot._leaf_cols(query["filters"]) - rot._PSEUDO_COLS)
6854
+ needed.update(sort["colId"] for sort in query["sorts"])
6855
+ needed.update(agg.get("colId") for agg in query.get("aggs", []) if agg.get("colId"))
6856
+ if query.get("groupBy"):
6857
+ needed.add(query["groupBy"])
6858
+ if query.get("group"):
6859
+ needed.add(query["group"]["by"])
6860
+ for leaf in rot._fe()._rank_leaves(query["filters"], {key: {} for key in needed}):
6861
+ _mode, group_field = rot._fe()._rank_scope(leaf, None)
6862
+ if group_field:
6863
+ needed.add(group_field)
6864
+ if query["search"].strip():
6865
+ from routes_grid import _source_fields
6866
+ needed.update(field["key"] for field in _source_fields(session, database))
6867
  related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
6868
  databases = {database}
6869
  databases.update(_binding_related_entry(session, database, via)["target"]
6870
  for via in sheet_mod.related_vias(scope_nodes))
6871
+ sources = _drill_tokens(session, databases)
6872
  with _drill_mirror(session, database, window, needed, databases) as (made, token):
6873
  cur, spec, cols = made
6874
  cols = dict(cols)
6875
  # Apply the aggregate's field wall before compiling either the count or the IDs.
6876
  hidden = perm_scope.hidden_keys(session.user, database, spec.get("fields") or [], st=session.runtime)
6877
+ _require_aggregate_field(session, binding, hidden=hidden)
6878
  if spec.get("scope_applied"):
6879
  wall_nodes, wall_conj = [], "and"
6880
  unavailable = (sheet_mod.filter_columns_named(scope_nodes) - set(cols)
 
6893
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
6894
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
6895
  scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
 
 
 
6896
  except sheet_mod.SheetRefusal as exc:
6897
  raise err(400, exc.code, exc.message)
6898
  except ValueError:
 
6901
  "connection. Edit the scope or ask an administrator to update the restriction.")
6902
  if spec.get("scope_applied") and limits:
6903
  raise err(409, "scope_not_applicable", "This cell's filter cannot be applied to this database.")
6904
+ cols = {key: value for key, value in cols.items() if key not in hidden}
6905
+ population_total = int(cur.execute(count_sql, count_params).fetchone()[0])
6906
+ source, where, params, order = _drill_read_sql(
6907
+ session, database, cur, spec, cols, count_sql, count_params, query, limits)
6908
+ narrowed = bool(query["filters"] or query["search"].strip() or query.get("group")
6909
+ or query.get("ids") is not None)
6910
+ total = int(cur.execute(f"SELECT count(*) FROM {source} WHERE {where}", params).fetchone()[0]) \
6911
+ if narrowed else population_total
6912
+ size = min(limit, datastore.WINDOW_MAX)
6913
+ summary = None
6914
+ if "aggs" in query:
6915
+ summary = rot._aggregate_over(cur, spec, cols, where, params, {"from_sql": source},
6916
+ query["aggs"], group_by=query.get("groupBy"), limits=limits,
6917
+ group_offset=query.get("groupOffset", 0), group_limit=query.get("groupLimit"))
6918
+ record_ids = []
6919
+ else:
6920
+ ids_sql = (f"SELECT {spec['id']} FROM {source} WHERE {where} ORDER BY {order} "
6921
+ "LIMIT ? OFFSET ?")
6922
+ record_ids = [int(row[0]) for row in cur.execute(ids_sql, [*params, size, offset]).fetchall()]
6923
+ try:
6924
+ wall_stamp = perm_scope.wall_stamp(session.tenant, session.uname, database, user=session.user)
6925
+ except Exception as exc:
6926
+ from core.request_timing import raise_if_resource_error
6927
+ raise_if_resource_error(exc)
6928
+ wall_stamp = None
6929
+ population_query = {key: value for key, value in query.items()
6930
+ if key not in {"groupOffset", "groupLimit"}}
6931
+ population = sheet_binding.fingerprint([session.tenant, session.uname, session.user,
6932
+ binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
6933
+ if all(sources.values()) and wall_stamp else None
6934
 
6935
  limits = list(limits) + list(spec.get("limits") or [])
6936
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
6937
+ "record_ids_scope": "page", "id_limit": None, "limits": limits,
6938
+ "population": population, "query": query}
6939
  if scope.get("view_id"):
6940
  out["view_id"] = scope["view_id"]
6941
  else:
6942
  out["filter"] = scope.get("filter")
6943
+ if summary is not None:
6944
+ out["summary"] = {"ok": True, **summary}
6945
+ else:
6946
+ out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
6947
+ out["page"]["totalUnfiltered"] = population_total
6948
+ if _drill_tokens(session, databases) != sources:
6949
+ raise err(409, "drill_population_changed", "These records changed while loading. Try again.")
6950
  out["token"] = token
6951
  return out
6952
 
api/routes_tables.py CHANGED
@@ -252,7 +252,12 @@ def _table_lend(runtime, table_key, defs_only=False):
252
  ut = _ut()
253
  if defs_only:
254
  # Metadata must enter through the store projection, even on entry-capable stores.
255
- return ut.lend_defs(runtime)
 
 
 
 
 
256
  if _ut()._entry_reader(runtime) is None:
257
  return ut.lend(runtime)
258
  entry = _table_entry(runtime, table_key)
@@ -578,6 +583,9 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
578
  if isinstance(documents, dict):
579
  context_store = _ut().lend(context_store, **{
580
  name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
 
 
 
581
  from copy import copy
582
  field_session = copy(session)
583
  field_session.runtime = context_store
@@ -621,9 +629,10 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
621
  if windowed:
622
  connected.add(database)
623
  elif database in ('customer_data', 'product_data'):
624
- from routes_grid import _builtin_history_context
625
- _, g, _ = _builtin_history_context(
626
- session, 'product' if database == 'product_data' else 'customer')
 
627
  else:
628
  g = None
629
  contexts[database] = g
@@ -636,8 +645,26 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
636
  contexts[database] = None
637
  except perm_scope.Unresolvable:
638
  contexts[database] = None
 
 
 
 
 
639
  return contexts[database]
640
 
 
 
 
 
 
 
 
 
 
 
 
 
 
641
  def partner(obj):
642
  nonlocal applications
643
  identity = json.dumps(obj, sort_keys=True)
@@ -688,9 +715,14 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
688
  if wanted:
689
  from routes_odoo_tables import _relation_rows
690
  try:
691
- contexts[database]['pids'] = frozenset(
692
- row['pid'] for row in _relation_rows(
693
- session, database, {'pid'}, ids=wanted, st=context_store))
 
 
 
 
 
694
  except HTTPException as exc:
695
  if exc.status_code not in (403, 404, 409):
696
  raise
@@ -704,7 +736,7 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
704
  continue
705
  source_pids[database] = frozenset(str(pid) for pid in contexts[database]['pids'])
706
  scoped = perm_scope.apply_row_scope(raw['records'], session.user, key, own['fields'],
707
- st=session.runtime, defn=own)
708
  for event in scoped:
709
  obj = event['object']
710
  clean = None
@@ -725,8 +757,8 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
725
  previous = pair.get('before')
726
  declared = {f['key'] for f in (g.get('defn') or {}).get('fields', g['fields'])}
727
  if field_key not in declared and isinstance(previous, dict):
728
- corpus = list(g['fields']) + [previous]
729
- hidden = _ut_hidden(session, database, corpus, st=session.runtime)
730
  if field_key not in hidden and field_key not in g.get('hidden', ()):
731
  visible = visible | {field_key}
732
  admitted = field_key in visible
@@ -737,7 +769,7 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
737
  and database not in connected
738
  and changes and all(pair.get('after') is None for pair in changes.values())
739
  and not perm_scope.row_scope_applies(
740
- session.user, database, st=session.runtime, defn=g['defn'])):
741
  admitted = True
742
  if admitted:
743
  clean = (changes if changes.keys() <= visible else
@@ -1283,7 +1315,7 @@ def scoped_pids(session: Session, table_key: str, limits=None, st=None):
1283
  def ut_write_ctx(session: Session, table_key: str):
1284
  """The events transport borrows the same rows-free assembly as workspace and Sheet."""
1285
  return _compute_ut_assembly(session, table_key, consume_corrections=False,
1286
- with_rows=False, include_usage=False)
1287
 
1288
 
1289
  def _ut_hidden(session, table_key, fields, st=None):
@@ -1367,7 +1399,9 @@ def _ut_shared_cells(session, table_key, pids):
1367
  from core import shared_overlay
1368
  try:
1369
  return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime)
1370
- except Exception: # noqa: BLE001
 
 
1371
  return {}
1372
 
1373
 
@@ -1718,6 +1752,15 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
1718
  consume_corrections: bool = False, with_rows: bool = True, st=None,
1719
  *, fields_only: bool = False, needed=None):
1720
  if not with_rows and st is None:
 
 
 
 
 
 
 
 
 
1721
  from routes_customers import grid_assembly
1722
  return grid_assembly(session, table_key, storage_key,
1723
  build=lambda borrowed: _compute_ut_assembly(
@@ -1727,10 +1770,54 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
1727
  fields_only=fields_only, needed=needed)
1728
 
1729
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1730
  def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
1731
  consume_corrections: bool = False, with_rows: bool = True, st=None,
1732
  *, fields_only: bool = False, needed=None, include_usage: bool = True,
1733
- include_measures: bool = True, include_views: bool = True):
 
1734
  """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
1735
  `/workspace` and the events route consume any of the three interchangeably.
1736
 
@@ -1750,9 +1837,8 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1750
  * every materialised `ut_*` database stops allocating a dict per row and sorting them on a
1751
  route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per
1752
  database switch (owner item 7).
1753
- ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers
1754
- the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved
1755
- on the write door.
1756
  """
1757
  import aios_grid
1758
  from core import grid_events
@@ -1765,8 +1851,13 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1765
  # reading that bucket once per question asked of it.
1766
  if st is None:
1767
  st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
 
1768
  if fields_only:
1769
  defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
 
 
 
 
1770
  fields_base = list(defn.get("fields") or [])
1771
  pids, rows_src = frozenset(), []
1772
  elif with_rows:
@@ -1774,7 +1865,14 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1774
  # a READ route passes one — see `scoped_pool`'s own note and contract C5.
1775
  pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
1776
  else:
1777
- pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st)
 
 
 
 
 
 
 
1778
  rows_src = []
1779
 
1780
  # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it
@@ -1791,10 +1889,18 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1791
  # that happen to carry a scoped `table`.
1792
  st=session.runtime,
1793
  scope_key=table_key, table=ops)
 
 
 
1794
  ws = grid_events.table_workspace(ctx, allowed_pids=pids,
1795
  consume_corrections=consume_corrections,
1796
  capture_document=True,
1797
  **({"include_views": False} if not include_views else {}))
 
 
 
 
 
1798
  workspace, fields, views, lists = aios_grid.workspace_wire(
1799
  ws, session.uname, set(pids), defs={}, scope_key=table_key,
1800
  storage_key=storage_key, fields_base=fields_base, st=st)
@@ -1932,7 +2038,9 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1932
  derived = aios_grid.cohort_cells(lists) if not fields_only else {}
1933
  materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
1934
  for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures,
1935
- getattr(session.runtime, 'measure_memo', {}))
 
 
1936
  if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
1937
  derived.setdefault(_pid, {}).update(_cells)
1938
 
@@ -2026,7 +2134,7 @@ def ut_measures(table_key, session, *, include_offer=True):
2026
  return []
2027
 
2028
 
2029
- def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None):
2030
  """`{pid: {field key: value}}` for this database's Metric columns.
2031
 
2032
  ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need
@@ -2059,11 +2167,12 @@ def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None)
2059
  out = {}
2060
  for (dfrom, dto), group in by_window.items():
2061
  keys = tuple(sorted({f["measure"]["key"] for f in group}))
2062
- mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys)
2063
  if mkey not in memo:
2064
  try:
2065
  memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom,
2066
- date_to=dto, team_id=team_id, offer=offer)
 
2067
  except Exception as e: # noqa: BLE001
2068
  _ut_measure_err("ut-measure-column", e)
2069
  transient = False
@@ -2246,38 +2355,18 @@ def _module_link_targets(session, meta=None, st=None):
2246
  @router.get("/tables")
2247
  def list_tables(brief: bool = Query(False),
2248
  session: Session = Depends(require_session)):
2249
- """This session's user tables — the list the '+ New database' surface renders.
2250
-
2251
- ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This
2252
- route was never ticketed and has the same shape `/nav` and `/automations` were fixed for:
2253
- `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on
2254
- tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant
2255
- with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked
2256
- about every table; it is handed the document this function already holds. See
2257
- `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not
2258
- available.
2259
-
2260
- `brief=1` is for a link-field picker. It needs the tables it may target and their schemas,
2261
- not row counts or record mutability. Reading a full tenant document to decorate that one
2262
- field editor would move every stored row through Neon for no visible result, so this branch
2263
- keeps the server-side permission wall but lends its rows-free projection instead.
2264
-
2265
- ⭐⭐ W41-T15 / R9 / C5 (2026-08-24) — AND `brief=1` NOW OFFERS THE TWO REGISTRY MODULES TOO
2266
- (`_module_link_targets`), because `_clean_link` accepts them as link targets and this list is
2267
- the only producer of a link bag in the product.
2268
-
2269
- ⛔ THE BRIEF BRANCH ONLY, and the asymmetry is deliberate. The full listing is the "+ New
2270
- database" management surface: it renders row counts, `createdBy` and a delete affordance over
2271
- things a tenant OWNS, and `customer_data` is a compiled registry row with its own nav entry
2272
- and no such lifecycle. Offering it there would put a database nobody can delete into the list
2273
- of databases you delete from. Two callers, two questions, one route.
2274
- """
2275
  from core.if_field import project_fields
2276
  ut = _ut()
2277
  meta = nav_meta(session)
2278
  out = []
2279
- tables = (ut.all_defs(st=session.runtime) if brief
2280
- else ut.all_tables(st=session.runtime))
 
 
 
 
 
2281
  lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables})
2282
  for key, t in sorted(tables.items(),
2283
  key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
@@ -2297,7 +2386,8 @@ def list_tables(brief: bool = Query(False),
2297
  "recordsMutable": ut.records_mutable(key, st=lent),
2298
  "createdBy": t.get("createdBy") or "",
2299
  "created": t.get("created") or "",
2300
- "rowCount": len(t.get("rows") or {})})
 
2301
  out.append(row)
2302
  if brief:
2303
  # ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
@@ -2710,6 +2800,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
2710
  'recommendation': 'Use Next to read older events.'}]}
2711
 
2712
  _ig_forward(session, table_key)
 
2713
  # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED.
2714
  # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE
2715
  # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy),
@@ -2721,17 +2812,17 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
2721
  # value it replaced (contract C5).
2722
  if _ut()._entry_reader(session.runtime) is None:
2723
  _lent = _table_lend(session.runtime, table_key)
2724
- g = ut_assembly(session, table_key, st=_lent, consume_corrections=False)
2725
  else:
2726
- from dataclasses import replace
2727
  from routes_customers import _AssemblyStore
2728
  # One request owns these borrowed documents. A cold memo build and its page share reads;
2729
  # a warm page reads records/shared cells afresh and lends Zone documents to both readers.
2730
  original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore)
2731
  else session.runtime)
2732
  if not isinstance(session.runtime, _AssemblyStore):
2733
- session = replace(session, runtime=_AssemblyStore(original_rt))
2734
- g = ut_assembly(session, table_key, with_rows=False)
2735
  assembly_rt = session.runtime
2736
  entry_key = (_ut().STORE_KEY, ("entry", str(table_key)))
2737
  if entry_key in assembly_rt.docs:
@@ -2743,7 +2834,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
2743
  _lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {})
2744
  # The page borrows the same envelope as workspace; only records and shared cells are fresh.
2745
  pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent)
2746
- g['pids'] &= pids
2747
  from core import perm_scope
2748
  g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src
2749
  if row['pid'] in g['pids']]
@@ -2805,8 +2896,11 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
2805
  # honestly instead of showing half a truncated brace.
2806
  # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole,
2807
  # so a tenant using `json` for a short config sees no change at all.
2808
- rows = aios_grid.rows_from_pool(
2809
- g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"])
 
 
 
2810
  rows = _ut().zone_cells(table_key, rows, allowed_row_ids=g["pids"], st=_lent)
2811
  # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
2812
  # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
 
252
  ut = _ut()
253
  if defs_only:
254
  # Metadata must enter through the store projection, even on entry-capable stores.
255
+ try:
256
+ return ut.lend_defs(runtime, schema_only=True)
257
+ except Exception as exc:
258
+ from core.request_timing import raise_if_resource_error
259
+ raise_if_resource_error(exc)
260
+ raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc
261
  if _ut()._entry_reader(runtime) is None:
262
  return ut.lend(runtime)
263
  entry = _table_entry(runtime, table_key)
 
583
  if isinstance(documents, dict):
584
  context_store = _ut().lend(context_store, **{
585
  name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
586
+ from routes_customers import _AssemblyStore
587
+ # Lend the projected catalogue outside the assembly reader's per-read copy boundary.
588
+ context_store = _ut().lend_defs(_AssemblyStore(context_store, schema_only=True), schema_only=True)
589
  from copy import copy
590
  field_session = copy(session)
591
  field_session.runtime = context_store
 
629
  if windowed:
630
  connected.add(database)
631
  elif database in ('customer_data', 'product_data'):
632
+ from routes_grid import _builtin_activity_metadata
633
+ g = _builtin_activity_metadata(field_session,
634
+ 'product' if database == 'product_data' else 'customer', st=context_store)
635
+ page_sources.add(database)
636
  else:
637
  g = None
638
  contexts[database] = g
 
645
  contexts[database] = None
646
  except perm_scope.Unresolvable:
647
  contexts[database] = None
648
+ if database in ('customer_data', 'product_data'):
649
+ from copy import deepcopy
650
+ assemblies = getattr(session, '_sheet_assemblies', None)
651
+ if isinstance(assemblies, dict):
652
+ assemblies[(database, 'fields')] = deepcopy(contexts[database])
653
  return contexts[database]
654
 
655
+ assemblies = getattr(session, '_sheet_assemblies', None)
656
+ if isinstance(assemblies, dict):
657
+ class ActivityBuiltinFields(dict):
658
+ def __contains__(self, slot):
659
+ # Sheet asks for a fields slot before reading any builtin binding values.
660
+ if (isinstance(slot, tuple) and len(slot) == 2 and slot[1] == 'fields'
661
+ and slot[0] in ('customer_data', 'product_data')
662
+ and not super().__contains__(slot)):
663
+ source(slot[0])
664
+ return super().__contains__(slot)
665
+
666
+ session._sheet_assemblies = ActivityBuiltinFields(assemblies)
667
+
668
  def partner(obj):
669
  nonlocal applications
670
  identity = json.dumps(obj, sort_keys=True)
 
715
  if wanted:
716
  from routes_odoo_tables import _relation_rows
717
  try:
718
+ if database in ('customer_data', 'product_data'):
719
+ from routes_grid import _builtin_activity_pids
720
+ contexts[database]['pids'] = _builtin_activity_pids(
721
+ session, database, wanted, contexts[database], st=context_store)
722
+ else:
723
+ contexts[database]['pids'] = frozenset(
724
+ row['pid'] for row in _relation_rows(
725
+ session, database, {'pid'}, ids=wanted, st=context_store))
726
  except HTTPException as exc:
727
  if exc.status_code not in (403, 404, 409):
728
  raise
 
736
  continue
737
  source_pids[database] = frozenset(str(pid) for pid in contexts[database]['pids'])
738
  scoped = perm_scope.apply_row_scope(raw['records'], session.user, key, own['fields'],
739
+ st=context_store, defn=own)
740
  for event in scoped:
741
  obj = event['object']
742
  clean = None
 
757
  previous = pair.get('before')
758
  declared = {f['key'] for f in (g.get('defn') or {}).get('fields', g['fields'])}
759
  if field_key not in declared and isinstance(previous, dict):
760
+ corpus = list((g.get('defn') or {}).get('fields', g['fields'])) + [previous]
761
+ hidden = _ut_hidden(session, database, corpus, st=context_store)
762
  if field_key not in hidden and field_key not in g.get('hidden', ()):
763
  visible = visible | {field_key}
764
  admitted = field_key in visible
 
769
  and database not in connected
770
  and changes and all(pair.get('after') is None for pair in changes.values())
771
  and not perm_scope.row_scope_applies(
772
+ session.user, database, st=context_store, defn=g['defn'])):
773
  admitted = True
774
  if admitted:
775
  clean = (changes if changes.keys() <= visible else
 
1315
  def ut_write_ctx(session: Session, table_key: str):
1316
  """The events transport borrows the same rows-free assembly as workspace and Sheet."""
1317
  return _compute_ut_assembly(session, table_key, consume_corrections=False,
1318
+ with_rows=False, include_usage=False, metadata_only=False)
1319
 
1320
 
1321
  def _ut_hidden(session, table_key, fields, st=None):
 
1399
  from core import shared_overlay
1400
  try:
1401
  return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime)
1402
+ except Exception as exc: # noqa: BLE001
1403
+ from core.request_timing import raise_if_resource_error
1404
+ raise_if_resource_error(exc)
1405
  return {}
1406
 
1407
 
 
1752
  consume_corrections: bool = False, with_rows: bool = True, st=None,
1753
  *, fields_only: bool = False, needed=None):
1754
  if not with_rows and st is None:
1755
+ if not fields_only:
1756
+ import routes_odoo_tables as connected
1757
+ # Reuse revision-validated schema bytes; classification must not reread native rows.
1758
+ definitions, _token = connected._page_defs(session)
1759
+ defn = _ut().get(table_key, st=definitions)
1760
+ if not _ut().materialises(table_key, st=definitions, defn=defn):
1761
+ # Source membership changes independently of stored and workspace revisions.
1762
+ return _compute_ut_assembly(session, table_key, storage_key, False, False,
1763
+ st=definitions, fields_only=False, needed=needed)
1764
  from routes_customers import grid_assembly
1765
  return grid_assembly(session, table_key, storage_key,
1766
  build=lambda borrowed: _compute_ut_assembly(
 
1770
  fields_only=fields_only, needed=needed)
1771
 
1772
 
1773
+ def _metadata_pids(session, table_key, ctx, st):
1774
+ """Validate only saved membership IDs through the connected table's ordinary row wall."""
1775
+ from core import grid_events
1776
+ from modules import cohort
1777
+ import routes_odoo_tables as connected
1778
+
1779
+ document = ctx.table.st.get(ctx.table.table_key) or {}
1780
+ if isinstance(st, _ut()._Lent):
1781
+ st._docs[ctx.table.table_key] = document
1782
+ candidates = set()
1783
+
1784
+ def collect(value):
1785
+ if isinstance(value, dict):
1786
+ for key, item in value.items():
1787
+ if key in {"memberPids", "members"} and isinstance(item, (list, tuple)):
1788
+ candidates.update(pid for pid in item if type(pid) is int and pid > 0)
1789
+ elif isinstance(item, (dict, list, tuple)):
1790
+ collect(item)
1791
+ elif isinstance(value, (list, tuple)):
1792
+ for item in value:
1793
+ collect(item)
1794
+
1795
+ collect(document.get(session.uname) or {})
1796
+ from core.table_store import ASSIGNED_KEY
1797
+ assigned = ((document.get(ASSIGNED_KEY) or {}).get("views") or {}).get(session.uname) or {}
1798
+ collect(assigned)
1799
+ collect(grid_events._granted_views(ctx, session.uname))
1800
+ cohorts = cohort.scoped(table_key, st=st)
1801
+ if isinstance(st, _ut()._Lent):
1802
+ st._docs[cohorts.key] = st.get(cohorts.key) or {}
1803
+ collect(cohorts.all_for(session.uname))
1804
+ connected.require_amazon_mirror(table_key, session, st=st)
1805
+ if not candidates:
1806
+ return frozenset()
1807
+ read = connected._compiled_read(table_key, session, st=st)
1808
+ try:
1809
+ rows = connected._relation_rows(session, table_key, [], ids=sorted(candidates), st=st,
1810
+ compiled=read)
1811
+ return frozenset(int(row["pid"]) for row in rows)
1812
+ finally:
1813
+ read["cur"].close()
1814
+
1815
+
1816
  def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = "",
1817
  consume_corrections: bool = False, with_rows: bool = True, st=None,
1818
  *, fields_only: bool = False, needed=None, include_usage: bool = True,
1819
+ include_measures: bool = True, include_views: bool = True,
1820
+ metadata_only: bool = True):
1821
  """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so
1822
  `/workspace` and the events route consume any of the three interchangeably.
1823
 
 
1837
  * every materialised `ut_*` database stops allocating a dict per row and sorting them on a
1838
  route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per
1839
  database switch (owner item 7).
1840
+ Connected metadata validates only explicitly saved membership IDs through the SQL row wall.
1841
+ Write contexts pass `metadata_only=False` and retain complete membership admission.
 
1842
  """
1843
  import aios_grid
1844
  from core import grid_events
 
1851
  # reading that bucket once per question asked of it.
1852
  if st is None:
1853
  st = _table_lend(session.runtime, table_key, defs_only=not with_rows)
1854
+ metadata_membership = False
1855
  if fields_only:
1856
  defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
1857
+ if getattr(st, 'schema_only', False):
1858
+ from copy import deepcopy
1859
+ # Detach selected definitions while the request lends its projected catalogue.
1860
+ defn = deepcopy(defn)
1861
  fields_base = list(defn.get("fields") or [])
1862
  pids, rows_src = frozenset(), []
1863
  elif with_rows:
 
1865
  # a READ route passes one — see `scoped_pool`'s own note and contract C5.
1866
  pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st)
1867
  else:
1868
+ defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
1869
+ metadata_membership = (metadata_only and table_key != "ut_activity" and
1870
+ not _ut().materialises(table_key, st=session.runtime, defn=defn))
1871
+ if metadata_membership:
1872
+ fields_base = list(defn.get("fields") or [])
1873
+ pids = frozenset()
1874
+ else:
1875
+ pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st)
1876
  rows_src = []
1877
 
1878
  # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it
 
1889
  # that happen to carry a scoped `table`.
1890
  st=session.runtime,
1891
  scope_key=table_key, table=ops)
1892
+ if metadata_membership:
1893
+ pids = _metadata_pids(session, table_key, ctx, st)
1894
+ ctx.allowed_pids = pids
1895
  ws = grid_events.table_workspace(ctx, allowed_pids=pids,
1896
  consume_corrections=consume_corrections,
1897
  capture_document=True,
1898
  **({"include_views": False} if not include_views else {}))
1899
+ if metadata_membership or fields_only:
1900
+ ws["views"] = {vid: dict(view, config={**(view.get("config") or {}),
1901
+ "memberPids": [pid for pid in view["config"]["memberPids"] if pid in pids]})
1902
+ if isinstance(view, dict) and isinstance((view.get("config") or {}).get("memberPids"), list)
1903
+ else view for vid, view in (ws.get("views") or {}).items()}
1904
  workspace, fields, views, lists = aios_grid.workspace_wire(
1905
  ws, session.uname, set(pids), defs={}, scope_key=table_key,
1906
  storage_key=storage_key, fields_base=fields_base, st=st)
 
2038
  derived = aios_grid.cohort_cells(lists) if not fields_only else {}
2039
  materialized_fields = fields if needed is None else [f for f in fields if f.get("key") in needed]
2040
  for _pid, _cells in (_ut_measure_cells(table_key, materialized_fields, pids, today, measures,
2041
+ getattr(session.runtime, 'measure_memo', {}),
2042
+ **({'cur': session.runtime._canonical_source_cursor}
2043
+ if getattr(session.runtime, '_canonical_source_cursor', None) is not None else {}))
2044
  if not fields_only and (needed is None or aios_grid.measure_fields_of(materialized_fields)) else {}).items():
2045
  derived.setdefault(_pid, {}).update(_cells)
2046
 
 
2134
  return []
2135
 
2136
 
2137
+ def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None, *, cur=None):
2138
  """`{pid: {field key: value}}` for this database's Metric columns.
2139
 
2140
  ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need
 
2167
  out = {}
2168
  for (dfrom, dto), group in by_window.items():
2169
  keys = tuple(sorted({f["measure"]["key"] for f in group}))
2170
+ mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys, tuple(sorted(pids)))
2171
  if mkey not in memo:
2172
  try:
2173
  memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom,
2174
+ date_to=dto, team_id=team_id, offer=offer, group_keys=pids,
2175
+ **({'cur': cur} if cur is not None else {}))
2176
  except Exception as e: # noqa: BLE001
2177
  _ut_measure_err("ut-measure-column", e)
2178
  transient = False
 
2355
  @router.get("/tables")
2356
  def list_tables(brief: bool = Query(False),
2357
  session: Session = Depends(require_session)):
2358
+ """List permitted definitions with exact stored counts or explicit unknown source populations."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2359
  from core.if_field import project_fields
2360
  ut = _ut()
2361
  meta = nav_meta(session)
2362
  out = []
2363
+ try:
2364
+ tables = ut.all_defs(st=session.runtime, schema_only=True)
2365
+ counts = {} if brief else ut.definition_counts(session.runtime)
2366
+ except Exception as exc:
2367
+ from core.request_timing import raise_if_resource_error
2368
+ raise_if_resource_error(exc)
2369
+ raise err(503, 'schema_unavailable', 'table definitions are unavailable') from exc
2370
  lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables})
2371
  for key, t in sorted(tables.items(),
2372
  key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()):
 
2386
  "recordsMutable": ut.records_mutable(key, st=lent),
2387
  "createdBy": t.get("createdBy") or "",
2388
  "created": t.get("created") or "",
2389
+ "rowCount": (counts.get(key) if ut.materialises(key, defn=t)
2390
+ and type(counts.get(key)) is int and counts[key] >= 0 else None)})
2391
  out.append(row)
2392
  if brief:
2393
  # ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and
 
2800
  'recommendation': 'Use Next to read older events.'}]}
2801
 
2802
  _ig_forward(session, table_key)
2803
+ source_needed = getattr(session.runtime, '_canonical_needed', None)
2804
  # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED.
2805
  # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE
2806
  # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy),
 
2812
  # value it replaced (contract C5).
2813
  if _ut()._entry_reader(session.runtime) is None:
2814
  _lent = _table_lend(session.runtime, table_key)
2815
+ g = ut_assembly(session, table_key, st=_lent, consume_corrections=False, needed=source_needed)
2816
  else:
2817
+ from registry_source import with_runtime
2818
  from routes_customers import _AssemblyStore
2819
  # One request owns these borrowed documents. A cold memo build and its page share reads;
2820
  # a warm page reads records/shared cells afresh and lends Zone documents to both readers.
2821
  original_rt = (session.runtime._runtime if isinstance(session.runtime, _AssemblyStore)
2822
  else session.runtime)
2823
  if not isinstance(session.runtime, _AssemblyStore):
2824
+ session = with_runtime(session, _AssemblyStore(original_rt))
2825
+ g = ut_assembly(session, table_key, with_rows=False, needed=source_needed)
2826
  assembly_rt = session.runtime
2827
  entry_key = (_ut().STORE_KEY, ("entry", str(table_key)))
2828
  if entry_key in assembly_rt.docs:
 
2834
  _lent = _ut().lend(assembly_rt, user_tables={str(table_key): entry} if entry else {})
2835
  # The page borrows the same envelope as workspace; only records and shared cells are fresh.
2836
  pids, rows_src, _fields_base, _defn = scoped_pool(session, table_key, st=_lent)
2837
+ g['pids'] = pids
2838
  from core import perm_scope
2839
  g['rows_src'] = [perm_scope.strip_row(row, g['hidden']) for row in rows_src
2840
  if row['pid'] in g['pids']]
 
2896
  # honestly instead of showing half a truncated brace.
2897
  # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole,
2898
  # so a tenant using `json` for a short config sees no change at all.
2899
+ projection_fields = (g['fields'] if source_needed is None
2900
+ else [field for field in g['fields'] if field['key'] in source_needed])
2901
+ rows = (aios_grid.rows_from_pool(
2902
+ g["rows_src"], projection_fields, _thin_json(projection_fields, merged, table_key), derived=g["derived"])
2903
+ if source_needed is None or projection_fields else [{'pid': row['pid']} for row in g['rows_src']])
2904
  rows = _ut().zone_cells(table_key, rows, allowed_row_ids=g["pids"], st=_lent)
2905
  # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
2906
  # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
api/sheet_warm.py CHANGED
@@ -11,13 +11,15 @@ from __future__ import annotations
11
 
12
  import atexit
13
  import os
 
14
  import threading
15
  import time
 
16
  from datetime import datetime, time as dtime, timedelta, timezone
17
  from time import perf_counter
18
 
19
  import routes_sheet
20
- from core import external_access, perms, shares, sheets, users
21
  from deps import Session
22
 
23
  #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
@@ -357,6 +359,146 @@ def _desk_skip(rt, tenant):
357
  return str(text or "").startswith("pg:") and token == seen
358
 
359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  def desk_run_one(rt, tenant):
361
  """Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
362
  row = routes_sheet._refresh_desk_claim(rt)
@@ -364,6 +506,7 @@ def desk_run_one(rt, tenant):
364
  return False
365
  started = perf_counter()
366
  error = None
 
367
  try:
368
  book = sheets.get_workbook(rt, row["workbook"])
369
  rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
@@ -371,9 +514,11 @@ def desk_run_one(rt, tenant):
371
  if session is None:
372
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
373
  else:
374
- routes_sheet._refresh_desk_run(session, row)
 
375
  except Exception as exc: # noqa: BLE001
376
  error = routes_sheet._refresh_desk_error(exc)
 
377
  routes_sheet._refresh_desk_finish(rt, row["id"], error)
378
  print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
379
  f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
 
11
 
12
  import atexit
13
  import os
14
+ import re
15
  import threading
16
  import time
17
+ import uuid
18
  from datetime import datetime, time as dtime, timedelta, timezone
19
  from time import perf_counter
20
 
21
  import routes_sheet
22
+ from core import external_access, perms, refresh_proof, shares, sheets, users
23
  from deps import Session
24
 
25
  #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
 
359
  return str(text or "").startswith("pg:") and token == seen
360
 
361
 
362
+ DESK_RESULT_PREFIX = "[sheet-desk-result] "
363
+ _DESK_PROOF_REASONS = frozenset((
364
+ "workbook_incomplete", "sheet_incomplete", "sheet_partial", "cell_malformed", "refusal_malformed",
365
+ "identity_mismatch", "original_workbook_missing", "workbook_mismatch", "result_identity_mismatch",
366
+ "inventory_changed", "execution_build_unknown", "attempt_malformed", "attempt_not_committed",
367
+ "viewer_only", "partial_scope", "held_or_unresolved", "editor_commit_unproven",
368
+ "workbook_empty", "authored_definitions_changed", "proof_capacity_exceeded",
369
+ "generated_manifest_invalid", "generated_output_missing", "generated_output_invalid",
370
+ ))
371
+
372
+
373
+ def desk_result_census(book):
374
+ """Validate a complete workbook and count its cells without retaining their values."""
375
+ identity = refresh_proof.snapshot(book)
376
+ return identity["census"], identity["inventory"], identity["unresolved"]
377
+
378
+
379
+ def _desk_input_identity(book):
380
+ try:
381
+ return refresh_proof.snapshot(book)
382
+ except refresh_proof.ProofCapacityError as exc:
383
+ return {"error": "proof_capacity_exceeded", "capacityCause": str(exc)}
384
+ except ValueError as exc:
385
+ return {"error": str(exc) if str(exc) in _DESK_PROOF_REASONS else "cell_malformed"}
386
+ except Exception: # noqa: BLE001
387
+ return {"error": "cell_malformed"}
388
+
389
+
390
+ def desk_result_capture(row, tenant, session, before, result, *, admitted=None):
391
+ """Describe only the original resolver return and the worker's admitted pre-run inventory."""
392
+ out = {"schema": "sheet-desk-result/v1", "source": "_refresh_desk_run.return",
393
+ "executionId": uuid.uuid4().hex,
394
+ "requestId": row.get("id") if isinstance(row.get("id"), str) else None,
395
+ "tenant": tenant if isinstance(tenant, str) else None,
396
+ "reader": row.get("reader") if isinstance(row.get("reader"), str) else None,
397
+ "workbook": row.get("workbook") if isinstance(row.get("workbook"), str) else None,
398
+ "version": os.environ.get("AIOS_VERSION", ""),
399
+ "complete": False, "persisted": False, "reason": None, "revision": None,
400
+ "priorRevision": None, "attempt": None, "census": None, "inventory": None, "sheets": None,
401
+ "authored": None, "priorAuthored": None, "recommendation": None, "capacityCause": None}
402
+ try:
403
+ if (session is None or row.get("tenant") != tenant or session.tenant != tenant
404
+ or session.uname != row.get("reader") or before.get("id") != row.get("workbook")
405
+ or session.user.get("username") != row.get("reader")
406
+ or session.user.get("tenant", tenant) != tenant
407
+ or any(key and key != tenant for key in
408
+ (getattr(session.runtime, "key", None),
409
+ getattr(getattr(session.runtime, "tenant", None), "key", None)))):
410
+ raise ValueError("identity_mismatch")
411
+ if not isinstance(result, dict) or not isinstance(result.get("workbook"), dict):
412
+ raise ValueError("original_workbook_missing")
413
+ book = result["workbook"]
414
+ if book.get("id") != row.get("workbook"):
415
+ raise ValueError("workbook_mismatch")
416
+ if (any(item.get("tenant", tenant) != tenant for item in (result, book))
417
+ or result.get("reader", session.uname) != session.uname):
418
+ raise ValueError("result_identity_mismatch")
419
+ prior = admitted if admitted is not None else _desk_input_identity(before)
420
+ if prior.get("error"):
421
+ out["capacityCause"] = prior.get("capacityCause")
422
+ raise ValueError(prior["error"])
423
+ returned = refresh_proof.snapshot(book)
424
+ out["census"], unresolved = returned["census"], returned["unresolved"]
425
+ out["inventory"] = refresh_proof.digest(returned["inventory"])
426
+ out["sheets"] = returned["sheets"]
427
+ out["authored"], out["priorAuthored"] = returned["authored"], prior["authored"]
428
+ out["revision"] = book.get("rev") if type(book.get("rev")) is int else None
429
+ out["priorRevision"] = prior["revision"] if type(prior["revision"]) is int else None
430
+ if (returned["inventory"] != prior["inventory"] or returned["sheets"] != prior["sheets"]):
431
+ raise ValueError("inventory_changed")
432
+ if returned["authored"] != prior["authored"]:
433
+ raise ValueError("authored_definitions_changed")
434
+ if not re.search(r"\b[0-9a-f]{7,40}\b", out["version"]):
435
+ raise ValueError("execution_build_unknown")
436
+ if result.get("attempt") is not None and not isinstance(result["attempt"], dict):
437
+ raise ValueError("attempt_malformed")
438
+ if isinstance(result.get("attempt"), dict):
439
+ attempt = result["attempt"]
440
+ if any(attempt.get(key) is not None and not isinstance(attempt[key], str) for key in ("id", "state")):
441
+ raise ValueError("attempt_malformed")
442
+ out["attempt"] = {key: attempt.get(key) for key in ("id", "state")}
443
+ if attempt.get("state") != "committed":
444
+ raise ValueError("attempt_not_committed")
445
+ role = routes_sheet._role_on_workbook(session, before)
446
+ if role not in ("owner", "edit") or "refreshedFor" in book:
447
+ raise ValueError("viewer_only")
448
+ args = row.get("args") or {}
449
+ if (args != {"kind": "workbook"} and args != {"kind": "scope", "scope": {"workbook": True}}):
450
+ raise ValueError("partial_scope")
451
+ report = result.get("binding_refresh")
452
+ statuses = report.get("databases") if isinstance(report, dict) else None
453
+ if (not isinstance(statuses, dict) or any(state not in ("resolved", "skipped")
454
+ for state in statuses.values()) or unresolved or result.get("reused")
455
+ or result.get("held") or book.get("held") or result.get("partial") or book.get("partial")
456
+ or result.get("viewer") or result.get("viewer_only")
457
+ or result.get("state") not in (None, "committed")
458
+ or result.get("outcome") not in (None, "committed")
459
+ or result.get("persisted") is False or book.get("persisted") is False
460
+ or result.get("ok") is False):
461
+ raise ValueError("held_or_unresolved")
462
+ if (type(book.get("rev")) is not int or type(prior["revision"]) is not int
463
+ or prior["revision"] < 0 or book["rev"] != prior["revision"] + 1
464
+ or not isinstance(book.get("refreshed_at"), str) or not book["refreshed_at"]
465
+ or (any(_is_bound(cell) for tab in book["sheets"] for cell in tab["cells"].values())
466
+ and not statuses)):
467
+ raise ValueError("editor_commit_unproven")
468
+ empty_noop = prior["census"]["cells"] == returned["census"]["cells"] == 0
469
+ out.update(complete=True, persisted=True,
470
+ reason="empty_noop" if empty_noop else "editor_revision_advanced")
471
+ refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
472
+ except refresh_proof.ProofCapacityError as exc:
473
+ out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
474
+ recommendation=refresh_proof.RECOMMENDATION, capacityCause=str(exc))
475
+ except (ValueError, TypeError, AttributeError) as exc:
476
+ reason = str(exc) if isinstance(exc, ValueError) else "capture_malformed"
477
+ out.update(complete=False, persisted=False)
478
+ out["reason"] = reason if reason in _DESK_PROOF_REASONS else "capture_malformed"
479
+ if out["reason"] == "proof_capacity_exceeded":
480
+ out["recommendation"] = refresh_proof.RECOMMENDATION
481
+ try:
482
+ refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
483
+ except (ValueError, TypeError):
484
+ out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
485
+ sheets=None, attempt=None, recommendation=refresh_proof.RECOMMENDATION,
486
+ capacityCause="log_line_capacity")
487
+ for key in ("requestId", "tenant", "reader", "workbook", "version"):
488
+ if isinstance(out[key], str) and len(out[key]) > 1024:
489
+ out[key] = None
490
+ return out
491
+
492
+
493
+ def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
494
+ """A logging failure supplies no proof and never changes the desk's business outcome."""
495
+ try:
496
+ capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
497
+ print(DESK_RESULT_PREFIX + refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES), flush=True)
498
+ except Exception: # noqa: BLE001
499
+ pass
500
+
501
+
502
  def desk_run_one(rt, tenant):
503
  """Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
504
  row = routes_sheet._refresh_desk_claim(rt)
 
506
  return False
507
  started = perf_counter()
508
  error = None
509
+ book, session, result, admitted = None, None, None, None
510
  try:
511
  book = sheets.get_workbook(rt, row["workbook"])
512
  rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
 
514
  if session is None:
515
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
516
  else:
517
+ admitted = _desk_input_identity(book)
518
+ result = routes_sheet._refresh_desk_run(session, row)
519
  except Exception as exc: # noqa: BLE001
520
  error = routes_sheet._refresh_desk_error(exc)
521
+ _desk_result_log(row, tenant, session, book, result, admitted=admitted)
522
  routes_sheet._refresh_desk_finish(rt, row["id"], error)
523
  print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
524
  f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
platform/core/activity_log.py CHANGED
@@ -5,6 +5,9 @@ from contextlib import contextmanager
5
  from contextvars import ContextVar
6
  from datetime import datetime, timezone
7
  import json
 
 
 
8
  import re
9
 
10
  DATABASE_KEY = 'ut_activity'
@@ -27,8 +30,8 @@ _READ_SCOPE = ContextVar('activity_read_scope', default=None)
27
 
28
  @contextmanager
29
  def read_scope(rt):
30
- """Share one outbox drain and bounded history pages within one read-only request."""
31
- state = {'runtime': rt, 'backend': None, 'drained': False, 'now': None,
32
  'pending': [], 'histories': {}}
33
  token = _READ_SCOPE.set(state)
34
  try:
@@ -228,9 +231,10 @@ def drain(rt, *, keys=None):
228
  pending = table.get(OUTBOX_KEY) or {}
229
  delivered = {}
230
  try:
231
- for key, entry in pending.items():
232
- if keys is not None and key not in keys:
233
- continue
 
234
  append(rt, **entry)
235
  delivered[key] = entry['_delivery']['version']
236
  finally:
@@ -249,6 +253,68 @@ def drain(rt, *, keys=None):
249
  return len(delivered)
250
 
251
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  def prune(rt, *, limit=PRUNE_BATCH, now=None):
253
  """Remove a bounded oldest-first batch strictly older than thirteen calendar months."""
254
  if type(limit) is not int or not 1 <= limit <= MAX_SCAN:
@@ -287,10 +353,12 @@ def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
287
  or type(cursor[1]) is not int or cursor[1] < 1):
288
  raise ValueError('Use the next_cursor returned by the activity reader.')
289
  before = (_time(cursor[0]).isoformat(), cursor[1])
290
- if scope is None or not scope['drained']:
291
- drain(rt)
292
- if scope is not None:
293
- scope['drained'], scope['now'] = True, now
 
 
294
  prefetched = None
295
  if scope is not None and whole_history:
296
  workbook = target[1]
@@ -310,6 +378,17 @@ def read(rt, *, can_read_object, object=None, by=None, since=None, until=None,
310
  batch = backend.page(target=target, actor=actor, since=since.isoformat(),
311
  until=until.isoformat(), before=before,
312
  limit=batch_limit)
 
 
 
 
 
 
 
 
 
 
 
313
  if not batch:
314
  exhausted = True
315
  break
@@ -422,10 +501,12 @@ class PgActivityStore:
422
  columns = _COLUMNS[1:]
423
  conflict = ''
424
  if delivery is not None:
 
 
425
  columns += ('delivery_key', 'delivery_version')
426
  values = (*values, *delivery)
427
  assignments = ', '.join(f'{column} = EXCLUDED.{column}' for column in columns
428
- if column != 'delivery_key')
429
  conflict = (f' ON CONFLICT (delivery_key) DO UPDATE SET {assignments} '
430
  f'WHERE {self.table}.delivery_version < EXCLUDED.delivery_version')
431
  stored = con.execute(
@@ -501,6 +582,26 @@ class PgActivityStore:
501
  events.append(_event(row[:-1], decoded_json=(obj_id, changes)))
502
  return events
503
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
504
  def workbook_pages(self, workbooks, *, since, until, limit):
505
  """Read bounded first history pages through the existing tenant and object-time index."""
506
  if not 1 <= len(workbooks) <= 16 or not 1 <= limit <= 256:
 
5
  from contextvars import ContextVar
6
  from datetime import datetime, timezone
7
  import json
8
+ import hashlib
9
+ import heapq
10
+ from itertools import islice
11
  import re
12
 
13
  DATABASE_KEY = 'ut_activity'
 
30
 
31
  @contextmanager
32
  def read_scope(rt):
33
+ """Share committed outbox input and bounded history pages within one read-only request."""
34
+ state = {'runtime': rt, 'backend': None, 'outbox': None, 'now': None,
35
  'pending': [], 'histories': {}}
36
  token = _READ_SCOPE.set(state)
37
  try:
 
231
  pending = table.get(OUTBOX_KEY) or {}
232
  delivered = {}
233
  try:
234
+ # A successful writer also retries older committed deliveries left by failed writers.
235
+ ordered = sorted(pending, key=lambda key: key not in keys) if keys is not None else pending
236
+ for key in ordered:
237
+ entry = pending[key]
238
  append(rt, **entry)
239
  delivered[key] = entry['_delivery']['version']
240
  finally:
 
253
  return len(delivered)
254
 
255
 
256
+ def _delivery_id(key):
257
+ """Reserve an exact JSON integer for a delivery before its first persistent insert."""
258
+ digest = hashlib.sha256(_text(key, 'delivery key').encode()).digest()
259
+ return (1 << 52) | (int.from_bytes(digest[:8], 'big') & ((1 << 52) - 1))
260
+
261
+
262
+ def _outbox(rt):
263
+ """Read only this tenant's private committed Activity entry."""
264
+ from core import user_tables
265
+ while isinstance(rt, user_tables._Lent):
266
+ rt = rt._runtime
267
+ target = rt._store() if callable(getattr(rt, '_store', None)) else rt
268
+ if callable(getattr(rt, 'get_entry', None)):
269
+ target = rt
270
+ reader = getattr(target, 'get_entry', None)
271
+ if callable(reader):
272
+ table = reader(user_tables.STORE_KEY, DATABASE_KEY) or {}
273
+ elif isinstance(target, FixtureAdapter) and not callable(getattr(target, 'get', None)):
274
+ return {}
275
+ else:
276
+ table = (target.get(user_tables.STORE_KEY) or {}).get(DATABASE_KEY) or {}
277
+ return table.get(OUTBOX_KEY) or {}
278
+
279
+
280
+ def _pending_page(backend, pending, *, target, actor, since, until, before, limit):
281
+ """Project a bounded pending page with persistent delivery identities and version deduplication."""
282
+ def events():
283
+ entries = iter(pending.items())
284
+ while chunk := list(islice(entries, 256)):
285
+ delivered = backend.deliveries([key for key, _ in chunk])
286
+ occupied = {receipt['id']: key for key, (_, receipt) in delivered.items()}
287
+ for key, entry in chunk:
288
+ delivery = entry['_delivery']
289
+ if delivery['key'] != key:
290
+ raise ValueError('Activity delivery key does not match its committed intent.')
291
+ prior = delivered.get(key)
292
+ stamp = _time(delivery['occurred_at']).isoformat()
293
+ kind, name, identity = _object(entry['object'])
294
+ event_id = prior[1]['id'] if prior is not None else _delivery_id(key)
295
+ if event_id in occupied and occupied[event_id] != key:
296
+ raise ValueError('Activity delivery identity conflicts with another event.')
297
+ if (not since <= stamp <= until or (actor is not None and entry['by'] != actor)
298
+ or (target is not None and (kind, name) != target[:2])
299
+ or (target is not None and target[2] is not None and identity != target[2])):
300
+ continue
301
+ if before is not None and (stamp, event_id) >= before:
302
+ continue
303
+ if prior is not None and prior[0] >= delivery['version']:
304
+ yield prior[1]
305
+ continue
306
+ changes, truncated, original_bytes = _bounded_changes(entry.get('changes'))
307
+ yield _event((event_id, stamp, entry['kind'], kind, name, identity,
308
+ entry['by'], entry.get('via'), entry.get('on_behalf_of'),
309
+ changes, truncated, original_bytes),
310
+ decoded_json=(_decode_json(identity), changes))
311
+
312
+ page = heapq.nlargest(limit + 1, events(), key=lambda event: (event['occurred_at'], event['id']))
313
+ if len({event['id'] for event in page}) != len(page):
314
+ raise ValueError('Activity pending deliveries have conflicting identities.')
315
+ return EventPage(page[:limit], has_more=len(page) > limit)
316
+
317
+
318
  def prune(rt, *, limit=PRUNE_BATCH, now=None):
319
  """Remove a bounded oldest-first batch strictly older than thirteen calendar months."""
320
  if type(limit) is not int or not 1 <= limit <= MAX_SCAN:
 
353
  or type(cursor[1]) is not int or cursor[1] < 1):
354
  raise ValueError('Use the next_cursor returned by the activity reader.')
355
  before = (_time(cursor[0]).isoformat(), cursor[1])
356
+ if scope is not None:
357
+ if scope['outbox'] is None:
358
+ scope['outbox'], scope['now'] = _outbox(rt), now
359
+ pending = scope['outbox']
360
+ else:
361
+ pending = _outbox(rt)
362
  prefetched = None
363
  if scope is not None and whole_history:
364
  workbook = target[1]
 
378
  batch = backend.page(target=target, actor=actor, since=since.isoformat(),
379
  until=until.isoformat(), before=before,
380
  limit=batch_limit)
381
+ if pending:
382
+ overlay = _pending_page(backend, pending, target=target, actor=actor,
383
+ since=since.isoformat(), until=until.isoformat(),
384
+ before=before, limit=batch_limit)
385
+ merged = {event['id']: event for event in batch}
386
+ merged.update((event['id'], event) for event in overlay)
387
+ ordered = heapq.nlargest(batch_limit + 1, merged.values(),
388
+ key=lambda event: (event['occurred_at'], event['id']))
389
+ batch = EventPage(ordered[:batch_limit], has_more=(len(ordered) > batch_limit
390
+ or getattr(batch, 'has_more', len(batch) == batch_limit)
391
+ or overlay.has_more))
392
  if not batch:
393
  exhausted = True
394
  break
 
501
  columns = _COLUMNS[1:]
502
  conflict = ''
503
  if delivery is not None:
504
+ columns = ('id', *columns)
505
+ values = (_delivery_id(delivery[0]), *values)
506
  columns += ('delivery_key', 'delivery_version')
507
  values = (*values, *delivery)
508
  assignments = ', '.join(f'{column} = EXCLUDED.{column}' for column in columns
509
+ if column not in ('id', 'delivery_key'))
510
  conflict = (f' ON CONFLICT (delivery_key) DO UPDATE SET {assignments} '
511
  f'WHERE {self.table}.delivery_version < EXCLUDED.delivery_version')
512
  stored = con.execute(
 
582
  events.append(_event(row[:-1], decoded_json=(obj_id, changes)))
583
  return events
584
 
585
+ def deliveries(self, keys):
586
+ """Resolve at most 256 committed delivery receipts without changing the store."""
587
+ if not keys or len(keys) > 256:
588
+ raise ValueError('Read between 1 and 256 delivery receipts.')
589
+ columns = ', '.join('CAST(changes AS TEXT) AS changes' if col == 'changes' else col
590
+ for col in _COLUMNS)
591
+ try:
592
+ with self._connection() as con:
593
+ reserved = [_delivery_id(key) for key in keys]
594
+ placeholders = ', '.join(['%s'] * len(keys))
595
+ values = con.execute(f'SELECT {columns}, delivery_key, delivery_version '
596
+ f'FROM {self.table} WHERE delivery_key IN '
597
+ f'({placeholders}) OR id IN ({placeholders})',
598
+ [*keys, *reserved]).fetchall()
599
+ except Exception as exc:
600
+ if getattr(exc, 'sqlstate', None) != '42P01':
601
+ raise
602
+ return {}
603
+ return {row[-2]: (row[-1], _event(row[:-2])) for row in values}
604
+
605
  def workbook_pages(self, workbooks, *, since, until, limit):
606
  """Read bounded first history pages through the existing tenant and object-time index."""
607
  if not 1 <= len(workbooks) <= 16 or not 1 <= limit <= 256:
platform/core/grid_events.py CHANGED
@@ -2083,6 +2083,10 @@ def handle_one(event, ctx):
2083
  shared_prior = {**base_field, **stored_choice,
2084
  'key': key, 'source': 'overlay', 'shared': True,
2085
  'custom': True, 'type': base_field['type']}
 
 
 
 
2086
  shared_field = True
2087
  if shared_field:
2088
  if not _may_edit_field_definition(ctx, key, shared_prior):
@@ -2095,6 +2099,13 @@ def handle_one(event, ctx):
2095
  # cannot turn an image field into another schema type while saving
2096
  # its permissions.
2097
  field = dict(shared_prior)
 
 
 
 
 
 
 
2098
  field['label'] = str(raw.get('label') or field.get('label') or 'Untitled')[:120]
2099
  field['note'] = str(raw.get('note') or '')[:2000]
2100
  fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
 
2083
  shared_prior = {**base_field, **stored_choice,
2084
  'key': key, 'source': 'overlay', 'shared': True,
2085
  'custom': True, 'type': base_field['type']}
2086
+ # Registry admission comes from the server context, even when the native
2087
+ # definition stores granted=False. The stored marker is persistence only.
2088
+ if base_field.get('granted') is True:
2089
+ shared_prior['granted'] = True
2090
  shared_field = True
2091
  if shared_field:
2092
  if not _may_edit_field_definition(ctx, key, shared_prior):
 
2099
  # cannot turn an image field into another schema type while saving
2100
  # its permissions.
2101
  field = dict(shared_prior)
2102
+ # The Product choice projection may derive granted=True from the registry.
2103
+ # Keep it for admission above, but persist only the native marker state.
2104
+ if spreadsheet_choice:
2105
+ if 'granted' in stored_choice:
2106
+ field['granted'] = stored_choice['granted']
2107
+ else:
2108
+ field.pop('granted', None)
2109
  field['label'] = str(raw.get('label') or field.get('label') or 'Untitled')[:120]
2110
  field['note'] = str(raw.get('note') or '')[:2000]
2111
  fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
platform/core/measure_resolve.py CHANGED
@@ -142,7 +142,8 @@ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp,
142
  return out
143
 
144
 
145
- def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None):
 
146
  """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
147
  handed to `rows_from_pool(derived=…)` beside the cohort cells.
148
 
@@ -171,7 +172,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
171
  if key not in memo:
172
  try:
173
  memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
174
- today, team_id, allowed_pids, related=related)
 
175
  except Exception as e:
176
  if on_error:
177
  on_error('measure-column', e)
 
142
  return out
143
 
144
 
145
+ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None,
146
+ *, cur=None):
147
  """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
148
  handed to `rows_from_pool(derived=…)` beside the cohort cells.
149
 
 
172
  if key not in memo:
173
  try:
174
  memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
175
+ today, team_id, allowed_pids, related=related,
176
+ **({'cur': cur} if cur is not None else {}))
177
  except Exception as e:
178
  if on_error:
179
  on_error('measure-column', e)
platform/core/odoo.py CHANGED
@@ -10,9 +10,16 @@ a single source of truth for the connection. Adds:
10
  NOTHING here writes to Odoo — the underlying client hard-blocks writes.
11
  """
12
  import os
 
13
  import sys
14
  import queue as _queue
15
  import threading
 
 
 
 
 
 
16
  from concurrent.futures import ThreadPoolExecutor
17
  from contextlib import contextmanager
18
  from functools import lru_cache
@@ -23,7 +30,176 @@ from pathlib import Path
23
  _ROOT = Path(__file__).resolve().parents[1] # platform/
24
  if str(_ROOT) not in sys.path:
25
  sys.path.insert(0, str(_ROOT))
26
- from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  # ---- business constants (TENANT #0's scope, kept as module attributes for its modules) ----
29
  # These are Royal Imports' config — the default a request gets when nothing binds a tenant.
@@ -151,16 +327,47 @@ _pool_n = 0
151
  _pool_lock = threading.Lock()
152
 
153
 
 
 
 
 
 
 
 
 
 
 
 
 
154
  def _checkout():
155
  global _pool_n
 
156
  try:
157
- return _pool_q.get_nowait()
158
  except _queue.Empty:
159
  with _pool_lock:
160
  grow = _pool_n < _POOL_MAX
161
  if grow:
162
  _pool_n += 1
163
- return OdooClient() if grow else _pool_q.get() # else block until one frees
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
 
165
 
166
  def _checkin(c):
@@ -205,7 +412,10 @@ def parallel(tasks):
205
  if not tasks:
206
  return []
207
  if len(tasks) == 1:
208
- return [tasks[0]()]
 
 
 
209
 
210
  caller_mode = doc_mode() # propagate the submitter's basis to each worker
211
  caller_ctx = _ctx() # and the tenant binding — a worker thread must
@@ -216,14 +426,17 @@ def parallel(tasks):
216
  _tlocal.doc_mode = caller_mode
217
  _tlocal.tenant_ctx = caller_ctx
218
  try:
219
- return fn()
 
 
220
  finally:
221
  _tlocal.client = None
222
  _tlocal.doc_mode = 'order'
223
  _tlocal.tenant_ctx = None
224
  _checkin(c)
225
  with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
226
- return list(ex.map(_run, tasks))
 
227
 
228
 
229
  def parallel_map(task_dict):
 
10
  NOTHING here writes to Odoo — the underlying client hard-blocks writes.
11
  """
12
  import os
13
+ import math
14
  import sys
15
  import queue as _queue
16
  import threading
17
+ import contextvars
18
+ import http.client
19
+ import io
20
+ import socket
21
+ import time
22
+ import xmlrpc.client
23
  from concurrent.futures import ThreadPoolExecutor
24
  from contextlib import contextmanager
25
  from functools import lru_cache
 
30
  _ROOT = Path(__file__).resolve().parents[1] # platform/
31
  if str(_ROOT) not in sys.path:
32
  sys.path.insert(0, str(_ROOT))
33
+ from odoo_client import OdooClient as _BaseOdooClient, _RPC_TIMEOUT
34
+ from core import request_timing as _timing
35
+
36
+
37
+ class OdooRequestTimeout(TimeoutError):
38
+ """An Odoo request transport exhausted its absolute RPC budget."""
39
+
40
+
41
+ def _rpc_remaining():
42
+ return _timing.remaining_budget(_RPC_TIMEOUT)
43
+
44
+
45
+ def _create_rpc_connection(address, timeout=None, source_address=None):
46
+ """Bound each address attempt while checking the budget around synchronous DNS resolution."""
47
+ _timing.remaining_budget()
48
+ addresses = socket.getaddrinfo(address[0], address[1], 0, socket.SOCK_STREAM)
49
+ _timing.remaining_budget()
50
+ error = None
51
+ for family, kind, protocol, _, target in addresses:
52
+ _timing.remaining_budget()
53
+ con = socket.socket(family, kind, protocol)
54
+ try:
55
+ con.settimeout(_rpc_remaining())
56
+ if source_address:
57
+ con.bind(source_address)
58
+ con.connect(target)
59
+ con.settimeout(_rpc_remaining())
60
+ return con
61
+ except BaseException as exc:
62
+ con.close()
63
+ if not isinstance(exc, OSError):
64
+ raise
65
+ error = exc
66
+ if error is not None:
67
+ raise error
68
+ raise OSError("Odoo DNS resolution returned no addresses")
69
+
70
+
71
+ class _SocketReader(io.RawIOBase):
72
+ def __init__(self, sock):
73
+ super().__init__()
74
+ self.sock = sock
75
+
76
+ def readable(self):
77
+ return True
78
+
79
+ def readinto(self, buffer):
80
+ return self.sock.recv_into(buffer)
81
+
82
+
83
+ class _BudgetSocket:
84
+ def __init__(self, sock):
85
+ self.sock = sock
86
+
87
+ def __getattr__(self, name):
88
+ return getattr(self.sock, name)
89
+
90
+ def _call(self, method, *args):
91
+ self.sock.settimeout(_rpc_remaining())
92
+ try:
93
+ result = getattr(self.sock, method)(*args)
94
+ except socket.timeout as exc:
95
+ if _timing.current_budget() is not None:
96
+ _timing.remaining_budget()
97
+ raise OdooRequestTimeout("Odoo RPC socket timed out") from exc
98
+ raise
99
+ _timing.remaining_budget()
100
+ return result
101
+
102
+ def recv_into(self, buffer):
103
+ return self._call("recv_into", buffer)
104
+
105
+ def sendall(self, data):
106
+ return self._call("sendall", data)
107
+
108
+ def makefile(self, mode, buffering=None):
109
+ if mode != "rb":
110
+ raise ValueError("Odoo transport only reads binary responses")
111
+ return io.BufferedReader(_SocketReader(self))
112
+
113
+
114
+ class _BudgetHTTPConnection(http.client.HTTPConnection):
115
+ def __init__(self, *args, **kwargs):
116
+ super().__init__(*args, **kwargs)
117
+ self._create_connection = _create_rpc_connection
118
+
119
+ def connect(self):
120
+ self.timeout = _rpc_remaining()
121
+ super().connect()
122
+ self.sock = _BudgetSocket(self.sock)
123
+ _timing.remaining_budget()
124
+
125
+
126
+ class _BudgetHTTPSConnection(http.client.HTTPSConnection):
127
+ def __init__(self, *args, **kwargs):
128
+ super().__init__(*args, **kwargs)
129
+ self._create_connection = _create_rpc_connection
130
+
131
+ def connect(self):
132
+ self.timeout = _rpc_remaining()
133
+ super().connect()
134
+ self.sock = _BudgetSocket(self.sock)
135
+ _timing.remaining_budget()
136
+
137
+
138
+ class _BudgetTransport(xmlrpc.client.Transport):
139
+ connection_type = _BudgetHTTPConnection
140
+
141
+ def make_connection(self, host):
142
+ _timing.remaining_budget()
143
+ if self._connection and host == self._connection[0]:
144
+ return self._connection[1]
145
+ chost, self._extra_headers, _ = self.get_host_info(host)
146
+ con = self.connection_type(chost, timeout=_rpc_remaining())
147
+ self._connection = host, con
148
+ return con
149
+
150
+ def single_request(self, *args, **kwargs):
151
+ # A single deadline covers headers and every body read, including trickling responses.
152
+ deadline = time.monotonic() + _RPC_TIMEOUT
153
+ try:
154
+ with _timing.budget_scope(deadline, OdooRequestTimeout):
155
+ return super().single_request(*args, **kwargs)
156
+ except _timing.RequestDeadlineExceeded:
157
+ self.close()
158
+ raise
159
+ except socket.timeout as exc:
160
+ self.close()
161
+ _timing.remaining_budget()
162
+ raise OdooRequestTimeout("Odoo RPC socket timed out") from exc
163
+ except BaseException:
164
+ self.close()
165
+ raise
166
+
167
+
168
+ class _BudgetSafeTransport(_BudgetTransport, xmlrpc.client.SafeTransport):
169
+ connection_type = _BudgetHTTPSConnection
170
+
171
+
172
+ def _bounded_proxy(uri):
173
+ transport = _BudgetSafeTransport() if uri.startswith("https") else _BudgetTransport()
174
+ return xmlrpc.client.ServerProxy(uri, transport=transport)
175
+
176
+
177
+ class OdooClient(_BaseOdooClient):
178
+ def _connect(self, _retried=False):
179
+ try:
180
+ with _bounded_proxy(f'{self.url}/xmlrpc/2/common') as common:
181
+ self.uid = common.authenticate(self.db, self.user, self.api_key, {})
182
+ except (http.client.HTTPException, ConnectionError, OSError):
183
+ if _retried or _timing.current_budget() is not None:
184
+ raise
185
+ time.sleep(3)
186
+ return self._connect(_retried=True)
187
+ if not self.uid:
188
+ raise RuntimeError("Odoo authentication failed; check credentials.")
189
+ self.models = _bounded_proxy(f'{self.url}/xmlrpc/2/object')
190
+
191
+ def execute_kw(self, model, method, args, kwargs=None):
192
+ if _timing.current_budget() is None:
193
+ return super().execute_kw(model, method, args, kwargs)
194
+ self._guard(method)
195
+ if not self._xlock.acquire(timeout=_rpc_remaining()):
196
+ _timing.remaining_budget()
197
+ raise OdooRequestTimeout("Odoo client lock timed out")
198
+ try:
199
+ _timing.remaining_budget()
200
+ return self.models.execute_kw(self.db, self.uid, self.api_key, model, method, args, kwargs or {})
201
+ finally:
202
+ self._xlock.release()
203
 
204
  # ---- business constants (TENANT #0's scope, kept as module attributes for its modules) ----
205
  # These are Royal Imports' config — the default a request gets when nothing binds a tenant.
 
327
  _pool_lock = threading.Lock()
328
 
329
 
330
+ class OdooPoolTimeout(TimeoutError):
331
+ """All read-only Odoo clients remained checked out past the acquisition budget."""
332
+
333
+
334
+ def _checkout_timeout_s():
335
+ try:
336
+ value = float(os.environ.get("AIOS_ODOO_POOL_TIMEOUT_S", "10"))
337
+ return max(.01, value) if math.isfinite(value) else 10.0
338
+ except ValueError:
339
+ return 10.0
340
+
341
+
342
  def _checkout():
343
  global _pool_n
344
+ _timing.remaining_budget()
345
  try:
346
+ client = _pool_q.get_nowait()
347
  except _queue.Empty:
348
  with _pool_lock:
349
  grow = _pool_n < _POOL_MAX
350
  if grow:
351
  _pool_n += 1
352
+ if grow:
353
+ try:
354
+ client = OdooClient()
355
+ _timing.remaining_budget()
356
+ return client
357
+ except BaseException:
358
+ with _pool_lock:
359
+ _pool_n -= 1
360
+ raise
361
+ try:
362
+ client = _pool_q.get(timeout=_timing.remaining_budget(_checkout_timeout_s()))
363
+ except _queue.Empty:
364
+ raise OdooPoolTimeout("Odoo read pool acquisition timed out") from None
365
+ try:
366
+ _timing.remaining_budget()
367
+ except BaseException:
368
+ _checkin(client)
369
+ raise
370
+ return client
371
 
372
 
373
  def _checkin(c):
 
412
  if not tasks:
413
  return []
414
  if len(tasks) == 1:
415
+ _timing.remaining_budget()
416
+ result = tasks[0]()
417
+ _timing.remaining_budget()
418
+ return [result]
419
 
420
  caller_mode = doc_mode() # propagate the submitter's basis to each worker
421
  caller_ctx = _ctx() # and the tenant binding — a worker thread must
 
426
  _tlocal.doc_mode = caller_mode
427
  _tlocal.tenant_ctx = caller_ctx
428
  try:
429
+ result = fn()
430
+ _timing.remaining_budget()
431
+ return result
432
  finally:
433
  _tlocal.client = None
434
  _tlocal.doc_mode = 'order'
435
  _tlocal.tenant_ctx = None
436
  _checkin(c)
437
  with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
438
+ contexts = [contextvars.copy_context() for _ in tasks]
439
+ return list(ex.map(lambda pair: pair[0].run(_run, pair[1]), zip(contexts, tasks)))
440
 
441
 
442
  def parallel_map(task_dict):
platform/core/refresh_proof.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pure, bounded identities for authored Sheet inputs and complete returned outputs."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import math
7
+ import re
8
+
9
+ MAX_CELLS = 100_000
10
+ MAX_TABS = 128
11
+ MAX_DOCUMENT_BYTES = 32 * 1024 * 1024
12
+ MAX_LINE_BYTES = 256 * 1024
13
+ MAX_LOG_BYTES = 8 * 1024 * 1024
14
+ MAX_LOG_ROWS = 4096
15
+ MAX_STATE_BYTES = 16 * 1024 * 1024
16
+ MAX_STATE_ROWS = 16_384
17
+ RECOMMENDATION = "Increase the named proof budget and collect a fresh complete proof."
18
+ _UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00"
19
+
20
+
21
+ class ProofCapacityError(ValueError):
22
+ """A complete proof cannot fit the named finite budget."""
23
+
24
+
25
+ def bounded_json(value, limit=MAX_DOCUMENT_BYTES):
26
+ """Check depth, nodes and scalar sizes before allocating encoded JSON."""
27
+ remaining, nodes = [limit], [0]
28
+
29
+ def check(item, depth=0):
30
+ nodes[0] += 1
31
+ if depth > 40 or nodes[0] > 1_000_000:
32
+ raise ProofCapacityError("json_structure_capacity")
33
+ if isinstance(item, str):
34
+ if len(item) > remaining[0]:
35
+ raise ProofCapacityError("json_byte_capacity")
36
+ cost = len(item.encode("utf-8")) + 2
37
+ elif isinstance(item, (dict, list, tuple)):
38
+ if len(item) > limit:
39
+ raise ProofCapacityError("json_structure_capacity")
40
+ cost = 2 + 2 * len(item)
41
+ elif item is None or type(item) in (bool, int, float):
42
+ if type(item) is int and item.bit_length() > 256:
43
+ raise ProofCapacityError("json_scalar_capacity")
44
+ if type(item) is float and not math.isfinite(item):
45
+ raise ValueError("json_nonfinite")
46
+ cost = 32
47
+ else:
48
+ raise ValueError("json_malformed")
49
+ remaining[0] -= cost
50
+ if remaining[0] < 0:
51
+ raise ProofCapacityError("json_byte_capacity")
52
+ if isinstance(item, dict):
53
+ for key, child in item.items():
54
+ if not isinstance(key, str):
55
+ raise ValueError("json_key_malformed")
56
+ check(key, depth + 1)
57
+ check(child, depth + 1)
58
+ elif isinstance(item, (list, tuple)):
59
+ for child in item:
60
+ check(child, depth + 1)
61
+
62
+ check(value)
63
+ chunks, size = [], 0
64
+ for chunk in json.JSONEncoder(sort_keys=True, separators=(",", ":"),
65
+ ensure_ascii=False, allow_nan=False).iterencode(value):
66
+ size += len(chunk.encode("utf-8"))
67
+ if size > limit:
68
+ raise ProofCapacityError("json_byte_capacity")
69
+ chunks.append(chunk)
70
+ return "".join(chunks)
71
+
72
+
73
+ def digest(value):
74
+ return hashlib.sha256(bounded_json(value).encode("utf-8")).hexdigest()
75
+
76
+
77
+ def _address(addr):
78
+ if not isinstance(addr, str) or not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]{0,6}", addr):
79
+ return False
80
+ col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
81
+ n = 0
82
+ for char in col:
83
+ n = n * 26 + ord(char) - 64
84
+ return n <= 16384 and int(row) <= 1048576
85
+
86
+
87
+ def _definition(cell):
88
+ kind = cell["kind"]
89
+ if kind == "literal":
90
+ return {key: cell[key] for key in ("kind", "text", "value") if key in cell}
91
+ if kind == "formula":
92
+ meta = cell.get("formulaMeta") or {}
93
+ return {"kind": kind, "formula": cell["formula"],
94
+ "formulaMeta": {k: v for k, v in meta.items()
95
+ if k not in ("spill", "inferredFormat")}}
96
+ if kind == "bound":
97
+ return {"kind": kind, "binding": cell["binding"]}
98
+ return {"kind": kind}
99
+
100
+
101
+ def _position(addr):
102
+ if not _address(addr):
103
+ raise ValueError("generated_manifest_invalid")
104
+ col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
105
+ number = 0
106
+ for char in col:
107
+ number = number * 26 + ord(char) - 64
108
+ return number, int(row)
109
+
110
+
111
+ def _cell_address(col, row):
112
+ letters = ""
113
+ while col:
114
+ col, digit = divmod(col - 1, 26)
115
+ letters = chr(65 + digit) + letters
116
+ return letters + str(row)
117
+
118
+
119
+ def _generated_manifest(tab, scope, remaining):
120
+ """Expand production spill extents and placed Grid blocks before accepting their children."""
121
+ expected = {}
122
+ for addr, root in tab["cells"].items():
123
+ if not isinstance(root, dict):
124
+ raise ValueError("cell_malformed")
125
+ meta = root.get("formulaMeta") or {}
126
+ if not isinstance(meta, dict):
127
+ raise ValueError("cell_malformed")
128
+ has_spill = "spill" in root or "spill" in meta
129
+ spill = root.get("spill", meta.get("spill"))
130
+ binding = root.get("binding") or {}
131
+ view = (root.get("kind") == "bound" and isinstance(binding, dict)
132
+ and binding.get("mode", binding.get("kind")) == "view")
133
+ block = binding.get("block") if view and binding.get("viewType") == "grid" else None
134
+ if not has_spill and block is None:
135
+ continue
136
+ col, row = _position(addr)
137
+ extent = spill if has_spill else block
138
+ if (not isinstance(extent, dict) or any(type(extent.get(k)) is not int or extent[k] < 1
139
+ for k in ("rows", "cols"))):
140
+ raise ValueError("generated_manifest_invalid")
141
+ rows, cols = extent["rows"], extent["cols"]
142
+ if col + cols - 1 > 16384 or row + rows - 1 > 1048576:
143
+ raise ValueError("generated_manifest_invalid")
144
+ if rows * cols > remaining - len(expected):
145
+ raise ProofCapacityError("generated_cell_capacity")
146
+ if has_spill:
147
+ if (root.get("kind") != "formula" or set(spill) != {"version", "origin", "rows", "cols", "ref"}
148
+ or type(spill.get("version")) is not int or spill["version"] != 1
149
+ or spill.get("origin") != scope + "!" + addr
150
+ or spill.get("ref") != addr + ":" + _cell_address(col + cols - 1, row + rows - 1)
151
+ or any(key in holder and holder[key] != spill for holder, key in
152
+ ((root, "spill"), (meta, "spill")))):
153
+ raise ValueError("generated_manifest_invalid")
154
+ else:
155
+ fields = binding.get("fields")
156
+ if (rows < 2 or not isinstance(fields, list) or len(fields) != cols
157
+ or any(not isinstance(field, str) or not field for field in fields)
158
+ or len(set(fields)) != len(fields)):
159
+ raise ValueError("generated_manifest_invalid")
160
+ for dy in range(rows):
161
+ for dx in range(cols):
162
+ child_addr = _cell_address(col + dx, row + dy)
163
+ if child_addr in expected:
164
+ raise ValueError("generated_manifest_invalid")
165
+ expected[child_addr] = (("spill", scope + "!" + addr) if has_spill else
166
+ ("view", addr, fields[dx], -1 if dy == 0 else -2 if dy == rows - 1 else dy - 1))
167
+ if child_addr not in tab["cells"]:
168
+ raise ValueError("generated_output_missing")
169
+ return expected
170
+
171
+
172
+ def snapshot(book):
173
+ """Bind authored definitions and revision while censusing every validated output cell."""
174
+ from core import sheets
175
+
176
+ if not isinstance(book, dict) or not isinstance(book.get("sheets"), list) or not book["sheets"]:
177
+ raise ValueError("workbook_incomplete")
178
+ if len(book["sheets"]) > MAX_TABS:
179
+ raise ProofCapacityError("sheet_capacity")
180
+ total = 0
181
+ for tab in book["sheets"]:
182
+ if isinstance(tab, dict) and isinstance(tab.get("cells"), dict):
183
+ total += len(tab["cells"])
184
+ if total > MAX_CELLS:
185
+ raise ProofCapacityError("cell_capacity")
186
+ # Bound the existing document before canonical cleaners copy any nested cell object.
187
+ bounded_json(book)
188
+ census = {"cells": 0, "bound": 0, "formulas": 0, "refusals": []}
189
+ rows, definitions, tabs, names, unresolved = [], [], set(), set(), False
190
+ generated_cells = 0
191
+ for index, tab in enumerate(book["sheets"]):
192
+ if (not isinstance(tab, dict) or not isinstance(tab.get("id"), str) or not tab["id"]
193
+ or tab["id"] in tabs or not isinstance(tab.get("cells"), dict)
194
+ or (tab.get("name") is not None and not isinstance(tab["name"], str))):
195
+ raise ValueError("sheet_incomplete")
196
+ tabs.add(tab["id"])
197
+ if "cellCount" in tab and (type(tab["cellCount"]) is not int or tab["cellCount"] != len(tab["cells"])):
198
+ raise ValueError("sheet_partial")
199
+ if census["cells"] + len(tab["cells"]) > MAX_CELLS:
200
+ raise ProofCapacityError("cell_capacity")
201
+ name = tab.get("name") or ""
202
+ scope = name.upper() if name and name.upper() not in names else (_UNADDRESSABLE_SHEET + str(index)).upper()
203
+ names.add(scope)
204
+ generated = _generated_manifest(tab, scope, MAX_CELLS - generated_cells)
205
+ generated_cells += len(generated)
206
+ for addr, cell in tab["cells"].items():
207
+ if (not _address(addr) or not isinstance(cell, dict)
208
+ or cell.get("kind") not in sheets.CELL_KINDS
209
+ or cell.get("state") not in sheets.CELL_STATES):
210
+ raise ValueError("cell_malformed")
211
+ try:
212
+ sheets.clean_cell(cell, addr)
213
+ except sheets.SheetRefusal as exc:
214
+ raise ValueError("cell_malformed") from exc
215
+ kind = cell["kind"]
216
+ if ((kind == "formula" and (not isinstance(cell.get("formula"), str)
217
+ or not cell["formula"].startswith("=") or len(cell["formula"]) < 2))
218
+ or (kind == "bound" and (not isinstance(cell.get("binding"), dict) or not cell["binding"]))
219
+ or (kind == "literal" and "value" not in cell and "text" not in cell)):
220
+ raise ValueError("cell_malformed")
221
+ census["cells"] += 1
222
+ census["bound"] += kind == "bound"
223
+ census["formulas"] += kind == "formula"
224
+ unresolved |= kind in ("bound", "formula") and cell["state"] not in ("fresh", "refused")
225
+ unresolved |= cell.get("staleReason") == "refresh_held"
226
+ if cell["state"] == "refused":
227
+ refusal, error = cell.get("refusal") or {}, cell.get("error") or {}
228
+ if not isinstance(refusal, dict) or not isinstance(error, dict):
229
+ raise ValueError("refusal_malformed")
230
+ code = refusal.get("code") or error.get("code") or "refused"
231
+ if not isinstance(code, str) or not code:
232
+ raise ValueError("refusal_malformed")
233
+ census["refusals"].append({"cell": "%s!%s" % (name or tab["id"], addr),
234
+ "kind": kind, "code": code})
235
+ authored = cell
236
+ meta = cell.get("formulaMeta") or {}
237
+ marker = meta.get("spillChild")
238
+ declared = generated.get(addr)
239
+ if marker is not None:
240
+ key = scope + "!" + addr
241
+ origin = marker.get("origin") if isinstance(marker, dict) else None
242
+ root_addr = origin.rpartition("!")[2] if isinstance(origin, str) else ""
243
+ root = tab["cells"].get(root_addr, {})
244
+ if (not isinstance(marker, dict) or type(marker.get("version")) is not int
245
+ or marker["version"] != 1 or marker.get("address") != key
246
+ or origin != scope + "!" + root_addr or origin == key
247
+ or cell.get("spillParent") != origin or root.get("kind") != "formula"
248
+ or set(marker) - {"version", "origin", "address", "base"}
249
+ or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
250
+ raise ValueError("cell_malformed")
251
+ base = marker.get("base")
252
+ if base and (not isinstance(base, dict) or base.get("kind") != "empty"
253
+ or any(base.get(k) is not None for k in ("value", "text", "formula", "binding"))):
254
+ raise ValueError("cell_malformed")
255
+ if meta.get("spillClear") is True:
256
+ if (declared is not None or cell.get("value") is not None or cell.get("display") != ""
257
+ or cell["state"] != "typed" or cell.get("refusal") or cell.get("error")):
258
+ raise ValueError("generated_output_invalid")
259
+ elif ("spillClear" in meta or declared != ("spill", origin)
260
+ or cell["state"] not in ("fresh", "refused")):
261
+ raise ValueError("generated_output_invalid")
262
+ authored = base if base else None
263
+ elif "spillParent" in cell or "spillClear" in meta:
264
+ raise ValueError("cell_malformed")
265
+ view = cell.get("viewChild")
266
+ if view is not None:
267
+ root = tab["cells"].get(view.get("origin"), {}) if isinstance(view, dict) else {}
268
+ binding = root.get("binding") or {}
269
+ if (marker is not None or not isinstance(view, dict) or not _address(view.get("origin"))
270
+ or view["origin"] == addr or not isinstance(view.get("field"), str) or not view["field"]
271
+ or type(view.get("line")) is not int or view["line"] < -2
272
+ or root.get("kind") != "bound" or not isinstance(binding, dict)
273
+ or (binding.get("mode") != "view" and binding.get("kind") != "view")
274
+ or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
275
+ raise ValueError("cell_malformed")
276
+ if declared != ("view", view["origin"], view["field"], view["line"]):
277
+ raise ValueError("generated_output_invalid")
278
+ authored = None
279
+ if declared is not None:
280
+ is_root = (declared[1] == (scope + "!" + addr if declared[0] == "spill" else addr))
281
+ if not is_root and ((declared[0] == "spill" and marker is None)
282
+ or (declared[0] == "view" and view is None)):
283
+ raise ValueError("generated_output_invalid")
284
+ if authored is not None:
285
+ rows.append({"sheet": tab["id"], "address": addr, "kind": authored["kind"]})
286
+ definitions.append({"sheet": tab["id"], "address": addr, "definition": _definition(authored)})
287
+ if census["cells"] and not rows:
288
+ raise ValueError("workbook_empty")
289
+ rows.sort(key=lambda row: (row["sheet"], row["address"]))
290
+ definitions.sort(key=lambda row: (row["sheet"], row["address"]))
291
+ authored = digest({"sheets": [(tab["id"], tab.get("name")) for tab in book["sheets"]],
292
+ "cells": definitions, "namedRanges": book.get("namedRanges"),
293
+ "dateSystem": book.get("dateSystem")})
294
+ return {"revision": book.get("rev"), "inventory": rows, "authored": authored,
295
+ "sheets": sorted(tabs), "census": census, "unresolved": bool(unresolved)}
platform/core/request_timing.py ADDED
@@ -0,0 +1,412 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Request-local measurements shared by async tasks and copied threadpool contexts."""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ import hashlib
6
+ import sys
7
+ import threading
8
+ import time
9
+ from collections import Counter
10
+ from contextlib import contextmanager
11
+ from contextvars import ContextVar
12
+ from functools import wraps
13
+
14
+ CURRENT = ContextVar("aios_request_timing", default=None)
15
+ READ_MEMO_SECONDS = 2.0
16
+ READ_MEMO_ENTRIES = 64
17
+ READ_MEMO_BYTES = 8 * 1024 * 1024
18
+ _STORE_SPAN = ContextVar("aios_store_timing_span", default=None)
19
+ _MEMO_READ = ContextVar("aios_request_memo_read", default=None)
20
+ _THREAD_LOCK = threading.Lock()
21
+ _THREADS = {}
22
+ _DEADLINE_ARM = None
23
+ _LOCK_PROBE_ARM = None
24
+ _BUDGET = ContextVar("aios_operation_budget", default=None)
25
+ _SQL_VALUES = re.compile(
26
+ r"--[^\n]*|/\*.*?\*/|\$([A-Za-z_][A-Za-z_0-9]*|)\$.*?\$\1\$|"
27
+ r"(?:[eEbBxX])?'(?:''|\\.|[^'])*'|\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b", re.S)
28
+
29
+
30
+ class Accumulator:
31
+ """Mutable request state, including measurements from anyio's copied contexts."""
32
+
33
+ def __init__(self):
34
+ self.lock = threading.RLock()
35
+ self.started = time.perf_counter()
36
+ self.closed = False
37
+ self.queue_ms = 0.0
38
+ self.store_ms = 0.0
39
+ self.store_reads = 0
40
+ self.store_bytes = 0
41
+ self.store_write_bytes = 0
42
+ self.store_operations = Counter()
43
+ self.store_keys = {}
44
+ self.duckdb_ms = 0.0
45
+ self.duckdb_count = 0
46
+ self.slowest_sql = ""
47
+ self.slowest_sql_ms = 0.0
48
+ self.response_bytes = 0
49
+ self.samples = Counter()
50
+ self.deadline = None
51
+ self.cancelled = False
52
+ self.read_memo = {}
53
+ self.read_memo_bytes = 0
54
+ self.read_memo_epoch = 0
55
+ self.read_memo_invalidated_at = 0.0
56
+ self.read_memo_ended = False
57
+
58
+ def snapshot(self):
59
+ with self.lock:
60
+ # The middleware takes its first snapshot before sending response headers.
61
+ clear_read_memo(self, end=True)
62
+ return {"queue_ms": self.queue_ms, "store_ms": self.store_ms,
63
+ "store_reads": self.store_reads, "store_bytes": self.store_bytes,
64
+ "store_write_bytes": self.store_write_bytes,
65
+ "store_operations": dict(self.store_operations),
66
+ "store_keys": {key: dict(value) for key, value in self.store_keys.items()},
67
+ "duckdb_ms": self.duckdb_ms, "duckdb_count": self.duckdb_count,
68
+ "slowest_sql": self.slowest_sql, "slowest_sql_ms": self.slowest_sql_ms,
69
+ "response_bytes": self.response_bytes}
70
+
71
+
72
+ @contextmanager
73
+ def request_scope(accumulator=None):
74
+ """Bind a mutable accumulator and restore the caller's context on every exit."""
75
+ acc = accumulator if accumulator is not None else Accumulator()
76
+ token = CURRENT.set(acc)
77
+ try:
78
+ yield acc
79
+ finally:
80
+ clear_read_memo(acc, end=True)
81
+ CURRENT.reset(token)
82
+
83
+
84
+ def clear_read_memo(acc=None, *, end=False):
85
+ """Invalidate in-flight memo fills as well as completed values in copied contexts."""
86
+ acc = CURRENT.get() if acc is None else acc
87
+ if acc is not None:
88
+ with acc.lock:
89
+ acc.read_memo.clear()
90
+ acc.read_memo_bytes = 0
91
+ acc.read_memo_epoch += 1
92
+ acc.read_memo_invalidated_at = time.monotonic()
93
+ acc.read_memo_ended |= end
94
+
95
+
96
+ def memo_checkpoint():
97
+ """Capture the active request epoch before a backing read or dependent fill."""
98
+ acc = CURRENT.get()
99
+ if acc is None:
100
+ return None
101
+ with acc.lock:
102
+ parent = _MEMO_READ.get()
103
+ if parent is not None and parent[0] is acc and parent[1] != acc.read_memo_epoch:
104
+ return None
105
+ if (acc.closed or acc.cancelled or acc.read_memo_ended
106
+ or time.perf_counter() - acc.started >= READ_MEMO_SECONDS):
107
+ clear_read_memo(acc, end=True)
108
+ return None
109
+ return acc, acc.read_memo_epoch
110
+
111
+
112
+ def memo_read(fn):
113
+ """Revoke request snapshots on failure and keep nested fills in their original epoch."""
114
+ @wraps(fn)
115
+ def guarded(*args, **kwargs):
116
+ acc = CURRENT.get()
117
+ if acc is None:
118
+ return fn(*args, **kwargs)
119
+ parent = _MEMO_READ.get()
120
+ checkpoint = parent if parent is not None and parent[0] is acc else memo_checkpoint()
121
+ token = _MEMO_READ.set(checkpoint)
122
+ try:
123
+ return fn(*args, **kwargs)
124
+ except BaseException:
125
+ clear_read_memo(acc)
126
+ raise
127
+ finally:
128
+ _MEMO_READ.reset(token)
129
+ return guarded
130
+
131
+
132
+ def memo_peek(key):
133
+ """Find immutable text only while this request's bounded memo remains active."""
134
+ checkpoint = memo_checkpoint()
135
+ if checkpoint is not None:
136
+ acc, epoch = checkpoint
137
+ with acc.lock:
138
+ if epoch == acc.read_memo_epoch and key in acc.read_memo:
139
+ return True, acc.read_memo[key]
140
+ return False, None
141
+
142
+
143
+ def memo_seed(key, value, checkpoint):
144
+ """Publish successful backing-read text without reviving an expired or invalidated epoch."""
145
+ if checkpoint is not None and (value is None or isinstance(value, str)):
146
+ acc, epoch = checkpoint
147
+ def key_size(part):
148
+ return sys.getsizeof(part) + (sum(key_size(item) for item in part)
149
+ if isinstance(part, tuple) else 0)
150
+ size = key_size(key) + sys.getsizeof(value)
151
+ with acc.lock:
152
+ if (CURRENT.get() is acc and epoch == acc.read_memo_epoch
153
+ and not acc.closed and not acc.cancelled
154
+ and not acc.read_memo_ended
155
+ and time.perf_counter() - acc.started < READ_MEMO_SECONDS
156
+ and key not in acc.read_memo
157
+ and len(acc.read_memo) < READ_MEMO_ENTRIES
158
+ and acc.read_memo_bytes + size <= READ_MEMO_BYTES):
159
+ acc.read_memo[key] = value
160
+ acc.read_memo_bytes += size
161
+
162
+
163
+ @memo_read
164
+ def memo_text(key, read, *, fresh=False):
165
+ """Reuse bounded immutable backing-read text during the short pre-header request phase."""
166
+ if fresh:
167
+ clear_read_memo()
168
+ return read()
169
+ checkpoint = memo_checkpoint()
170
+ found, value = memo_peek(key)
171
+ if found:
172
+ return value
173
+ value = read()
174
+ memo_seed(key, value, checkpoint)
175
+ return value
176
+
177
+
178
+ @contextmanager
179
+ def thread_scope():
180
+ """Associate only this operation's thread with the current request for sampling."""
181
+ acc = CURRENT.get()
182
+ if acc is None or acc.closed:
183
+ yield
184
+ return
185
+ tid = threading.get_ident()
186
+ with _THREAD_LOCK:
187
+ previous = _THREADS.get(tid)
188
+ _THREADS[tid] = acc
189
+ try:
190
+ yield
191
+ finally:
192
+ with _THREAD_LOCK:
193
+ if previous is None:
194
+ _THREADS.pop(tid, None)
195
+ else:
196
+ _THREADS[tid] = previous
197
+
198
+
199
+ def associated_threads():
200
+ with _THREAD_LOCK:
201
+ return dict(_THREADS)
202
+
203
+
204
+ class RequestDeadlineExceeded(TimeoutError):
205
+ """A request exhausted its analytical query budget."""
206
+
207
+
208
+ def current_budget():
209
+ """Return the earliest absolute deadline and its named failure."""
210
+ budget = _BUDGET.get()
211
+ acc = CURRENT.get()
212
+ if acc is not None:
213
+ if acc.cancelled:
214
+ raise RequestDeadlineExceeded("request cancelled")
215
+ if acc.deadline is not None and (budget is None or acc.deadline <= budget[0]):
216
+ budget = (acc.deadline, RequestDeadlineExceeded)
217
+ return budget
218
+
219
+
220
+ def remaining_budget(limit=None):
221
+ """Check the absolute budget before lending its remaining seconds to a blocking operation."""
222
+ budget = current_budget()
223
+ if budget is None:
224
+ return limit
225
+ remaining = budget[0] - time.monotonic()
226
+ if remaining <= 0:
227
+ raise budget[1]("absolute operation deadline exceeded")
228
+ return remaining if limit is None else min(limit, remaining)
229
+
230
+
231
+ @contextmanager
232
+ def budget_scope(deadline, error=RequestDeadlineExceeded):
233
+ """Narrow a controlled operation's budget without extending its caller's deadline."""
234
+ parent = current_budget()
235
+ token = _BUDGET.set(parent if parent is not None and parent[0] <= deadline else (deadline, error))
236
+ try:
237
+ remaining_budget()
238
+ yield
239
+ remaining_budget()
240
+ finally:
241
+ _BUDGET.reset(token)
242
+
243
+
244
+ def raise_if_resource_error(error):
245
+ """Keep request resource refusals out of readiness fallbacks."""
246
+ if CURRENT.get() is None:
247
+ return
248
+ known = {
249
+ "RequestDeadlineExceeded", "DeadlineCapacityExceeded", "PoolTimeout", "TooManyRequests",
250
+ "QueryCanceled", "LockNotAvailable", "SharedReadTimeout", "OdooPoolTimeout",
251
+ "OperationalError", "InterfaceError", "IdleInTransactionSessionTimeout",
252
+ "OutOfMemoryException", "ConnectionTimeout", "PostgresOperationTimeout", "OdooRequestTimeout"}
253
+ seen, probe = set(), error
254
+ while probe is not None and id(probe) not in seen:
255
+ seen.add(id(probe))
256
+ if type(probe).__name__ in known:
257
+ raise error
258
+ probe = probe.__cause__ or probe.__context__
259
+
260
+
261
+ def configure_deadlines(arm):
262
+ """Install the runtime's registered statement watchdog without an upward import."""
263
+ global _DEADLINE_ARM
264
+ _DEADLINE_ARM = arm
265
+
266
+
267
+ def configure_lock_probes(arm):
268
+ """Install bounded lock diagnostics supplied by the runtime."""
269
+ global _LOCK_PROBE_ARM
270
+ _LOCK_PROBE_ARM = arm
271
+
272
+
273
+ @contextmanager
274
+ def lock_wait_probe(waiter_pid, wait_s, diagnose):
275
+ """Sample blockers while the row transaction waits, using a separate connection."""
276
+ handle = _LOCK_PROBE_ARM(waiter_pid, wait_s, diagnose) if (
277
+ waiter_pid is not None and _LOCK_PROBE_ARM is not None) else None
278
+ try:
279
+ yield handle
280
+ finally:
281
+ if handle is not None:
282
+ handle.close()
283
+
284
+
285
+ @contextmanager
286
+ def duckdb_operation(connection):
287
+ """Arm only the active cursor call and disarm before its connection can be reused."""
288
+ acc = CURRENT.get()
289
+ arm = _DEADLINE_ARM
290
+ if acc is None or acc.deadline is None or arm is None:
291
+ yield
292
+ return
293
+ if acc.cancelled or time.monotonic() >= acc.deadline:
294
+ raise RequestDeadlineExceeded("request query deadline exceeded")
295
+ handle = arm(connection, acc)
296
+ error = None
297
+ try:
298
+ yield
299
+ except BaseException as exc:
300
+ error = exc
301
+ raise
302
+ finally:
303
+ expired = handle.close()
304
+ if expired and (error is None or type(error).__name__ == "InterruptException"):
305
+ raise RequestDeadlineExceeded("request query deadline exceeded") from error
306
+
307
+
308
+ def text_bytes(value):
309
+ """Measure document payload bytes without serializing decoded caller objects."""
310
+ if isinstance(value, str):
311
+ return len(value.encode("utf-8"))
312
+ if isinstance(value, (bytes, bytearray, memoryview)):
313
+ return len(value)
314
+ return 0
315
+
316
+
317
+ def record_store(operation, keys, milliseconds=0.0, *, bytes_read=0, bytes_written=0, count=1):
318
+ """Record exclusive operation time and measured payload bytes, without document values."""
319
+ acc = CURRENT.get()
320
+ if acc is None or acc.closed:
321
+ return
322
+ keys = [keys] if isinstance(keys, str) else list(keys)
323
+ keys = keys or ["<none>"]
324
+ with acc.lock:
325
+ if acc.closed:
326
+ return
327
+ acc.store_ms += milliseconds
328
+ if count:
329
+ acc.store_operations[operation] += count
330
+ acc.store_reads += count if operation != "update" else 0
331
+ acc.store_bytes += bytes_read
332
+ acc.store_write_bytes += bytes_written
333
+ for key in keys:
334
+ key = str(key)
335
+ if not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", key):
336
+ prefix = re.split(r"[/.:]", key, maxsplit=1)[0]
337
+ prefix = prefix if re.fullmatch(r"[a-z][a-z_]{0,31}", prefix) else "document"
338
+ key = prefix + "#" + hashlib.sha256(key.encode("utf-8")).hexdigest()[:12]
339
+ if key not in acc.store_keys and len(acc.store_keys) >= 64:
340
+ key = "<other>"
341
+ row = acc.store_keys.setdefault(key, {"count": 0, "ms": 0.0, "bytes": 0, "write_bytes": 0})
342
+ row["count"] += count
343
+ row["ms"] += milliseconds / len(keys)
344
+ row["bytes"] += bytes_read / len(keys)
345
+ row["write_bytes"] += bytes_written / len(keys)
346
+
347
+
348
+ def store_operation(operation, *, many=False):
349
+ """Measure caller wait, acquisition and parse once, excluding nested store operation time."""
350
+ def decorate(fn):
351
+ @wraps(fn)
352
+ def measured(*args, **kwargs):
353
+ if CURRENT.get() is None:
354
+ return fn(*args, **kwargs)
355
+ keys = args[0] if args else kwargs.get("names" if many else "name", "<none>")
356
+ if many:
357
+ keys = tuple(dict.fromkeys(str(key) for key in (keys or ()) if str(key)))
358
+ if args:
359
+ args = (keys,) + args[1:]
360
+ else:
361
+ kwargs["names"] = keys
362
+ else:
363
+ keys = [str(keys)]
364
+ parent = _STORE_SPAN.get()
365
+ span = [0.0]
366
+ token = _STORE_SPAN.set(span)
367
+ started = time.perf_counter()
368
+ try:
369
+ with thread_scope():
370
+ return fn(*args, **kwargs)
371
+ except BaseException:
372
+ clear_read_memo()
373
+ raise
374
+ finally:
375
+ elapsed = (time.perf_counter() - started) * 1000
376
+ _STORE_SPAN.reset(token)
377
+ if parent is not None:
378
+ parent[0] += elapsed
379
+ record_store(operation, keys, max(0.0, elapsed - span[0]))
380
+ return measured
381
+ return decorate
382
+
383
+
384
+ def sql_text(statement):
385
+ """Keep bounded SQL structure and remove inline literals and comments."""
386
+ if not isinstance(statement, str):
387
+ return "<statement>"
388
+ return " ".join(_SQL_VALUES.sub("?", statement).split())[:300]
389
+
390
+
391
+ def record_duckdb(statement, milliseconds, *, count=1, statement_ms=None):
392
+ """Record execution or fetch time without accepting a parameter argument."""
393
+ acc = CURRENT.get()
394
+ if acc is None or acc.closed:
395
+ return
396
+ with acc.lock:
397
+ if acc.closed:
398
+ return
399
+ acc.duckdb_ms += milliseconds
400
+ acc.duckdb_count += count
401
+ duration = milliseconds if statement_ms is None else statement_ms
402
+ if duration >= acc.slowest_sql_ms:
403
+ acc.slowest_sql_ms = duration
404
+ acc.slowest_sql = sql_text(statement)
405
+
406
+
407
+ def record_queue(milliseconds):
408
+ acc = CURRENT.get()
409
+ if acc is not None and not acc.closed:
410
+ with acc.lock:
411
+ if not acc.closed:
412
+ acc.queue_ms += milliseconds
platform/core/store_pg.py CHANGED
@@ -48,12 +48,14 @@ import tempfile
48
  import threading
49
  import time
50
  from collections import OrderedDict
 
51
  from functools import lru_cache
52
  from pathlib import Path
53
  from urllib.parse import urlparse
54
 
55
  import core.data_binding as data_binding
56
  from core import freeze_notice
 
57
  from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
58
 
59
  _URL_ENV = 'DATABASE_URL'
@@ -341,11 +343,18 @@ def _holder_pool_class(base, timeout_error):
341
  super().__init__(*a, **kw)
342
 
343
  def getconn(self, timeout=None):
 
 
344
  try:
345
  conn = super().getconn(timeout=timeout)
346
  except timeout_error:
347
  self._log_exhausted()
348
  raise
 
 
 
 
 
349
  label = _holder_label()
350
  with self._holders_lock:
351
  self._holders[id(conn)] = (time.monotonic(), label)
@@ -356,6 +365,19 @@ def _holder_pool_class(base, timeout_error):
356
  self._holders.pop(id(conn), None)
357
  super().putconn(conn)
358
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
  def _log_exhausted(self):
360
  now = time.monotonic()
361
  with self._holders_lock:
@@ -381,6 +403,67 @@ def _lock_timeout_s():
381
  return max(1.0, value)
382
 
383
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
384
  def _pool():
385
  """The psycopg connection pool, opened once per process.
386
 
@@ -399,6 +482,7 @@ def _pool():
399
  return _POOL['pool']
400
  try:
401
  from psycopg_pool import ConnectionPool, PoolTimeout
 
402
  except ImportError as e: # pragma: no cover - depends on B-3
403
  raise RuntimeError(
404
  'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
@@ -415,11 +499,17 @@ def _pool():
415
  # real requests, not be held awake by this process's idle pool for psycopg's 10-minute
416
  # default. Thirty seconds still reuses a normal request burst without creating a resident
417
  # database client.
418
- options = {'kwargs': {'sslrootcert': sslrootcert}} if sslrootcert is not None else {}
419
- # The subclass only records holders and names them on PoolTimeout. Sizes are unchanged.
 
 
 
 
 
420
  pool_cls = _holder_pool_class(ConnectionPool, PoolTimeout)
421
- _POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=10.0,
422
- max_idle=30.0, **options)
 
423
  _POOL['url'] = url
424
  _POOL['sslrootcert'] = sslrootcert
425
  return _POOL['pool']
@@ -434,6 +524,7 @@ def available():
434
  """
435
  if not _url():
436
  return False
 
437
  if _POOL.get('ok'):
438
  return True
439
  try:
@@ -441,7 +532,8 @@ def available():
441
  con.execute('SELECT 1')
442
  _POOL['ok'] = True
443
  return True
444
- except Exception:
 
445
  return False
446
 
447
 
@@ -477,7 +569,7 @@ def _q(template, kind, tenant_slug=None):
477
  #: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
478
  #: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
479
  #: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
480
- #: ⚠ NOT A CACHE: nothing outlives the SELECT that produced it, so `fresh` still means nothing here.
481
  #: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
482
  #: finishes, and a read never joins a flight started under an older generation.
483
  #: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
@@ -488,34 +580,57 @@ _READ_FLIGHT_LOCK = threading.Lock()
488
 
489
 
490
  class _ReadFlight:
491
- __slots__ = ('done', 'text', 'error', 'generation')
492
 
493
  def __init__(self, generation):
494
  self.done = threading.Event()
495
  self.text = None
496
  self.error = None
497
  self.generation = generation
 
 
 
 
 
 
 
 
 
498
 
499
 
500
  def _bump_write_generation(name, tenant_slug):
 
501
  key = (_effective_slug(tenant_slug), str(name))
502
  with _READ_FLIGHT_LOCK:
503
  _WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
 
 
 
 
504
 
505
 
506
- def _shared_read(name, drops, tenant_slug, fetch_text):
507
- """`fetch_text()` once for every concurrent identical read. -> the value's JSON text, or None."""
508
  doc = (_effective_slug(tenant_slug), str(name))
509
- key = doc + (drops,)
 
 
 
 
 
510
  with _READ_FLIGHT_LOCK:
511
  generation = _WRITE_GENERATION.get(doc, 0)
512
  flight = _READ_FLIGHTS.get(key)
513
- leader = flight is None or flight.generation != generation
 
514
  if leader:
515
  flight = _ReadFlight(generation)
516
  _READ_FLIGHTS[key] = flight
517
  if not leader:
518
- flight.done.wait()
 
 
 
519
  if flight.error is not None:
520
  # ⛔ THE LEADER'S OWN EXCEPTION, never a wrapper: routes and retries classify store
521
  # failures by TYPE (`PoolTimeout`/`OperationalError` -> 503 "try again", a transient
@@ -524,7 +639,10 @@ def _shared_read(name, drops, tenant_slug, fetch_text):
524
  raise flight.error
525
  return flight.text
526
  try:
527
- flight.text = fetch_text()
 
 
 
528
  except BaseException as exc:
529
  flight.error = exc
530
  raise
@@ -538,26 +656,172 @@ def _shared_read(name, drops, tenant_slug, fetch_text):
538
 
539
  def _parsed(text):
540
  """The caller's own object from the shared JSON text (`None` = the key is absent)."""
541
- if text is None:
542
- return {}
543
- value = json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text
544
- return dict(value) if isinstance(value, dict) else value
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
545
 
546
 
547
  def get(name, fresh=False, tenant_slug=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
548
  """The value stored under `name`, or `{}` when the key is genuinely absent.
549
 
550
- Ordinary reads share identical SELECTs already in flight. A fresh read starts its own SELECT
551
- so deletion readback cannot join a query begun before another process's committed mutation.
552
  """
 
 
 
 
 
 
553
  def fetch():
554
- with _pool().connection() as con:
 
 
 
 
 
 
555
  row = con.execute(
556
- _q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
557
- (str(name),)).fetchone()
558
- return row[0] if row else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
559
 
560
- return _parsed(fetch() if fresh else _shared_read(name, None, tenant_slug, fetch))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
561
 
562
 
563
  _PROJECTION_TEXT = OrderedDict()
@@ -569,8 +833,10 @@ _PROJECTION_TEXT_LIMIT = 16 * 1024 * 1024
569
  def _remember_projection(key, revision, stamp, text):
570
  """Retain bounded immutable content whose revision must pass the next database read."""
571
  global _PROJECTION_TEXT_BYTES
 
572
  size = len(text.encode('utf-8')) if isinstance(text, str) else 0
573
  with _PROJECTION_TEXT_LOCK:
 
574
  previous = _PROJECTION_TEXT.pop(key, None)
575
  if previous is not None:
576
  _PROJECTION_TEXT_BYTES -= previous[3]
@@ -584,6 +850,27 @@ def _remember_projection(key, revision, stamp, text):
584
 
585
 
586
  def get_projection(name, drop=(), tenant_slug=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
587
  """`core.store.get_projection` with the DROP performed inside Postgres.
588
 
589
  `user_tables` is one JSONB value, and tenant #0's stored ``rows`` have measured at 28.6 MB
@@ -597,11 +884,11 @@ def get_projection(name, drop=(), tenant_slug=None):
597
  so marking those dictionaries needs no second JSON serialization and parse. The ``CASE`` retains the generic store contract:
598
  non-object JSON values are returned untouched rather than making ``jsonb_each`` error.
599
  """
600
- import core.store as _hf # noqa: PLC0415 — lazy: _project lives with the class
601
-
602
  drops = tuple(str(d) for d in drop)
603
  pool = _pool()
604
  content_key = (pool, _effective_slug(tenant_slug), str(name), tuple(sorted(drops)))
 
 
605
 
606
  def fetch():
607
  with _PROJECTION_TEXT_LOCK:
@@ -612,9 +899,13 @@ def get_projection(name, drop=(), tenant_slug=None):
612
  con.execute("SET LOCAL work_mem = '64MB'")
613
  if not drops:
614
  row = con.execute(
615
- _q('SELECT value::text FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
 
616
  (str(name),)).fetchone()
617
- return row[0] if row else None
 
 
 
618
  row = con.execute(
619
  _q('SELECT kv.rev, kv.updated_at, CASE '
620
  'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ELSE (CASE '
@@ -629,23 +920,21 @@ def get_projection(name, drop=(), tenant_slug=None):
629
  list(drops), str(name))).fetchone()
630
  # The SELECT always checks the live row. Only its unchanged JSON reconstruction is reused.
631
  text = row[2] if row else None
 
 
632
  if row and text is None:
633
  if cached is None or row[:2] != cached[:2]:
634
  raise RuntimeError('A projected read returned no content for its current revision')
635
  text = cached[2]
636
  _remember_projection(content_key, row[0] if row else None, row[1] if row else None, text)
637
- return text
638
 
639
  # D-2640: the projection is the read 14 concurrent backends were making; share it too.
640
- text = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch)
641
- data = _parsed(text)
642
- dropped = frozenset(drops)
643
- if not isinstance(text, (str, bytes, bytearray)):
644
- return _hf._project(data, dropped)
645
- if not isinstance(data, dict):
646
- return data
647
- return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped)
648
- if isinstance(value, dict) else value for key, value in data.items()}
649
 
650
 
651
  def _get_workbook_projection(name, *, metadata_only=False, tab_ids=None, cells=None,
@@ -769,6 +1058,7 @@ def _amazon_backup_policy(*, trigger_at, checked_at, tenant_slug=None):
769
  'cause': 'provider_backup_expiry_verification_unavailable'}
770
 
771
 
 
772
  def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
773
  """Several projected documents in one round trip, keyed by their requested store key.
774
 
@@ -782,7 +1072,10 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
782
  if not keys:
783
  return {}
784
  drops = tuple(str(d) for d in drop)
785
- with _pool().connection() as con:
 
 
 
786
  if timeout_s is not None:
787
  timeout_ms = max(1, int(float(timeout_s) * 1000))
788
  # Transaction-local: a pooled connection must not lend this caller's budget to the
@@ -791,16 +1084,34 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
791
  con.execute("SELECT set_config('statement_timeout', %s, true)",
792
  (str(timeout_ms),))
793
  rows = con.execute(
794
- _q('SELECT kv.key, CASE '
795
  "WHEN jsonb_typeof(kv.value) = 'object' THEN "
796
  "COALESCE((SELECT jsonb_object_agg(e.entry_key, "
797
  "CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
798
  'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
799
  'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
800
- "'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv "
801
  'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
802
  (list(drops), keys)).fetchall()
803
- found = {str(key): value for key, value in rows}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
804
  dropped = frozenset(drops)
805
  return {
806
  key: _hf._project(
@@ -832,17 +1143,221 @@ def _get_entry(name, entry, tenant_slug=None):
832
  return dict(value) if isinstance(value, dict) else None
833
 
834
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
835
  def _get_many(names, tenant_slug=None):
836
  """Several ordinary documents in one round trip, keyed by requested store key."""
837
  keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
838
  if not keys:
839
  return {}
840
- with _pool().connection() as con:
 
 
 
841
  rows = con.execute(
842
- _q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])',
843
  'store_kv', tenant_slug),
844
  (keys,)).fetchall()
845
- found = {str(key): value for key, value in rows}
 
 
 
 
 
 
 
 
 
 
 
 
 
846
  return {
847
  key: (dict(found[key]) if isinstance(found.get(key), dict)
848
  else (found[key] if key in found else {}))
@@ -957,6 +1472,7 @@ def put(name, data, tenant_slug=None):
957
  return data
958
 
959
 
 
960
  def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=None):
961
  """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
962
 
@@ -1006,7 +1522,36 @@ def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=
1006
 
1007
 
1008
  def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
1009
- """`update`'s transaction body (split out so the D-2640 generation bump wraps all of it)."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1010
  from psycopg.errors import LockNotAvailable
1011
  wait_s = _lock_timeout_s()
1012
  with _pool().connection() as con:
@@ -1015,21 +1560,28 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
1015
  # lock gives its pooled connection back after `wait_s` instead of holding it forever.
1016
  con.execute("SELECT set_config('lock_timeout', %s, true)",
1017
  (f'{int(wait_s * 1000)}ms',))
1018
- try:
1019
- row = con.execute(
1020
- _q('SELECT value, rev FROM {tbl} WHERE key = %s FOR UPDATE',
1021
- 'store_kv', tenant_slug),
1022
- (str(name),)).fetchone()
1023
- except LockNotAvailable:
1024
- print(f'[store-pg] lock wait over {wait_s:g}s on '
1025
- f'{_effective_slug(tenant_slug)}/{name}', flush=True)
1026
- raise
 
 
 
 
 
 
 
1027
  if expect_rev is not None:
1028
  current = int(row[1]) if row and row[1] is not None else 0
1029
  if current != int(expect_rev):
1030
  from core.view_audit import ViewRevisionStale
1031
  raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev))
1032
- data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
1033
  result = fn(data)
1034
  data = result if result is not None else data
1035
  payload = _payload(name, data, con, tenant_slug)
@@ -1038,6 +1590,8 @@ def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
1038
  'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
1039
  'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
1040
  (str(name), payload))
 
 
1041
  # ⚠ A LIST, OR ONE DICT, AND THE LIST IS NOT A CONVENIENCE. Moving views between
1042
  # folders is ONE workspace write that changes several views' placements, so the
1043
  # trail for it is several rows or it is wrong; a single-row keyword would have
@@ -1096,6 +1650,46 @@ def delete_path(path_in_repo, tenant_slug=None):
1096
 
1097
 
1098
  def revision(name, tenant_slug=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1099
  """`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6.
1100
 
1101
  ⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and
@@ -1115,8 +1709,12 @@ def revision(name, tenant_slug=None):
1115
  """
1116
  with _pool().connection() as con:
1117
  row = con.execute(
1118
- _q('SELECT rev, updated_at FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
 
 
1119
  (str(name),)).fetchone()
 
 
1120
  rev = int(row[0]) if row and row[0] is not None else 0
1121
  stamp = row[1] if row else None
1122
  # ⚠ EPOCH SECONDS ON BOTH BACKENDS. `core.store` records `time.time()`; this column is a
@@ -1163,8 +1761,8 @@ class PgStore:
1163
  """`core.store.Store`'s interface for ONE tenant's schema.
1164
 
1165
  Deliberately a thin binder over the module functions rather than a reimplementation: every
1166
- behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh`
1167
- ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift
1168
  between the two entry points.
1169
 
1170
  ⭐⭐ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and
@@ -1214,6 +1812,27 @@ class PgStore:
1214
  return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
1215
  timeout_s=timeout_s)
1216
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1217
  def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None,
1218
  expected_rev=None):
1219
  return _get_workbook_projection(name, metadata_only=metadata_only, tab_ids=tab_ids,
@@ -1244,6 +1863,11 @@ class PgStore:
1244
  """Bound multi-key read used by request-scoped read snapshots."""
1245
  return _get_many(names, tenant_slug=self.tenant_slug)
1246
 
 
 
 
 
 
1247
  def get_entry(self, name, entry):
1248
  """Bound single-entry read; None means absent, and the caller falls back to `get`."""
1249
  return _get_entry(name, entry, tenant_slug=self.tenant_slug)
 
48
  import threading
49
  import time
50
  from collections import OrderedDict
51
+ from contextlib import contextmanager
52
  from functools import lru_cache
53
  from pathlib import Path
54
  from urllib.parse import urlparse
55
 
56
  import core.data_binding as data_binding
57
  from core import freeze_notice
58
+ from core import request_timing as _request_timing
59
  from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
60
 
61
  _URL_ENV = 'DATABASE_URL'
 
343
  super().__init__(*a, **kw)
344
 
345
  def getconn(self, timeout=None):
346
+ timeout = _request_timing.remaining_budget(
347
+ timeout if timeout is not None else getattr(self, 'timeout', _pg_limits()['acquire']))
348
  try:
349
  conn = super().getconn(timeout=timeout)
350
  except timeout_error:
351
  self._log_exhausted()
352
  raise
353
+ try:
354
+ _request_timing.remaining_budget()
355
+ except BaseException:
356
+ super().putconn(conn)
357
+ raise
358
  label = _holder_label()
359
  with self._holders_lock:
360
  self._holders[id(conn)] = (time.monotonic(), label)
 
365
  self._holders.pop(id(conn), None)
366
  super().putconn(conn)
367
 
368
+ @contextmanager
369
+ def connection(self, timeout=None):
370
+ # SET LOCAL survives transaction pooling and rolls back with the caller's writes.
371
+ with super().connection(timeout=timeout) as con:
372
+ _request_timing.remaining_budget()
373
+ with con.transaction():
374
+ limits = _pg_limits()
375
+ con.execute("SELECT set_config('statement_timeout', %s, true), "
376
+ "set_config('idle_in_transaction_session_timeout', %s, true)",
377
+ (f"{int(limits['statement'] * 1000)}ms",
378
+ f"{int(limits['idle'] * 1000)}ms"))
379
+ yield con
380
+
381
  def _log_exhausted(self):
382
  now = time.monotonic()
383
  with self._holders_lock:
 
403
  return max(1.0, value)
404
 
405
 
406
+ def _bound_env(name, default, minimum=.01):
407
+ try:
408
+ value = float(os.environ.get(name, default))
409
+ return max(minimum, value) if math.isfinite(value) else float(default)
410
+ except ValueError:
411
+ return float(default)
412
+
413
+
414
+ def _pg_limits():
415
+ worker = os.environ.get('AIOS_PROCESS_ROLE') == 'worker'
416
+ return {
417
+ 'connect': int(_bound_env('AIOS_PG_CONNECT_TIMEOUT_S', 5, 2)),
418
+ 'acquire': _bound_env('AIOS_PG_POOL_TIMEOUT_S', 10),
419
+ 'waiting': int(_bound_env('AIOS_PG_MAX_WAITING', 16, 1)),
420
+ 'statement': _bound_env('AIOS_PG_STATEMENT_TIMEOUT_S', 300 if worker else 30),
421
+ 'idle': _bound_env('AIOS_PG_IDLE_TRANSACTION_TIMEOUT_S', 300 if worker else 60),
422
+ 'network_ms': int(_bound_env('AIOS_PG_TCP_USER_TIMEOUT_MS', 15000, 1)),
423
+ }
424
+
425
+
426
+ def _read_timeout_s():
427
+ limits = _pg_limits()
428
+ return limits['acquire'] + limits['statement'] + limits['network_ms'] / 1000
429
+
430
+
431
+ class PostgresOperationTimeout(TimeoutError):
432
+ """A PostgreSQL socket operation exhausted its client-side limit."""
433
+
434
+
435
+ def _checked_pg_wait(connection, gen):
436
+ """Check each libpq poll and discard expired connections without asynchronous cancellation."""
437
+ deadline = time.monotonic() + _pg_limits()['statement']
438
+ try:
439
+ with _request_timing.budget_scope(deadline, PostgresOperationTimeout):
440
+ ready = None
441
+ while True:
442
+ _request_timing.remaining_budget()
443
+ try:
444
+ state = gen.send(ready)
445
+ except StopIteration as result:
446
+ _request_timing.remaining_budget()
447
+ return result.value
448
+ ready = yield state
449
+ except (PostgresOperationTimeout, SharedReadTimeout, _request_timing.RequestDeadlineExceeded):
450
+ # No cancel packet can arrive after this socket is returned to the pool.
451
+ connection.close()
452
+ raise
453
+ finally:
454
+ gen.close()
455
+
456
+
457
+ @lru_cache(maxsize=None)
458
+ def _budget_connection_class(base):
459
+ class BudgetConnection(base):
460
+ def wait(self, gen, interval=.01):
461
+ # libpq yields on socket waits, including BEGIN, settings, fetch and COMMIT.
462
+ return super().wait(_checked_pg_wait(self, gen), interval=.01)
463
+
464
+ return BudgetConnection
465
+
466
+
467
  def _pool():
468
  """The psycopg connection pool, opened once per process.
469
 
 
482
  return _POOL['pool']
483
  try:
484
  from psycopg_pool import ConnectionPool, PoolTimeout
485
+ from psycopg import Connection
486
  except ImportError as e: # pragma: no cover - depends on B-3
487
  raise RuntimeError(
488
  'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
 
499
  # real requests, not be held awake by this process's idle pool for psycopg's 10-minute
500
  # default. Thirty seconds still reuses a normal request burst without creating a resident
501
  # database client.
502
+ limits = _pg_limits()
503
+ connection_kwargs = dict(connect_timeout=limits['connect'], keepalives=1,
504
+ keepalives_idle=30, keepalives_interval=10, keepalives_count=3,
505
+ tcp_user_timeout=limits['network_ms'])
506
+ if sslrootcert is not None:
507
+ connection_kwargs['sslrootcert'] = sslrootcert
508
+ # The pool applies transaction settings and lends connections with checked socket polls.
509
  pool_cls = _holder_pool_class(ConnectionPool, PoolTimeout)
510
+ _POOL['pool'] = pool_cls(url, min_size=0, max_size=8, open=True, timeout=limits['acquire'],
511
+ max_idle=30.0, max_waiting=limits['waiting'], kwargs=connection_kwargs,
512
+ connection_class=_budget_connection_class(Connection))
513
  _POOL['url'] = url
514
  _POOL['sslrootcert'] = sslrootcert
515
  return _POOL['pool']
 
524
  """
525
  if not _url():
526
  return False
527
+ _request_timing.remaining_budget()
528
  if _POOL.get('ok'):
529
  return True
530
  try:
 
532
  con.execute('SELECT 1')
533
  _POOL['ok'] = True
534
  return True
535
+ except Exception as exc:
536
+ _request_timing.raise_if_resource_error(exc)
537
  return False
538
 
539
 
 
569
  #: copy, drove the node to 94% swap and 100% CPU. A burst of requests arriving during a sync asks for
570
  #: the SAME document. So a read that finds an identical read already in flight (same tenant, key and
571
  #: projection) waits for it and parses the same TEXT itself, instead of opening another backend.
572
+ #: Completed reads may retain immutable text, validated by a new SELECT before reuse.
573
  #: ⚠ READ-YOUR-WRITES HOLDS: every `put`/`update` in this process bumps the key's generation when it
574
  #: finishes, and a read never joins a flight started under an older generation.
575
  #: ⚠ Each caller gets its OWN parsed object (the shared thing is an immutable str), because callers
 
580
 
581
 
582
  class _ReadFlight:
583
+ __slots__ = ('done', 'text', 'error', 'generation', 'deadline', 'started')
584
 
585
  def __init__(self, generation):
586
  self.done = threading.Event()
587
  self.text = None
588
  self.error = None
589
  self.generation = generation
590
+ self.started = time.monotonic()
591
+ self.deadline = self.started + _read_timeout_s()
592
+ budget = _request_timing.current_budget()
593
+ if budget is not None:
594
+ self.deadline = min(self.deadline, budget[0])
595
+
596
+
597
+ class SharedReadTimeout(TimeoutError):
598
+ """A coalesced follower exhausted the original leader's read budget."""
599
 
600
 
601
  def _bump_write_generation(name, tenant_slug):
602
+ _request_timing.clear_read_memo()
603
  key = (_effective_slug(tenant_slug), str(name))
604
  with _READ_FLIGHT_LOCK:
605
  _WRITE_GENERATION[key] = _WRITE_GENERATION.get(key, 0) + 1
606
+ with _WHOLE_TEXT_LOCK:
607
+ for content_key in list(_WHOLE_TEXT):
608
+ if content_key[1:] == key:
609
+ _forget_whole(content_key)
610
 
611
 
612
+ def _shared_read(name, drops, tenant_slug, fetch_text, *, pool=None):
613
+ """Share one immutable store-controlled result across concurrent identical reads."""
614
  doc = (_effective_slug(tenant_slug), str(name))
615
+ key = doc + (drops,) if pool is None else (pool,) + doc + (drops,)
616
+ acc = _request_timing.CURRENT.get()
617
+ invalidated_at = 0.0
618
+ if acc is not None:
619
+ with acc.lock:
620
+ invalidated_at = acc.read_memo_invalidated_at
621
  with _READ_FLIGHT_LOCK:
622
  generation = _WRITE_GENERATION.get(doc, 0)
623
  flight = _READ_FLIGHTS.get(key)
624
+ leader = (flight is None or flight.generation != generation
625
+ or flight.started <= invalidated_at)
626
  if leader:
627
  flight = _ReadFlight(generation)
628
  _READ_FLIGHTS[key] = flight
629
  if not leader:
630
+ with _request_timing.budget_scope(flight.deadline, SharedReadTimeout):
631
+ if not flight.done.wait(_request_timing.remaining_budget()):
632
+ _request_timing.remaining_budget()
633
+ raise SharedReadTimeout('shared store read deadline exceeded')
634
  if flight.error is not None:
635
  # ⛔ THE LEADER'S OWN EXCEPTION, never a wrapper: routes and retries classify store
636
  # failures by TYPE (`PoolTimeout`/`OperationalError` -> 503 "try again", a transient
 
639
  raise flight.error
640
  return flight.text
641
  try:
642
+ # Only store-controlled reads use this callback. Python decoding is cooperative.
643
+ with _request_timing.budget_scope(flight.deadline, SharedReadTimeout):
644
+ text = fetch_text()
645
+ flight.text = text
646
  except BaseException as exc:
647
  flight.error = exc
648
  raise
 
656
 
657
  def _parsed(text):
658
  """The caller's own object from the shared JSON text (`None` = the key is absent)."""
659
+ try:
660
+ _request_timing.remaining_budget()
661
+ value = {} if text is None else (json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text)
662
+ value = dict(value) if isinstance(value, dict) else value
663
+ _request_timing.remaining_budget()
664
+ return value
665
+ except BaseException:
666
+ _request_timing.clear_read_memo()
667
+ raise
668
+
669
+
670
+ def _parse_read(name, operation, text):
671
+ """Keep independent parsing in H1 store time without counting another backing read."""
672
+ if _request_timing.CURRENT.get() is None:
673
+ return _parsed(text)
674
+ started = time.perf_counter()
675
+ try:
676
+ return _parsed(text)
677
+ finally:
678
+ elapsed = (time.perf_counter() - started) * 1000
679
+ parent = _request_timing._STORE_SPAN.get()
680
+ if parent is not None:
681
+ parent[0] += elapsed
682
+ _request_timing.record_store(operation, str(name), elapsed, count=0)
683
+
684
+
685
+ def _request_key(name, shape, tenant_slug, pool=None):
686
+ """Address a read shape by pool, tenant and local write generation."""
687
+ pool = _pool() if pool is None else pool
688
+ doc = (_effective_slug(tenant_slug), str(name))
689
+ with _READ_FLIGHT_LOCK:
690
+ generation = _WRITE_GENERATION.get(doc, 0)
691
+ return (pool,) + doc + (shape, generation)
692
+
693
+
694
+ @_request_timing.memo_read
695
+ def _request_text(name, shape, tenant_slug, read, *, fresh=False):
696
+ """Share only this request's validated backing read, including independently acquired handles."""
697
+ if _request_timing.CURRENT.get() is None:
698
+ return read()
699
+ return _request_timing.memo_text(_request_key(name, shape, tenant_slug), read, fresh=fresh)
700
+
701
+
702
+ def _revision_value(raw_rev, stamp):
703
+ rev = int(raw_rev) if raw_rev is not None else 0
704
+ return {'rev': rev, 'updated_at': stamp.timestamp() if hasattr(stamp, 'timestamp') else None,
705
+ 'token': f'pg:{rev}'}
706
+
707
+
708
+ def _seed_read_token(key, checkpoint, revision, stamp):
709
+ """A successful content read validates both revision columns for every request read shape."""
710
+ _request_timing.memo_seed(key, json.dumps(_revision_value(revision, stamp)), checkpoint)
711
+
712
+
713
+ def _validated_content(name, drops, tenant_slug):
714
+ """Reuse retained content only against this request's checked revision AND timestamp."""
715
+ checkpoint = _request_timing.memo_checkpoint()
716
+ if checkpoint is None:
717
+ return False, None
718
+ key = _request_key(name, 'revision', tenant_slug)
719
+ found, text = _request_timing.memo_peek(key)
720
+ if not found:
721
+ return False, None
722
+ token = _parsed(text)
723
+ if token.get('updated_at') is None:
724
+ return False, None
725
+ cache_key = key[:3] if not drops else (*key[:3], drops)
726
+ cache, lock = ((_WHOLE_TEXT, _WHOLE_TEXT_LOCK) if not drops else
727
+ (_PROJECTION_TEXT, _PROJECTION_TEXT_LOCK))
728
+ with lock:
729
+ cached = cache.get(cache_key)
730
+ if cached is not None and _revision_value(*cached[:2]) == token:
731
+ shape = 'whole' if drops is None else ('projection', drops)
732
+ content_key = (*key[:3], shape, key[-1])
733
+ _request_timing.memo_seed(content_key, cached[2], checkpoint)
734
+ return _request_timing.memo_peek(content_key)
735
+ return False, None
736
 
737
 
738
  def get(name, fresh=False, tenant_slug=None):
739
+ """Parse an independent document from this request's backing read or a new validated read."""
740
+ def read():
741
+ found, text = (False, None) if fresh or str(name).endswith('user_tables') else (
742
+ _validated_content(name, None, tenant_slug))
743
+ return text if found else _get_text(name, fresh=fresh, tenant_slug=tenant_slug)
744
+ # Whole row documents never become request metadata snapshots.
745
+ text = read() if str(name).endswith('user_tables') and not fresh else _request_text(
746
+ name, 'whole', tenant_slug, read, fresh=fresh)
747
+ return _parse_read(name, 'get', text)
748
+
749
+
750
+ @_request_timing.store_operation("get")
751
+ def _get_text(name, fresh=False, tenant_slug=None):
752
  """The value stored under `name`, or `{}` when the key is genuinely absent.
753
 
754
+ Ordinary reads validate both live tokens before reusing immutable text and share concurrent
755
+ SELECTs. A fresh read bypasses both the content token and any existing flight.
756
  """
757
+ pool = _pool()
758
+ doc = (_effective_slug(tenant_slug), str(name))
759
+ content_key = (pool,) + doc
760
+ token_key = _request_key(name, 'revision', tenant_slug, pool)
761
+ checkpoint = None if fresh else _request_timing.memo_checkpoint()
762
+
763
  def fetch():
764
+ with _READ_FLIGHT_LOCK:
765
+ generation = _WRITE_GENERATION.get(doc, 0)
766
+ with _WHOLE_TEXT_LOCK:
767
+ cached = _WHOLE_TEXT.get(content_key) if not fresh else None
768
+ if cached is not None:
769
+ _WHOLE_TEXT.move_to_end(content_key)
770
+ with pool.connection() as con:
771
  row = con.execute(
772
+ _q('SELECT kv.rev, kv.updated_at, CASE '
773
+ 'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL '
774
+ 'ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s',
775
+ 'store_kv', tenant_slug),
776
+ (cached[0] if cached else None, cached[1] if cached else None,
777
+ str(name))).fetchone()
778
+ text = row[2] if row else None
779
+ _request_timing.record_store("get", str(name), bytes_read=_request_timing.text_bytes(text), count=0)
780
+ if row and text is None:
781
+ if cached is None or row[:2] != cached[:2]:
782
+ raise RuntimeError('A whole read returned no content for its current revision')
783
+ text = cached[2]
784
+ _remember_whole(content_key, generation, row[0] if row else None,
785
+ row[1] if row else None, text)
786
+ return text, row[0] if row else None, row[1] if row else None
787
+
788
+ text, rev, stamp = fetch() if fresh else _shared_read(name, None, tenant_slug, fetch, pool=pool)
789
+ _seed_read_token(token_key, checkpoint, rev, stamp)
790
+ return text
791
+
792
+
793
+ _WHOLE_TEXT = OrderedDict()
794
+ _WHOLE_TEXT_LOCK = threading.Lock()
795
+ _WHOLE_TEXT_BYTES = 0
796
+ _WHOLE_TEXT_LIMIT = 128 * 1024 * 1024
797
+ _WHOLE_TEXT_ENTRIES = 64
798
+
799
 
800
+ def _forget_whole(key):
801
+ """Remove one retained string while holding `_WHOLE_TEXT_LOCK`."""
802
+ global _WHOLE_TEXT_BYTES
803
+ previous = _WHOLE_TEXT.pop(key, None)
804
+ if previous is not None:
805
+ _WHOLE_TEXT_BYTES -= previous[3]
806
+
807
+
808
+ def _remember_whole(key, generation, revision, stamp, text):
809
+ """Retain bounded immutable text only while its local write generation remains current."""
810
+ global _WHOLE_TEXT_BYTES
811
+ _request_timing.remaining_budget()
812
+ size = sys.getsizeof(text) if isinstance(text, str) else 0
813
+ with _READ_FLIGHT_LOCK:
814
+ if _WRITE_GENERATION.get(key[1:], 0) != generation:
815
+ return
816
+ with _WHOLE_TEXT_LOCK:
817
+ _request_timing.remaining_budget()
818
+ _forget_whole(key)
819
+ if revision is not None and stamp is not None and 0 < size <= _WHOLE_TEXT_LIMIT:
820
+ _WHOLE_TEXT[key] = (revision, stamp, text, size)
821
+ _WHOLE_TEXT_BYTES += size
822
+ while _WHOLE_TEXT and (len(_WHOLE_TEXT) > _WHOLE_TEXT_ENTRIES
823
+ or _WHOLE_TEXT_BYTES > _WHOLE_TEXT_LIMIT):
824
+ _forget_whole(next(iter(_WHOLE_TEXT)))
825
 
826
 
827
  _PROJECTION_TEXT = OrderedDict()
 
833
  def _remember_projection(key, revision, stamp, text):
834
  """Retain bounded immutable content whose revision must pass the next database read."""
835
  global _PROJECTION_TEXT_BYTES
836
+ _request_timing.remaining_budget()
837
  size = len(text.encode('utf-8')) if isinstance(text, str) else 0
838
  with _PROJECTION_TEXT_LOCK:
839
+ _request_timing.remaining_budget()
840
  previous = _PROJECTION_TEXT.pop(key, None)
841
  if previous is not None:
842
  _PROJECTION_TEXT_BYTES -= previous[3]
 
850
 
851
 
852
  def get_projection(name, drop=(), tenant_slug=None):
853
+ """Parse independent projected definitions from the current request's backing read."""
854
+ import core.store as _hf
855
+ drops = tuple(sorted(set(str(d) for d in drop)))
856
+ can_memo = not str(name).endswith('user_tables') or 'rows' in drops
857
+ def read():
858
+ found, text = _validated_content(name, drops, tenant_slug) if can_memo else (False, None)
859
+ return text if found else _get_projection_text(name, drops, tenant_slug)
860
+ text = (read() if not can_memo else
861
+ _request_text(name, ('projection', drops), tenant_slug, read))
862
+ data = _parse_read(name, 'get_projection', text)
863
+ dropped = frozenset(drops)
864
+ if not isinstance(text, (str, bytes, bytearray)):
865
+ return _hf._project(data, dropped)
866
+ if not isinstance(data, dict):
867
+ return data
868
+ return {key: _hf._Projected({k: v for k, v in value.items() if k not in dropped}, dropped)
869
+ if isinstance(value, dict) else value for key, value in data.items()}
870
+
871
+
872
+ @_request_timing.store_operation("get_projection")
873
+ def _get_projection_text(name, drop=(), tenant_slug=None):
874
  """`core.store.get_projection` with the DROP performed inside Postgres.
875
 
876
  `user_tables` is one JSONB value, and tenant #0's stored ``rows`` have measured at 28.6 MB
 
884
  so marking those dictionaries needs no second JSON serialization and parse. The ``CASE`` retains the generic store contract:
885
  non-object JSON values are returned untouched rather than making ``jsonb_each`` error.
886
  """
 
 
887
  drops = tuple(str(d) for d in drop)
888
  pool = _pool()
889
  content_key = (pool, _effective_slug(tenant_slug), str(name), tuple(sorted(drops)))
890
+ token_key = _request_key(name, 'revision', tenant_slug, pool)
891
+ checkpoint = _request_timing.memo_checkpoint()
892
 
893
  def fetch():
894
  with _PROJECTION_TEXT_LOCK:
 
899
  con.execute("SET LOCAL work_mem = '64MB'")
900
  if not drops:
901
  row = con.execute(
902
+ _q('SELECT kv.rev, kv.updated_at, kv.value::text FROM {tbl} AS kv WHERE kv.key = %s',
903
+ 'store_kv', tenant_slug),
904
  (str(name),)).fetchone()
905
+ text = row[2] if row else None
906
+ _request_timing.record_store("get_projection", str(name),
907
+ bytes_read=_request_timing.text_bytes(text), count=0)
908
+ return text, row[0] if row else None, row[1] if row else None
909
  row = con.execute(
910
  _q('SELECT kv.rev, kv.updated_at, CASE '
911
  'WHEN kv.rev = %s AND kv.updated_at = %s THEN NULL ELSE (CASE '
 
920
  list(drops), str(name))).fetchone()
921
  # The SELECT always checks the live row. Only its unchanged JSON reconstruction is reused.
922
  text = row[2] if row else None
923
+ _request_timing.record_store("get_projection", str(name),
924
+ bytes_read=_request_timing.text_bytes(text), count=0)
925
  if row and text is None:
926
  if cached is None or row[:2] != cached[:2]:
927
  raise RuntimeError('A projected read returned no content for its current revision')
928
  text = cached[2]
929
  _remember_projection(content_key, row[0] if row else None, row[1] if row else None, text)
930
+ return text, row[0] if row else None, row[1] if row else None
931
 
932
  # D-2640: the projection is the read 14 concurrent backends were making; share it too.
933
+ text, rev, stamp = _shared_read(name, tuple(sorted(drops)), tenant_slug, fetch, pool=pool)
934
+ _seed_read_token(token_key, checkpoint, rev, stamp)
935
+ if not drops and not str(name).endswith('user_tables'):
936
+ _request_timing.memo_seed((*token_key[:3], 'whole', token_key[-1]), text, checkpoint)
937
+ return text
 
 
 
 
938
 
939
 
940
  def _get_workbook_projection(name, *, metadata_only=False, tab_ids=None, cells=None,
 
1058
  'cause': 'provider_backup_expiry_verification_unavailable'}
1059
 
1060
 
1061
+ @_request_timing.store_operation("get_projections", many=True)
1062
  def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
1063
  """Several projected documents in one round trip, keyed by their requested store key.
1064
 
 
1072
  if not keys:
1073
  return {}
1074
  drops = tuple(str(d) for d in drop)
1075
+ pool = _pool()
1076
+ request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys}
1077
+ checkpoint = _request_timing.memo_checkpoint()
1078
+ with pool.connection() as con:
1079
  if timeout_s is not None:
1080
  timeout_ms = max(1, int(float(timeout_s) * 1000))
1081
  # Transaction-local: a pooled connection must not lend this caller's budget to the
 
1084
  con.execute("SELECT set_config('statement_timeout', %s, true)",
1085
  (str(timeout_ms),))
1086
  rows = con.execute(
1087
+ _q('SELECT kv.key, (CASE '
1088
  "WHEN jsonb_typeof(kv.value) = 'object' THEN "
1089
  "COALESCE((SELECT jsonb_object_agg(e.entry_key, "
1090
  "CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
1091
  'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
1092
  'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
1093
+ "'{{}}'::jsonb) ELSE kv.value END)::text, kv.rev, kv.updated_at FROM {tbl} AS kv "
1094
  'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
1095
  (list(drops), keys)).fetchall()
1096
+ found = {}
1097
+ for key, value, rev, stamp in rows:
1098
+ _request_timing.record_store("get_projections", str(key),
1099
+ bytes_read=_request_timing.text_bytes(value), count=0)
1100
+ found[str(key)] = _parsed(value)
1101
+ _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp)
1102
+ if not str(key).endswith('user_tables') or 'rows' in drops:
1103
+ shape_key = (*request_keys[str(key)][:3], ('projection', tuple(sorted(set(drops)))),
1104
+ request_keys[str(key)][-1])
1105
+ _request_timing.memo_seed(shape_key, value, checkpoint)
1106
+ if not drops:
1107
+ _request_timing.memo_seed((*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1]),
1108
+ value, checkpoint)
1109
+ for key in keys:
1110
+ if key not in found:
1111
+ _seed_read_token(request_keys[key], checkpoint, None, None)
1112
+ if not key.endswith('user_tables') or 'rows' in drops:
1113
+ _request_timing.memo_seed((*request_keys[key][:3], ('projection', tuple(sorted(set(drops)))),
1114
+ request_keys[key][-1]), None, checkpoint)
1115
  dropped = frozenset(drops)
1116
  return {
1117
  key: _hf._project(
 
1143
  return dict(value) if isinstance(value, dict) else None
1144
 
1145
 
1146
+ @_request_timing.store_operation("json_paths")
1147
+ def _get_json_paths(name, paths, tenant_slug=None):
1148
+ """Read explicit nested JSON values without transferring their containing document."""
1149
+ import itertools
1150
+ paths = [list(map(str, path)) for path in itertools.islice(iter(paths), 4097)]
1151
+ if len(paths) > 4096:
1152
+ raise ValueError("A JSON path read accepts at most 4096 paths per batch")
1153
+ if not paths:
1154
+ return []
1155
+ with _pool().connection() as con:
1156
+ row = con.execute(_q(
1157
+ "SELECT ARRAY(SELECT (kv.value #> ARRAY(SELECT jsonb_array_elements_text(p.path)))::text "
1158
+ "FROM jsonb_array_elements(%s::jsonb) WITH ORDINALITY AS p(path, n) ORDER BY p.n) "
1159
+ "FROM {tbl} AS kv WHERE kv.key = %s", 'store_kv', tenant_slug),
1160
+ (json.dumps(paths), str(name))).fetchone()
1161
+ return [json.loads(value) if value is not None else None for value in row[0]] if row else [None] * len(paths)
1162
+
1163
+
1164
+ def _iter_json_items(name, path=(), *, batch_size=1024, tenant_slug=None):
1165
+ """Stream one JSON object through a server cursor within the existing checked connection."""
1166
+ import uuid
1167
+ size = int(batch_size)
1168
+ if not 1 <= size <= 4096:
1169
+ raise ValueError('A JSON object batch accepts between 1 and 4096 entries')
1170
+ with _pool().connection() as con:
1171
+ with con.cursor(name='registry_' + uuid.uuid4().hex) as cur:
1172
+ cur.execute(_q(
1173
+ 'SELECT jsonb_typeof(kv.value #> %s::text[]), e.key, e.value::text FROM {tbl} AS kv '
1174
+ 'LEFT JOIN LATERAL jsonb_each(CASE '
1175
+ "WHEN jsonb_typeof(kv.value #> %s::text[]) = 'object' "
1176
+ "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) AS e ON TRUE WHERE kv.key = %s",
1177
+ 'store_kv', tenant_slug), (list(map(str, path)), list(map(str, path)),
1178
+ list(map(str, path)), str(name)))
1179
+ found = False
1180
+ while True:
1181
+ _request_timing.remaining_budget()
1182
+ rows = cur.fetchmany(size)
1183
+ if not rows:
1184
+ if not found:
1185
+ raise RuntimeError('canonical_source_unavailable: the stored document disappeared')
1186
+ return
1187
+ found = True
1188
+ if any(kind not in (None, 'null', 'object') for kind, _key, _value in rows):
1189
+ raise RuntimeError('canonical_source_invalid: the stored rows are not an object')
1190
+ _request_timing.record_store('json_items', str(name),
1191
+ bytes_read=sum(len(value.encode('utf-8')) for _kind, _key, value in rows if value is not None))
1192
+ yield [(key, json.loads(value)) for _kind, key, value in rows if key is not None]
1193
+ rows.clear()
1194
+
1195
+
1196
+ def _iter_json_values(name, path=(), *, drop=(), batch_size=1024, tenant_slug=None):
1197
+ """Stream an array with selected object operands removed before transport."""
1198
+ import uuid
1199
+ size = int(batch_size)
1200
+ if not 1 <= size <= 4096:
1201
+ raise ValueError('A JSON array batch accepts between 1 and 4096 values')
1202
+ with _pool().connection() as con:
1203
+ with con.cursor(name='canonical_' + uuid.uuid4().hex) as cur:
1204
+ cur.execute(_q("SELECT e.n-1, (CASE WHEN jsonb_typeof(e.v)='object' "
1205
+ "THEN e.v - %s::text[] ELSE e.v END)::text FROM {tbl} kv "
1206
+ "CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
1207
+ "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) "
1208
+ 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug),
1209
+ (list(drop), list(map(str, path)), list(map(str, path)), str(name)))
1210
+ while True:
1211
+ _request_timing.remaining_budget()
1212
+ rows = cur.fetchmany(size)
1213
+ if not rows:
1214
+ return
1215
+ _request_timing.record_store('json_values', str(name),
1216
+ bytes_read=sum(len(value.encode('utf-8')) for _, value in rows))
1217
+ yield [(str(index), json.loads(value)) for index, value in rows]
1218
+ rows.clear()
1219
+
1220
+
1221
+ class _ViewWorkspace(dict):
1222
+ pass
1223
+
1224
+
1225
+ def _get_view_workspace(name, tenant_slug=None):
1226
+ """Saved-view metadata without authored memberships crossing the JSON decoder."""
1227
+ with _pool().connection() as con:
1228
+ row = con.execute(_q("SELECT kv.rev, kv.updated_at, CASE WHEN jsonb_typeof(kv.value)='object' THEN "
1229
+ "coalesce((SELECT jsonb_object_agg(u.k, CASE WHEN jsonb_typeof(u.v)='object' THEN "
1230
+ "(u.v - 'overlays' - 'trash') || CASE WHEN jsonb_typeof(u.v->'views')='object' THEN "
1231
+ "jsonb_build_object('views', coalesce((SELECT jsonb_object_agg(v.k, CASE "
1232
+ "WHEN jsonb_typeof(v.v->'config')='object' THEN jsonb_set(v.v, '{{config}}', "
1233
+ "CASE WHEN jsonb_typeof(v.v->'config'->'memberPids')='array' THEN "
1234
+ "(v.v->'config') - 'memberPids' ELSE v.v->'config' END) ELSE v.v END) FROM jsonb_each(u.v->'views') v(k,v)), "
1235
+ "'{{}}'::jsonb)) ELSE '{{}}'::jsonb END ELSE u.v END) FROM jsonb_each(kv.value) u(k,v)), "
1236
+ "'{{}}'::jsonb) ELSE kv.value END::text FROM {tbl} kv WHERE kv.key=%s",
1237
+ 'store_kv', tenant_slug), (str(name),)).fetchone()
1238
+ text = row[2] if row else None
1239
+ _request_timing.record_store('view_workspace', str(name), bytes_read=_request_timing.text_bytes(text), count=1)
1240
+ parsed = _parse_read(name, 'view_workspace', text)
1241
+ if not isinstance(parsed, dict):
1242
+ return parsed
1243
+ result = _ViewWorkspace(parsed)
1244
+ result.source_revision = row[:2] if row else None
1245
+ return result
1246
+
1247
+
1248
+ def _iter_view_members(name, path, revision, *, batch_size=2048, tenant_slug=None):
1249
+ """Stream the exact metadata generation, refusing disappearance or concurrent edits."""
1250
+ import uuid
1251
+ size = int(batch_size)
1252
+ if not 1 <= size <= 4096:
1253
+ raise ValueError('A view membership batch accepts between 1 and 4096 values')
1254
+ with _pool().connection() as con:
1255
+ with con.cursor(name='view_members_' + uuid.uuid4().hex) as cur:
1256
+ cur.execute(_q("SELECT kv.rev, kv.updated_at, e.n, e.v::text FROM {tbl} kv "
1257
+ "LEFT JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
1258
+ "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) ON TRUE "
1259
+ 'WHERE kv.key=%s ORDER BY e.n', 'store_kv', tenant_slug),
1260
+ (list(map(str, path)), list(map(str, path)), str(name)))
1261
+ found = False
1262
+ while True:
1263
+ _request_timing.remaining_budget()
1264
+ rows = cur.fetchmany(size)
1265
+ if not rows:
1266
+ if not found:
1267
+ raise RuntimeError('canonical_source_unavailable: the saved-view document disappeared')
1268
+ return
1269
+ found = True
1270
+ if any(tuple(row[:2]) != tuple(revision or ()) for row in rows):
1271
+ raise RuntimeError('canonical_source_unavailable: the saved view changed during this read')
1272
+ _request_timing.record_store('view_members', str(name),
1273
+ bytes_read=sum(len(value.encode('utf-8')) for _, _, _, value in rows if value is not None))
1274
+ yield [(str(index - 1), json.loads(value)) for _, _, index, value in rows if index is not None]
1275
+ rows.clear()
1276
+
1277
+
1278
+ def _get_json_object(name, path=(), *, drop=(), omit=(), tenant_slug=None):
1279
+ """Read nested object definitions while removing their population operands in SQL."""
1280
+ with _pool().connection() as con:
1281
+ row = con.execute(_q("SELECT coalesce(jsonb_object_agg(e.k, CASE WHEN jsonb_typeof(e.v)='object' "
1282
+ "THEN e.v - %s::text[] ELSE e.v END), '{{}}'::jsonb)::text FROM {tbl} kv "
1283
+ "CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='object' "
1284
+ "THEN kv.value #> %s::text[] ELSE '{{}}'::jsonb END) e(k,v) WHERE kv.key=%s AND NOT e.k=ANY(%s::text[])",
1285
+ 'store_kv', tenant_slug), (list(drop), list(map(str, path)), list(map(str, path)), str(name), list(omit))).fetchone()
1286
+ _request_timing.record_store('json_object', str(name), bytes_read=_request_timing.text_bytes(row[0]), count=1)
1287
+ return json.loads(row[0])
1288
+
1289
+
1290
+ def _get_json_members(name, path, ids, *, id_key=None, source_limit=None, tenant_slug=None):
1291
+ """Read complete selected membership without transferring an entire stored array."""
1292
+ wanted = list(dict.fromkeys(str(int(pid)) for pid in ids))
1293
+ if len(wanted) > 4096:
1294
+ raise ValueError('A membership read accepts at most 4096 identities')
1295
+ if not wanted:
1296
+ return []
1297
+ with _pool().connection() as con:
1298
+ rows = con.execute(_q("WITH members AS (SELECT e.v, e.n, CASE WHEN %s::text IS NULL "
1299
+ "THEN e.v ELSE e.v -> %s END ident FROM {tbl} kv "
1300
+ "CROSS JOIN LATERAL jsonb_array_elements(CASE WHEN jsonb_typeof(kv.value #> %s::text[])='array' "
1301
+ "THEN kv.value #> %s::text[] ELSE '[]'::jsonb END) WITH ORDINALITY e(v,n) "
1302
+ 'WHERE kv.key=%s AND (%s::bigint IS NULL OR e.n <= %s)), '
1303
+ "normalized AS (SELECT v,n,CASE WHEN jsonb_typeof(ident)='number' THEN trunc((ident #>> '{{}}')::numeric)::text "
1304
+ "WHEN ident='true'::jsonb THEN '1' WHEN ident='false'::jsonb THEN '0' ELSE trim(ident #>> '{{}}') END id FROM members), "
1305
+ "selected AS (SELECT v,n,CASE WHEN id ~ '^[+-]?[0-9]+$' THEN id::numeric::text END id FROM normalized), "
1306
+ 'ranked AS (SELECT v,n,row_number() OVER (PARTITION BY id ORDER BY n) r FROM selected WHERE id=ANY(%s::text[])) '
1307
+ 'SELECT v::text FROM ranked WHERE r=1 ORDER BY n', 'store_kv', tenant_slug),
1308
+ (id_key, id_key, list(map(str, path)), list(map(str, path)), str(name),
1309
+ source_limit, source_limit, wanted)).fetchall()
1310
+ _request_timing.record_store('json_members', str(name), bytes_read=sum(len(row[0]) for row in rows), count=1)
1311
+ return [json.loads(row[0]) for row in rows]
1312
+
1313
+
1314
+ @_request_timing.store_operation("definition_counts")
1315
+ def _definition_counts_text(name, tenant_slug=None):
1316
+ """Count stored row keys in Postgres without returning row values."""
1317
+ with _pool().connection() as con:
1318
+ row = con.execute(_q(
1319
+ "SELECT COALESCE(json_object_agg(e.key, CASE "
1320
+ "WHEN jsonb_typeof(e.value->'rows') = 'object' THEN "
1321
+ "(SELECT count(*) FROM jsonb_object_keys(e.value->'rows')) "
1322
+ "WHEN NOT (e.value ? 'rows') OR e.value->'rows' = 'null'::jsonb THEN 0 "
1323
+ "ELSE NULL END), '{{}}'::json)::text FROM {tbl} AS kv "
1324
+ "CROSS JOIN LATERAL jsonb_each(CASE WHEN jsonb_typeof(kv.value) = 'object' "
1325
+ "THEN kv.value ELSE '{{}}'::jsonb END) AS e WHERE kv.key = %s",
1326
+ 'store_kv', tenant_slug), (str(name),)).fetchone()
1327
+ text = row[0] if row else None
1328
+ _request_timing.record_store('definition_counts', str(name),
1329
+ bytes_read=_request_timing.text_bytes(text), count=0)
1330
+ return text
1331
+
1332
+
1333
+ @_request_timing.store_operation('get_many', many=True)
1334
  def _get_many(names, tenant_slug=None):
1335
  """Several ordinary documents in one round trip, keyed by requested store key."""
1336
  keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
1337
  if not keys:
1338
  return {}
1339
+ pool = _pool()
1340
+ request_keys = {key: _request_key(key, 'revision', tenant_slug, pool) for key in keys}
1341
+ checkpoint = _request_timing.memo_checkpoint()
1342
+ with pool.connection() as con:
1343
  rows = con.execute(
1344
+ _q('SELECT key, value::text, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])',
1345
  'store_kv', tenant_slug),
1346
  (keys,)).fetchall()
1347
+ found = {}
1348
+ for key, text, rev, stamp in rows:
1349
+ found[str(key)] = _parsed(text)
1350
+ _request_timing.record_store('get_many', str(key), bytes_read=_request_timing.text_bytes(text), count=0)
1351
+ _seed_read_token(request_keys[str(key)], checkpoint, rev, stamp)
1352
+ if not str(key).endswith('user_tables'):
1353
+ shape_key = (*request_keys[str(key)][:3], 'whole', request_keys[str(key)][-1])
1354
+ _request_timing.memo_seed(shape_key, text, checkpoint)
1355
+ for key in keys:
1356
+ if key not in found:
1357
+ _seed_read_token(request_keys[key], checkpoint, None, None)
1358
+ if not key.endswith('user_tables'):
1359
+ _request_timing.memo_seed((*request_keys[key][:3], 'whole', request_keys[key][-1]),
1360
+ None, checkpoint)
1361
  return {
1362
  key: (dict(found[key]) if isinstance(found.get(key), dict)
1363
  else (found[key] if key in found else {}))
 
1472
  return data
1473
 
1474
 
1475
+ @_request_timing.store_operation("update")
1476
  def update(name, fn, flush='sync', tenant_slug=None, expect_rev=None, audit_row=None):
1477
  """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
1478
 
 
1522
 
1523
 
1524
  def _update_locked(name, fn, tenant_slug, expect_rev, audit_row):
1525
+ """Diagnose lock refusal only after the aborted row transaction has rolled back."""
1526
+ from psycopg.errors import LockNotAvailable
1527
+ try:
1528
+ return _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row)
1529
+ except LockNotAvailable as exc:
1530
+ if not getattr(exc, '_aios_lock_diagnosed', False):
1531
+ _log_lock_blockers(getattr(exc, '_aios_waiter_pid', None))
1532
+ raise
1533
+
1534
+
1535
+ def _log_lock_blockers(waiter_pid):
1536
+ """Read blocker identities on a separate transaction without logging statement values."""
1537
+ if waiter_pid is None:
1538
+ return
1539
+ try:
1540
+ with _pool().connection(timeout=1.0) as diagnostic:
1541
+ with diagnostic.transaction():
1542
+ diagnostic.execute("SELECT set_config('statement_timeout', '1000ms', true)")
1543
+ rows = diagnostic.execute(
1544
+ 'SELECT a.pid, a.query FROM pg_stat_activity AS a '
1545
+ 'WHERE a.pid = ANY(pg_blocking_pids(%s))', (int(waiter_pid),)).fetchall()
1546
+ blockers = [{'pid': int(pid), 'query': _request_timing.sql_text(query)} for pid, query in rows]
1547
+ print(f'[store-pg] lock blockers for pid {int(waiter_pid)}: '
1548
+ f'{json.dumps(blockers, ensure_ascii=True)}', flush=True)
1549
+ except Exception as exc:
1550
+ print(f'[store-pg] lock blocker diagnosis unavailable ({type(exc).__name__})', flush=True)
1551
+
1552
+
1553
+ def _update_locked_body(name, fn, tenant_slug, expect_rev, audit_row):
1554
+ """Run the callback and write under the existing row lock and transaction."""
1555
  from psycopg.errors import LockNotAvailable
1556
  wait_s = _lock_timeout_s()
1557
  with _pool().connection() as con:
 
1560
  # lock gives its pooled connection back after `wait_s` instead of holding it forever.
1561
  con.execute("SELECT set_config('lock_timeout', %s, true)",
1562
  (f'{int(wait_s * 1000)}ms',))
1563
+ pid_row = con.execute('SELECT pg_backend_pid()').fetchone()
1564
+ waiter_pid = pid_row[0] if pid_row else None
1565
+ with _request_timing.lock_wait_probe(waiter_pid, wait_s, _log_lock_blockers) as probe:
1566
+ try:
1567
+ row = con.execute(
1568
+ _q('SELECT value::text, rev FROM {tbl} WHERE key = %s FOR UPDATE',
1569
+ 'store_kv', tenant_slug),
1570
+ (str(name),)).fetchone()
1571
+ except LockNotAvailable as exc:
1572
+ exc._aios_waiter_pid = waiter_pid
1573
+ exc._aios_lock_diagnosed = probe is not None and probe.fired
1574
+ print(f'[store-pg] lock wait over {wait_s:g}s on '
1575
+ f'{_effective_slug(tenant_slug)}/{name}', flush=True)
1576
+ raise
1577
+ _request_timing.record_store("update", str(name),
1578
+ bytes_read=_request_timing.text_bytes(row[0] if row else None), count=0)
1579
  if expect_rev is not None:
1580
  current = int(row[1]) if row and row[1] is not None else 0
1581
  if current != int(expect_rev):
1582
  from core.view_audit import ViewRevisionStale
1583
  raise ViewRevisionStale(rev_current=current, rev_sent=int(expect_rev))
1584
+ data = _parsed(row[0] if row else None)
1585
  result = fn(data)
1586
  data = result if result is not None else data
1587
  payload = _payload(name, data, con, tenant_slug)
 
1590
  'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
1591
  'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
1592
  (str(name), payload))
1593
+ _request_timing.record_store("update", str(name),
1594
+ bytes_written=_request_timing.text_bytes(payload), count=0)
1595
  # ⚠ A LIST, OR ONE DICT, AND THE LIST IS NOT A CONVENIENCE. Moving views between
1596
  # folders is ONE workspace write that changes several views' placements, so the
1597
  # trail for it is several rows or it is wrong; a single-row keyword would have
 
1650
 
1651
 
1652
  def revision(name, tenant_slug=None):
1653
+ """Return an independent revision object, reusing only this request's backing validation."""
1654
+ return _parsed(_request_text(name, 'revision', tenant_slug,
1655
+ lambda: json.dumps(_revision(name, tenant_slug))))
1656
+
1657
+
1658
+ @_request_timing.memo_read
1659
+ def _revision_many(names, tenant_slug=None):
1660
+ """Batch only unchecked keys and lend complete tokens to every request read shape."""
1661
+ keys = tuple(sorted(set(map(str, names))))
1662
+ checkpoint = _request_timing.memo_checkpoint()
1663
+ request_keys = {name: _request_key(name, 'revision', tenant_slug) for name in keys}
1664
+ found, missing = {}, []
1665
+ for name, key in request_keys.items():
1666
+ hit, text = _request_timing.memo_peek(key)
1667
+ if hit:
1668
+ found[name] = _parsed(text)
1669
+ else:
1670
+ missing.append(name)
1671
+ if missing:
1672
+ read = _read_revisions(missing, tenant_slug)
1673
+ for name in missing:
1674
+ token = read.get(name, _revision_value(None, None))
1675
+ _request_timing.memo_seed(request_keys[name], json.dumps(token), checkpoint)
1676
+ found[name] = token
1677
+ return {name: found[name] for name in keys}
1678
+
1679
+
1680
+ @_request_timing.store_operation('revisions', many=True)
1681
+ def _read_revisions(names, tenant_slug=None):
1682
+ with _pool().connection() as con:
1683
+ rows = con.execute(_q('SELECT key, rev, updated_at FROM {tbl} WHERE key = ANY(%s::text[])',
1684
+ 'store_kv', tenant_slug), (list(names),)).fetchall()
1685
+ found = {}
1686
+ for name, raw_rev, stamp in rows:
1687
+ found[name] = _revision_value(raw_rev, stamp)
1688
+ return found
1689
+
1690
+
1691
+ @_request_timing.store_operation("revision")
1692
+ def _revision(name, tenant_slug=None):
1693
  """`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6.
1694
 
1695
  ⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and
 
1709
  """
1710
  with _pool().connection() as con:
1711
  row = con.execute(
1712
+ _q('SELECT revision.rev, revision.updated_at, octet_length(revision.rev::text) + '
1713
+ 'octet_length(revision.updated_at::text) '
1714
+ 'FROM (SELECT rev, updated_at FROM {tbl} WHERE key = %s) AS revision', 'store_kv', tenant_slug),
1715
  (str(name),)).fetchone()
1716
+ _request_timing.record_store("revision", str(name), bytes_read=int(row[2] or 0) if row and len(row) > 2 else 0,
1717
+ count=0)
1718
  rev = int(row[0]) if row and row[0] is not None else 0
1719
  stamp = row[1] if row else None
1720
  # ⚠ EPOCH SECONDS ON BOTH BACKENDS. `core.store` records `time.time()`; this column is a
 
1761
  """`core.store.Store`'s interface for ONE tenant's schema.
1762
 
1763
  Deliberately a thin binder over the module functions rather than a reimplementation: every
1764
+ behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, fresh reads
1765
+ bypassing text reuse) is stated once, up there, and cannot drift
1766
  between the two entry points.
1767
 
1768
  ⭐⭐ AND THAT IS WHY THE D-353 GUARD IS NOT REPEATED HERE. `put`, `update`, `upload_bytes` and
 
1812
  return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
1813
  timeout_s=timeout_s)
1814
 
1815
+ def get_json_paths(self, name, paths):
1816
+ return _get_json_paths(name, paths, tenant_slug=self.tenant_slug)
1817
+
1818
+ def iter_json_items(self, name, path=(), *, batch_size=1024):
1819
+ return _iter_json_items(name, path, batch_size=batch_size, tenant_slug=self.tenant_slug)
1820
+
1821
+ def iter_json_values(self, name, path=(), *, drop=(), batch_size=1024):
1822
+ return _iter_json_values(name, path, drop=drop, batch_size=batch_size, tenant_slug=self.tenant_slug)
1823
+
1824
+ def get_json_object(self, name, path=(), *, drop=(), omit=()):
1825
+ return _get_json_object(name, path, drop=drop, omit=omit, tenant_slug=self.tenant_slug)
1826
+
1827
+ def get_view_workspace(self, name):
1828
+ return _get_view_workspace(name, tenant_slug=self.tenant_slug)
1829
+
1830
+ def iter_view_members(self, name, path, revision, *, batch_size=2048):
1831
+ return _iter_view_members(name, path, revision, batch_size=batch_size, tenant_slug=self.tenant_slug)
1832
+
1833
+ def get_json_members(self, name, path, ids, *, id_key=None, source_limit=None):
1834
+ return _get_json_members(name, path, ids, id_key=id_key, source_limit=source_limit, tenant_slug=self.tenant_slug)
1835
+
1836
  def get_workbook_projection(self, name, *, metadata_only=False, tab_ids=None, cells=None,
1837
  expected_rev=None):
1838
  return _get_workbook_projection(name, metadata_only=metadata_only, tab_ids=tab_ids,
 
1863
  """Bound multi-key read used by request-scoped read snapshots."""
1864
  return _get_many(names, tenant_slug=self.tenant_slug)
1865
 
1866
+ def get_definition_counts(self, name):
1867
+ """Exact stored counts only; a read-through source's population is not measured here."""
1868
+ return _parsed(_request_text(name, 'definition_counts', self.tenant_slug,
1869
+ lambda: _definition_counts_text(name, self.tenant_slug)))
1870
+
1871
  def get_entry(self, name, entry):
1872
  """Bound single-entry read; None means absent, and the caller falls back to `get`."""
1873
  return _get_entry(name, entry, tenant_slug=self.tenant_slug)
platform/core/user_tables.py CHANGED
@@ -1916,8 +1916,11 @@ def all_tables(st=None):
1916
  ⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that
1917
  only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below.
1918
  """
 
 
 
1919
  try:
1920
- return _with_activity(_st(st).get(STORE_KEY))
1921
  except Exception:
1922
  return {}
1923
 
@@ -1927,7 +1930,27 @@ def all_tables(st=None):
1927
  DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit')
1928
 
1929
 
1930
- def all_defs(st=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1931
  """{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row.
1932
 
1933
  ⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for
@@ -1940,8 +1963,9 @@ def all_defs(st=None):
1940
  from a database that genuinely has no rows, and the caller least able to notice is the one that
1941
  wanted rows. If you get that error you wanted `all_tables`.
1942
 
1943
- ⚠ Falls back to `all_tables` on ANY failure, including a store handle too old to project (a
1944
- fake store installed by a gate, a `_Lent` handle serving a document somebody else read). The
 
1945
  fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for
1946
  — which is the one direction a fallback here is allowed to be wrong in.
1947
 
@@ -1952,7 +1976,7 @@ def all_defs(st=None):
1952
  databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]).
1953
  Nothing had noticed because the projection does not fail on a healthy store — a fallback is
1954
  dead code until the day it is the only code running. **Every failure now falls back to the
1955
- whole read**, which keeps `all_tables`' own `except: return {}` as the single backstop instead
1956
  of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on
1957
  this function, so an empty answer here stops being a blank nav and becomes a 404 on every
1958
  database in the tenant.
@@ -1960,13 +1984,22 @@ def all_defs(st=None):
1960
  # Schema-only handles cannot use the historical whole-reader fallback described above:
1961
  # their caller must distinguish an unavailable definition snapshot from an empty one.
1962
  reader = _st(st)
 
 
1963
  try:
1964
- return _with_activity(reader.get_projection(STORE_KEY, drop=DEF_ONLY_DROPS))
1965
- except Exception:
 
 
 
 
1966
  # A schema-only request promises not to materialize rows, including on an outage.
1967
- # Its existing route maps this failure to a named unavailable response. Ordinary
1968
- # full readers retain the correct-but-slow fallback below.
1969
- if getattr(reader, 'schema_only', False):
 
 
 
1970
  raise
1971
  # ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
1972
  # only ever ONE of the ways this can fail, and singling it out is what made the other ways
@@ -1982,6 +2015,8 @@ def _entry_reader(st):
1982
  """Use an entry door only when it belongs to the same reader as the bound snapshot."""
1983
  # The API's request reader defines get_entry even when its wrapped snapshot does not.
1984
  # Judge that runtime's own static capability and owner, then retain the request read door.
 
 
1985
  bound = st
1986
  if type(st).__name__ == '_AssemblyStore':
1987
  st = st._runtime
@@ -5430,6 +5465,15 @@ class _Lent:
5430
  self._runtime, self._docs = runtime, dict(docs or {})
5431
 
5432
  def get(self, name, fresh=False):
 
 
 
 
 
 
 
 
 
5433
  if fresh or (name not in self._docs and name not in _LENDABLE
5434
  and not str(name).endswith('__shared')):
5435
  return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
@@ -5470,7 +5514,7 @@ def lend(st=None, **docs):
5470
  return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None})
5471
 
5472
 
5473
- def lend_defs(st=None):
5474
  """`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows.
5475
 
5476
  ⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02
@@ -5489,7 +5533,12 @@ def lend_defs(st=None):
5489
  ⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY
5490
  should take the whole read once rather than this plus that.
5491
  """
5492
- return lend(st, **{STORE_KEY: all_defs(st)})
 
 
 
 
 
5493
 
5494
 
5495
  def _utc_now():
 
1916
  ⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that
1917
  only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below.
1918
  """
1919
+ reader = _st(st)
1920
+ if getattr(reader, 'schema_only', False):
1921
+ return all_defs(reader, schema_only=True)
1922
  try:
1923
+ return _with_activity(reader.get(STORE_KEY))
1924
  except Exception:
1925
  return {}
1926
 
 
1930
  DEF_ONLY_DROPS = ('rows', '_activity_outbox', '_rowCreationOrder', '_rowAudit')
1931
 
1932
 
1933
+ def definition_counts(st=None):
1934
+ """Exact stored counts from a bound capability, or unknown without reading rows."""
1935
+ reader = _st(st)
1936
+ # Snapshot wrappers cannot borrow live counts from a different document owner.
1937
+ if isinstance(reader, _Lent) or type(reader).__name__ == '_AssemblyStore':
1938
+ return {}
1939
+ backend = reader._store() if callable(getattr(reader, '_store', None)) else reader
1940
+ if callable(getattr(backend, '_d', None)):
1941
+ backend = backend._d()
1942
+ count = getattr(backend, 'get_definition_counts', None)
1943
+ if not callable(count):
1944
+ return {}
1945
+ name = reader.store_key(STORE_KEY) if callable(getattr(reader, 'store_key', None)) else STORE_KEY
1946
+ counts = count(name)
1947
+ if not isinstance(counts, dict):
1948
+ from core.shared_overlay import SchemaUnavailable
1949
+ raise SchemaUnavailable('stored row counts are unavailable')
1950
+ return counts
1951
+
1952
+
1953
+ def all_defs(st=None, *, schema_only=False):
1954
  """{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row.
1955
 
1956
  ⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for
 
1963
  from a database that genuinely has no rows, and the caller least able to notice is the one that
1964
  wanted rows. If you get that error you wanted `all_tables`.
1965
 
1966
+ Schema-only requests propagate read failures and never fall back to a whole read.
1967
+ Other readers fall back to `all_tables` on ANY failure, including a store handle too old to
1968
+ project (a fake store installed by a gate, a `_Lent` handle serving an already-read document). The
1969
  fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for
1970
  — which is the one direction a fallback here is allowed to be wrong in.
1971
 
 
1976
  databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]).
1977
  Nothing had noticed because the projection does not fail on a healthy store — a fallback is
1978
  dead code until the day it is the only code running. **Every failure now falls back to the
1979
+ whole read for ordinary readers**, which keeps `all_tables`' own `except: return {}` as the single backstop instead
1980
  of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on
1981
  this function, so an empty answer here stops being a blank nav and becomes a 404 on every
1982
  database in the tenant.
 
1984
  # Schema-only handles cannot use the historical whole-reader fallback described above:
1985
  # their caller must distinguish an unavailable definition snapshot from an empty one.
1986
  reader = _st(st)
1987
+ from core import request_timing
1988
+ schema_only = schema_only or getattr(reader, 'schema_only', False) or request_timing.CURRENT.get() is not None
1989
  try:
1990
+ project = getattr(reader, 'get_projection', None)
1991
+ if schema_only and not callable(project):
1992
+ from core.shared_overlay import SchemaUnavailable
1993
+ raise SchemaUnavailable('table definitions require a projected store reader')
1994
+ return _with_activity(project(STORE_KEY, drop=DEF_ONLY_DROPS))
1995
+ except Exception as exc:
1996
  # A schema-only request promises not to materialize rows, including on an outage.
1997
+ # Preserve the original failure type for request callers to classify.
1998
+ if schema_only:
1999
+ request_timing.raise_if_resource_error(exc)
2000
+ if request_timing.CURRENT.get() is not None:
2001
+ from core.shared_overlay import SchemaUnavailable
2002
+ raise SchemaUnavailable('table definitions are unavailable') from exc
2003
  raise
2004
  # ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was
2005
  # only ever ONE of the ways this can fail, and singling it out is what made the other ways
 
2015
  """Use an entry door only when it belongs to the same reader as the bound snapshot."""
2016
  # The API's request reader defines get_entry even when its wrapped snapshot does not.
2017
  # Judge that runtime's own static capability and owner, then retain the request read door.
2018
+ if getattr(st, 'schema_only', False):
2019
+ return None
2020
  bound = st
2021
  if type(st).__name__ == '_AssemblyStore':
2022
  st = st._runtime
 
5465
  self._runtime, self._docs = runtime, dict(docs or {})
5466
 
5467
  def get(self, name, fresh=False):
5468
+ if fresh:
5469
+ from core.request_timing import clear_read_memo
5470
+ clear_read_memo()
5471
+ if name == STORE_KEY and getattr(self, 'schema_only', False):
5472
+ if fresh:
5473
+ self._docs.pop(name, None)
5474
+ if fresh or name not in self._docs:
5475
+ self._docs[name] = all_defs(self._runtime, schema_only=True)
5476
+ return self._docs[name]
5477
  if fresh or (name not in self._docs and name not in _LENDABLE
5478
  and not str(name).endswith('__shared')):
5479
  return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
 
5514
  return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None})
5515
 
5516
 
5517
+ def lend_defs(st=None, *, schema_only=False):
5518
  """`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows.
5519
 
5520
  ⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02
 
5533
  ⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY
5534
  should take the whole read once rather than this plus that.
5535
  """
5536
+ reader = _st(st)
5537
+ strict = schema_only or getattr(reader, 'schema_only', False)
5538
+ lent = lend(reader, **{STORE_KEY: all_defs(reader, schema_only=strict)})
5539
+ if strict:
5540
+ lent.schema_only = True
5541
+ return lent
5542
 
5543
 
5544
  def _utc_now():
platform/harness/canonical_source.py ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Bounded, read-only operands for the existing compiled customer and product projections."""
2
+ import contextvars
3
+ import itertools
4
+ import json
5
+ import uuid
6
+ from contextlib import contextmanager
7
+
8
+ from core import odoo as _odoo
9
+
10
+ BATCH = 1024
11
+ _ACTIVE = contextvars.ContextVar('canonical_source', default=None)
12
+
13
+
14
+ def code_sql(alias):
15
+ raw = f"json_extract_string({alias}.body, '/default_code')"
16
+ return (f"CASE WHEN {raw} IS NULL OR {raw} = '' OR json_type({alias}.body, '/default_code') = 'BOOLEAN' "
17
+ f"THEN 'pid:' || json_extract_string({alias}.body, '/id') ELSE trim({raw}) END")
18
+
19
+
20
+ class TierProjection(list):
21
+ """Retain displayed tiers and the complete overflow receipt from SQL."""
22
+ def __init__(self):
23
+ super().__init__()
24
+ self.overflow = {}
25
+
26
+
27
+ class Rows:
28
+ """Reiterate a SQL snapshot without retaining its Python records."""
29
+
30
+ def __init__(self, owner, table):
31
+ self.owner, self.table = owner, table
32
+
33
+ def __iter__(self):
34
+ last = -1
35
+ while True:
36
+ batch = self.owner.cur.execute(
37
+ f'SELECT ordinal, body FROM "{self.table}" WHERE ordinal > ? ORDER BY ordinal LIMIT ?',
38
+ [last, BATCH]).fetchall()
39
+ if not batch:
40
+ return
41
+ for _, body in batch:
42
+ yield json.loads(body)
43
+ last = batch[-1][0]
44
+ batch.clear()
45
+
46
+ def __len__(self):
47
+ return self.owner.cur.execute(f'SELECT count(*) FROM "{self.table}"').fetchone()[0]
48
+
49
+ def __bool__(self):
50
+ return bool(len(self))
51
+
52
+
53
+ class Source:
54
+ """Own bounded source operands and release every temporary snapshot on exit."""
55
+
56
+ def __init__(self, cur, kind, identities, store=None):
57
+ self.cur, self.kind, self.identities, self.store = cur, kind, list(identities), store
58
+ self.tables = []
59
+ self.cache = {}
60
+ self.failure = None
61
+
62
+ def snapshot(self, records):
63
+ name = 'canonical_' + uuid.uuid4().hex
64
+ self.cur.execute(f'CREATE TEMP TABLE "{name}" (ordinal BIGINT, body JSON)')
65
+ self.tables.append(name)
66
+ records, offset = iter(records), 0
67
+ try:
68
+ while True:
69
+ batch = list(itertools.islice(records, BATCH))
70
+ if not batch:
71
+ break
72
+ self.cur.execute(f'INSERT INTO "{name}" SELECT unnest(?::BIGINT[]), unnest(?::JSON[])',
73
+ [list(range(offset, offset + len(batch))), [json.dumps(r, default=str) for r in batch]])
74
+ offset += len(batch)
75
+ batch.clear()
76
+ except BaseException as exc:
77
+ self.failure = exc
78
+ raise
79
+ finally:
80
+ close = getattr(records, 'close', None)
81
+ if close:
82
+ close()
83
+ return Rows(self, name)
84
+
85
+ def domain(self, model, domain):
86
+ domain = list(domain or [])
87
+ if self.kind == 'customer':
88
+ field = {'res.partner': 'id', 'sale.order': 'partner_id',
89
+ 'sale.order.line': 'order_partner_id', 'account.move': 'partner_id'}.get(model)
90
+ # Explicit partner lookup domains also resolve agents and must retain their ids.
91
+ if model == 'res.partner' and any(isinstance(d, (tuple, list)) and d[0] in {'id', 'name'}
92
+ for d in domain):
93
+ field = None
94
+ if field:
95
+ domain.append((field, 'in', self.identities))
96
+ elif self.kind == 'product' and model == 'product.product':
97
+ # Match the canonical stripped code, including every duplicate and archived variant.
98
+ codes = list(self.identities)
99
+ ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()]
100
+ terms = [('default_code', '=like', '%' + code + '%') for code in codes]
101
+ terms.append(('id', 'in', ids))
102
+ domain += ['|'] * (len(terms) - 1) + terms
103
+ elif self.kind == 'product' and model in {'sale.order.line', 'stock.quant'}:
104
+ codes = list(self.identities)
105
+ ids = [int(c[4:]) for c in self.identities if c.startswith('pid:') and c[4:].isdigit()]
106
+ terms = [('product_id.default_code', '=like', '%' + code + '%') for code in codes]
107
+ terms.append(('product_id', 'in', ids))
108
+ domain += ['|'] * (len(terms) - 1) + terms
109
+ return domain
110
+
111
+ def products(self):
112
+ key = ('products',)
113
+ if key not in self.cache:
114
+ self.cache[key] = self.search_read('product.product', [('active', 'in', [True, False])],
115
+ ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
116
+ return self.cache[key]
117
+
118
+ def sku_revenue(self, groups):
119
+ products = self.products()
120
+ code = code_sql('p')
121
+ rows = self.cur.execute(f"""SELECT {code} code,
122
+ first(json_extract_string(g.body, '/product_id/1') ORDER BY g.ordinal),
123
+ sum(coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0)),
124
+ sum(coalesce(try_cast(json_extract_string(g.body, '/product_uom_qty') AS DOUBLE), 0)),
125
+ count(DISTINCT nullif(json_extract_string(g.body, '/order_id/0'), '0'))
126
+ FROM "{groups.table}" g JOIN "{products.table}" p
127
+ ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id')
128
+ WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL
129
+ GROUP BY code""").fetchall()
130
+ return {code: {'name': name, 'rev': rev, 'qty': qty, 'orders': orders}
131
+ for code, name, rev, qty, orders in rows}
132
+
133
+ def customer_mix(self, groups):
134
+ def products():
135
+ last = 0
136
+ while True:
137
+ ids = [r[0] for r in self.cur.execute(f"""SELECT DISTINCT
138
+ try_cast(json_extract_string(body, '/product_id/0') AS BIGINT) id
139
+ FROM "{groups.table}" WHERE id > ? ORDER BY id LIMIT ?""", [last, BATCH]).fetchall()]
140
+ if not ids:
141
+ return
142
+ yield from _odoo.search_read('product.product', [('id', 'in', ids)],
143
+ ['id', 'name', 'categ_id'], limit=BATCH, order='id asc')
144
+ last = ids[-1]
145
+ prods = self.snapshot(products())
146
+ query = f"""WITH lines AS (
147
+ SELECT try_cast(json_extract_string(g.body, '/order_partner_id/0') AS BIGINT) pid,
148
+ json_extract_string(g.body, '/product_id/0') product,
149
+ coalesce(nullif(trim(list_last(string_split(json_extract_string(p.body, '/categ_id/1'), ' / '))), ''), '(none)') cat,
150
+ coalesce(try_cast(json_extract_string(g.body, '/price_subtotal') AS DOUBLE), 0) val,
151
+ g.ordinal, json_extract_string(p.body, '/name') AS product_name
152
+ FROM "{groups.table}" g LEFT JOIN "{prods.table}" p
153
+ ON json_extract_string(g.body, '/product_id/0') = json_extract_string(p.body, '/id')
154
+ WHERE json_extract_string(g.body, '/product_id/0') IS NOT NULL),
155
+ cats AS (SELECT pid, cat, sum(val) val, min(ordinal) ordinal FROM lines GROUP BY pid, cat),
156
+ topcat AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM cats),
157
+ skus AS (SELECT pid, product, first(product_name ORDER BY ordinal) AS product_name, sum(val) val,
158
+ min(ordinal) ordinal FROM lines GROUP BY pid, product),
159
+ topsku AS (SELECT *, row_number() OVER (PARTITION BY pid ORDER BY val DESC, ordinal) rn FROM skus),
160
+ totals AS (SELECT pid, count(DISTINCT product) n, sum(val) total FROM lines GROUP BY pid)
161
+ SELECT t.pid, c.cat, CASE WHEN t.total != 0 THEN c.val / t.total END, t.n,
162
+ coalesce(s.product_name, '(none)') FROM totals t JOIN topcat c ON t.pid=c.pid AND c.rn=1
163
+ JOIN topsku s ON t.pid=s.pid AND s.rn=1"""
164
+ return {pid: {'top_category': cat, 'top_category_pct': pct, 'sku_count': n, 'top_sku': name}
165
+ for pid, cat, pct, n, name in self.cur.execute(query).fetchall()}
166
+
167
+ def categories(self):
168
+ rows = self.products()
169
+ cats = self.search_read('product.category', [], ['id', 'complete_name'])
170
+ return {code: category for code, category in self.cur.execute(f"""SELECT
171
+ {code_sql('p')} code,
172
+ first(coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''),
173
+ '(uncategorized)') ORDER BY p.ordinal)
174
+ FROM "{rows.table}" p LEFT JOIN "{cats.table}" c
175
+ ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id')
176
+ WHERE json_extract_string(p.body, '/active') = 'true'
177
+ AND json_type(p.body, '/default_code') = 'VARCHAR' AND json_extract_string(p.body, '/default_code') != ''
178
+ GROUP BY code""").fetchall()}
179
+
180
+ def backlog(self, groups):
181
+ products = self.products()
182
+ return {code: qty for code, qty in self.cur.execute(f"""WITH per AS (
183
+ SELECT json_extract_string(body, '/product_id/0') id,
184
+ greatest(0, coalesce(try_cast(json_extract_string(body, '/product_uom_qty') AS DOUBLE), 0)
185
+ - coalesce(try_cast(json_extract_string(body, '/qty_delivered') AS DOUBLE), 0)) qty
186
+ FROM "{groups.table}")
187
+ SELECT {code_sql('p')} code,
188
+ sum(per.qty) FROM per JOIN "{products.table}" p
189
+ ON per.id=json_extract_string(p.body, '/id')
190
+ WHERE per.id IS NOT NULL AND per.qty > 1e-9 GROUP BY code""").fetchall()}
191
+
192
+ def inventory_frame(self, t):
193
+ from core import periods
194
+ from modules import inventory
195
+ import pandas as pd
196
+ lf, lt = periods.ltm(t)
197
+ quant = self.read_group('stock.quant', [('location_id.usage', '=', 'internal')],
198
+ ['product_id', 'quantity:sum'], ['product_id'], lazy=False)
199
+ sales = self.read_group('sale.order.line', _odoo.sale_line_domain(lf, lt, all_channels=True),
200
+ ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
201
+ products = self.search_read('product.product', [('default_code', '!=', False)],
202
+ ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
203
+ cats = self.search_read('product.category', [], ['id', 'complete_name'])
204
+ query = f"""WITH raw AS (SELECT p.ordinal,
205
+ trim(json_extract_string(p.body, '/default_code')) sku,
206
+ json_extract_string(p.body, '/name') AS product_name,
207
+ coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''), '(uncategorized)') category,
208
+ json_extract_string(p.body, '/type') = 'product' storable,
209
+ coalesce(try_cast(json_extract_string(p.body, '/standard_price') AS DOUBLE), 0) AS cost,
210
+ coalesce(try_cast(json_extract_string(q.body, '/quantity') AS DOUBLE), 0) on_hand,
211
+ coalesce(try_cast(json_extract_string(s.body, '/product_uom_qty') AS DOUBLE), 0) qty_ltm,
212
+ coalesce(try_cast(json_extract_string(s.body, '/price_subtotal') AS DOUBLE), 0) rev_ltm
213
+ FROM "{products.table}" p
214
+ LEFT JOIN "{quant.table}" q ON json_extract_string(p.body, '/id')=json_extract_string(q.body, '/product_id/0')
215
+ LEFT JOIN "{sales.table}" s ON json_extract_string(p.body, '/id')=json_extract_string(s.body, '/product_id/0')
216
+ LEFT JOIN "{cats.table}" c ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id'))
217
+ SELECT sku, first(product_name ORDER BY ordinal) AS name, first(category ORDER BY ordinal) category,
218
+ bool_or(storable) storable, sum(on_hand) on_hand, sum(on_hand*cost) inv_value,
219
+ sum(qty_ltm*cost) cogs_ltm, sum(qty_ltm) qty_ltm, sum(rev_ltm) rev_ltm
220
+ FROM raw GROUP BY sku"""
221
+ result = self.cur.execute(query)
222
+ names = [d[0] for d in result.description]
223
+ return pd.DataFrame(result.fetchall(), columns=names), (lf, lt)
224
+
225
+ def tiers(self, prods):
226
+ from core import periods
227
+ from modules import products
228
+ lists = self.search_read('product.pricelist', [], ['id', 'name'])
229
+ today = periods.today().isoformat()
230
+ rules = self.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'),
231
+ ('applied_on', 'in', ['0_product_variant', '1_product']),
232
+ '|', ('date_start', '=', False), ('date_start', '<=', today),
233
+ '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)],
234
+ ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
235
+ 'fixed_price', 'packaging_id', 'packaging_price'])
236
+ normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
237
+ canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
238
+ prods = products.active_products() if prods is None else prods
239
+ declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
240
+ query = f"""WITH eligible AS (
241
+ SELECT {code_sql('p')} code, p.ordinal product_order, r.ordinal rule_order,
242
+ trim(json_extract_string(l.body, '/name')) pricelist,
243
+ try_cast(json_extract_string(r.body, '/canonical_price') AS DOUBLE) price,
244
+ json_extract_string(r.body, '/canonical_unit') unit
245
+ FROM "{prods.table}" p JOIN "{normalized.table}" r ON (
246
+ (json_extract_string(r.body, '/applied_on')='0_product_variant'
247
+ AND json_extract_string(r.body, '/product_id/0')=json_extract_string(p.body, '/id')) OR
248
+ (json_extract_string(r.body, '/applied_on')='1_product'
249
+ AND json_extract_string(r.body, '/product_tmpl_id/0')=json_extract_string(p.body, '/product_tmpl_id/0')
250
+ AND NOT EXISTS (SELECT 1 FROM "{normalized.table}" v
251
+ WHERE json_extract_string(v.body, '/applied_on')='0_product_variant'
252
+ AND json_extract_string(v.body, '/pricelist_id/0')=json_extract_string(r.body, '/pricelist_id/0')
253
+ AND json_extract_string(v.body, '/product_id/0')=json_extract_string(p.body, '/id'))))
254
+ JOIN "{lists.table}" l ON json_extract_string(l.body, '/id')=json_extract_string(r.body, '/pricelist_id/0')),
255
+ identified AS (SELECT *, min(product_order) OVER (PARTITION BY code) first_product,
256
+ row_number() OVER (PARTITION BY code, pricelist, price, unit ORDER BY product_order, rule_order) ident FROM eligible),
257
+ merged AS (SELECT * FROM identified WHERE product_order=first_product OR ident=1),
258
+ ranked AS (SELECT *, row_number() OVER (PARTITION BY code ORDER BY price, lower(unit), lower(pricelist), product_order, rule_order) pooled,
259
+ row_number() OVER (PARTITION BY code, pricelist ORDER BY price, lower(unit), product_order, rule_order) tier,
260
+ count(*) OVER (PARTITION BY code, pricelist) n FROM merged)
261
+ SELECT code, pricelist, price, unit, n FROM ranked
262
+ WHERE pooled <= 3 OR (pricelist IN (SELECT unnest(?::VARCHAR[])) AND tier <= ?)
263
+ ORDER BY code, price, lower(unit), lower(pricelist), product_order, rule_order"""
264
+ result = self.cur.execute(query, [declared, products.PRICELIST_TIER_DEPTH])
265
+ out = {}
266
+ while True:
267
+ batch = result.fetchmany(BATCH)
268
+ if not batch:
269
+ break
270
+ for code, name, price, unit, n in batch:
271
+ entry = out.setdefault(code, TierProjection())
272
+ entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
273
+ if name in declared and n > products.PRICELIST_TIER_DEPTH:
274
+ entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
275
+ return out, {'bounded': True}
276
+
277
+ def search_read(self, model, domain=None, fields=None, **kw):
278
+ domain = self.domain(model, domain)
279
+ # Keyset traversal avoids a connector's default limit and never silently caps connected rows.
280
+ kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset', 'order'}}
281
+ def records():
282
+ last = 0
283
+ while True:
284
+ page = _odoo.search_read(model, domain + [('id', '>', last)],
285
+ list(dict.fromkeys(['id', *(['default_code'] if self.kind == 'product' and model == 'product.product' else []),
286
+ *(fields or [])])), limit=BATCH, order='id asc', **kw)
287
+ if not page:
288
+ return
289
+ nxt = max(int(r['id']) for r in page)
290
+ if nxt <= last:
291
+ raise RuntimeError('canonical_source_invalid: source keyset did not advance')
292
+ if self.kind == 'product' and model == 'product.product':
293
+ selected = set(self.identities)
294
+ for row in page:
295
+ code = str(row['default_code']).strip() if row.get('default_code') else f"pid:{row['id']}"
296
+ if code in selected:
297
+ yield row
298
+ else:
299
+ yield from page
300
+ last = nxt
301
+ page.clear()
302
+ return self.snapshot(records())
303
+
304
+ def read_group(self, model, domain, fields, groupby, **kw):
305
+ domain = self.domain(model, domain)
306
+ kw = {k: v for k, v in kw.items() if k not in {'limit', 'offset'}}
307
+ def records():
308
+ offset = 0
309
+ while True:
310
+ options = dict(kw, offset=offset, limit=BATCH)
311
+ if groupby:
312
+ options.setdefault('orderby', ','.join(groupby))
313
+ page = _odoo.get_odoo().execute_kw(model, 'read_group', [domain, fields, groupby], options)
314
+ if not page:
315
+ return
316
+ for row in page:
317
+ yield {key: value for key, value in row.items()
318
+ if not key.startswith('__') or key == '__count'}
319
+ offset += len(page)
320
+ page.clear()
321
+ return self.snapshot(records())
322
+
323
+ def search_count(self, model, domain=None, **kw):
324
+ if self.kind == 'product' and model == 'product.product':
325
+ return len(self.search_read(model, domain, ['id', 'default_code']))
326
+ return _odoo.get_odoo().search_count(model, self.domain(model, domain), **kw)
327
+
328
+ def agent_members(self, name):
329
+ if name == '(none)':
330
+ return self.search_read('res.partner', [('customer_rank', '>', 0),
331
+ ('agent_ids', '=', False), ('active', 'in', [True, False])], ['id'])
332
+ agents = self.search_read('res.partner', [('name', '=', name),
333
+ ('active', 'in', [True, False])], ['id'])
334
+ def records():
335
+ source = iter(agents)
336
+ while True:
337
+ batch = list(itertools.islice(source, BATCH))
338
+ if not batch:
339
+ return
340
+ rows = self.search_read('res.partner', [('agent_ids', 'in', [r['id'] for r in batch]),
341
+ ('active', 'in', [True, False])], ['id'])
342
+ try:
343
+ yield from rows
344
+ finally:
345
+ self.cur.execute(f'DROP TABLE IF EXISTS "{rows.table}"')
346
+ self.tables.remove(rows.table)
347
+ return self.snapshot(records())
348
+
349
+ def close(self):
350
+ for table in reversed(self.tables):
351
+ self.cur.execute(f'DROP TABLE IF EXISTS "{table}"')
352
+ self.tables.clear()
353
+ self.cache.clear()
354
+
355
+
356
+ def active():
357
+ return _ACTIVE.get()
358
+
359
+
360
+ @contextmanager
361
+ def bounded(cur, kind, identities, store=None):
362
+ source = Source(cur, kind, identities, store)
363
+ token = _ACTIVE.set(source)
364
+ try:
365
+ yield source
366
+ if source.failure is not None:
367
+ raise RuntimeError('canonical_source_unavailable: ' + str(source.failure)) from source.failure
368
+ finally:
369
+ _ACTIVE.reset(token)
370
+ source.close()
371
+
372
+
373
+ class _Odoo:
374
+ def __getattr__(self, name):
375
+ source = active()
376
+ if source is not None and name in {'search_read', 'read_group', 'search_count'}:
377
+ return getattr(source, name)
378
+ if source is not None and name == 'get_odoo':
379
+ return lambda: self
380
+ return getattr(_odoo, name)
381
+
382
+
383
+ odoo = _Odoo()
platform/harness/datastore.py CHANGED
@@ -17,18 +17,23 @@ Design rules:
17
  - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
18
  """
19
  import datetime as dt
 
20
  import json
21
  import math
22
  import os
 
23
  import re
24
  import sys
25
  import time
 
 
26
  from contextlib import nullcontext
27
  from pathlib import Path
28
 
29
  import duckdb
30
 
31
  import core.odoo as O
 
32
 
33
  _STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
34
 
@@ -136,16 +141,46 @@ def path_for(tenant_key):
136
 
137
 
138
  def apply_limits(con):
139
- """Cap one DuckDB instance from AIOS_DUCKDB_MEMORY and AIOS_DUCKDB_THREADS. Unset keeps the default."""
140
- memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or "").strip()
141
- threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or "").strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
142
  if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
143
  con.execute(f"SET memory_limit='{memory}'")
144
  if threads.isdigit() and int(threads) > 0:
145
  con.execute(f"SET threads={int(threads)}")
 
 
146
  return con
147
 
148
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  def use_path(path):
150
  """Repoint the store at `path`, CLOSING the process singleton first.
151
 
@@ -180,6 +215,7 @@ def use_path(path):
180
  _READY["ok"] = False
181
  _GENERATION[0] += 1 # every other thread's cached cursor is now stale
182
  DB_PATH = Path(path)
 
183
  return DB_PATH
184
 
185
  # Entity specs: Odoo model → store table. m2o fields land as <name>_id + <name>_name.
@@ -505,7 +541,7 @@ def snapshot_status(tenant_key, source, *, rt=None):
505
  result.update(saved)
506
  if not path.exists():
507
  return result
508
- con = ro_cursor() if path.resolve() == DB_PATH.resolve() else duckdb.connect(str(path))
509
  try:
510
  manifest = _publication(con, source)
511
  if manifest:
@@ -826,11 +862,23 @@ _GENERATION = [0]
826
  #: Bumped after every mirror snapshot install (`MirrorStore._install_snapshot`). D-2670: the
827
  #: Sheet change token caches its table scan per generation, so an install must move this.
828
  _MIRROR_INSTALLS = [0]
 
 
 
 
 
 
 
 
 
 
 
829
 
830
 
831
  def note_mirror_install():
832
  """Mark that a mirror install committed or rolled back, so cached change tokens recompute."""
833
  _MIRROR_INSTALLS[0] += 1
 
834
 
835
 
836
  def mirror_generation():
@@ -868,9 +916,98 @@ def _instance():
868
  return _INSTANCE["con"]
869
 
870
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
871
  def ro_cursor():
872
  """A read cursor on the shared instance — the contention-free way to read the store."""
873
- return _instance().cursor()
874
 
875
 
876
  def ready():
@@ -891,7 +1028,8 @@ def ready():
891
  cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
892
  finally:
893
  cur.close()
894
- except Exception:
 
895
  return False
896
  # ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN
897
  # OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every
@@ -943,7 +1081,8 @@ def entity_live(key):
943
  columns_ready = _required_columns_ready(cur, key, ENTITIES.get(key) or {})
944
  finally:
945
  cur.close()
946
- except Exception: # noqa: BLE001
 
947
  return False
948
  return bool(row) and row[0] == "live" and columns_ready
949
 
@@ -975,7 +1114,8 @@ def ro_con():
975
  try:
976
  cur.execute("SELECT 1") # cheap liveness probe
977
  return cur
978
- except Exception:
 
979
  _RO_TLS.cur = None # stale → reopen below
980
  if not ready():
981
  raise RuntimeError("the tenant data store is completing its FIRST sync on this "
@@ -1034,11 +1174,335 @@ _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
1034
 
1035
  #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
1036
  _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1037
 
1038
 
1039
  def window(table=None, select=None, where=None, params=(), order_by=None,
1040
  offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
1041
- page_first=False):
1042
  """One page of a mirror table PLUS the true count of everything the same predicate matches.
1043
 
1044
  Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
@@ -1065,6 +1529,9 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
1065
 
1066
  `page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
1067
  the page's rows before the projection runs, when only the projection holds a subquery.
 
 
 
1068
  """
1069
  if bool(table) == bool(from_sql):
1070
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
@@ -1097,10 +1564,22 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
1097
  # ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate:
1098
  # if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than
1099
  # quietly returning a good count beside a broken page.
1100
- total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0)
1101
  grp = str(group_by or "").strip()
1102
  fold = f" GROUP BY {grp}" if grp else ""
1103
- if (page_first and table and not grp and _SUBQUERY.search(proj)
 
 
 
 
 
 
 
 
 
 
 
 
1104
  and not _SUBQUERY.search(f"{pred} {order}")):
1105
  # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
1106
  # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
@@ -1336,7 +1815,7 @@ def _prepare_seed_mirrors(path, rt):
1336
  if rt is None:
1337
  return
1338
  from harness import mirror_store
1339
- con = duckdb.connect(str(path))
1340
  try:
1341
  _ensure_sync_state(con)
1342
  for connector in tuple(_connector_spec.MIRRORS):
@@ -1412,6 +1891,8 @@ def connect():
1412
  cur = _candidate_connection()
1413
  if cur is None:
1414
  cur = _instance().cursor()
 
 
1415
  _ensure_sync_state(cur)
1416
  return cur
1417
 
 
17
  - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
18
  """
19
  import datetime as dt
20
+ import hashlib
21
  import json
22
  import math
23
  import os
24
+ import pickle
25
  import re
26
  import sys
27
  import time
28
+ import tempfile
29
+ from collections import OrderedDict
30
  from contextlib import nullcontext
31
  from pathlib import Path
32
 
33
  import duckdb
34
 
35
  import core.odoo as O
36
+ from core import request_timing as _request_timing
37
 
38
  _STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
39
 
 
141
 
142
 
143
  def apply_limits(con):
144
+ """Bound each instance's memory, execution threads and local spill space."""
145
+ web = os.environ.get("AIOS_PROCESS_ROLE") == "web"
146
+ memory = str(os.environ.get("AIOS_DUCKDB_MEMORY") or (
147
+ os.environ.get("AIOS_WEB_DUCKDB_MEMORY") or "6GB" if web else "4GB")).strip()
148
+ threads = str(os.environ.get("AIOS_DUCKDB_THREADS") or (
149
+ os.environ.get("AIOS_WEB_DUCKDB_THREADS") or "2" if web else "1")).strip()
150
+ spill = str(os.environ.get("AIOS_DUCKDB_MAX_TEMP_SIZE") or "4GB").strip()
151
+ temp = Path(os.environ.get("AIOS_DUCKDB_TEMP_DIR") or (
152
+ Path(tempfile.gettempdir()) / f"aios-duckdb-{os.getpid()}"))
153
+ rows = con.execute("PRAGMA database_list").fetchall()
154
+ identity = next((str(row[2]) for row in rows if row[2]), str(id(con)))
155
+ temp = temp / hashlib.sha256(identity.encode()).hexdigest()[:16]
156
+ temp.mkdir(parents=True, exist_ok=True, mode=0o700)
157
+ if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
158
+ raise ValueError("Invalid AIOS_DUCKDB_MEMORY")
159
+ if not threads.isdigit() or int(threads) < 1:
160
+ raise ValueError("Invalid AIOS_DUCKDB_THREADS")
161
+ if not re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", spill, re.IGNORECASE):
162
+ raise ValueError("Invalid AIOS_DUCKDB_MAX_TEMP_SIZE")
163
  if memory and re.fullmatch(r"\d+(\.\d+)?\s*[KMGT]i?B", memory, re.IGNORECASE):
164
  con.execute(f"SET memory_limit='{memory}'")
165
  if threads.isdigit() and int(threads) > 0:
166
  con.execute(f"SET threads={int(threads)}")
167
+ con.execute("SET temp_directory='" + str(temp).replace("'", "''") + "'")
168
+ con.execute(f"SET max_temp_directory_size='{spill}'")
169
  return con
170
 
171
 
172
+ def bounded_connection(con):
173
+ """Apply instance budgets and request deadlines to a temporary analytical connection."""
174
+ try:
175
+ return _TimedCursor(apply_limits(con))
176
+ except BaseException:
177
+ try:
178
+ con.close()
179
+ except Exception:
180
+ pass
181
+ raise
182
+
183
+
184
  def use_path(path):
185
  """Repoint the store at `path`, CLOSING the process singleton first.
186
 
 
215
  _READY["ok"] = False
216
  _GENERATION[0] += 1 # every other thread's cached cursor is now stale
217
  DB_PATH = Path(path)
218
+ _invalidate_counts()
219
  return DB_PATH
220
 
221
  # Entity specs: Odoo model → store table. m2o fields land as <name>_id + <name>_name.
 
541
  result.update(saved)
542
  if not path.exists():
543
  return result
544
+ con = ro_cursor() if path.resolve() == DB_PATH.resolve() else _TimedCursor(apply_limits(duckdb.connect(str(path))))
545
  try:
546
  manifest = _publication(con, source)
547
  if manifest:
 
862
  #: Bumped after every mirror snapshot install (`MirrorStore._install_snapshot`). D-2670: the
863
  #: Sheet change token caches its table scan per generation, so an install must move this.
864
  _MIRROR_INSTALLS = [0]
865
+ _MIRROR_WRITES = [0]
866
+ _ACTIVE_WRITES = [0]
867
+ _COUNT_CACHE = OrderedDict()
868
+ _COUNT_CACHE_MAX = 256
869
+ _COUNT_CACHE_LOCK = _thr.RLock()
870
+
871
+
872
+ def _invalidate_counts():
873
+ with _COUNT_CACHE_LOCK:
874
+ _MIRROR_WRITES[0] += 1
875
+ _COUNT_CACHE.clear()
876
 
877
 
878
  def note_mirror_install():
879
  """Mark that a mirror install committed or rolled back, so cached change tokens recompute."""
880
  _MIRROR_INSTALLS[0] += 1
881
+ _invalidate_counts()
882
 
883
 
884
  def mirror_generation():
 
916
  return _INSTANCE["con"]
917
 
918
 
919
+ class _TimedCursor:
920
+ """Keep cursor chaining and thread-local identity while timing execute and result conversion."""
921
+
922
+ _FETCH = {"fetchone", "fetchall", "fetchmany", "fetchdf", "fetch_df", "df", "fetchnumpy",
923
+ "fetch_arrow_table", "fetch_record_batch", "arrow", "pl", "fetch_df_chunk"}
924
+
925
+ def __init__(self, cursor):
926
+ self._cursor = cursor
927
+ self._statement = ""
928
+ self._statement_ms = 0.0
929
+ self._request = None
930
+ self._in_transaction = False
931
+
932
+ def _call(self, name, *args, **kwargs):
933
+ acc = _request_timing.CURRENT.get()
934
+ execute = name in {"execute", "executemany"}
935
+ writes = False
936
+ transaction = None
937
+ if execute:
938
+ statement = args[0] if args else kwargs.get("query", "<statement>")
939
+ self._statement = _request_timing.sql_text(statement) if acc is not None else ""
940
+ self._statement_ms = 0.0
941
+ self._request = acc
942
+ try:
943
+ statements = self._cursor.extract_statements(statement)
944
+ writes = any(s.type.name not in {"SELECT", "TRANSACTION"} for s in statements)
945
+ for parsed in statements:
946
+ if parsed.type.name == "TRANSACTION":
947
+ transaction = not re.match(r"\s*(COMMIT|ROLLBACK|END)\b", parsed.query, re.I)
948
+ except Exception as exc:
949
+ _request_timing.raise_if_resource_error(exc)
950
+ # Unclassified statements cannot establish a reusable read snapshot.
951
+ writes, self._in_transaction = True, True
952
+ if transaction is not None:
953
+ # A failed statement batch may leave its BEGIN active.
954
+ self._in_transaction = True
955
+ if writes:
956
+ with _COUNT_CACHE_LOCK:
957
+ _ACTIVE_WRITES[0] += 1
958
+ _invalidate_counts()
959
+ started = time.perf_counter()
960
+ try:
961
+ with _request_timing.thread_scope(), _request_timing.duckdb_operation(self._cursor):
962
+ result = getattr(self._cursor, name)(*args, **kwargs)
963
+ if transaction is not None:
964
+ self._in_transaction = transaction
965
+ return self if result is self._cursor else result
966
+ finally:
967
+ if writes:
968
+ with _COUNT_CACHE_LOCK:
969
+ _ACTIVE_WRITES[0] -= 1
970
+ _invalidate_counts()
971
+ if acc is not None and self._request is acc:
972
+ elapsed = (time.perf_counter() - started) * 1000
973
+ self._statement_ms += elapsed
974
+ _request_timing.record_duckdb(self._statement, elapsed, count=int(execute),
975
+ statement_ms=self._statement_ms)
976
+
977
+ def execute(self, *args, **kwargs):
978
+ return self._call("execute", *args, **kwargs)
979
+
980
+ def executemany(self, *args, **kwargs):
981
+ return self._call("executemany", *args, **kwargs)
982
+
983
+ def cursor(self, *args, **kwargs):
984
+ return _TimedCursor(self._cursor.cursor(*args, **kwargs))
985
+
986
+ def __getattr__(self, name):
987
+ if name in self._FETCH:
988
+ return lambda *args, **kwargs: self._call(name, *args, **kwargs)
989
+ return getattr(self._cursor, name)
990
+
991
+ def __enter__(self):
992
+ self._cursor.__enter__()
993
+ return self
994
+
995
+ def __exit__(self, *args):
996
+ return self._cursor.__exit__(*args)
997
+
998
+ def __iter__(self):
999
+ return self
1000
+
1001
+ def __next__(self):
1002
+ row = self.fetchone()
1003
+ if row is None:
1004
+ raise StopIteration
1005
+ return row
1006
+
1007
+
1008
  def ro_cursor():
1009
  """A read cursor on the shared instance — the contention-free way to read the store."""
1010
+ return _TimedCursor(_instance().cursor())
1011
 
1012
 
1013
  def ready():
 
1028
  cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
1029
  finally:
1030
  cur.close()
1031
+ except Exception as exc:
1032
+ _request_timing.raise_if_resource_error(exc)
1033
  return False
1034
  # ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN
1035
  # OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every
 
1081
  columns_ready = _required_columns_ready(cur, key, ENTITIES.get(key) or {})
1082
  finally:
1083
  cur.close()
1084
+ except Exception as exc: # noqa: BLE001
1085
+ _request_timing.raise_if_resource_error(exc)
1086
  return False
1087
  return bool(row) and row[0] == "live" and columns_ready
1088
 
 
1114
  try:
1115
  cur.execute("SELECT 1") # cheap liveness probe
1116
  return cur
1117
+ except Exception as exc:
1118
+ _request_timing.raise_if_resource_error(exc)
1119
  _RO_TLS.cur = None # stale → reopen below
1120
  if not ready():
1121
  raise RuntimeError("the tenant data store is completing its FIRST sync on this "
 
1174
 
1175
  #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
1176
  _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
1177
+ _ORDINAL_ORDER = re.compile(
1178
+ r"(?:\(\s*)*[+-]?\d[\d_]*(?:\s*\))*\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
1179
+ _ALL_ORDER = re.compile(r"ALL\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
1180
+
1181
+
1182
+ def _sql_terms(sql):
1183
+ """Split trusted SQL at unquoted, top-level commas, or decline unfamiliar syntax."""
1184
+ depth, quote, start, index = 0, None, 0, 0
1185
+ terms = []
1186
+ while index < len(sql):
1187
+ char = sql[index]
1188
+ if quote:
1189
+ if char == "\\":
1190
+ return None
1191
+ if char == quote:
1192
+ if index + 1 < len(sql) and sql[index + 1] == quote:
1193
+ index += 2
1194
+ continue
1195
+ quote = None
1196
+ elif char in "'\"":
1197
+ quote = char
1198
+ elif (not (char.isalnum() or char.isspace() or char in "_.,()+-*/%=<>!|:&~^")
1199
+ or sql[index:index + 2] in ("--", "/*", "*/", "->", "=>", ":=")):
1200
+ return None
1201
+ elif char == "(":
1202
+ depth += 1
1203
+ elif char == ")":
1204
+ depth -= 1
1205
+ if depth < 0:
1206
+ return None
1207
+ elif char == "," and depth == 0:
1208
+ terms.append(sql[start:index].strip())
1209
+ start = index + 1
1210
+ index += 1
1211
+ if quote or depth:
1212
+ return None
1213
+ terms.append(sql[start:].strip())
1214
+ return terms if all(terms) else None
1215
+
1216
+
1217
+ def _sql_tokens(sql):
1218
+ """Use DuckDB's lexer for identifier quoting and expression comparisons."""
1219
+ tokens = duckdb.tokenize(sql)
1220
+ encoded = sql.encode("utf-8")
1221
+ result = [(encoded[start:tokens[i + 1][0] if i + 1 < len(tokens) else len(encoded)].decode("utf-8").strip(), kind)
1222
+ for i, (start, kind) in enumerate(tokens)]
1223
+ if any(text.upper() in ("COLUMNS", "EXCLUDE", "REPLACE", "USING") for text, _kind in result):
1224
+ raise ValueError("Unrecognized page ordering syntax")
1225
+ return result
1226
+
1227
+
1228
+ def _order_needs_projection(order, projection=""):
1229
+ """Keep positional, ALL, alias-dependent and unfamiliar orders after projection."""
1230
+ terms = _sql_terms(order)
1231
+ if terms is None or any(_ORDINAL_ORDER.fullmatch(t) or _ALL_ORDER.fullmatch(t) for t in terms):
1232
+ return True
1233
+ selected = _sql_terms(projection) if projection else []
1234
+ if selected is None:
1235
+ return True
1236
+ try:
1237
+ fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
1238
+ def normalized(tokens):
1239
+ return [(text[1:-1].replace('""', '"').translate(fold) if text.startswith('"')
1240
+ else text if text.startswith("'") else text.translate(fold), kind) for text, kind in tokens]
1241
+ ordered = normalized(_sql_tokens(order))
1242
+ aliases = {}
1243
+ for term in selected:
1244
+ tokens = _sql_tokens(term)
1245
+ depth, top = 0, []
1246
+ for i, (text, _kind) in enumerate(tokens):
1247
+ if text == ")":
1248
+ depth -= 1
1249
+ if depth == 0:
1250
+ top.append(i)
1251
+ if text == "(":
1252
+ depth += 1
1253
+ if not top or any(tokens[i][0] in ("*", ":", ":=") for i in top):
1254
+ return True
1255
+ last = top[-1]
1256
+ if len(top) >= 2 and tokens[top[-2]][0].upper() == "AS":
1257
+ expression = tokens[:top[-2]]
1258
+ elif (last > 0 and tokens[last][1] == duckdb.token_type.identifier
1259
+ and tokens[last - 1][0] != "."):
1260
+ expression = tokens[:last]
1261
+ else:
1262
+ continue
1263
+ alias = normalized([tokens[last]])[0][0]
1264
+ if tokens[last][0].startswith("'"):
1265
+ alias = tokens[last][0][1:-1].replace("''", "'").translate(fold)
1266
+ aliases.setdefault(alias, []).append(normalized(expression))
1267
+ for alias, expressions in aliases.items():
1268
+ covered = set()
1269
+ # A repeated source expression is safe, including generated casts and passthrough aliases.
1270
+ if all(expr == expressions[0] for expr in expressions):
1271
+ expr = expressions[0]
1272
+ for i in range(len(ordered) - len(expr) + 1):
1273
+ if expr and ordered[i:i + len(expr)] == expr:
1274
+ covered.update(range(i, i + len(expr)))
1275
+ for i, token in enumerate(ordered):
1276
+ if (token[0] == alias and token[1] in (duckdb.token_type.identifier, duckdb.token_type.keyword)
1277
+ and i not in covered
1278
+ and (i == 0 or ordered[i - 1][0] != ".")
1279
+ and (i + 1 == len(ordered) or ordered[i + 1][0] not in (".", "("))):
1280
+ return True
1281
+ except (duckdb.Error, AttributeError, ValueError, UnicodeError):
1282
+ return True
1283
+ return False
1284
+
1285
+
1286
+ def _physical_page_order(order, page_key):
1287
+ """Make physical pages total without changing ALL or unfamiliar SQL ordering semantics."""
1288
+ terms = _sql_terms(order)
1289
+ if terms is None or any(_ALL_ORDER.fullmatch(term) for term in terms):
1290
+ return order
1291
+ try:
1292
+ _sql_tokens(order)
1293
+ except (duckdb.Error, AttributeError, ValueError):
1294
+ return order
1295
+ return f"{order}, {page_key} ASC"
1296
+
1297
+
1298
+ def _page_is_row_local(projection, order, cur):
1299
+ """Prove outer expressions scalar with DuckDB's parser and bounded catalogue lookups."""
1300
+ metadata, verified, expanding, bound = {}, set(), set(), set()
1301
+
1302
+ def parse(expressions, ordering=""):
1303
+ sql = "SELECT " + expressions + (" ORDER BY " + ordering if ordering else "")
1304
+ parsed = json.loads(cur.execute("SELECT json_serialize_sql(?)", [sql]).fetchone()[0])
1305
+ if parsed.get("error") or len(parsed["statements"]) != 1:
1306
+ raise ValueError("Unrecognized page expression")
1307
+ node = parsed["statements"][0]["node"]
1308
+ if (node["type"] != "SELECT_NODE" or node["from_table"]["type"] != "EMPTY"
1309
+ or node["cte_map"]["map"] or node["group_expressions"] or node["group_sets"]
1310
+ or node["aggregate_handling"] != "STANDARD_HANDLING"
1311
+ or any(node[key] is not None for key in ("where_clause", "having", "sample", "qualify"))
1312
+ or any(m["type"] != "ORDER_MODIFIER" for m in node["modifiers"])):
1313
+ raise ValueError("Projection changes the outer population")
1314
+ return [node["select_list"], node["modifiers"]]
1315
+
1316
+ def aggregate_is_local(value, scope):
1317
+ columns = set()
1318
+
1319
+ def collect(node):
1320
+ if isinstance(node, list):
1321
+ return all(collect(v) for v in node)
1322
+ if not isinstance(node, dict):
1323
+ return True
1324
+ kind = node.get("class")
1325
+ if kind == "COLUMN_REF":
1326
+ columns.add(tuple(node["column_names"]))
1327
+ elif kind is not None and kind not in {
1328
+ "CONSTANT", "CAST", "CASE", "COMPARISON", "CONJUNCTION",
1329
+ "OPERATOR", "FUNCTION", "PARAMETER"}:
1330
+ return False
1331
+ return all(collect(v) for v in node.values())
1332
+
1333
+ if not collect(value):
1334
+ return False
1335
+ if not columns:
1336
+ return True
1337
+ if scope is None or scope[1] or scope[0]["type"] != "SELECT_NODE":
1338
+ return False
1339
+ source = scope[0]["from_table"]
1340
+ if (source["type"] != "BASE_TABLE" or source.get("column_name_alias")
1341
+ or source.get("at_clause") or source.get("sample")):
1342
+ return False
1343
+ alias = source.get("alias") or source["table_name"]
1344
+ fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
1345
+ if any(len(c) != 2 or c[0].translate(fold) != alias.translate(fold) for c in columns):
1346
+ return False
1347
+ quote = lambda name: '"' + name.replace('"', '""') + '"'
1348
+ table = '.'.join(quote(source[key]) for key in ("catalog_name", "schema_name", "table_name")
1349
+ if source.get(key))
1350
+ selected = ','.join('.'.join(map(quote, c)) for c in sorted(columns))
1351
+ sql = f"DESCRIBE SELECT {selected} FROM {table} AS {quote(alias)}"
1352
+ if sql not in bound:
1353
+ # A matching alias alone cannot prove that a missing column will not bind outward.
1354
+ cur.execute(sql).fetchall()
1355
+ bound.add(sql)
1356
+ return True
1357
+
1358
+ def references(value, required, nested, scalar=True, scope=None):
1359
+ if isinstance(value, list):
1360
+ return all(references(v, required, nested, scalar, scope) for v in value)
1361
+ if not isinstance(value, dict):
1362
+ return True
1363
+ if "cte_map" in value:
1364
+ scope = (value, bool(value["cte_map"]["map"]) or bool(scope and scope[1]))
1365
+ kind = value.get("class")
1366
+ if scalar and kind == "SUBQUERY":
1367
+ # Nested aggregates must bind locally even inside a syntactic scalar subquery.
1368
+ return (value["subquery_type"] == "SCALAR" and value.get("child") is None
1369
+ and references(value["subquery"], required, nested, False, scope))
1370
+ if scalar and kind is not None:
1371
+ if kind not in {"COLUMN_REF", "CONSTANT", "CAST", "CASE", "COMPARISON",
1372
+ "CONJUNCTION", "OPERATOR", "FUNCTION", "PARAMETER"}:
1373
+ return False
1374
+ if kind == "OPERATOR" and value["type"] not in {
1375
+ "OPERATOR_COALESCE", "OPERATOR_NOT", "OPERATOR_IS_NULL",
1376
+ "OPERATOR_IS_NOT_NULL", "COMPARE_IN", "COMPARE_NOT_IN"}:
1377
+ return False
1378
+ if "function_name" in value:
1379
+ if value.get("catalog") or value.get("schema", "") not in ("", "main"):
1380
+ return False
1381
+ if scalar and (value.get("distinct") or value.get("filter") is not None
1382
+ or value.get("export_state") or value.get("order_bys", {}).get("orders")):
1383
+ return False
1384
+ name = value["function_name"].lower()
1385
+ required[name] = required.get(name, False) or scalar
1386
+ if not scalar:
1387
+ nested.append((name, value, scope))
1388
+ return all(references(v, required, nested, scalar, scope) for v in value.values())
1389
+
1390
+ def prove(expressions, ordering=""):
1391
+ required, nested = {}, []
1392
+ if not references(parse(expressions, ordering), required, nested):
1393
+ return False
1394
+ missing = sorted(set(required) - metadata.keys())
1395
+ if missing:
1396
+ rows = cur.execute(
1397
+ "SELECT lower(function_name), function_type, internal, macro_definition "
1398
+ "FROM duckdb_functions() WHERE lower(function_name) = ANY(?)", [missing]).fetchall()
1399
+ metadata.update((name, []) for name in missing)
1400
+ for name, function_type, internal, definition in rows:
1401
+ metadata[name].append((function_type, internal, definition))
1402
+ for name, scalar in required.items():
1403
+ if (name, scalar) in verified:
1404
+ continue
1405
+ if not metadata[name] or name in expanding or len(expanding) >= 16:
1406
+ return False
1407
+ for function_type, internal, definition in metadata[name]:
1408
+ if function_type == "macro":
1409
+ # Only engine-owned macros with independently scalar bodies are admitted.
1410
+ if not internal or not definition:
1411
+ return False
1412
+ expanding.add(name)
1413
+ try:
1414
+ if not prove(definition):
1415
+ return False
1416
+ finally:
1417
+ expanding.remove(name)
1418
+ elif function_type not in ({"scalar"} if scalar else {"scalar", "aggregate", "table"}):
1419
+ return False
1420
+ verified.add((name, scalar))
1421
+ for name, value, scope in nested:
1422
+ if (value.get("class") != "WINDOW"
1423
+ and any(kind == "aggregate" for kind, _internal, _definition in metadata[name])
1424
+ and not aggregate_is_local(value, scope)):
1425
+ return False
1426
+ return True
1427
+
1428
+ try:
1429
+ return prove(projection, order)
1430
+ except Exception as exc:
1431
+ # A parser/catalogue refusal never authorizes moving the original projection.
1432
+ _request_timing.raise_if_resource_error(exc)
1433
+ return False
1434
+
1435
+
1436
+ def mirror_file_marks(cur):
1437
+ """Identify the cursor's actual file and WAL, or decline an unnamed mirror."""
1438
+ row = cur.execute("SELECT path FROM duckdb_databases() "
1439
+ "WHERE database_name = current_database()").fetchone()
1440
+ path = row[0] if row else None
1441
+ if not path or path == ":memory:":
1442
+ return None
1443
+ marks = []
1444
+ for name in (path, path + ".wal"):
1445
+ try:
1446
+ stat = os.stat(name)
1447
+ marks.append((stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns))
1448
+ except OSError:
1449
+ marks.append(None)
1450
+ return (str(Path(path).resolve()), tuple(marks)) if marks[0] is not None else None
1451
+
1452
+
1453
+ def _count_revision(cur):
1454
+ """Cache only managed, autocommit reads of one persistent mirror without local relations."""
1455
+ if not isinstance(cur, _TimedCursor) or cur._in_transaction or _ACTIVE_WRITES[0]:
1456
+ return None
1457
+ generation = mirror_generation(), _MIRROR_WRITES[0]
1458
+ marks = mirror_file_marks(cur)
1459
+ if marks is None:
1460
+ return None
1461
+ unsafe = cur.execute(
1462
+ "SELECT (SELECT count(*) FROM duckdb_databases() WHERE NOT internal) <> 1 "
1463
+ "OR EXISTS (SELECT 1 FROM duckdb_tables() WHERE temporary AND NOT internal) "
1464
+ "OR EXISTS (SELECT 1 FROM duckdb_views() WHERE temporary AND NOT internal)").fetchone()[0]
1465
+ return None if unsafe or generation != (mirror_generation(), _MIRROR_WRITES[0]) else (marks, generation)
1466
+
1467
+
1468
+ def count_rows(table=None, where=None, params=(), cur=None, from_sql=None):
1469
+ """Return the exact matching row population, reusing only an unchanged mirror revision."""
1470
+ # Callers supply deterministic compiled SQL, as they do for window().
1471
+ if bool(table) == bool(from_sql):
1472
+ raise ValueError("datastore.count_rows: pass exactly one of `table` or `from_sql`")
1473
+ name = str(from_sql or table).strip()
1474
+ if table and not _IDENT_OK.fullmatch(name):
1475
+ raise ValueError(f"datastore.count_rows: {table!r} is not a table identifier")
1476
+ pred = str(where or "").strip()
1477
+ sql = f"SELECT count(*) FROM {name}" + (f" WHERE {pred}" if pred else "")
1478
+ params = tuple(params or ())
1479
+ cur = cur if cur is not None else ro_con()
1480
+ revision = _count_revision(cur)
1481
+ key = None
1482
+ if revision is not None:
1483
+ try:
1484
+ # SQL and typed bindings remain private, including large IN predicates.
1485
+ key = revision, hashlib.sha256(pickle.dumps((sql, params), protocol=5)).digest()
1486
+ except (TypeError, ValueError, AttributeError, OverflowError, pickle.PickleError):
1487
+ pass
1488
+ if key is not None:
1489
+ with _COUNT_CACHE_LOCK:
1490
+ if key in _COUNT_CACHE:
1491
+ _COUNT_CACHE.move_to_end(key)
1492
+ return _COUNT_CACHE[key]
1493
+ total = int(cur.execute(sql, params).fetchone()[0] or 0)
1494
+ if key is not None and _count_revision(cur) == revision:
1495
+ with _COUNT_CACHE_LOCK:
1496
+ _COUNT_CACHE[key] = total
1497
+ _COUNT_CACHE.move_to_end(key)
1498
+ while len(_COUNT_CACHE) > _COUNT_CACHE_MAX:
1499
+ _COUNT_CACHE.popitem(last=False)
1500
+ return total
1501
 
1502
 
1503
  def window(table=None, select=None, where=None, params=(), order_by=None,
1504
  offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
1505
+ page_first=False, page_key=None):
1506
  """One page of a mirror table PLUS the true count of everything the same predicate matches.
1507
 
1508
  Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
 
1529
 
1530
  `page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
1531
  the page's rows before the projection runs, when only the projection holds a subquery.
1532
+ `page_key` opts a physical table into a narrow, materialised key page before projection.
1533
+ It must uniquely identify source rows, independently of the displayed record identity.
1534
+ Proven row-local pages end with this physical key ascending. Fallback SQL retains its order.
1535
  """
1536
  if bool(table) == bool(from_sql):
1537
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
 
1564
  # ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate:
1565
  # if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than
1566
  # quietly returning a good count beside a broken page.
1567
+ total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
1568
  grp = str(group_by or "").strip()
1569
  fold = f" GROUP BY {grp}" if grp else ""
1570
+ needs_projection = _order_needs_projection(order, proj) if page_first else True
1571
+ row_local = (page_first and table and not grp and not needs_projection
1572
+ and _page_is_row_local(proj, order, cur))
1573
+ if row_local and page_key:
1574
+ order = _physical_page_order(order, page_key)
1575
+ if row_local and page_key:
1576
+ # Only keys cross the materialisation boundary; identity order still uses the full population.
1577
+ got = cur.execute(
1578
+ f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
1579
+ f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
1580
+ f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
1581
+ f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
1582
+ elif (row_local and not page_key and _SUBQUERY.search(proj)
1583
  and not _SUBQUERY.search(f"{pred} {order}")):
1584
  # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
1585
  # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
 
1815
  if rt is None:
1816
  return
1817
  from harness import mirror_store
1818
+ con = apply_limits(duckdb.connect(str(path)))
1819
  try:
1820
  _ensure_sync_state(con)
1821
  for connector in tuple(_connector_spec.MIRRORS):
 
1891
  cur = _candidate_connection()
1892
  if cur is None:
1893
  cur = _instance().cursor()
1894
+ if _request_timing.CURRENT.get() is not None and not isinstance(cur, _TimedCursor):
1895
+ cur = _TimedCursor(cur)
1896
  _ensure_sync_state(cur)
1897
  return cur
1898
 
platform/harness/filter_sql.py CHANGED
@@ -715,9 +715,13 @@ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql
715
  if member_ids:
716
  if not id_sql:
717
  raise ValueError('member_ids requires id_sql (the row-identity column)')
718
- ids = [int(x) for x in member_ids]
719
- sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
720
- params = list(params) + ids
 
 
 
 
721
  return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
722
 
723
 
 
715
  if member_ids:
716
  if not id_sql:
717
  raise ValueError('member_ids requires id_sql (the row-identity column)')
718
+ if isinstance(member_ids, SqlMembership):
719
+ sql = f"({sql} OR {id_sql} IN ({member_ids.sql}))"
720
+ params = list(params) + list(member_ids.params)
721
+ else:
722
+ ids = [int(x) for x in member_ids]
723
+ sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
724
+ params = list(params) + ids
725
  return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
726
 
727
 
platform/harness/measure_filter.py CHANGED
@@ -406,7 +406,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None, related=None):
406
  "which would be a different question with an authoritative-looking answer")
407
  con = _sem._store_con()
408
  try:
409
- totals = _totals(con, key, rule.get('window'), today, team_id, related)
410
  finally:
411
  con.close()
412
  # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision,
@@ -523,7 +523,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
523
  # compare in Python — the same shape `resolve_pair` uses, and for the same reason.
524
  con = _sem._store_con()
525
  try:
526
- totals = _totals(con, key, rule.get('window'), today, team_id, related)
527
  finally:
528
  con.close()
529
  # No pool means we can only speak about customers a base has seen. WITH a pool, everyone
@@ -535,6 +535,9 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
535
 
536
  frm, id_col, agg, where_sql, params = _query_parts(
537
  key, rule.get('window'), today, team_id, related)
 
 
 
538
  sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
539
  f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
540
 
@@ -559,7 +562,7 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None):
559
  return _related_answer(matched, related)
560
 
561
 
562
- def _totals(con, key, window, today, team_id, related=None):
563
  """`{pid: value}` for one (measure, window), COMPOSITES included.
564
 
565
  For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
@@ -573,13 +576,16 @@ def _totals(con, key, window, today, team_id, related=None):
573
  """
574
  if is_composite(key):
575
  bases, _ = _sem._expand_measures([key])
576
- parts = {b: _totals(con, b, window, today, team_id, related) for b in bases}
577
  pids = set()
578
  for got in parts.values():
579
  pids |= set(got)
580
  return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
581
  for pid in pids}
582
  frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related)
 
 
 
583
  sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
584
  # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
585
  # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and
@@ -620,8 +626,8 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None, related=None):
620
  pool = _pool_of(allowed_pids)
621
  con = _sem._store_con()
622
  try:
623
- left = _totals(con, lkey, rule.get('window'), today, team_id, related)
624
- right = _totals(con, rkey, rhs.get('window'), today, team_id, related)
625
  finally:
626
  con.close()
627
  # With no pool we can only speak about customers at least one side has seen — the zero-vs-
@@ -636,7 +642,7 @@ def _zero_satisfies(op, value):
636
  return _CMP[op](0.0, value)
637
 
638
 
639
- def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None):
640
  """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN
641
  version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
642
 
@@ -669,16 +675,22 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
669
  f"unresolvable window {window!r} for the {key!r} column: refusing rather than "
670
  f"defaulting to all time")
671
  related = _related_scope(related, [key])
 
 
 
 
 
672
  pool = _pool_of(allowed_pids)
673
- con = _sem._store_con()
674
  try:
675
- totals = _totals(con, key, window, today, team_id, related)
676
  guard = None
677
  metric = _sem.metrics().get(key, {})
678
  if related is not None and not related.refusal and metric.get('agg') == 'ratio':
679
- guard = _totals(con, metric['denominator'], window, today, team_id, related)
680
  finally:
681
- con.close()
 
682
  scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
683
  mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
684
  nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
 
406
  "which would be a different question with an authoritative-looking answer")
407
  con = _sem._store_con()
408
  try:
409
+ totals = _totals(con, key, rule.get('window'), today, team_id, related, pool)
410
  finally:
411
  con.close()
412
  # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision,
 
523
  # compare in Python — the same shape `resolve_pair` uses, and for the same reason.
524
  con = _sem._store_con()
525
  try:
526
+ totals = _totals(con, key, rule.get('window'), today, team_id, related, pool)
527
  finally:
528
  con.close()
529
  # No pool means we can only speak about customers a base has seen. WITH a pool, everyone
 
535
 
536
  frm, id_col, agg, where_sql, params = _query_parts(
537
  key, rule.get('window'), today, team_id, related)
538
+ if pool is not None:
539
+ where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))"
540
+ params = [*params, list(pool)]
541
  sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
542
  f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
543
 
 
562
  return _related_answer(matched, related)
563
 
564
 
565
+ def _totals(con, key, window, today, team_id, related=None, allowed_pids=None):
566
  """`{pid: value}` for one (measure, window), COMPOSITES included.
567
 
568
  For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
 
576
  """
577
  if is_composite(key):
578
  bases, _ = _sem._expand_measures([key])
579
+ parts = {b: _totals(con, b, window, today, team_id, related, allowed_pids) for b in bases}
580
  pids = set()
581
  for got in parts.values():
582
  pids |= set(got)
583
  return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
584
  for pid in pids}
585
  frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related)
586
+ if allowed_pids is not None:
587
+ where_sql += f" AND {id_col} IN (SELECT unnest(?::BIGINT[]))"
588
+ params = [*params, list(allowed_pids)]
589
  sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
590
  # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
591
  # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and
 
626
  pool = _pool_of(allowed_pids)
627
  con = _sem._store_con()
628
  try:
629
+ left = _totals(con, lkey, rule.get('window'), today, team_id, related, pool)
630
+ right = _totals(con, rkey, rhs.get('window'), today, team_id, related, pool)
631
  finally:
632
  con.close()
633
  # With no pool we can only speak about customers at least one side has seen — the zero-vs-
 
642
  return _CMP[op](0.0, value)
643
 
644
 
645
+ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None, *, cur=None):
646
  """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN
647
  version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
648
 
 
675
  f"unresolvable window {window!r} for the {key!r} column: refusing rather than "
676
  f"defaulting to all time")
677
  related = _related_scope(related, [key])
678
+ if cur is None:
679
+ from harness.canonical_source import active
680
+ source = active()
681
+ if source is not None:
682
+ cur = source.cur
683
  pool = _pool_of(allowed_pids)
684
+ con = cur if cur is not None else _sem._store_con()
685
  try:
686
+ totals = _totals(con, key, window, today, team_id, related, pool)
687
  guard = None
688
  metric = _sem.metrics().get(key, {})
689
  if related is not None and not related.refusal and metric.get('agg') == 'ratio':
690
+ guard = _totals(con, metric['denominator'], window, today, team_id, related, pool)
691
  finally:
692
+ if cur is None:
693
+ con.close()
694
  scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
695
  mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
696
  nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
platform/harness/mirror_tokens.py CHANGED
@@ -13,7 +13,6 @@ does it. Until then that function RAISES, which `database_change_token` already
13
  prove unchanged, therefore resolve" -- the safe direction.
14
  """
15
  from __future__ import annotations
16
- import os
17
  import re
18
  import threading
19
  import time
@@ -28,20 +27,9 @@ _TABLE_ROWS_LOCK = threading.Lock()
28
 
29
 
30
  def _file_marks(cur):
31
- """Size and mtime of the mirror file and its WAL, or None when the mirror has no file."""
32
- row = cur.execute("SELECT path FROM duckdb_databases() "
33
- "WHERE database_name = current_database()").fetchone()
34
- path = row[0] if row else None
35
- if not path or path == ":memory:":
36
- return None
37
- marks = []
38
- for name in (path, path + ".wal"):
39
- try:
40
- stat = os.stat(name)
41
- marks.append([stat.st_size, stat.st_mtime_ns])
42
- except OSError:
43
- marks.append(None)
44
- return marks
45
 
46
 
47
  def mirror_change_token(st, database):
 
13
  prove unchanged, therefore resolve" -- the safe direction.
14
  """
15
  from __future__ import annotations
 
16
  import re
17
  import threading
18
  import time
 
27
 
28
 
29
  def _file_marks(cur):
30
+ """The shared actual-file and WAL revision, or None for an unnamed mirror."""
31
+ from harness import datastore
32
+ return datastore.mirror_file_marks(cur)
 
 
 
 
 
 
 
 
 
 
 
33
 
34
 
35
  def mirror_change_token(st, database):
platform/harness/semantic.py CHANGED
@@ -256,11 +256,17 @@ def render_scope(sql):
256
  return out
257
 
258
 
 
 
 
 
 
 
259
  @functools.lru_cache(maxsize=1)
260
  def _model():
261
  topics, metrics = {}, {}
262
  for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
263
- d = yaml.safe_load(f.read_text(encoding="utf-8"))
264
  if not d or "key" not in d:
265
  raise ModelError(f"topic file {f.name} lacks a 'key'")
266
  if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
@@ -268,7 +274,7 @@ def _model():
268
  # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises
269
  # a clear error if a live resolution is attempted on it
270
  topics[d["key"]] = d
271
- metric_docs = [(f.name, yaml.safe_load(f.read_text(encoding="utf-8")))
272
  for f in sorted((MODEL_DIR / "metrics").glob("*.yml"))]
273
  # Metric libraries may own new fact topics without changing legacy topic definitions.
274
  for name, doc in metric_docs:
@@ -1652,7 +1658,7 @@ def _check_shipped_links(date_from, date_to):
1652
  def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
1653
  team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
1654
  filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None,
1655
- *, runtime=None, user=None):
1656
  """The semantic query over the tenant store — the engine behind the Analyst's
1657
  run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
1658
  t = _model()["topics"].get(topic)
@@ -1810,6 +1816,16 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
1810
  where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
1811
  params.extend(ids)
1812
 
 
 
 
 
 
 
 
 
 
 
1813
  # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
1814
  # this is the nested, 11-operator contract the table UI emits. Compiled by
1815
  # harness/filter_sql, which is held in lock-step with the TS engine and the validator by
@@ -1846,13 +1862,14 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
1846
  # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap).
1847
  sql += f" LIMIT {limit + 1}"
1848
 
1849
- con = _store_con()
1850
  try:
1851
- cur = con.execute(sql, params)
1852
- cols = [d[0] for d in cur.description]
1853
- rows = [dict(zip(cols, r)) for r in cur.fetchall()]
1854
  finally:
1855
- con.close()
 
1856
  for row in rows: # ratio/derived post-compute per group
1857
  row.update(foreign_vals) # scalar cross-topic components
1858
  for k in requested:
@@ -2391,7 +2408,8 @@ def _one_binding_offer(topic, b, strict=False, ready=None):
2391
 
2392
 
2393
  def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None,
2394
- exclude_services=False, offer=None, related=None):
 
2395
  """`{group key: {measure key: value}}` for an entity topic's lookback columns.
2396
 
2397
  ONE grouped query for all `keys` together — they share a topic, a scope and a window, so
@@ -2421,6 +2439,10 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
2421
  `_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin
2422
  nobody measured, under a row the user has no reason to doubt.
2423
  """
 
 
 
 
2424
  offer = offer if offer is not None else entity_measures(topic)
2425
  by_key = {m["key"]: m for m in offer}
2426
  want = [k for k in keys if k in by_key]
@@ -2439,7 +2461,8 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
2439
  for (_src, _dim, _variant), ks in groups.items():
2440
  answer = entity_measure_values(
2441
  topic, ks, date_from=date_from, date_to=date_to, team_id=team_id,
2442
- exclude_services=exclude_services, offer=offer, related=related)
 
2443
  if related is not None:
2444
  merged.refusals.extend(getattr(answer, 'refusals', ()))
2445
  for gk, cell in answer.items():
@@ -2458,7 +2481,9 @@ def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=Non
2458
  base_of = {k: (by_key[k].get("base_key") or k) for k in want}
2459
  res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from,
2460
  date_to=date_to, team_id=team_id, exclude_services=exclude_services,
2461
- limit=MAX_GROUPS, scope_variant=scope_variant, related=related)
 
 
2462
  if res.get("refusals"):
2463
  from core import relations
2464
  return relations.MeasureAnswer(refusals=[dict(marker, measure=k)
@@ -2532,7 +2557,7 @@ def entity_measure_leaves(nodes, admitted):
2532
 
2533
 
2534
  def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None,
2535
- exclude_services=False):
2536
  """`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row.
2537
 
2538
  ⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty
@@ -2578,7 +2603,8 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe
2578
  else:
2579
  vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1],
2580
  team_id=team_id, exclude_services=exclude_services,
2581
- offer=offer)
 
2582
  cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c}
2583
  return cache[sig]
2584
 
@@ -2595,7 +2621,7 @@ def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offe
2595
  left = values_for(mkey, rule.get("window"))
2596
  if left is None:
2597
  continue
2598
- pool = list(keys_by_id or left)
2599
  zero = m["empty"] == "zero"
2600
  rhs = rule.get("rhs")
2601
  try:
 
256
  return out
257
 
258
 
259
+ def _load_model_yaml(text):
260
+ """Parse model documents with the safe C loader when available."""
261
+ loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
262
+ return yaml.load(text, Loader=loader)
263
+
264
+
265
  @functools.lru_cache(maxsize=1)
266
  def _model():
267
  topics, metrics = {}, {}
268
  for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
269
+ d = _load_model_yaml(f.read_text(encoding="utf-8"))
270
  if not d or "key" not in d:
271
  raise ModelError(f"topic file {f.name} lacks a 'key'")
272
  if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
 
274
  # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises
275
  # a clear error if a live resolution is attempted on it
276
  topics[d["key"]] = d
277
+ metric_docs = [(f.name, _load_model_yaml(f.read_text(encoding="utf-8")))
278
  for f in sorted((MODEL_DIR / "metrics").glob("*.yml"))]
279
  # Metric libraries may own new fact topics without changing legacy topic definitions.
280
  for name, doc in metric_docs:
 
1658
  def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
1659
  team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
1660
  filter_tree=None, filter_conj="and", today=None, scope_variant=None, related=None,
1661
+ *, runtime=None, user=None, cur=None, group_keys=None):
1662
  """The semantic query over the tenant store — the engine behind the Analyst's
1663
  run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
1664
  t = _model()["topics"].get(topic)
 
1816
  where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
1817
  params.extend(ids)
1818
 
1819
+ if group_keys is not None:
1820
+ if len(gb) != 1:
1821
+ raise ModelError('Selected entity keys require exactly one grouping dimension')
1822
+ values = list(group_keys)
1823
+ if values:
1824
+ where.append(f"{dims[gb[0]]['col']} IN ({','.join('?' for _ in values)})")
1825
+ params.extend(values)
1826
+ else:
1827
+ where.append('FALSE')
1828
+
1829
  # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
1830
  # this is the nested, 11-operator contract the table UI emits. Compiled by
1831
  # harness/filter_sql, which is held in lock-step with the TS engine and the validator by
 
1862
  # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap).
1863
  sql += f" LIMIT {limit + 1}"
1864
 
1865
+ con = cur if cur is not None else _store_con()
1866
  try:
1867
+ result = con.execute(sql, params)
1868
+ cols = [d[0] for d in result.description]
1869
+ rows = [dict(zip(cols, r)) for r in result.fetchall()]
1870
  finally:
1871
+ if cur is None:
1872
+ con.close()
1873
  for row in rows: # ratio/derived post-compute per group
1874
  row.update(foreign_vals) # scalar cross-topic components
1875
  for k in requested:
 
2408
 
2409
 
2410
  def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None,
2411
+ exclude_services=False, offer=None, related=None, *, group_keys=None,
2412
+ cur=None):
2413
  """`{group key: {measure key: value}}` for an entity topic's lookback columns.
2414
 
2415
  ONE grouped query for all `keys` together — they share a topic, a scope and a window, so
 
2439
  `_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin
2440
  nobody measured, under a row the user has no reason to doubt.
2441
  """
2442
+ if group_keys is not None:
2443
+ group_keys = tuple(group_keys)
2444
+ if not group_keys:
2445
+ return {}
2446
  offer = offer if offer is not None else entity_measures(topic)
2447
  by_key = {m["key"]: m for m in offer}
2448
  want = [k for k in keys if k in by_key]
 
2461
  for (_src, _dim, _variant), ks in groups.items():
2462
  answer = entity_measure_values(
2463
  topic, ks, date_from=date_from, date_to=date_to, team_id=team_id,
2464
+ exclude_services=exclude_services, offer=offer, related=related,
2465
+ group_keys=group_keys, cur=cur)
2466
  if related is not None:
2467
  merged.refusals.extend(getattr(answer, 'refusals', ()))
2468
  for gk, cell in answer.items():
 
2481
  base_of = {k: (by_key[k].get("base_key") or k) for k in want}
2482
  res = store_query(src, [base_of[k] for k in want], group_by=[dim], date_from=date_from,
2483
  date_to=date_to, team_id=team_id, exclude_services=exclude_services,
2484
+ limit=MAX_GROUPS, scope_variant=scope_variant, related=related,
2485
+ **({'group_keys': group_keys} if group_keys is not None else {}),
2486
+ **({'cur': cur} if cur is not None else {}))
2487
  if res.get("refusals"):
2488
  from core import relations
2489
  return relations.MeasureAnswer(refusals=[dict(marker, measure=k)
 
2557
 
2558
 
2559
  def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None,
2560
+ exclude_services=False, *, cur=None):
2561
  """`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row.
2562
 
2563
  ⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty
 
2603
  else:
2604
  vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1],
2605
  team_id=team_id, exclude_services=exclude_services,
2606
+ offer=offer, group_keys=keys_by_id,
2607
+ **({'cur': cur} if cur is not None else {}))
2608
  cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c}
2609
  return cache[sig]
2610
 
 
2621
  left = values_for(mkey, rule.get("window"))
2622
  if left is None:
2623
  continue
2624
+ pool = list(left if keys_by_id is None else keys_by_id)
2625
  zero = m["empty"] == "zero"
2626
  rhs = rule.get("rhs")
2627
  try:
platform/modules/ar.py CHANGED
@@ -10,7 +10,7 @@ import math
10
  import datetime as dt
11
  from pathlib import Path
12
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
13
- import core.odoo as O
14
  import core.periods as P
15
  import core.store as store
16
 
@@ -32,7 +32,7 @@ DTP_KEY = 'ar_days_to_pay'
32
  DTP_MAX_AGE_HOURS = 20
33
 
34
 
35
- def days_to_pay_snapshot():
36
  """`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises.
37
 
38
  Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the
@@ -40,8 +40,11 @@ def days_to_pay_snapshot():
40
  derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about.
41
  """
42
  try:
43
- snap = store.get(DTP_KEY) or {}
44
- days = snap.get('days') or {}
 
 
 
45
  out = {}
46
  for k, v in days.items():
47
  try:
@@ -123,15 +126,18 @@ def _open_docs(t=None):
123
  ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
124
  'amount_total', 'amount_residual_signed'])
125
  today = t or P.today()
126
- for d in docs:
127
- due = d.get('invoice_date_due')
128
- try:
129
- d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0
130
- except Exception:
131
- d['days_overdue'] = 0
132
- d['bucket'] = _bucket(d['days_overdue'])
133
- d['open'] = d.get('amount_residual_signed') or 0.0
134
- return docs
 
 
 
135
 
136
 
137
  def _ltm_invoiced(t=None):
 
10
  import datetime as dt
11
  from pathlib import Path
12
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
13
+ from harness.canonical_source import odoo as O
14
  import core.periods as P
15
  import core.store as store
16
 
 
32
  DTP_MAX_AGE_HOURS = 20
33
 
34
 
35
+ def days_to_pay_snapshot(*, pids=None, st=None):
36
  """`{pid: avg_days}` from the last snapshot — CHEAP, and it never raises.
37
 
38
  Returns `{}` when the snapshot is absent, unreadable or malformed. A blank column is the
 
40
  derived statistic is missing, and `days_to_pay` is a statistic, not a fact the page is about.
41
  """
42
  try:
43
+ if pids is not None and st is not None:
44
+ days = st.selected_values(DTP_KEY, ['days'], pids)
45
+ else:
46
+ snap = store.get(DTP_KEY) or {}
47
+ days = snap.get('days') or {}
48
  out = {}
49
  for k, v in days.items():
50
  try:
 
126
  ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
127
  'amount_total', 'amount_residual_signed'])
128
  today = t or P.today()
129
+ def enriched():
130
+ for d in docs:
131
+ due = d.get('invoice_date_due')
132
+ try:
133
+ d['days_overdue'] = max((today - dt.date.fromisoformat(due)).days, 0) if due else 0
134
+ except Exception:
135
+ d['days_overdue'] = 0
136
+ d['bucket'] = _bucket(d['days_overdue'])
137
+ d['open'] = d.get('amount_residual_signed') or 0.0
138
+ yield d
139
+ from harness.canonical_source import active
140
+ return enriched() if active() is not None else list(enriched())
141
 
142
 
143
  def _ltm_invoiced(t=None):
platform/modules/backorders.py CHANGED
@@ -11,7 +11,7 @@ to dealers. BU-filterable.
11
  """
12
  import datetime as dt
13
 
14
- import core.odoo as O
15
  import core.periods as P
16
  import modules.procurement as proc
17
 
@@ -72,6 +72,9 @@ def sku_backlog(t=None, team_id=None):
72
  groups = O.read_group('sale.order.line', domain,
73
  ['product_uom_qty:sum', 'qty_delivered:sum'], ['product_id'],
74
  lazy=False)
 
 
 
75
  out = {}
76
  for group in groups:
77
  pid = O.m2o_id(group.get('product_id'))
@@ -85,7 +88,8 @@ def sku_backlog(t=None, team_id=None):
85
 
86
  open_by_pid = _by_product(list(common))
87
  pre_by_pid = _by_product(list(common) + [('order_id.commitment_date', '>', end_of_day)])
88
- pids = sorted(set(open_by_pid) | set(pre_by_pid))
 
89
  codes = {}
90
  for ch in _chunk(pids, 500):
91
  for product in O.search_read('product.product',
@@ -101,7 +105,8 @@ def sku_backlog(t=None, team_id=None):
101
  out[code] = out.get(code, 0.0) + qty
102
  return out
103
 
104
- pre_by_code, open_by_code = _by_code(pre_by_pid), _by_code(open_by_pid)
 
105
  report = {
106
  'as_of': t.isoformat(),
107
  'team_ids': teams,
 
11
  """
12
  import datetime as dt
13
 
14
+ from harness.canonical_source import odoo as O
15
  import core.periods as P
16
  import modules.procurement as proc
17
 
 
72
  groups = O.read_group('sale.order.line', domain,
73
  ['product_uom_qty:sum', 'qty_delivered:sum'], ['product_id'],
74
  lazy=False)
75
+ from harness.canonical_source import active
76
+ if active() is not None:
77
+ return active().backlog(groups)
78
  out = {}
79
  for group in groups:
80
  pid = O.m2o_id(group.get('product_id'))
 
88
 
89
  open_by_pid = _by_product(list(common))
90
  pre_by_pid = _by_product(list(common) + [('order_id.commitment_date', '>', end_of_day)])
91
+ from harness.canonical_source import active
92
+ pids = [] if active() is not None else sorted(set(open_by_pid) | set(pre_by_pid))
93
  codes = {}
94
  for ch in _chunk(pids, 500):
95
  for product in O.search_read('product.product',
 
105
  out[code] = out.get(code, 0.0) + qty
106
  return out
107
 
108
+ pre_by_code, open_by_code = ((pre_by_pid, open_by_pid) if active() is not None else
109
+ (_by_code(pre_by_pid), _by_code(open_by_pid)))
110
  report = {
111
  'as_of': t.isoformat(),
112
  'team_ids': teams,
platform/modules/customer_data.py CHANGED
@@ -16,7 +16,7 @@ no link (owner/CFO/admin) see the whole book.
16
  import re
17
  from urllib.parse import quote_plus
18
 
19
- import core.odoo as O
20
  import core.periods as P
21
  import core.store as store
22
  import core.table_store as table_store
@@ -117,7 +117,7 @@ AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60',
117
  '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
118
 
119
 
120
- def _ar_attrs(t=None):
121
  """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
122
 
123
  Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
@@ -160,7 +160,9 @@ def _ar_attrs(t=None):
160
  }
161
  except Exception:
162
  pass
163
- for pid, avg in ar.days_to_pay_snapshot().items():
 
 
164
  # A customer with an AR row but no settled invoice in the window keeps `None` — blank,
165
  # not 0. "Pays in 0 days" is a claim, and a false one.
166
  out.setdefault(pid, {})['days_to_pay'] = avg
@@ -197,6 +199,10 @@ def _mix_attrs(pids=None, t=None):
197
  keep = set(pids) if pids else None
198
  rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
199
  ['order_partner_id', 'product_id'], lazy=False)
 
 
 
 
200
  prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
201
  prod_ids.discard(None)
202
  cats = {}
@@ -502,6 +508,28 @@ def _dba_attrs(pids, t):
502
  return {}
503
 
504
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
505
  def _book_pids(team_id=None, agent_pids=None):
506
  """⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
507
 
@@ -548,36 +576,7 @@ def _book_pids(team_id=None, agent_pids=None):
548
  the whole permissioning model rests on.
549
  """
550
  try:
551
- if team_id:
552
- # Scoped: partners with confirmed orders in THIS BU, ever. `read_group` on partner_id
553
- # rather than a search_read of orders — the group is the distinct set, and it is one
554
- # round trip instead of paging tens of thousands of order rows.
555
- rows = O.read_group('sale.order',
556
- [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
557
- ['partner_id'], ['partner_id'], lazy=False)
558
- book = {O.m2o_id(r.get('partner_id')) for r in rows}
559
- book.discard(None)
560
- else:
561
- rank = O.search_read('res.partner',
562
- [('customer_rank', '>', 0),
563
- ('active', 'in', [True, False])],
564
- ['id'], limit=200000)
565
- # `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or
566
- # have had its rank reset, and an account that demonstrably bought from us belongs in
567
- # the book whatever its flags say now. An ORDER is a fact; a rank is a setting.
568
- rows = O.read_group('sale.order',
569
- [('state', 'in', ('sale', 'done'))],
570
- ['partner_id'], ['partner_id'], lazy=False)
571
- ever = {O.m2o_id(r.get('partner_id')) for r in rows}
572
- ever.discard(None)
573
- docs = O.read_group(
574
- 'account.move',
575
- [('state', '=', 'posted'),
576
- ('move_type', 'in', ('out_invoice', 'out_refund'))],
577
- ['partner_id'], ['partner_id'], lazy=False)
578
- documented = {O.m2o_id(r.get('partner_id')) for r in docs}
579
- documented.discard(None)
580
- book = {r['id'] for r in rank} | ever | documented
581
  # An agent filter NARROWS the book to that agent's partners — never widens it. The
582
  # intersection is what keeps an agent-login user inside their own book while still
583
  # gaining every no-order account assigned to them, which is the whole point of item 27.
@@ -630,8 +629,85 @@ def identity_pids(agent_name=None, team_id=None, t=None):
630
  return set(_pool_identity(agent_name, team_id, t)["pids"])
631
 
632
 
633
- def _pool_build(agent_name, team_id, limit, fast=False):
634
- identity = _pool_identity(agent_name, team_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
635
  t = identity["today"]
636
  pids = identity["agentPids"]
637
  this, last, ltm = identity["this"], identity["last"], identity["ltm"]
@@ -644,8 +720,9 @@ def _pool_build(agent_name, team_id, limit, fast=False):
644
  key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
645
  + (last.get(p) or {}).get('rev', 0.0)))
646
  all_pids = set(ranked[:limit])
647
- attrs = cust._partner_attrs(list(all_pids))
648
- invoice_days = {} if fast else cust._invoice_cadence_bulk(t, partner_ids=all_pids)
 
649
  if fast:
650
  # C1 fast phase: the slow families stay empty — their columns render blank behind the
651
  # toolbar's partial indicator until the full build swaps in.
@@ -654,21 +731,25 @@ def _pool_build(agent_name, team_id, limit, fast=False):
654
  # The three families the owner asked for (2026-07-27). Each degrades to {} rather than
655
  # raising: a customer table that cannot render because AR is momentarily unreachable is
656
  # a worse failure than one with a blank column.
657
- ar_a = _ar_attrs(t)
658
- mix_a = _mix_attrs(all_pids, t)
659
- sp_a = _salesperson_attrs(all_pids, t)
 
 
 
660
  # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
661
  # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
662
  # client's map view discloses the count.
663
- geo_a = map_mod.coords_for(list(all_pids))
 
664
  # C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
665
- dba_a = _dba_attrs(all_pids, t)
666
  # customers whose only activity sits in the older half of the 24-month cadence window
667
  # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
668
  # the win-back targets this union exists for, would render as '?')
669
  unnamed = [pid for pid in all_pids
670
  if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
671
- or (ltm.get(pid) or {}).get('name'))]
672
  extra_names = ({r['id']: r['name'] for r in
673
  O.search_read('res.partner', [('id', 'in', unnamed),
674
  ('active', 'in', [True, False])], ['name'])}
@@ -763,7 +844,7 @@ def _pool_build(agent_name, team_id, limit, fast=False):
763
  # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
764
  # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
765
  # so letting it write would mark ~1,450 living customers archived on every cold load.
766
- if not fast:
767
  rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
768
  rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
769
  return {'rows': rows, 'total': total}
@@ -772,7 +853,7 @@ def _pool_build(agent_name, team_id, limit, fast=False):
772
  LEDGER_KEY = 'customer_ledger'
773
 
774
 
775
- def _reconcile_ledger(rows, total, agent_name, team_id):
776
  """Remember every customer we have ever seen, and keep serving the ones Odoo stopped
777
  returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
778
  so stored in our platform even if deleted in Odoo").
@@ -791,14 +872,14 @@ def _reconcile_ledger(rows, total, agent_name, team_id):
791
  they cannot prove they should see.
792
  """
793
  try:
794
- ledger = dict(store.get(LEDGER_KEY) or {})
795
  except Exception:
796
  return rows, total # a ledger we cannot read must not break the table
797
  today = str(P.today())
798
  live = {r['pid'] for r in rows}
799
  unscoped = agent_name is None and team_id is None
800
 
801
- if unscoped:
802
  for r in rows:
803
  e = ledger.get(str(r['pid'])) or {}
804
  e.update({'name': r['customer'], 'last_seen': today})
@@ -849,7 +930,7 @@ def _reconcile_ledger(rows, total, agent_name, team_id):
849
  'days_since': None, 'days_since_invoice': None,
850
  'typical_gap_days': None, 'overdue_days': None,
851
  })
852
- if store.available():
853
  try:
854
  store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
855
  except Exception:
 
16
  import re
17
  from urllib.parse import quote_plus
18
 
19
+ from harness.canonical_source import odoo as O
20
  import core.periods as P
21
  import core.store as store
22
  import core.table_store as table_store
 
117
  '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
118
 
119
 
120
+ def _ar_attrs(t=None, *, pids=None, st=None):
121
  """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
122
 
123
  Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
 
160
  }
161
  except Exception:
162
  pass
163
+ snapshot = (ar.days_to_pay_snapshot() if pids is None and st is None else
164
+ ar.days_to_pay_snapshot(pids=pids, st=st))
165
+ for pid, avg in snapshot.items():
166
  # A customer with an AR row but no settled invoice in the window keeps `None` — blank,
167
  # not 0. "Pays in 0 days" is a claim, and a false one.
168
  out.setdefault(pid, {})['days_to_pay'] = avg
 
199
  keep = set(pids) if pids else None
200
  rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
201
  ['order_partner_id', 'product_id'], lazy=False)
202
+ from harness.canonical_source import active
203
+ source = active()
204
+ if source is not None:
205
+ return source.customer_mix(rows)
206
  prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
207
  prod_ids.discard(None)
208
  cats = {}
 
508
  return {}
509
 
510
 
511
+ def _book_records(team_id=None):
512
+ """Yield the canonical rank, confirmed-order and posted-document identity union."""
513
+ if team_id:
514
+ groups = O.read_group('sale.order',
515
+ [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
516
+ ['partner_id'], ['partner_id'], lazy=False)
517
+ for row in groups:
518
+ pid = O.m2o_id(row.get('partner_id'))
519
+ if pid is not None:
520
+ yield {'id': pid}
521
+ return
522
+ yield from O.search_read('res.partner', [('customer_rank', '>', 0),
523
+ ('active', 'in', [True, False])], ['id'], limit=200000)
524
+ for model, domain in [
525
+ ('sale.order', [('state', 'in', ('sale', 'done'))]),
526
+ ('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))])]:
527
+ for row in O.read_group(model, domain, ['partner_id'], ['partner_id'], lazy=False):
528
+ pid = O.m2o_id(row.get('partner_id'))
529
+ if pid is not None:
530
+ yield {'id': pid}
531
+
532
+
533
  def _book_pids(team_id=None, agent_pids=None):
534
  """⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
535
 
 
576
  the whole permissioning model rests on.
577
  """
578
  try:
579
+ book = {r['id'] for r in _book_records(team_id)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
580
  # An agent filter NARROWS the book to that agent's partners — never widens it. The
581
  # intersection is what keeps an agent-login user inside their own book while still
582
  # gaining every no-order account assigned to them, which is the whole point of item 27.
 
629
  return set(_pool_identity(agent_name, team_id, t)["pids"])
630
 
631
 
632
+ def source_batches(cur, *, st, agent_name=None, team_id=None, ids=None, needed=None,
633
+ ledger_batches=()):
634
+ """Stream the canonical customer identity union and project only one batch at a time."""
635
+ from harness.canonical_source import bounded, BATCH
636
+ with bounded(cur, 'identity' if ids is None else 'customer', ids or [], st) as identity:
637
+ live = identity.snapshot(_book_records(team_id))
638
+ where = ''
639
+ if agent_name and agent_name not in {'All', 'All agents'}:
640
+ members = identity.agent_members(agent_name)
641
+ where = (f"WHERE json_extract_string(body, '/id') IN "
642
+ f"(SELECT json_extract_string(body, '/id') FROM \"{members.table}\")")
643
+ population = identity.snapshot({'id': r[0], 'live': True} for r in ())
644
+ cur.execute(f'INSERT INTO "{population.table}" SELECT row_number() OVER (), '
645
+ f"json_object('id', try_cast(json_extract_string(body, '/id') AS BIGINT), 'live', true) "
646
+ f'FROM "{live.table}" {where} GROUP BY json_extract_string(body, \'/id\')')
647
+ if not agent_name and team_id is None:
648
+ for batch in ledger_batches:
649
+ records = []
650
+ for key, entry in batch:
651
+ try:
652
+ pid = int(key)
653
+ except (TypeError, ValueError):
654
+ continue
655
+ if ids is None or pid in ids:
656
+ records.append({'id': pid, 'live': False, 'entry': entry})
657
+ if records:
658
+ cur.execute(f'INSERT INTO "{population.table}" SELECT 0, j FROM unnest(?::JSON[]) r(j) '
659
+ f"WHERE try_cast(json_extract_string(j, '/id') AS BIGINT) NOT IN "
660
+ f"(SELECT try_cast(json_extract_string(body, '/id') AS BIGINT) FROM \"{live.table}\")",
661
+ [[__import__('json').dumps(r) for r in records]])
662
+ records.clear()
663
+ # Sort the complete live/retained union once; pages seek a physical key, not JSON.
664
+ cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
665
+ f"SELECT ordinal, body, try_cast(json_extract_string(body, '/id') AS BIGINT) pid "
666
+ f'FROM "{population.table}" ORDER BY pid')
667
+ last = None
668
+ while True:
669
+ rows = cur.execute(f"SELECT pid, "
670
+ f"json_extract_string(body, '/live') = 'true', json_extract(body, '/entry') "
671
+ f'FROM "{population.table}" WHERE (?::BIGINT IS NULL OR pid > ?) ORDER BY pid LIMIT ?',
672
+ [last, last, BATCH]).fetchall()
673
+ if not rows:
674
+ return
675
+ live_ids = [pid for pid, present, _ in rows if present]
676
+ ledger = {str(pid): __import__('json').loads(entry) for pid, present, entry in rows if not present}
677
+ with bounded(cur, 'customer', live_ids, st):
678
+ if needed is not None and not needed:
679
+ projected = [dict(cust.audit_attr_defaults(), pid=pid) for pid in live_ids]
680
+ else:
681
+ projected = _pool_build(agent_name, team_id, None, selected=live_ids,
682
+ needed=needed, st=st)['rows'] if live_ids else []
683
+ projected, _ = _reconcile_ledger(projected, len(projected), agent_name, team_id,
684
+ ledger=ledger, write=False)
685
+ yield projected
686
+ last = rows[-1][0]
687
+ projected.clear()
688
+ rows.clear()
689
+
690
+
691
+ def _pool_build(agent_name, team_id, limit, fast=False, *, selected=None, needed=None, st=None):
692
+ from harness.canonical_source import active
693
+ source = active()
694
+ if selected is None:
695
+ identity = _pool_identity(agent_name, team_id)
696
+ else:
697
+ t = P.today()
698
+ pids = set(selected)
699
+ wants = lambda keys: needed is None or bool(set(keys) & set(needed))
700
+ yf, yt = P.ytd(t)
701
+ lf, lt = P.ytd_last_year(t)
702
+ mf, mt = P.ltm(t)
703
+ identity = {'today': t, 'agentPids': pids, 'pids': pids,
704
+ 'this': cust._cust_rev(yf, yt, team_id, pids) if wants(
705
+ ['customer', 'status', 'revenue_ytd', 'yoy_pct', 'at_risk']) else {},
706
+ 'last': cust._cust_rev(lf, lt, team_id, pids) if wants(
707
+ ['customer', 'status', 'revenue_ly', 'yoy_pct', 'at_risk']) else {},
708
+ 'ltm': cust._cust_rev(mf, mt, team_id, pids) if wants(['customer', 'ltm_rev']) else {},
709
+ 'cadence': cust._cadence_bulk(t, team_id, agent_pids=pids) if wants([
710
+ 'orders_24m', 'aov', 'last_order', 'days_since', 'typical_gap_days', 'overdue_days']) else {}}
711
  t = identity["today"]
712
  pids = identity["agentPids"]
713
  this, last, ltm = identity["this"], identity["last"], identity["ltm"]
 
720
  key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
721
  + (last.get(p) or {}).get('rev', 0.0)))
722
  all_pids = set(ranked[:limit])
723
+ attrs = cust._partner_attrs(list(all_pids), **({'needed': needed} if selected is not None else {}))
724
+ invoice_days = {} if fast or (needed is not None and 'days_since_invoice' not in needed) \
725
+ else cust._invoice_cadence_bulk(t, partner_ids=all_pids)
726
  if fast:
727
  # C1 fast phase: the slow families stay empty — their columns render blank behind the
728
  # toolbar's partial indicator until the full build swaps in.
 
731
  # The three families the owner asked for (2026-07-27). Each degrades to {} rather than
732
  # raising: a customer table that cannot render because AR is momentarily unreachable is
733
  # a worse failure than one with a blank column.
734
+ wants = lambda keys: needed is None or bool(set(keys) & set(needed))
735
+ ar_a = (_ar_attrs(t, pids=all_pids, st=st) if source is not None else _ar_attrs(t)) \
736
+ if wants(['ar_open', 'ar_overdue', 'ar_outstanding', 'ar_exposure', 'days_to_pay',
737
+ *AR_BUCKET_FIELDS.values()]) else {}
738
+ mix_a = _mix_attrs(all_pids, t) if wants(['top_category', 'top_category_pct', 'sku_count', 'top_sku']) else {}
739
+ sp_a = _salesperson_attrs(all_pids, t) if wants(['salesperson']) else {}
740
  # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
741
  # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
742
  # client's map view discloses the count.
743
+ geo_a = map_mod.coords_for(list(all_pids), **({'st': st} if source is not None else {})) \
744
+ if wants(['lat', 'lon']) else {}
745
  # C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
746
+ dba_a = _dba_attrs(all_pids, t) if wants(['dba']) else {}
747
  # customers whose only activity sits in the older half of the 24-month cadence window
748
  # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
749
  # the win-back targets this union exists for, would render as '?')
750
  unnamed = [pid for pid in all_pids
751
  if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
752
+ or (ltm.get(pid) or {}).get('name'))] if needed is None or 'customer' in needed else []
753
  extra_names = ({r['id']: r['name'] for r in
754
  O.search_read('res.partner', [('id', 'in', unnamed),
755
  ('active', 'in', [True, False])], ['name'])}
 
844
  # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
845
  # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
846
  # so letting it write would mark ~1,450 living customers archived on every cold load.
847
+ if not fast and selected is None:
848
  rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
849
  rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
850
  return {'rows': rows, 'total': total}
 
853
  LEDGER_KEY = 'customer_ledger'
854
 
855
 
856
+ def _reconcile_ledger(rows, total, agent_name, team_id, *, ledger=None, write=True):
857
  """Remember every customer we have ever seen, and keep serving the ones Odoo stopped
858
  returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
859
  so stored in our platform even if deleted in Odoo").
 
872
  they cannot prove they should see.
873
  """
874
  try:
875
+ ledger = dict(store.get(LEDGER_KEY) or {}) if ledger is None else dict(ledger)
876
  except Exception:
877
  return rows, total # a ledger we cannot read must not break the table
878
  today = str(P.today())
879
  live = {r['pid'] for r in rows}
880
  unscoped = agent_name is None and team_id is None
881
 
882
+ if unscoped and write:
883
  for r in rows:
884
  e = ledger.get(str(r['pid'])) or {}
885
  e.update({'name': r['customer'], 'last_seen': today})
 
930
  'days_since': None, 'days_since_invoice': None,
931
  'typical_gap_days': None, 'overdue_days': None,
932
  })
933
+ if write and store.available():
934
  try:
935
  store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
936
  except Exception:
platform/modules/customers.py CHANGED
@@ -14,7 +14,7 @@ import datetime as dt
14
  import statistics
15
  from pathlib import Path
16
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
17
- import core.odoo as O
18
  import core.periods as P
19
  import modules.sales as sales_mod
20
 
@@ -49,7 +49,10 @@ def _cust_group_store(date_from, date_to, team_id, agent_pids, select):
49
  params += list(agent_pids)
50
  if O.doc_mode() == 'invoice':
51
  w.append("invoice_status = 'invoiced'")
52
- return DS.ro_con().execute(
 
 
 
53
  f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} "
54
  "FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id "
55
  "WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall()
@@ -534,10 +537,13 @@ def _invoice_cadence_bulk(t=None, partner_ids=None):
534
  if USE_STORE:
535
  try:
536
  from harness import semantic as sem
 
 
537
  result = sem.store_query(
538
  'customer_invoices', ['customer_days_since_invoice'], group_by=['partner'],
539
  filters={'partner': partner_ids} if partner_ids is not None else None,
540
- limit=sem.MAX_GROUPS, today=t.isoformat())
 
541
  if result.get('truncated'):
542
  raise sem.ModelError('The invoice cadence group set was truncated.')
543
  return {int(row['partner_id']): row['customer_days_since_invoice']
@@ -650,7 +656,7 @@ def _audit_attrs(record):
650
  return out
651
 
652
 
653
- def _partner_attrs(pids):
654
  """{pid: {street, street2, city, state, country, agent, zip, payment_terms, customer_since,
655
  tags, pricelist}} — the customer attributes the sum-level rollups slice by and the Customer
656
  table displays.
@@ -667,15 +673,26 @@ def _partner_attrs(pids):
667
  pids = list(pids)
668
  if not pids:
669
  return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
670
  # Odoo's active_test context otherwise hides archived partners even when their ids are named.
671
  # The explicit two-state domain makes `archived` an ordinary source attribute rather than a
672
  # conclusion inferred from a missing row.
673
  rows = O.search_read('res.partner', [('id', 'in', pids),
674
  ('active', 'in', [True, False])],
675
- ['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids',
676
- 'zip', 'property_payment_term_id', 'create_date', 'category_id',
677
- 'property_product_pricelist', 'active']
678
- + [field for field, _kind in AUDIT_ATTRS.values()])
679
  aids = {a for r in rows for a in (r.get('agent_ids') or [])}
680
  anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])}
681
  if aids else {})
 
14
  import statistics
15
  from pathlib import Path
16
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
17
+ from harness.canonical_source import odoo as O
18
  import core.periods as P
19
  import modules.sales as sales_mod
20
 
 
49
  params += list(agent_pids)
50
  if O.doc_mode() == 'invoice':
51
  w.append("invoice_status = 'invoiced'")
52
+ from harness.canonical_source import active
53
+ source = active()
54
+ con = source.cur if source is not None else DS.ro_con()
55
+ return con.execute(
56
  f"SELECT o.partner_id, coalesce(p.name, '#' || o.partner_id), {select} "
57
  "FROM sale_order o LEFT JOIN res_partner p ON p.id = o.partner_id "
58
  "WHERE " + " AND ".join(w) + " GROUP BY 1, 2", params).fetchall()
 
537
  if USE_STORE:
538
  try:
539
  from harness import semantic as sem
540
+ from harness.canonical_source import active
541
+ source = active()
542
  result = sem.store_query(
543
  'customer_invoices', ['customer_days_since_invoice'], group_by=['partner'],
544
  filters={'partner': partner_ids} if partner_ids is not None else None,
545
+ limit=sem.MAX_GROUPS, today=t.isoformat(),
546
+ **({'cur': source.cur} if source is not None else {}))
547
  if result.get('truncated'):
548
  raise sem.ModelError('The invoice cadence group set was truncated.')
549
  return {int(row['partner_id']): row['customer_days_since_invoice']
 
656
  return out
657
 
658
 
659
+ def _partner_attrs(pids, *, needed=None):
660
  """{pid: {street, street2, city, state, country, agent, zip, payment_terms, customer_since,
661
  tags, pricelist}} — the customer attributes the sum-level rollups slice by and the Customer
662
  table displays.
 
673
  pids = list(pids)
674
  if not pids:
675
  return {}
676
+ columns = ['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids',
677
+ 'zip', 'property_payment_term_id', 'create_date', 'category_id',
678
+ 'property_product_pricelist', 'active'] + [field for field, _kind in AUDIT_ATTRS.values()]
679
+ if needed is not None:
680
+ operands = {'state': ['state_id'], 'country': ['country_id'], 'agent': ['agent_ids'],
681
+ 'payment_terms': ['property_payment_term_id'], 'customer_since': ['create_date'],
682
+ '_created': ['create_date'], 'created_on': ['create_date'], 'tags': ['category_id'],
683
+ 'pricelist': ['property_product_pricelist'], 'archived': ['active'],
684
+ 'odoo_status': ['active'], 'maps_url': ['street', 'street2', 'city', 'state_id', 'country_id', 'zip']}
685
+ operands.update({key: [field] for key, (field, _kind) in AUDIT_ATTRS.items()})
686
+ wanted = {field for key in needed for field in operands.get(key, [key])}
687
+ columns = [field for field in columns if field in wanted]
688
+ if not columns:
689
+ return {}
690
  # Odoo's active_test context otherwise hides archived partners even when their ids are named.
691
  # The explicit two-state domain makes `archived` an ordinary source attribute rather than a
692
  # conclusion inferred from a missing row.
693
  rows = O.search_read('res.partner', [('id', 'in', pids),
694
  ('active', 'in', [True, False])],
695
+ columns)
 
 
 
696
  aids = {a for r in rows for a in (r.get('agent_ids') or [])}
697
  anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])}
698
  if aids else {})
platform/modules/inventory.py CHANGED
@@ -9,7 +9,7 @@ import sys
9
  from pathlib import Path
10
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
11
  import pandas as pd
12
- import core.odoo as O
13
  import core.periods as P
14
 
15
 
@@ -273,13 +273,17 @@ def sku_inventory(target_days=180, carry_pct=_CARRY_PCT, t=None):
273
  brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand,
274
  inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value,
275
  carry_cost, turns, target_units, target_days}}."""
276
- df, _ = _build(t)
277
- df = _enrich(df, target_days)
278
- g = df.groupby('sku').agg(
 
 
 
 
279
  name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'),
280
  on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'),
281
  cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'),
282
- ).reset_index()
283
  out = {}
284
  for r in g.itertuples(index=False):
285
  daily = r.qty_ltm / 365.0
 
9
  from pathlib import Path
10
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
11
  import pandas as pd
12
+ from harness.canonical_source import odoo as O
13
  import core.periods as P
14
 
15
 
 
273
  brand-tagged, so this is brand-independent. Returns {code: {name, category, storable, on_hand,
274
  inv_value, unit_cost, qty_ltm, dos (None=never sells through), bucket, excess_units, excess_value,
275
  carry_cost, turns, target_units, target_days}}."""
276
+ from harness.canonical_source import active
277
+ if active() is not None:
278
+ g, _ = active().inventory_frame(t or P.today())
279
+ else:
280
+ df, _ = _build(t)
281
+ df = _enrich(df, target_days)
282
+ g = df.groupby('sku').agg(
283
  name=('name', 'first'), category=('category', 'first'), storable=('storable', 'max'),
284
  on_hand=('on_hand', 'sum'), inv_value=('inv_value', 'sum'),
285
  cogs_ltm=('cogs_ltm', 'sum'), qty_ltm=('qty_ltm', 'sum'), rev_ltm=('rev_ltm', 'sum'),
286
+ ).reset_index()
287
  out = {}
288
  for r in g.itertuples(index=False):
289
  daily = r.qty_ltm / 365.0
platform/modules/map.py CHANGED
@@ -18,7 +18,7 @@ import csv
18
  import io
19
  import hashlib
20
 
21
- import core.odoo as O
22
  import core.periods as P
23
  import core.store as store
24
  import modules.sales as sales_mod
@@ -174,7 +174,7 @@ def customer_locations(t=None, team_id=None, agent_pids=None):
174
  bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])}
175
 
176
 
177
- def coords_for(pids):
178
  """{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode
179
  CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map
180
  VIEW reads these off the pool rows; the standalone Map page retired the same day, and
@@ -182,7 +182,10 @@ def coords_for(pids):
182
  if not pids:
183
  return {}
184
  try:
185
- cache = store.get(_GEO_NS) if store.available() else {}
 
 
 
186
  parts = O.search_read(
187
  'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])],
188
  ['street', 'street2', 'city', 'state_id', 'zip',
 
18
  import io
19
  import hashlib
20
 
21
+ from harness.canonical_source import odoo as O
22
  import core.periods as P
23
  import core.store as store
24
  import modules.sales as sales_mod
 
174
  bad=[r for r in located if not _in_bounds(r['lat'], r['lon'])])}
175
 
176
 
177
+ def coords_for(pids, *, st=None):
178
  """{pid: {'lat','lon'}} for the given partners — Odoo's own coordinates, else the geocode
179
  CACHE (read-only reuse; NEVER triggers geocoding). Wave-7 W11: the Customer table's Map
180
  VIEW reads these off the pool rows; the standalone Map page retired the same day, and
 
182
  if not pids:
183
  return {}
184
  try:
185
+ if st is None:
186
+ cache = store.get(_GEO_NS) if store.available() else {}
187
+ else:
188
+ cache = st.selected_values(_GEO_NS, [], pids)
189
  parts = O.search_read(
190
  'res.partner', [('id', 'in', list(pids)), ('active', 'in', [True, False])],
191
  ['street', 'street2', 'city', 'state_id', 'zip',
platform/modules/product_data.py CHANGED
@@ -622,6 +622,45 @@ def identity_rows():
622
  return rows
623
 
624
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
625
  def _fba_by_code(cat):
626
  """`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows.
627
 
@@ -667,7 +706,7 @@ def _fba_by_code(cat):
667
  return out, report
668
 
669
 
670
- def pool(team_id=None, t=None, raw_fields=()):
671
  """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
672
 
673
  CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
@@ -683,10 +722,14 @@ def pool(team_id=None, t=None, raw_fields=()):
683
  consolidated = team_id is None
684
  # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
685
  # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
686
- inventory_snapshot = _inventory_by_code(t)
 
 
 
 
687
  inventory_source_available = isinstance(inventory_snapshot, dict)
688
  inv = inventory_snapshot if inventory_source_available else {}
689
- bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
690
  sup = supplier_master()
691
  # ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build:
692
  # tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the
@@ -697,15 +740,19 @@ def pool(team_id=None, t=None, raw_fields=()):
697
  _prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both
698
  except Exception: # noqa: BLE001
699
  _prods = None
700
- tiers, _tier_report = products.tier_prices_by_code(_prods)
701
- pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id)
 
 
 
702
  cat = _catalogue_by_code()
703
  # ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
704
  # same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
705
  # will sell. Degrades to `{}` on a read failure, matching every other column source here — a
706
  # blank cover gap is honest, a product grid that will not render is not.
707
  try:
708
- fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id)
 
709
  except Exception: # noqa: BLE001
710
  fwd_demand, _fwd_report = {}, {"error": "forward demand unavailable"}
711
  # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
@@ -716,8 +763,9 @@ def pool(team_id=None, t=None, raw_fields=()):
716
  # because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds
717
  # units, not brands, and pro-rating them onto a BU would print a number no Odoo query could
718
  # reproduce. Blank, never zero, when the tenant has no FBA warehouse.
719
- fba_by_code, _fba_report = _fba_by_code(cat)
720
- rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
 
721
  raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p
722
  for p in (_prods or []) if p.get("default_code") or p.get("id")}
723
 
 
622
  return rows
623
 
624
 
625
+ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
626
+ """Stream catalogue identities, refuse global hash collisions and project bounded code batches."""
627
+ from harness.canonical_source import bounded, BATCH
628
+ with bounded(cur, 'identity', [], st) as identity:
629
+ def codes():
630
+ for product in identity.search_read('product.product', [('active', '=', True)], ['id', 'default_code']):
631
+ code = str(product['default_code']).strip() if product.get('default_code') else f"pid:{product['id']}"
632
+ yield {'code': code, 'pid': sku_pid(code)}
633
+ raw = identity.snapshot(codes())
634
+ collision = cur.execute(f"SELECT json_extract_string(body, '/pid') pid "
635
+ f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
636
+ if collision:
637
+ raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
638
+ selected = '' if ids is None else " WHERE try_cast(json_extract_string(body, '/pid') AS BIGINT) IN (SELECT unnest(?::BIGINT[]))"
639
+ params = [] if ids is None else [list(ids)]
640
+ population = identity.snapshot(())
641
+ cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
642
+ f'{selected} GROUP BY body', params)
643
+ # Keep the tracked snapshot name while materialising its sorted, typed seek key.
644
+ cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
645
+ f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
646
+ f'FROM "{population.table}" ORDER BY pid')
647
+ last = -1
648
+ while True:
649
+ codes = cur.execute(f"SELECT pid, "
650
+ f"json_extract_string(body, '/code') FROM \"{population.table}\" "
651
+ 'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
652
+ if not codes:
653
+ return
654
+ with bounded(cur, 'product', [code for _, code in codes], st):
655
+ rows = ([{'pid': pid, 'code': code} for pid, code in codes]
656
+ if needed is not None and not needed else
657
+ pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
658
+ yield rows
659
+ last = codes[-1][0]
660
+ rows.clear()
661
+ codes.clear()
662
+
663
+
664
  def _fba_by_code(cat):
665
  """`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows.
666
 
 
706
  return out, report
707
 
708
 
709
+ def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
710
  """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
711
 
712
  CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
 
722
  consolidated = team_id is None
723
  # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
724
  # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
725
+ wants = lambda keys: needed is None or bool(set(keys) & set(needed))
726
+ inventory_needed = wants(['on_hand', 'unit_cost', 'inv_value', 'qty_ltm', 'dos', 'stock_bucket',
727
+ 'inventory_value_per_quantity', 'inventory_value_per_quantity_state', 'cover_gap_units',
728
+ 'cover_gap_d', 'buy_now'])
729
+ inventory_snapshot = _inventory_by_code(t) if inventory_needed else {}
730
  inventory_source_available = isinstance(inventory_snapshot, dict)
731
  inv = inventory_snapshot if inventory_source_available else {}
732
+ bu_share = {} if consolidated or not inventory_needed else _bu_ltm_share(t, team_id)
733
  sup = supplier_master()
734
  # ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build:
735
  # tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the
 
740
  _prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both
741
  except Exception: # noqa: BLE001
742
  _prods = None
743
+ tiers, _tier_report = products.tier_prices_by_code(_prods) if wants(
744
+ [f'price_{n}' for n in range(1, 4)] + [f'unit_{n}' for n in range(1, 4)] +
745
+ [key for price, unit, _name, _n in products.PRICELIST_TIER_COLUMNS for key in (price, unit)]) else ({}, {})
746
+ pre_book, open_backlog, _backlog_report = backorders.sku_backlog(t=t, team_id=team_id) \
747
+ if wants(['pre_book_qty', 'open_backlog_qty']) else ({}, {}, {})
748
  cat = _catalogue_by_code()
749
  # ⭐ THE BUY LIST'S DEMAND HORIZON (owner ruling 2026-08-19). BU-shaped through `team_id`, the
750
  # same way the revenue columns are: a Fisch reader's reorder quantity must answer what FISCH
751
  # will sell. Degrades to `{}` on a read failure, matching every other column source here — a
752
  # blank cover gap is honest, a product grid that will not render is not.
753
  try:
754
+ fwd_demand, _fwd_report = products.forward_demand_by_code(t=t, team_id=team_id) \
755
+ if wants(['demand_fwd', 'cover_gap_units', 'cover_gap_d', 'buy_now']) else ({}, {})
756
  except Exception: # noqa: BLE001
757
  fwd_demand, _fwd_report = {}, {"error": "forward demand unavailable"}
758
  # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
 
763
  # because it is keyed off it, and UNSCOPED for the same reason `on_hand` is: a warehouse holds
764
  # units, not brands, and pro-rating them onto a BU would print a number no Odoo query could
765
  # reproduce. Blank, never zero, when the tenant has no FBA warehouse.
766
+ fba_by_code, _fba_report = _fba_by_code(cat) if inventory_needed or wants(['fba']) else ({}, {})
767
+ rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} if wants(
768
+ ['rev_ytd', 'rev_ly', 'yoy_pct', 'qty_ytd', 'orders_ytd']) else {}
769
  raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p
770
  for p in (_prods or []) if p.get("default_code") or p.get("id")}
771
 
platform/modules/products.py CHANGED
@@ -20,7 +20,7 @@ import datetime as dt
20
  from pathlib import Path
21
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
  from functools import lru_cache
23
- import core.odoo as O
24
  import core.periods as P
25
  import modules.sales as sales_mod
26
 
@@ -65,6 +65,10 @@ def _sku_rev(date_from, date_to, team_id=None, all_channels=False):
65
  all_channels=all_channels),
66
  ['price_subtotal:sum', 'product_uom_qty:sum'],
67
  ['product_id', 'order_id'], lazy=False)
 
 
 
 
68
  codes = _code_map()
69
  out = {}
70
  orders_by_code = {}
@@ -246,6 +250,9 @@ def velocity_leaders(t=None, limit=25, team_id=None):
246
  def _code_category():
247
  """code -> category name. Built from _code_map (pid->code) + the sales category map
248
  (pid->category); the first record carrying a code sets that code's category."""
 
 
 
249
  codes = _code_map()
250
  cats = sales_mod._product_cat()
251
  out = {}
@@ -630,6 +637,8 @@ def tier_cap_overflow(tiers):
630
  fourth tier is dropped — and a dropped tier that nobody counts is the silent cap the rule
631
  forbids. `validate_price_and_unit_cells` turns this into a named, reported figure.
632
  """
 
 
633
  counts = {}
634
  for tier in tiers or ():
635
  counts[tier.get("pricelist")] = counts.get(tier.get("pricelist"), 0) + 1
@@ -685,6 +694,9 @@ def tier_prices_by_code(prods=None):
685
  2,605 price rows against **0 customers and 0 orders**, so a SKU reading several pairs is
686
  catalogue-true and commercially misleading. The entry keeps the list NAME for provenance.
687
  """
 
 
 
688
  report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0,
689
  "rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0,
690
  "rules_with_packaging": 0}
 
20
  from pathlib import Path
21
  sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
22
  from functools import lru_cache
23
+ from harness.canonical_source import odoo as O
24
  import core.periods as P
25
  import modules.sales as sales_mod
26
 
 
65
  all_channels=all_channels),
66
  ['price_subtotal:sum', 'product_uom_qty:sum'],
67
  ['product_id', 'order_id'], lazy=False)
68
+ from harness.canonical_source import active
69
+ source = active()
70
+ if source is not None:
71
+ return source.sku_revenue(g)
72
  codes = _code_map()
73
  out = {}
74
  orders_by_code = {}
 
250
  def _code_category():
251
  """code -> category name. Built from _code_map (pid->code) + the sales category map
252
  (pid->category); the first record carrying a code sets that code's category."""
253
+ from harness.canonical_source import active
254
+ if active() is not None:
255
+ return active().categories()
256
  codes = _code_map()
257
  cats = sales_mod._product_cat()
258
  out = {}
 
637
  fourth tier is dropped — and a dropped tier that nobody counts is the silent cap the rule
638
  forbids. `validate_price_and_unit_cells` turns this into a named, reported figure.
639
  """
640
+ if hasattr(tiers, 'overflow'):
641
+ return dict(tiers.overflow)
642
  counts = {}
643
  for tier in tiers or ():
644
  counts[tier.get("pricelist")] = counts.get(tier.get("pricelist"), 0) + 1
 
694
  2,605 price rows against **0 customers and 0 orders**, so a SKU reading several pairs is
695
  catalogue-true and commercially misleading. The entry keeps the list NAME for provenance.
696
  """
697
+ from harness.canonical_source import active
698
+ if active() is not None:
699
+ return active().tiers(prods)
700
  report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0,
701
  "rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0,
702
  "rules_with_packaging": 0}
platform/modules/sheet.py CHANGED
@@ -1179,7 +1179,7 @@ def _related_predicate(related, columns, *, today=None, cohort_sets=None):
1179
  where = [f"({part})" for part in (rspec.get("where"), related.get("where")) if part]
1180
  params, limits = [], []
1181
 
1182
- def leaf_limits(nodes):
1183
  # Checking every leaf also catches a dropped condition on a column which a
1184
  # different, valid condition used. The legacy column-set check cannot see it.
1185
  out = []
@@ -1187,19 +1187,21 @@ def _related_predicate(related, columns, *, today=None, cohort_sets=None):
1187
  if not isinstance(node, dict):
1188
  raise ValueError("malformed related filter condition")
1189
  if isinstance(node.get("children"), list):
1190
- out.extend(leaf_limits(node["children"]))
1191
  else:
1192
  if "children" in node or not node.get("colId"):
1193
  raise ValueError("malformed related filter condition")
1194
- part = compile_scope([node], "and", rcols, today=today, cohort_sets=cohort_sets)
1195
- out.extend(widen_limits([node], rcols, part))
1196
  return out
1197
 
1198
  for nodes, conj, is_wall in (
1199
  (related.get("wall_nodes"), related.get("wall_conj", "and"), True),
1200
  (related["scope_nodes"], related["scope_conj"], False)):
1201
- compiled = compile_scope(nodes, conj, rcols, today=today, cohort_sets=cohort_sets)
1202
- dropped = leaf_limits(nodes)
 
 
1203
  if is_wall and dropped:
1204
  raise ValueError("unapplied related row wall")
1205
  limits.extend({**item, "subject": f"{related['database']}: {item['subject']}"}
 
1179
  where = [f"({part})" for part in (rspec.get("where"), related.get("where")) if part]
1180
  params, limits = [], []
1181
 
1182
+ def leaf_limits(nodes, cmap):
1183
  # Checking every leaf also catches a dropped condition on a column which a
1184
  # different, valid condition used. The legacy column-set check cannot see it.
1185
  out = []
 
1187
  if not isinstance(node, dict):
1188
  raise ValueError("malformed related filter condition")
1189
  if isinstance(node.get("children"), list):
1190
+ out.extend(leaf_limits(node["children"], cmap))
1191
  else:
1192
  if "children" in node or not node.get("colId"):
1193
  raise ValueError("malformed related filter condition")
1194
+ part = compile_scope([node], "and", cmap, today=today, cohort_sets=cohort_sets)
1195
+ out.extend(widen_limits([node], cmap, part))
1196
  return out
1197
 
1198
  for nodes, conj, is_wall in (
1199
  (related.get("wall_nodes"), related.get("wall_conj", "and"), True),
1200
  (related["scope_nodes"], related["scope_conj"], False)):
1201
+ # A wall reads the reader's hidden columns too; authored conditions never do.
1202
+ cmap = (related.get("wall_columns") or rcols) if is_wall else rcols
1203
+ compiled = compile_scope(nodes, conj, cmap, today=today, cohort_sets=cohort_sets)
1204
+ dropped = leaf_limits(nodes, cmap)
1205
  if is_wall and dropped:
1206
  raise ValueError("unapplied related row wall")
1207
  limits.extend({**item, "subject": f"{related['database']}: {item['subject']}"}
requirements.txt CHANGED
@@ -15,13 +15,13 @@
15
  # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the
16
  # import lives inside the handler, and the gate now asserts this line exists here.
17
  fastapi>=0.139
18
- starlette>=1.3.1
19
  uvicorn[standard]>=0.30
20
  python-dotenv>=1.2.2
21
  pandas>=2.0
22
  requests>=2.28
23
  huggingface_hub>=0.20
24
- duckdb>=1.5
25
  pyyaml>=6.0
26
  pillow>=10.0
27
  beautifulsoup4>=4.12
 
15
  # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the
16
  # import lives inside the handler, and the gate now asserts this line exists here.
17
  fastapi>=0.139
18
+ starlette==1.3.1
19
  uvicorn[standard]>=0.30
20
  python-dotenv>=1.2.2
21
  pandas>=2.0
22
  requests>=2.28
23
  huggingface_hub>=0.20
24
+ duckdb==1.5.4
25
  pyyaml>=6.0
26
  pillow>=10.0
27
  beautifulsoup4>=4.12
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 27f1f5a90c0f4ceb5ae13bc834f4eb2e52d92512e72d6a2924151e39bad85159
 
1
+ a7a3bbeebf18bb7116b669ba2c9dddd8b93014e7ecae45679ca8eb23bbc10f8d
web/src/automation/AutomationBuilder.tsx CHANGED
@@ -2360,7 +2360,8 @@ function TriggerProps({
2360
  <option value="">Select a database…</option>
2361
  {tables.map((t) => (
2362
  <option key={t.key} value={t.key}>
2363
- {t.label} ({t.rowCount} {t.rowCount === 1 ? "row" : "rows"})
 
2364
  </option>
2365
  ))}
2366
  </select>
 
2360
  <option value="">Select a database…</option>
2361
  {tables.map((t) => (
2362
  <option key={t.key} value={t.key}>
2363
+ {t.label}
2364
+ {t.rowCount != null && ` (${t.rowCount} ${t.rowCount === 1 ? "row" : "rows"})`}
2365
  </option>
2366
  ))}
2367
  </select>
web/src/automation/AutomationDetail.tsx CHANGED
@@ -1597,7 +1597,8 @@ function AutomationDetail({
1597
  ) : null}
1598
  {tables.map((t) => (
1599
  <option key={t.key} value={t.key}>
1600
- {t.label} ({t.rowCount} {t.rowCount === 1 ? "row" : "rows"})
 
1601
  </option>
1602
  ))}
1603
  </select>
 
1597
  ) : null}
1598
  {tables.map((t) => (
1599
  <option key={t.key} value={t.key}>
1600
+ {t.label}
1601
+ {t.rowCount != null && ` (${t.rowCount} ${t.rowCount === 1 ? "row" : "rows"})`}
1602
  </option>
1603
  ))}
1604
  </select>