fsanyoto commited on
Commit
c70f19a
·
verified ·
1 Parent(s): 80d35f3

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "1930f5a925",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "52ab0bfe7b",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 1930f5a925
 
1
+ 52ab0bfe7b
api/amazon_relational.py CHANGED
@@ -238,8 +238,14 @@ def _format(col, duck_type):
238
  return None
239
 
240
 
241
- def mirror_columns(cur, table):
242
  """[(name, type)] for a mirror table, or [] when it has never been synced."""
 
 
 
 
 
 
243
  try:
244
  return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
245
  except Exception: # noqa: BLE001
@@ -369,7 +375,7 @@ def _parent_lookup_sql(cur):
369
  return resolved
370
 
371
 
372
- def lookups(cur, key, names=None):
373
  """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
374
 
375
  `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
@@ -385,6 +391,23 @@ def lookups(cur, key, names=None):
385
  sku = sku if sku in names else None
386
  if not asin and not sku:
387
  return []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
  listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
389
  parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
390
  latest = _latest_listings_sql(listing_names)
@@ -529,14 +552,15 @@ def _quoted(name):
529
  AMAZON_STORED_BYTES = 4 * 1024 * 1024
530
 
531
 
532
- def _read_through(cur, key, count):
533
  """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
534
  ut = _odoo._ut()
535
  if key in READ_THROUGH or count > ut.MAX_ROWS:
536
  return True
537
  if not count:
538
  return False
539
- names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])]
 
540
  sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
541
  # Include the stored id, refreshed stamp and row-object punctuation, with the same
542
  # ASCII escaping as the document writer. The largest sampled row is conservative.
@@ -590,9 +614,9 @@ def _integer(value):
590
  return None if value in (None, "") else int(value)
591
 
592
 
593
- def grid_source(cur, key):
594
  """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
595
- columns = mirror_columns(cur, SOURCE[key])
596
  if not columns:
597
  return missing_source(key)
598
  names = {name for name, _dtype in columns}
@@ -645,7 +669,7 @@ def grid_source(cur, key):
645
  cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
646
  # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
647
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
648
- for lookup in lookups(cur, key, names):
649
  cols[lookup["key"]] = (lookup["expr"], _text)
650
  spec = {"table": SOURCE[key], "where": "", "id": ident,
651
  "needs_excluded": False, "cols": cols}
@@ -654,6 +678,19 @@ def grid_source(cur, key):
654
  return spec
655
 
656
 
 
 
 
 
 
 
 
 
 
 
 
 
 
657
  def grid_sources(cur=None):
658
  """Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
659
  own_cursor = cur is None
@@ -773,7 +810,7 @@ def wall_link_expression(source, predicate, identity):
773
  f"WHERE {predicate}) AS _wall_links)")
774
 
775
 
776
- def fill_link_cells(rows, fields, read_targets):
777
  """Populate derived Amazon links with one read per distinct target on this page.
778
 
779
  The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
@@ -782,17 +819,42 @@ def fill_link_cells(rows, fields, read_targets):
782
  Neither path stores history rows or caps the matching target set.
783
  """
784
  edges = link_edges(fields)
785
- requests = {}
786
  for _field, target, frm, on in edges:
787
  values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
788
  requests.setdefault(target, {}).setdefault(on, set()).update(values)
 
 
 
 
789
  indexes = {}
 
790
  for target, matches in requests.items():
791
  indexes[target] = {column: {} for column in matches}
792
- for row in read_targets(target, matches) if any(matches.values()) else ():
793
- for column in matches:
794
- if row.get(column) not in (None, ""):
795
- indexes[target][column].setdefault(str(row[column]), set()).add(int(row["pid"]))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
796
  for field, target, frm, on in edges:
797
  for row in rows:
798
  row[field["key"]] = ",".join(str(pid) for pid in sorted(
 
238
  return None
239
 
240
 
241
+ def mirror_columns(cur, table, *, strict=False):
242
  """[(name, type)] for a mirror table, or [] when it has never been synced."""
243
+ if strict:
244
+ # A successful empty metadata query proves absence on this cursor only.
245
+ return cur.execute(
246
+ "SELECT column_name, data_type FROM information_schema.columns "
247
+ "WHERE table_catalog = current_database() AND table_schema = current_schema() "
248
+ "AND table_name = ? ORDER BY ordinal_position", [table]).fetchall()
249
  try:
250
  return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
251
  except Exception: # noqa: BLE001
 
375
  return resolved
376
 
377
 
378
+ def lookups(cur, key, names=None, *, allowed=True):
379
  """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
380
 
381
  `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
 
391
  sku = sku if sku in names else None
392
  if not asin and not sku:
393
  return []
394
+ if not allowed:
395
+ # Keep declared cells blank without inspecting a denied lookup dependency.
396
+ keys = []
397
+ if asin:
398
+ if "parent_asin" not in names:
399
+ keys.append((LOOKUP_PARENT, asin, asin_via))
400
+ if not sku:
401
+ keys.append((LOOKUP_SKU, asin, asin_via))
402
+ elif sku:
403
+ if not names & _NATIVE_ASIN:
404
+ keys.append((LOOKUP_ASIN, sku, sku_via))
405
+ if "parent_asin" not in names:
406
+ keys.append((LOOKUP_PARENT, sku, sku_via))
407
+ return [{"key": name, "label": _LOOKUP_LABELS[name], "after": after,
408
+ "expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None,
409
+ "description": _LOOKUP_DESCRIPTIONS[name]}
410
+ for name, after, via in keys if name not in names]
411
  listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
412
  parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
413
  latest = _latest_listings_sql(listing_names)
 
552
  AMAZON_STORED_BYTES = 4 * 1024 * 1024
553
 
554
 
555
+ def _read_through(cur, key, count, *, columns=None):
556
  """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
557
  ut = _odoo._ut()
558
  if key in READ_THROUGH or count > ut.MAX_ROWS:
559
  return True
560
  if not count:
561
  return False
562
+ names = [name for name, _dtype in (columns if columns is not None
563
+ else mirror_columns(cur, SOURCE[key]))]
564
  sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
565
  # Include the stored id, refreshed stamp and row-object punctuation, with the same
566
  # ASCII escaping as the document writer. The largest sampled row is conservative.
 
614
  return None if value in (None, "") else int(value)
615
 
616
 
617
+ def grid_source(cur, key, *, columns=None, allow_lookups=True):
618
  """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
619
+ columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
620
  if not columns:
621
  return missing_source(key)
622
  names = {name for name, _dtype in columns}
 
669
  cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
670
  # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
671
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
672
+ for lookup in lookups(cur, key, names, allowed=allow_lookups):
673
  cols[lookup["key"]] = (lookup["expr"], _text)
674
  spec = {"table": SOURCE[key], "where": "", "id": ident,
675
  "needs_excluded": False, "cols": cols}
 
678
  return spec
679
 
680
 
681
+ def page_source(cur, key, definition=None, *, allow_lookups=True):
682
+ """Resolve one target's storage policy on the admitted cursor without provider discovery."""
683
+ columns = mirror_columns(cur, SOURCE[key], strict=True)
684
+ mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough"))
685
+ if not columns:
686
+ return missing_source(key) if mandatory else None
687
+ if not mandatory:
688
+ count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
689
+ if not _read_through(cur, key, count, columns=columns):
690
+ return None
691
+ return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
692
+
693
+
694
  def grid_sources(cur=None):
695
  """Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
696
  own_cursor = cur is None
 
810
  f"WHERE {predicate}) AS _wall_links)")
811
 
812
 
813
+ def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refusal=None):
814
  """Populate derived Amazon links with one read per distinct target on this page.
815
 
816
  The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
 
819
  Neither path stores history rows or caps the matching target set.
820
  """
821
  edges = link_edges(fields)
822
+ requests, weights = {}, {}
823
  for _field, target, frm, on in edges:
824
  values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
825
  requests.setdefault(target, {}).setdefault(on, set()).update(values)
826
+ for row in rows:
827
+ if row.get(frm) not in (None, ""):
828
+ slot = (target, on, str(row[frm]))
829
+ weights[slot] = weights.get(slot, 0) + 1
830
  indexes = {}
831
+ remaining = max_bytes
832
  for target, matches in requests.items():
833
  indexes[target] = {column: {} for column in matches}
834
+ iterator = iter(read_targets(target, matches) if any(matches.values()) else ())
835
+ try:
836
+ for row in iterator:
837
+ pid = int(row["pid"])
838
+ for column, wanted in matches.items():
839
+ value = str(row[column]) if row.get(column) not in (None, "") else None
840
+ if value not in wanted:
841
+ continue
842
+ bucket = indexes[target][column].get(value)
843
+ if bucket is not None and pid in bucket:
844
+ continue
845
+ if remaining is not None:
846
+ # Charge every output occurrence before allocating its distinct ID.
847
+ cost = (len(str(pid)) + (1 if bucket else 2)) * weights[target, column, value]
848
+ if cost > remaining:
849
+ raise capacity_refusal()
850
+ remaining -= cost
851
+ if bucket is None:
852
+ bucket = indexes[target][column][value] = set()
853
+ bucket.add(pid)
854
+ finally:
855
+ close = getattr(iterator, "close", None)
856
+ if close is not None:
857
+ close()
858
  for field, target, frm, on in edges:
859
  for row in rows:
860
  row[field["key"]] = ",".join(str(pid) for pid in sorted(
api/routes_odoo_tables.py CHANGED
@@ -778,25 +778,19 @@ GRID_SOURCES = {}
778
  _SOURCE_PROVIDERS = []
779
  _SOURCE_ANALYSES = {}
780
 
781
- #: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
782
- #: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
783
- #: keys to the shared dict as it went, and returned the dict to any caller that found it
784
- #: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
785
- #: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
786
- #: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
787
- #: this lock and publishes it in one `dict.update`; a reader waits on the lock until
788
- #: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
789
- #: Amazon fallback) happens only after a build has been published, so it can never stand in
790
- #: for one either.
791
  _SOURCES_LOCK = threading.RLock()
792
  _SOURCES_BUILT = False
793
- _SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
 
 
794
 
795
 
796
  def invalidate_sources():
797
  """Forget measured bindings after a connector publishes a new mirror schema."""
798
- global _SOURCES_BUILT
799
  with _SOURCES_LOCK:
 
800
  _SOURCES_BUILT = False
801
  GRID_SOURCES.clear()
802
 
@@ -848,54 +842,54 @@ def _sources():
848
  importing that module at file-import time would drag the Odoo layer into every process that
849
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
850
 
851
- ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
852
- lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
853
- and so does one that holds only admissions made since the last invalidation.
854
  """
855
- global _SOURCES_BUILT, _SOURCES_BUILDER
856
  if _SOURCES_BUILT and GRID_SOURCES:
857
  return GRID_SOURCES
858
  with _SOURCES_LOCK:
859
- if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
860
  return GRID_SOURCES
861
- _SOURCES_BUILDER = threading.get_ident()
862
- try:
863
- built = {}
864
- for build in list(_SOURCE_PROVIDERS):
865
- try:
866
- extra = build() or {}
867
- except Exception: # noqa: BLE001
868
- # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
869
- # grids must not go dark because a newer connector's module is unhappy, and
870
- # vice versa.
871
- continue
872
- if extra:
873
- built.update(extra)
874
- _ut().register_connected(*extra)
875
- # A key admitted since the last invalidation was measured at least as recently as
876
- # this build, so it stands; everything else comes from the build. ONE update, so a
877
- # lock-free reader of the dict sees the old entry or the new one, never a gap.
878
  GRID_SOURCES.update({**built, **GRID_SOURCES})
879
  _SOURCES_BUILT = True
880
- finally:
881
- _SOURCES_BUILDER = None
 
 
882
  return GRID_SOURCES
883
 
884
 
885
- def _learn_amazon_sources(cur=None):
886
- """Publish the complete current Amazon binding set, including restored missing entries."""
887
- _sources()
 
 
 
888
  if cur is None:
889
  extra = _amazon_sources()
890
  else:
891
  import amazon_relational
892
  extra = amazon_relational.grid_sources(cur)
893
- if not extra:
894
- return
895
  with _SOURCES_LOCK:
896
- # One known key cannot certify every history table. Replace old missing bindings as
897
- # one publication, so the first rows AND aggregates request sees the restored schema.
898
  GRID_SOURCES.update(extra)
 
899
  _ut().register_connected(*extra)
900
 
901
 
@@ -1298,13 +1292,7 @@ def _amazon_sources():
1298
 
1299
 
1300
  def require_amazon_mirror(table_key, session, definition=None, st=None):
1301
- """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
1302
-
1303
- The provider contributes no keys on an Odoo-only mirror. A persisted read-through
1304
- definition still must refuse there instead of falling through to its empty stored rows.
1305
- S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
1306
- window route stops paying a second definitions projection for this one check.
1307
- """
1308
  if not str(table_key).startswith("ut_amzn_"):
1309
  return
1310
  import amazon_relational as amazon
@@ -1322,28 +1310,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
1322
  if mismatch:
1323
  raise err(409, "cross_tenant_store", mismatch)
1324
  try:
1325
- spec = _sources().get(table_key)
1326
- if (spec is not None or definition.get("readThrough")
1327
- or table_key in amazon.READ_THROUGH):
1328
- # Check current table presence even if a binding was cached before a restart.
1329
- current = _page_source_for(cur, table_key, st)
1330
- if current is None:
1331
- current = amazon.grid_source(cur, table_key)
1332
- if current.get("missing"):
1333
- raise err(503, "amazon_mirror_not_loaded", current["missing"])
1334
- # A dynamically oversized snapshot may have a durable stub before this process
1335
- # has measured it. Admit its binding without changing another source's registry.
1336
- # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
1337
- # key. Admitted into an empty or Amazon-less registry, this one key used to make
1338
- # `_sources()` skip its build, so the grid resolved while its link targets
1339
- # (orders, read-through) were dispatched as stored tables: a first-page 500.
1340
- if spec is None or spec.get("missing"):
1341
- _learn_amazon_sources(cur)
1342
- with _SOURCES_LOCK:
1343
- _sources()
1344
- held = GRID_SOURCES.get(table_key)
1345
- if held is None or held.get("missing"):
1346
- GRID_SOURCES[table_key] = current
1347
  finally:
1348
  cur.close()
1349
 
@@ -1559,7 +1526,27 @@ def _degrade(expr, have_by_alias, default_alias=""):
1559
  return expr if bare.lower() in have else "NULL"
1560
 
1561
 
1562
- def _source_for(cur, table_key):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1563
  """The resolved spec for one connected grid, or None when this grid is not read-through yet.
1564
 
1565
  ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
@@ -1586,25 +1573,15 @@ def _source_for(cur, table_key):
1586
  # table and turns Products ordered into the wrong 404.
1587
  if module_cache_source(key) is not None:
1588
  return None
 
 
 
 
1589
  spec = _sources().get(key)
1590
- if (not spec or spec.get("missing")) and key.startswith("ut_amzn_"):
1591
- # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
1592
- # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
1593
- # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
1594
- # read-through Amazon grid answered "no read-through binding". S70-T12 re-asks the
1595
- # complete Amazon provider on this restored cursor, including cached missing entries.
1596
- _learn_amazon_sources(cur)
1597
- spec = GRID_SOURCES.get(key)
1598
  if not spec:
1599
  return None
1600
  if spec.get("missing"):
1601
  raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1602
- if key.startswith("ut_amzn_"):
1603
- import amazon_relational as amazon
1604
- if key in amazon.SOURCE:
1605
- spec = amazon.grid_source(cur, key)
1606
- if spec.get("missing"):
1607
- raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1608
  from harness import datastore
1609
  if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
1610
  try:
@@ -1662,19 +1639,14 @@ def _source_for(cur, table_key):
1662
  return {**spec, "cols": cols}
1663
 
1664
 
1665
- def _page_source_for(cur, table_key, st=None):
1666
- """Reuse source resolution only inside the page's explicitly lent definition snapshot."""
1667
- from harness import datastore
1668
- plans = None
1669
- if isinstance(st, _ut()._Lent):
1670
- plans = st.__dict__.setdefault("_page_sources", {})
1671
- key = (table_key, datastore.mirror_generation())
1672
- if plans is not None and key in plans:
1673
- return plans[key]
1674
- spec = _source_for(cur, table_key)
1675
- if plans is not None:
1676
- plans[key] = spec
1677
- return spec
1678
 
1679
 
1680
  #: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
@@ -1912,13 +1884,14 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
1912
  targets[key] = None
1913
  return None
1914
  definition = _ut().get(key, st=handle)
1915
- target_spec = _page_source_for(cur, key, handle)
1916
  if target_spec is None and str(key).startswith("ut_amzn_"):
1917
  # Small Amazon targets (notably listings) are stored snapshots, but
1918
  # their raw mirror still answers the same automatic link in SQL.
1919
  import amazon_relational
1920
  if key in amazon_relational.SOURCE:
1921
- target_spec = amazon_relational.grid_source(cur, key)
 
1922
  if not definition or target_spec is None or target_spec.get("missing"):
1923
  refuse(field)
1924
  target_fields = definition.get("fields") or []
@@ -2408,21 +2381,25 @@ def _related_source_value(session, entry, record):
2408
  return None
2409
 
2410
 
2411
- def _materialised_table(table_key):
2412
- """Storage dispatch, independent of the browser's fallible status roster.
2413
-
2414
- ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
2415
- (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
2416
- the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
2417
- """
2418
  if module_cache_source(table_key) is not None:
2419
  return True
2420
- if table_key in _sources():
2421
- return False
2422
  if str(table_key).startswith("ut_amzn_"):
2423
- _learn_amazon_sources()
2424
- return table_key not in GRID_SOURCES
2425
- return True
 
 
 
 
 
 
 
 
 
 
 
2426
 
2427
 
2428
  #: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
@@ -2520,7 +2497,7 @@ def _page_defs(session):
2520
  return lent, token
2521
 
2522
 
2523
- def _stored_link_rows(session, table, token=_UNREAD):
2524
  """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
2525
 
2526
  ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
@@ -2534,7 +2511,8 @@ def _stored_link_rows(session, table, token=_UNREAD):
2534
 
2535
  rt = session.runtime
2536
  if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
2537
- or module_cache_source(table) is not None or not _materialised_table(table)):
 
2538
  return perm_scope._read(table, session.user, rt)
2539
  return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
2540
  keep=lambda v: bool(v[1]),
@@ -2567,7 +2545,7 @@ class _RelationRows:
2567
 
2568
  def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
2569
  traversal_of=None, st=None, rev=..., normalize_match=False,
2570
- compiled=None, stream=False): # `...` IS `_UNREAD`
2571
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
2572
 
2573
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
@@ -2585,7 +2563,10 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2585
  import core.relations as relations
2586
 
2587
  columns = set(columns) | {relations.IDENTITY_COL}
2588
- if not _materialised_table(table):
 
 
 
2589
  read = compiled if compiled is not None else (
2590
  _compiled_read(table, session, traversal_intermediate=True, st=st)
2591
  if traversal_of is not None else _compiled_read(table, session, st=st))
@@ -2636,21 +2617,22 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2636
  else:
2637
  sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
2638
  def batches():
2639
- cursor = read["cur"].execute(sql, params)
2640
- while True:
2641
- batch = cursor.fetchmany(2048)
2642
- if not batch:
2643
- return
2644
- for row in batch:
2645
- yield dict(zip(names, row))
2646
- batch.clear()
 
 
 
 
2647
  return _RelationRows(batches) if stream else list(batches())
2648
 
2649
  # This is an internal join, never a cell response. Apply the same permanent wall on the
2650
  # complete contract before taking any join value, including a join on a hidden column.
2651
- if traversal_of is None and not perm_scope.may_read(
2652
- session.user, table, st=st if st is not None else session.runtime):
2653
- raise err(403, "forbidden_table", "This account cannot open the linked database.")
2654
  if module_cache_source(table) is not None:
2655
  session.require(table)
2656
  # The revision memo serves a caller that handed this request's token (the page's link fills,
@@ -2661,7 +2643,7 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
2661
  fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
2662
  if definition is not None else perm_scope._read(table, session.user, session.runtime))
2663
  else:
2664
- fields, rows = _stored_link_rows(session, table, token=rev)
2665
  if ids is not None:
2666
  wanted = {str(value) for value in ids}
2667
  rows = [row for row in rows if str(row.get("pid")) in wanted]
@@ -3701,7 +3683,12 @@ def resolved_fields(table_key, defn):
3701
  fields = rel.namespaced_fields(mk_fields(), connection)
3702
  break
3703
  if fields is None:
3704
- if table_key not in _sources():
 
 
 
 
 
3705
  raise err(404, "not_connected", "that database is not a connected grid")
3706
  fields = (defn or {}).get("fields") or []
3707
  if not fields:
@@ -3806,7 +3793,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
3806
  if mismatch:
3807
  raise err(409, "cross_tenant_store", mismatch)
3808
 
3809
- spec = _page_source_for(cur, table_key, st)
3810
  if spec is None:
3811
  from core import perm_scope
3812
  rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
@@ -4413,13 +4400,10 @@ def odoo_table_rows(table_key: str,
4413
  "limit, filters, sorts and search in the query string.")
4414
  ids = _record_ids(body)
4415
 
4416
- # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
4417
- # asks it instead of reading its own definitions projection (one store round trip fewer on
4418
- # every Amazon page). Not reordered past it: the check may register this grid's binding,
4419
- # which is what `_materialised_table` below then reads.
4420
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4421
- require_amazon_mirror(table_key, session, st=page[0] if page else None)
4422
- if _materialised_table(table_key):
4423
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
4424
  sorts, search, related, groupBy, groupKey, ids=ids)
4425
 
@@ -4496,12 +4480,15 @@ def odoo_table_rows(table_key: str,
4496
  if not _may_open(session, target, st=page_st):
4497
  return []
4498
  return _relation_rows(session, target, list(matches), matches=matches,
4499
- st=page_st, rev=page_rev)
 
 
4500
 
4501
  # Fields are the current reader's contract; target rows cross the same wall
4502
  # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
4503
- amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets)
4504
- overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
 
4505
 
4506
  # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
4507
  #
@@ -4526,8 +4513,6 @@ def odoo_table_rows(table_key: str,
4526
  for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
4527
  overlays.setdefault(pid, {}).update(cells)
4528
  if_keys = {field["key"] for field in fields if field.get("type") == "if"}
4529
- for row in rows_src:
4530
- overlays[str(row["pid"])].update({key: row.get(key, "") for key in if_keys})
4531
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
4532
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
4533
  # in an overlay column that reads a hidden mirror column was outside its reach - and a formula
@@ -4544,7 +4529,13 @@ def odoo_table_rows(table_key: str,
4544
  rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
4545
  overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
4546
  for pid, cells in overlays.items()}
4547
- rows = aios_grid.rows_from_pool(rows_src, fields, overlays)
 
 
 
 
 
 
4548
 
4549
  unfiltered = win["total"] if where == base_where else datastore.count_rows(
4550
  where=base_where, params=tuple(wall_params), cur=cur, **frm)
@@ -4581,6 +4572,13 @@ MAX_AGGREGATES = 200
4581
  LINK_VALUE_MAX_BYTES = 1_048_576
4582
 
4583
 
 
 
 
 
 
 
 
4584
  def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
4585
  """Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
4586
  read = read if read is not None else _compiled_read(target, session, st=st)
@@ -4601,9 +4599,7 @@ def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=
4601
  required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
4602
  f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
4603
  if required > remaining:
4604
- raise err(503, 'relation_value_capacity',
4605
- f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
4606
- 'Reduce the page size or open the related database with filters.')
4607
  # The byte check bounds both the aggregate and its Python result. IDs are never a
4608
  # Python population; DISTINCT and numeric ordering preserve the existing wire value.
4609
  cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
@@ -4653,9 +4649,7 @@ def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
4653
  # Count repeated cells as well: several page rows may share one target key.
4654
  remaining -= len(value) + (2 if value else 0)
4655
  if remaining < 0:
4656
- raise err(503, 'relation_value_capacity',
4657
- f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
4658
- 'Reduce the page size or open the related database with filters.')
4659
  row[field['key']] = value
4660
 
4661
 
@@ -4912,9 +4906,9 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
4912
  raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
4913
 
4914
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4915
- require_amazon_mirror(table_key, session, st=page[0] if page else None)
4916
 
4917
- if _materialised_table(table_key):
4918
  with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
4919
  import odoo_default_scope
4920
  if (odoo_default_scope.applies(table_key)
 
778
  _SOURCE_PROVIDERS = []
779
  _SOURCE_ANALYSES = {}
780
 
781
+ # Provider SQL runs outside the publication lock. Invalidation rejects an older build.
 
 
 
 
 
 
 
 
 
782
  _SOURCES_LOCK = threading.RLock()
783
  _SOURCES_BUILT = False
784
+ _SOURCES_BUILDER = None # compatibility for callers that reset legacy build state
785
+ _SOURCES_BUILDING = threading.local()
786
+ _SOURCES_EPOCH = 0
787
 
788
 
789
  def invalidate_sources():
790
  """Forget measured bindings after a connector publishes a new mirror schema."""
791
+ global _SOURCES_BUILT, _SOURCES_EPOCH
792
  with _SOURCES_LOCK:
793
+ _SOURCES_EPOCH += 1
794
  _SOURCES_BUILT = False
795
  GRID_SOURCES.clear()
796
 
 
842
  importing that module at file-import time would drag the Odoo layer into every process that
843
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
844
 
845
+ Providers build a local dictionary outside the publication lock.
 
 
846
  """
847
+ global _SOURCES_BUILT
848
  if _SOURCES_BUILT and GRID_SOURCES:
849
  return GRID_SOURCES
850
  with _SOURCES_LOCK:
851
+ if (_SOURCES_BUILT and GRID_SOURCES) or getattr(_SOURCES_BUILDING, "active", False):
852
  return GRID_SOURCES
853
+ epoch = _SOURCES_EPOCH
854
+ from harness import datastore
855
+ generation = datastore.mirror_generation()
856
+ _SOURCES_BUILDING.active = True
857
+ try:
858
+ built = {}
859
+ for build in list(_SOURCE_PROVIDERS):
860
+ try:
861
+ extra = build() or {}
862
+ except Exception: # noqa: BLE001
863
+ continue
864
+ built.update(extra)
865
+ with _SOURCES_LOCK:
866
+ if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
867
+ return GRID_SOURCES
 
 
868
  GRID_SOURCES.update({**built, **GRID_SOURCES})
869
  _SOURCES_BUILT = True
870
+ if built:
871
+ _ut().register_connected(*built)
872
+ finally:
873
+ _SOURCES_BUILDING.active = False
874
  return GRID_SOURCES
875
 
876
 
877
+ def _learn_amazon_sources(cur=None, *, tenant=None):
878
+ """Refresh discovery for non-page callers without retaining negative results."""
879
+ with _SOURCES_LOCK:
880
+ epoch = _SOURCES_EPOCH
881
+ from harness import datastore
882
+ generation = datastore.mirror_generation()
883
  if cur is None:
884
  extra = _amazon_sources()
885
  else:
886
  import amazon_relational
887
  extra = amazon_relational.grid_sources(cur)
 
 
888
  with _SOURCES_LOCK:
889
+ if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
890
+ return
891
  GRID_SOURCES.update(extra)
892
+ if extra:
893
  _ut().register_connected(*extra)
894
 
895
 
 
1292
 
1293
 
1294
  def require_amazon_mirror(table_key, session, definition=None, st=None):
1295
+ """Admit one Amazon grid and return its binding, or None for a stored snapshot."""
 
 
 
 
 
 
1296
  if not str(table_key).startswith("ut_amzn_"):
1297
  return
1298
  import amazon_relational as amazon
 
1310
  if mismatch:
1311
  raise err(409, "cross_tenant_store", mismatch)
1312
  try:
1313
+ return _amazon_page_source(cur, table_key, definition, allow_lookups=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1314
  finally:
1315
  cur.close()
1316
 
 
1526
  return expr if bare.lower() in have else "NULL"
1527
 
1528
 
1529
+ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_lookups=True):
1530
+ """Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
1531
+ import amazon_relational as amazon
1532
+
1533
+ try:
1534
+ if binding:
1535
+ columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
1536
+ spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
1537
+ else:
1538
+ spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
1539
+ except Exception as exc: # noqa: BLE001
1540
+ from core.request_timing import raise_if_resource_error
1541
+ raise_if_resource_error(exc)
1542
+ raise err(503, "amazon_source_unavailable",
1543
+ "This Amazon database could not be resolved. Retry the request.") from exc
1544
+ if spec is not None and spec.get("missing"):
1545
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1546
+ return spec
1547
+
1548
+
1549
+ def _source_for(cur, table_key, *, allow_lookups=True):
1550
  """The resolved spec for one connected grid, or None when this grid is not read-through yet.
1551
 
1552
  ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
 
1573
  # table and turns Products ordered into the wrong 404.
1574
  if module_cache_source(key) is not None:
1575
  return None
1576
+ if key.startswith("ut_amzn_"):
1577
+ import amazon_relational as amazon
1578
+ if key in amazon.SOURCE:
1579
+ return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
1580
  spec = _sources().get(key)
 
 
 
 
 
 
 
 
1581
  if not spec:
1582
  return None
1583
  if spec.get("missing"):
1584
  raise err(503, "amazon_mirror_not_loaded", spec["missing"])
 
 
 
 
 
 
1585
  from harness import datastore
1586
  if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
1587
  try:
 
1639
  return {**spec, "cols": cols}
1640
 
1641
 
1642
+ def _page_source_for(cur, table_key, st=None, *, session=None):
1643
+ """Plan admitted lookups afresh because an autocommit cursor can change snapshots."""
1644
+ if str(table_key).startswith("ut_amzn_") and session is not None:
1645
+ import amazon_relational as amazon
1646
+ from routes_grid import _may_open
1647
+ allowed = _may_open(session, amazon.LISTINGS, st=st)
1648
+ return _source_for(cur, table_key, allow_lookups=allowed)
1649
+ return _source_for(cur, table_key)
 
 
 
 
 
1650
 
1651
 
1652
  #: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
 
1884
  targets[key] = None
1885
  return None
1886
  definition = _ut().get(key, st=handle)
1887
+ target_spec = _page_source_for(cur, key, handle, session=session)
1888
  if target_spec is None and str(key).startswith("ut_amzn_"):
1889
  # Small Amazon targets (notably listings) are stored snapshots, but
1890
  # their raw mirror still answers the same automatic link in SQL.
1891
  import amazon_relational
1892
  if key in amazon_relational.SOURCE:
1893
+ target_spec = amazon_relational.grid_source(cur, key,
1894
+ allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle))
1895
  if not definition or target_spec is None or target_spec.get("missing"):
1896
  refuse(field)
1897
  target_fields = definition.get("fields") or []
 
2381
  return None
2382
 
2383
 
2384
+ def _materialised_table(table_key, *, cur=None, st=None):
2385
+ """Dispatch one target against its cursor without treating registry misses as evidence."""
 
 
 
 
 
2386
  if module_cache_source(table_key) is not None:
2387
  return True
 
 
2388
  if str(table_key).startswith("ut_amzn_"):
2389
+ from harness import datastore
2390
+ import amazon_relational as amazon
2391
+ if table_key not in amazon.SOURCE:
2392
+ return True
2393
+ own_cursor = cur is None
2394
+ cur = datastore.ro_cursor() if own_cursor else cur
2395
+ try:
2396
+ definition = _ut().get(table_key, st=st) if st is not None else None
2397
+ spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
2398
+ return spec is None
2399
+ finally:
2400
+ if own_cursor:
2401
+ cur.close()
2402
+ return table_key not in _sources()
2403
 
2404
 
2405
  #: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
 
2497
  return lent, token
2498
 
2499
 
2500
+ def _stored_link_rows(session, table, token=_UNREAD, *, materialised=None):
2501
  """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
2502
 
2503
  ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
 
2511
 
2512
  rt = session.runtime
2513
  if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
2514
+ or module_cache_source(table) is not None
2515
+ or not (materialised if materialised is not None else _materialised_table(table))):
2516
  return perm_scope._read(table, session.user, rt)
2517
  return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
2518
  keep=lambda v: bool(v[1]),
 
2545
 
2546
  def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
2547
  traversal_of=None, st=None, rev=..., normalize_match=False,
2548
+ compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD`
2549
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
2550
 
2551
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
 
2563
  import core.relations as relations
2564
 
2565
  columns = set(columns) | {relations.IDENTITY_COL}
2566
+ if traversal_of is None and not perm_scope.may_read(
2567
+ session.user, table, st=st if st is not None else session.runtime):
2568
+ raise err(403, "forbidden_table", "This account cannot open the linked database.")
2569
+ if not (materialised if materialised is not None else _materialised_table(table)):
2570
  read = compiled if compiled is not None else (
2571
  _compiled_read(table, session, traversal_intermediate=True, st=st)
2572
  if traversal_of is not None else _compiled_read(table, session, st=st))
 
2617
  else:
2618
  sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
2619
  def batches():
2620
+ try:
2621
+ cursor = read["cur"].execute(sql, params)
2622
+ while True:
2623
+ batch = cursor.fetchmany(2048)
2624
+ if not batch:
2625
+ return
2626
+ for row in batch:
2627
+ yield dict(zip(names, row))
2628
+ batch.clear()
2629
+ finally:
2630
+ if stream and compiled is None:
2631
+ read["cur"].close()
2632
  return _RelationRows(batches) if stream else list(batches())
2633
 
2634
  # This is an internal join, never a cell response. Apply the same permanent wall on the
2635
  # complete contract before taking any join value, including a join on a hidden column.
 
 
 
2636
  if module_cache_source(table) is not None:
2637
  session.require(table)
2638
  # The revision memo serves a caller that handed this request's token (the page's link fills,
 
2643
  fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
2644
  if definition is not None else perm_scope._read(table, session.user, session.runtime))
2645
  else:
2646
+ fields, rows = _stored_link_rows(session, table, token=rev, materialised=True)
2647
  if ids is not None:
2648
  wanted = {str(value) for value in ids}
2649
  rows = [row for row in rows if str(row.get("pid")) in wanted]
 
3683
  fields = rel.namespaced_fields(mk_fields(), connection)
3684
  break
3685
  if fields is None:
3686
+ if str(table_key).startswith("ut_amzn_"):
3687
+ import amazon_relational as amazon
3688
+ connected = table_key in amazon.SOURCE
3689
+ else:
3690
+ connected = table_key in _sources()
3691
+ if not connected:
3692
  raise err(404, "not_connected", "that database is not a connected grid")
3693
  fields = (defn or {}).get("fields") or []
3694
  if not fields:
 
3793
  if mismatch:
3794
  raise err(409, "cross_tenant_store", mismatch)
3795
 
3796
+ spec = _page_source_for(cur, table_key, st, session=session)
3797
  if spec is None:
3798
  from core import perm_scope
3799
  rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
 
4400
  "limit, filters, sorts and search in the query string.")
4401
  ids = _record_ids(body)
4402
 
4403
+ # Admission resolves this Amazon database without publishing a process-wide binding.
 
 
 
4404
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4405
+ admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4406
+ if (admitted is None if page is not None else _materialised_table(table_key)):
4407
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
4408
  sorts, search, related, groupBy, groupKey, ids=ids)
4409
 
 
4480
  if not _may_open(session, target, st=page_st):
4481
  return []
4482
  return _relation_rows(session, target, list(matches), matches=matches,
4483
+ st=page_st, rev=page_rev,
4484
+ stream=True,
4485
+ materialised=_materialised_table(target, cur=cur, st=page_st))
4486
 
4487
  # Fields are the current reader's contract; target rows cross the same wall
4488
  # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
4489
+ amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets,
4490
+ max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal)
4491
+ overlays = {}
4492
 
4493
  # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
4494
  #
 
4513
  for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
4514
  overlays.setdefault(pid, {}).update(cells)
4515
  if_keys = {field["key"] for field in fields if field.get("type") == "if"}
 
 
4516
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
4517
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
4518
  # in an overlay column that reads a hidden mirror column was outside its reach - and a formula
 
4529
  rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
4530
  overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
4531
  for pid, cells in overlays.items()}
4532
+ rows = []
4533
+ for row in rows_src:
4534
+ # Physical payloads stay separate even when business identities share a pid.
4535
+ cells = {key: value for key, value in row.items() if key != "pid"}
4536
+ cells.update(overlays.get(str(row["pid"]), {}))
4537
+ cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden})
4538
+ rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells}))
4539
 
4540
  unfiltered = win["total"] if where == base_where else datastore.count_rows(
4541
  where=base_where, params=tuple(wall_params), cur=cur, **frm)
 
4572
  LINK_VALUE_MAX_BYTES = 1_048_576
4573
 
4574
 
4575
+ def _link_capacity_refusal():
4576
+ """Name the complete-value capacity and the action that reduces it."""
4577
+ return err(503, 'relation_value_capacity',
4578
+ f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
4579
+ 'Reduce the page size or open the related database with filters.')
4580
+
4581
+
4582
  def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
4583
  """Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
4584
  read = read if read is not None else _compiled_read(target, session, st=st)
 
4599
  required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
4600
  f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
4601
  if required > remaining:
4602
+ raise _link_capacity_refusal()
 
 
4603
  # The byte check bounds both the aggregate and its Python result. IDs are never a
4604
  # Python population; DISTINCT and numeric ordering preserve the existing wire value.
4605
  cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
 
4649
  # Count repeated cells as well: several page rows may share one target key.
4650
  remaining -= len(value) + (2 if value else 0)
4651
  if remaining < 0:
4652
+ raise _link_capacity_refusal()
 
 
4653
  row[field['key']] = value
4654
 
4655
 
 
4906
  raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
4907
 
4908
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4909
+ admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4910
 
4911
+ if (admitted is None if page is not None else _materialised_table(table_key)):
4912
  with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
4913
  import odoo_default_scope
4914
  if (odoo_default_scope.applies(table_key)
api/routes_tables.py CHANGED
@@ -584,7 +584,8 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
584
  context_store = _ut().lend(context_store, **{
585
  name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
586
  from routes_customers import _AssemblyStore
587
- context_store = _AssemblyStore(context_store, schema_only=True)
 
588
  from copy import copy
589
  field_session = copy(session)
590
  field_session.runtime = context_store
@@ -1853,6 +1854,10 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
1853
  metadata_membership = False
1854
  if fields_only:
1855
  defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
 
 
 
 
1856
  fields_base = list(defn.get("fields") or [])
1857
  pids, rows_src = frozenset(), []
1858
  elif with_rows:
 
584
  context_store = _ut().lend(context_store, **{
585
  name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
586
  from routes_customers import _AssemblyStore
587
+ # Lend the projected catalogue outside the assembly reader's per-read copy boundary.
588
+ context_store = _ut().lend_defs(_AssemblyStore(context_store, schema_only=True), schema_only=True)
589
  from copy import copy
590
  field_session = copy(session)
591
  field_session.runtime = context_store
 
1854
  metadata_membership = False
1855
  if fields_only:
1856
  defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
1857
+ if getattr(st, 'schema_only', False):
1858
+ from copy import deepcopy
1859
+ # Detach selected definitions while the request lends its projected catalogue.
1860
+ defn = deepcopy(defn)
1861
  fields_base = list(defn.get("fields") or [])
1862
  pids, rows_src = frozenset(), []
1863
  elif with_rows:
api/sheet_warm.py CHANGED
@@ -11,13 +11,15 @@ from __future__ import annotations
11
 
12
  import atexit
13
  import os
 
14
  import threading
15
  import time
 
16
  from datetime import datetime, time as dtime, timedelta, timezone
17
  from time import perf_counter
18
 
19
  import routes_sheet
20
- from core import external_access, perms, shares, sheets, users
21
  from deps import Session
22
 
23
  #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
@@ -357,6 +359,146 @@ def _desk_skip(rt, tenant):
357
  return str(text or "").startswith("pg:") and token == seen
358
 
359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  def desk_run_one(rt, tenant):
361
  """Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
362
  row = routes_sheet._refresh_desk_claim(rt)
@@ -364,6 +506,7 @@ def desk_run_one(rt, tenant):
364
  return False
365
  started = perf_counter()
366
  error = None
 
367
  try:
368
  book = sheets.get_workbook(rt, row["workbook"])
369
  rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
@@ -371,9 +514,11 @@ def desk_run_one(rt, tenant):
371
  if session is None:
372
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
373
  else:
374
- routes_sheet._refresh_desk_run(session, row)
 
375
  except Exception as exc: # noqa: BLE001
376
  error = routes_sheet._refresh_desk_error(exc)
 
377
  routes_sheet._refresh_desk_finish(rt, row["id"], error)
378
  print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
379
  f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
 
11
 
12
  import atexit
13
  import os
14
+ import re
15
  import threading
16
  import time
17
+ import uuid
18
  from datetime import datetime, time as dtime, timedelta, timezone
19
  from time import perf_counter
20
 
21
  import routes_sheet
22
+ from core import external_access, perms, refresh_proof, shares, sheets, users
23
  from deps import Session
24
 
25
  #: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
 
359
  return str(text or "").startswith("pg:") and token == seen
360
 
361
 
362
+ DESK_RESULT_PREFIX = "[sheet-desk-result] "
363
+ _DESK_PROOF_REASONS = frozenset((
364
+ "workbook_incomplete", "sheet_incomplete", "sheet_partial", "cell_malformed", "refusal_malformed",
365
+ "identity_mismatch", "original_workbook_missing", "workbook_mismatch", "result_identity_mismatch",
366
+ "inventory_changed", "execution_build_unknown", "attempt_malformed", "attempt_not_committed",
367
+ "viewer_only", "partial_scope", "held_or_unresolved", "editor_commit_unproven",
368
+ "workbook_empty", "authored_definitions_changed", "proof_capacity_exceeded",
369
+ "generated_manifest_invalid", "generated_output_missing", "generated_output_invalid",
370
+ ))
371
+
372
+
373
+ def desk_result_census(book):
374
+ """Validate a complete workbook and count its cells without retaining their values."""
375
+ identity = refresh_proof.snapshot(book)
376
+ return identity["census"], identity["inventory"], identity["unresolved"]
377
+
378
+
379
+ def _desk_input_identity(book):
380
+ try:
381
+ return refresh_proof.snapshot(book)
382
+ except refresh_proof.ProofCapacityError as exc:
383
+ return {"error": "proof_capacity_exceeded", "capacityCause": str(exc)}
384
+ except ValueError as exc:
385
+ return {"error": str(exc) if str(exc) in _DESK_PROOF_REASONS else "cell_malformed"}
386
+ except Exception: # noqa: BLE001
387
+ return {"error": "cell_malformed"}
388
+
389
+
390
+ def desk_result_capture(row, tenant, session, before, result, *, admitted=None):
391
+ """Describe only the original resolver return and the worker's admitted pre-run inventory."""
392
+ out = {"schema": "sheet-desk-result/v1", "source": "_refresh_desk_run.return",
393
+ "executionId": uuid.uuid4().hex,
394
+ "requestId": row.get("id") if isinstance(row.get("id"), str) else None,
395
+ "tenant": tenant if isinstance(tenant, str) else None,
396
+ "reader": row.get("reader") if isinstance(row.get("reader"), str) else None,
397
+ "workbook": row.get("workbook") if isinstance(row.get("workbook"), str) else None,
398
+ "version": os.environ.get("AIOS_VERSION", ""),
399
+ "complete": False, "persisted": False, "reason": None, "revision": None,
400
+ "priorRevision": None, "attempt": None, "census": None, "inventory": None, "sheets": None,
401
+ "authored": None, "priorAuthored": None, "recommendation": None, "capacityCause": None}
402
+ try:
403
+ if (session is None or row.get("tenant") != tenant or session.tenant != tenant
404
+ or session.uname != row.get("reader") or before.get("id") != row.get("workbook")
405
+ or session.user.get("username") != row.get("reader")
406
+ or session.user.get("tenant", tenant) != tenant
407
+ or any(key and key != tenant for key in
408
+ (getattr(session.runtime, "key", None),
409
+ getattr(getattr(session.runtime, "tenant", None), "key", None)))):
410
+ raise ValueError("identity_mismatch")
411
+ if not isinstance(result, dict) or not isinstance(result.get("workbook"), dict):
412
+ raise ValueError("original_workbook_missing")
413
+ book = result["workbook"]
414
+ if book.get("id") != row.get("workbook"):
415
+ raise ValueError("workbook_mismatch")
416
+ if (any(item.get("tenant", tenant) != tenant for item in (result, book))
417
+ or result.get("reader", session.uname) != session.uname):
418
+ raise ValueError("result_identity_mismatch")
419
+ prior = admitted if admitted is not None else _desk_input_identity(before)
420
+ if prior.get("error"):
421
+ out["capacityCause"] = prior.get("capacityCause")
422
+ raise ValueError(prior["error"])
423
+ returned = refresh_proof.snapshot(book)
424
+ out["census"], unresolved = returned["census"], returned["unresolved"]
425
+ out["inventory"] = refresh_proof.digest(returned["inventory"])
426
+ out["sheets"] = returned["sheets"]
427
+ out["authored"], out["priorAuthored"] = returned["authored"], prior["authored"]
428
+ out["revision"] = book.get("rev") if type(book.get("rev")) is int else None
429
+ out["priorRevision"] = prior["revision"] if type(prior["revision"]) is int else None
430
+ if (returned["inventory"] != prior["inventory"] or returned["sheets"] != prior["sheets"]):
431
+ raise ValueError("inventory_changed")
432
+ if returned["authored"] != prior["authored"]:
433
+ raise ValueError("authored_definitions_changed")
434
+ if not re.search(r"\b[0-9a-f]{7,40}\b", out["version"]):
435
+ raise ValueError("execution_build_unknown")
436
+ if result.get("attempt") is not None and not isinstance(result["attempt"], dict):
437
+ raise ValueError("attempt_malformed")
438
+ if isinstance(result.get("attempt"), dict):
439
+ attempt = result["attempt"]
440
+ if any(attempt.get(key) is not None and not isinstance(attempt[key], str) for key in ("id", "state")):
441
+ raise ValueError("attempt_malformed")
442
+ out["attempt"] = {key: attempt.get(key) for key in ("id", "state")}
443
+ if attempt.get("state") != "committed":
444
+ raise ValueError("attempt_not_committed")
445
+ role = routes_sheet._role_on_workbook(session, before)
446
+ if role not in ("owner", "edit") or "refreshedFor" in book:
447
+ raise ValueError("viewer_only")
448
+ args = row.get("args") or {}
449
+ if (args != {"kind": "workbook"} and args != {"kind": "scope", "scope": {"workbook": True}}):
450
+ raise ValueError("partial_scope")
451
+ report = result.get("binding_refresh")
452
+ statuses = report.get("databases") if isinstance(report, dict) else None
453
+ if (not isinstance(statuses, dict) or any(state not in ("resolved", "skipped")
454
+ for state in statuses.values()) or unresolved or result.get("reused")
455
+ or result.get("held") or book.get("held") or result.get("partial") or book.get("partial")
456
+ or result.get("viewer") or result.get("viewer_only")
457
+ or result.get("state") not in (None, "committed")
458
+ or result.get("outcome") not in (None, "committed")
459
+ or result.get("persisted") is False or book.get("persisted") is False
460
+ or result.get("ok") is False):
461
+ raise ValueError("held_or_unresolved")
462
+ if (type(book.get("rev")) is not int or type(prior["revision"]) is not int
463
+ or prior["revision"] < 0 or book["rev"] != prior["revision"] + 1
464
+ or not isinstance(book.get("refreshed_at"), str) or not book["refreshed_at"]
465
+ or (any(_is_bound(cell) for tab in book["sheets"] for cell in tab["cells"].values())
466
+ and not statuses)):
467
+ raise ValueError("editor_commit_unproven")
468
+ empty_noop = prior["census"]["cells"] == returned["census"]["cells"] == 0
469
+ out.update(complete=True, persisted=True,
470
+ reason="empty_noop" if empty_noop else "editor_revision_advanced")
471
+ refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
472
+ except refresh_proof.ProofCapacityError as exc:
473
+ out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
474
+ recommendation=refresh_proof.RECOMMENDATION, capacityCause=str(exc))
475
+ except (ValueError, TypeError, AttributeError) as exc:
476
+ reason = str(exc) if isinstance(exc, ValueError) else "capture_malformed"
477
+ out.update(complete=False, persisted=False)
478
+ out["reason"] = reason if reason in _DESK_PROOF_REASONS else "capture_malformed"
479
+ if out["reason"] == "proof_capacity_exceeded":
480
+ out["recommendation"] = refresh_proof.RECOMMENDATION
481
+ try:
482
+ refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
483
+ except (ValueError, TypeError):
484
+ out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
485
+ sheets=None, attempt=None, recommendation=refresh_proof.RECOMMENDATION,
486
+ capacityCause="log_line_capacity")
487
+ for key in ("requestId", "tenant", "reader", "workbook", "version"):
488
+ if isinstance(out[key], str) and len(out[key]) > 1024:
489
+ out[key] = None
490
+ return out
491
+
492
+
493
+ def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
494
+ """A logging failure supplies no proof and never changes the desk's business outcome."""
495
+ try:
496
+ capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
497
+ print(DESK_RESULT_PREFIX + refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES), flush=True)
498
+ except Exception: # noqa: BLE001
499
+ pass
500
+
501
+
502
  def desk_run_one(rt, tenant):
503
  """Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
504
  row = routes_sheet._refresh_desk_claim(rt)
 
506
  return False
507
  started = perf_counter()
508
  error = None
509
+ book, session, result, admitted = None, None, None, None
510
  try:
511
  book = sheets.get_workbook(rt, row["workbook"])
512
  rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
 
514
  if session is None:
515
  error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
516
  else:
517
+ admitted = _desk_input_identity(book)
518
+ result = routes_sheet._refresh_desk_run(session, row)
519
  except Exception as exc: # noqa: BLE001
520
  error = routes_sheet._refresh_desk_error(exc)
521
+ _desk_result_log(row, tenant, session, book, result, admitted=admitted)
522
  routes_sheet._refresh_desk_finish(rt, row["id"], error)
523
  print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
524
  f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
platform/core/refresh_proof.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pure, bounded identities for authored Sheet inputs and complete returned outputs."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import math
7
+ import re
8
+
9
+ MAX_CELLS = 100_000
10
+ MAX_TABS = 128
11
+ MAX_DOCUMENT_BYTES = 32 * 1024 * 1024
12
+ MAX_LINE_BYTES = 256 * 1024
13
+ MAX_LOG_BYTES = 8 * 1024 * 1024
14
+ MAX_LOG_ROWS = 4096
15
+ MAX_STATE_BYTES = 16 * 1024 * 1024
16
+ MAX_STATE_ROWS = 16_384
17
+ RECOMMENDATION = "Increase the named proof budget and collect a fresh complete proof."
18
+ _UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00"
19
+
20
+
21
+ class ProofCapacityError(ValueError):
22
+ """A complete proof cannot fit the named finite budget."""
23
+
24
+
25
+ def bounded_json(value, limit=MAX_DOCUMENT_BYTES):
26
+ """Check depth, nodes and scalar sizes before allocating encoded JSON."""
27
+ remaining, nodes = [limit], [0]
28
+
29
+ def check(item, depth=0):
30
+ nodes[0] += 1
31
+ if depth > 40 or nodes[0] > 1_000_000:
32
+ raise ProofCapacityError("json_structure_capacity")
33
+ if isinstance(item, str):
34
+ if len(item) > remaining[0]:
35
+ raise ProofCapacityError("json_byte_capacity")
36
+ cost = len(item.encode("utf-8")) + 2
37
+ elif isinstance(item, (dict, list, tuple)):
38
+ if len(item) > limit:
39
+ raise ProofCapacityError("json_structure_capacity")
40
+ cost = 2 + 2 * len(item)
41
+ elif item is None or type(item) in (bool, int, float):
42
+ if type(item) is int and item.bit_length() > 256:
43
+ raise ProofCapacityError("json_scalar_capacity")
44
+ if type(item) is float and not math.isfinite(item):
45
+ raise ValueError("json_nonfinite")
46
+ cost = 32
47
+ else:
48
+ raise ValueError("json_malformed")
49
+ remaining[0] -= cost
50
+ if remaining[0] < 0:
51
+ raise ProofCapacityError("json_byte_capacity")
52
+ if isinstance(item, dict):
53
+ for key, child in item.items():
54
+ if not isinstance(key, str):
55
+ raise ValueError("json_key_malformed")
56
+ check(key, depth + 1)
57
+ check(child, depth + 1)
58
+ elif isinstance(item, (list, tuple)):
59
+ for child in item:
60
+ check(child, depth + 1)
61
+
62
+ check(value)
63
+ chunks, size = [], 0
64
+ for chunk in json.JSONEncoder(sort_keys=True, separators=(",", ":"),
65
+ ensure_ascii=False, allow_nan=False).iterencode(value):
66
+ size += len(chunk.encode("utf-8"))
67
+ if size > limit:
68
+ raise ProofCapacityError("json_byte_capacity")
69
+ chunks.append(chunk)
70
+ return "".join(chunks)
71
+
72
+
73
+ def digest(value):
74
+ return hashlib.sha256(bounded_json(value).encode("utf-8")).hexdigest()
75
+
76
+
77
+ def _address(addr):
78
+ if not isinstance(addr, str) or not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]{0,6}", addr):
79
+ return False
80
+ col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
81
+ n = 0
82
+ for char in col:
83
+ n = n * 26 + ord(char) - 64
84
+ return n <= 16384 and int(row) <= 1048576
85
+
86
+
87
+ def _definition(cell):
88
+ kind = cell["kind"]
89
+ if kind == "literal":
90
+ return {key: cell[key] for key in ("kind", "text", "value") if key in cell}
91
+ if kind == "formula":
92
+ meta = cell.get("formulaMeta") or {}
93
+ return {"kind": kind, "formula": cell["formula"],
94
+ "formulaMeta": {k: v for k, v in meta.items()
95
+ if k not in ("spill", "inferredFormat")}}
96
+ if kind == "bound":
97
+ return {"kind": kind, "binding": cell["binding"]}
98
+ return {"kind": kind}
99
+
100
+
101
+ def _position(addr):
102
+ if not _address(addr):
103
+ raise ValueError("generated_manifest_invalid")
104
+ col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
105
+ number = 0
106
+ for char in col:
107
+ number = number * 26 + ord(char) - 64
108
+ return number, int(row)
109
+
110
+
111
+ def _cell_address(col, row):
112
+ letters = ""
113
+ while col:
114
+ col, digit = divmod(col - 1, 26)
115
+ letters = chr(65 + digit) + letters
116
+ return letters + str(row)
117
+
118
+
119
+ def _generated_manifest(tab, scope, remaining):
120
+ """Expand production spill extents and placed Grid blocks before accepting their children."""
121
+ expected = {}
122
+ for addr, root in tab["cells"].items():
123
+ if not isinstance(root, dict):
124
+ raise ValueError("cell_malformed")
125
+ meta = root.get("formulaMeta") or {}
126
+ if not isinstance(meta, dict):
127
+ raise ValueError("cell_malformed")
128
+ has_spill = "spill" in root or "spill" in meta
129
+ spill = root.get("spill", meta.get("spill"))
130
+ binding = root.get("binding") or {}
131
+ view = (root.get("kind") == "bound" and isinstance(binding, dict)
132
+ and binding.get("mode", binding.get("kind")) == "view")
133
+ block = binding.get("block") if view and binding.get("viewType") == "grid" else None
134
+ if not has_spill and block is None:
135
+ continue
136
+ col, row = _position(addr)
137
+ extent = spill if has_spill else block
138
+ if (not isinstance(extent, dict) or any(type(extent.get(k)) is not int or extent[k] < 1
139
+ for k in ("rows", "cols"))):
140
+ raise ValueError("generated_manifest_invalid")
141
+ rows, cols = extent["rows"], extent["cols"]
142
+ if col + cols - 1 > 16384 or row + rows - 1 > 1048576:
143
+ raise ValueError("generated_manifest_invalid")
144
+ if rows * cols > remaining - len(expected):
145
+ raise ProofCapacityError("generated_cell_capacity")
146
+ if has_spill:
147
+ if (root.get("kind") != "formula" or set(spill) != {"version", "origin", "rows", "cols", "ref"}
148
+ or type(spill.get("version")) is not int or spill["version"] != 1
149
+ or spill.get("origin") != scope + "!" + addr
150
+ or spill.get("ref") != addr + ":" + _cell_address(col + cols - 1, row + rows - 1)
151
+ or any(key in holder and holder[key] != spill for holder, key in
152
+ ((root, "spill"), (meta, "spill")))):
153
+ raise ValueError("generated_manifest_invalid")
154
+ else:
155
+ fields = binding.get("fields")
156
+ if (rows < 2 or not isinstance(fields, list) or len(fields) != cols
157
+ or any(not isinstance(field, str) or not field for field in fields)
158
+ or len(set(fields)) != len(fields)):
159
+ raise ValueError("generated_manifest_invalid")
160
+ for dy in range(rows):
161
+ for dx in range(cols):
162
+ child_addr = _cell_address(col + dx, row + dy)
163
+ if child_addr in expected:
164
+ raise ValueError("generated_manifest_invalid")
165
+ expected[child_addr] = (("spill", scope + "!" + addr) if has_spill else
166
+ ("view", addr, fields[dx], -1 if dy == 0 else -2 if dy == rows - 1 else dy - 1))
167
+ if child_addr not in tab["cells"]:
168
+ raise ValueError("generated_output_missing")
169
+ return expected
170
+
171
+
172
+ def snapshot(book):
173
+ """Bind authored definitions and revision while censusing every validated output cell."""
174
+ from core import sheets
175
+
176
+ if not isinstance(book, dict) or not isinstance(book.get("sheets"), list) or not book["sheets"]:
177
+ raise ValueError("workbook_incomplete")
178
+ if len(book["sheets"]) > MAX_TABS:
179
+ raise ProofCapacityError("sheet_capacity")
180
+ total = 0
181
+ for tab in book["sheets"]:
182
+ if isinstance(tab, dict) and isinstance(tab.get("cells"), dict):
183
+ total += len(tab["cells"])
184
+ if total > MAX_CELLS:
185
+ raise ProofCapacityError("cell_capacity")
186
+ # Bound the existing document before canonical cleaners copy any nested cell object.
187
+ bounded_json(book)
188
+ census = {"cells": 0, "bound": 0, "formulas": 0, "refusals": []}
189
+ rows, definitions, tabs, names, unresolved = [], [], set(), set(), False
190
+ generated_cells = 0
191
+ for index, tab in enumerate(book["sheets"]):
192
+ if (not isinstance(tab, dict) or not isinstance(tab.get("id"), str) or not tab["id"]
193
+ or tab["id"] in tabs or not isinstance(tab.get("cells"), dict)
194
+ or (tab.get("name") is not None and not isinstance(tab["name"], str))):
195
+ raise ValueError("sheet_incomplete")
196
+ tabs.add(tab["id"])
197
+ if "cellCount" in tab and (type(tab["cellCount"]) is not int or tab["cellCount"] != len(tab["cells"])):
198
+ raise ValueError("sheet_partial")
199
+ if census["cells"] + len(tab["cells"]) > MAX_CELLS:
200
+ raise ProofCapacityError("cell_capacity")
201
+ name = tab.get("name") or ""
202
+ scope = name.upper() if name and name.upper() not in names else (_UNADDRESSABLE_SHEET + str(index)).upper()
203
+ names.add(scope)
204
+ generated = _generated_manifest(tab, scope, MAX_CELLS - generated_cells)
205
+ generated_cells += len(generated)
206
+ for addr, cell in tab["cells"].items():
207
+ if (not _address(addr) or not isinstance(cell, dict)
208
+ or cell.get("kind") not in sheets.CELL_KINDS
209
+ or cell.get("state") not in sheets.CELL_STATES):
210
+ raise ValueError("cell_malformed")
211
+ try:
212
+ sheets.clean_cell(cell, addr)
213
+ except sheets.SheetRefusal as exc:
214
+ raise ValueError("cell_malformed") from exc
215
+ kind = cell["kind"]
216
+ if ((kind == "formula" and (not isinstance(cell.get("formula"), str)
217
+ or not cell["formula"].startswith("=") or len(cell["formula"]) < 2))
218
+ or (kind == "bound" and (not isinstance(cell.get("binding"), dict) or not cell["binding"]))
219
+ or (kind == "literal" and "value" not in cell and "text" not in cell)):
220
+ raise ValueError("cell_malformed")
221
+ census["cells"] += 1
222
+ census["bound"] += kind == "bound"
223
+ census["formulas"] += kind == "formula"
224
+ unresolved |= kind in ("bound", "formula") and cell["state"] not in ("fresh", "refused")
225
+ unresolved |= cell.get("staleReason") == "refresh_held"
226
+ if cell["state"] == "refused":
227
+ refusal, error = cell.get("refusal") or {}, cell.get("error") or {}
228
+ if not isinstance(refusal, dict) or not isinstance(error, dict):
229
+ raise ValueError("refusal_malformed")
230
+ code = refusal.get("code") or error.get("code") or "refused"
231
+ if not isinstance(code, str) or not code:
232
+ raise ValueError("refusal_malformed")
233
+ census["refusals"].append({"cell": "%s!%s" % (name or tab["id"], addr),
234
+ "kind": kind, "code": code})
235
+ authored = cell
236
+ meta = cell.get("formulaMeta") or {}
237
+ marker = meta.get("spillChild")
238
+ declared = generated.get(addr)
239
+ if marker is not None:
240
+ key = scope + "!" + addr
241
+ origin = marker.get("origin") if isinstance(marker, dict) else None
242
+ root_addr = origin.rpartition("!")[2] if isinstance(origin, str) else ""
243
+ root = tab["cells"].get(root_addr, {})
244
+ if (not isinstance(marker, dict) or type(marker.get("version")) is not int
245
+ or marker["version"] != 1 or marker.get("address") != key
246
+ or origin != scope + "!" + root_addr or origin == key
247
+ or cell.get("spillParent") != origin or root.get("kind") != "formula"
248
+ or set(marker) - {"version", "origin", "address", "base"}
249
+ or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
250
+ raise ValueError("cell_malformed")
251
+ base = marker.get("base")
252
+ if base and (not isinstance(base, dict) or base.get("kind") != "empty"
253
+ or any(base.get(k) is not None for k in ("value", "text", "formula", "binding"))):
254
+ raise ValueError("cell_malformed")
255
+ if meta.get("spillClear") is True:
256
+ if (declared is not None or cell.get("value") is not None or cell.get("display") != ""
257
+ or cell["state"] != "typed" or cell.get("refusal") or cell.get("error")):
258
+ raise ValueError("generated_output_invalid")
259
+ elif ("spillClear" in meta or declared != ("spill", origin)
260
+ or cell["state"] not in ("fresh", "refused")):
261
+ raise ValueError("generated_output_invalid")
262
+ authored = base if base else None
263
+ elif "spillParent" in cell or "spillClear" in meta:
264
+ raise ValueError("cell_malformed")
265
+ view = cell.get("viewChild")
266
+ if view is not None:
267
+ root = tab["cells"].get(view.get("origin"), {}) if isinstance(view, dict) else {}
268
+ binding = root.get("binding") or {}
269
+ if (marker is not None or not isinstance(view, dict) or not _address(view.get("origin"))
270
+ or view["origin"] == addr or not isinstance(view.get("field"), str) or not view["field"]
271
+ or type(view.get("line")) is not int or view["line"] < -2
272
+ or root.get("kind") != "bound" or not isinstance(binding, dict)
273
+ or (binding.get("mode") != "view" and binding.get("kind") != "view")
274
+ or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
275
+ raise ValueError("cell_malformed")
276
+ if declared != ("view", view["origin"], view["field"], view["line"]):
277
+ raise ValueError("generated_output_invalid")
278
+ authored = None
279
+ if declared is not None:
280
+ is_root = (declared[1] == (scope + "!" + addr if declared[0] == "spill" else addr))
281
+ if not is_root and ((declared[0] == "spill" and marker is None)
282
+ or (declared[0] == "view" and view is None)):
283
+ raise ValueError("generated_output_invalid")
284
+ if authored is not None:
285
+ rows.append({"sheet": tab["id"], "address": addr, "kind": authored["kind"]})
286
+ definitions.append({"sheet": tab["id"], "address": addr, "definition": _definition(authored)})
287
+ if census["cells"] and not rows:
288
+ raise ValueError("workbook_empty")
289
+ rows.sort(key=lambda row: (row["sheet"], row["address"]))
290
+ definitions.sort(key=lambda row: (row["sheet"], row["address"]))
291
+ authored = digest({"sheets": [(tab["id"], tab.get("name")) for tab in book["sheets"]],
292
+ "cells": definitions, "namedRanges": book.get("namedRanges"),
293
+ "dateSystem": book.get("dateSystem")})
294
+ return {"revision": book.get("rev"), "inventory": rows, "authored": authored,
295
+ "sheets": sorted(tabs), "census": census, "unresolved": bool(unresolved)}
platform/harness/datastore.py CHANGED
@@ -1174,7 +1174,263 @@ _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
1174
 
1175
  #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
1176
  _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
1177
- _ORDINAL_ORDER = re.compile(r"(?:^|,)\s*\d+\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?\s*(?:,|$)", re.I)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1178
 
1179
 
1180
  def mirror_file_marks(cur):
@@ -1275,6 +1531,7 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
1275
  the page's rows before the projection runs, when only the projection holds a subquery.
1276
  `page_key` opts a physical table into a narrow, materialised key page before projection.
1277
  It must uniquely identify source rows, independently of the displayed record identity.
 
1278
  """
1279
  if bool(table) == bool(from_sql):
1280
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
@@ -1310,14 +1567,19 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
1310
  total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
1311
  grp = str(group_by or "").strip()
1312
  fold = f" GROUP BY {grp}" if grp else ""
1313
- if page_first and page_key and table and not grp and not _ORDINAL_ORDER.search(order):
 
 
 
 
 
1314
  # Only keys cross the materialisation boundary; identity order still uses the full population.
1315
  got = cur.execute(
1316
  f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
1317
  f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
1318
  f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
1319
  f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
1320
- elif (page_first and not page_key and table and not grp and _SUBQUERY.search(proj)
1321
  and not _SUBQUERY.search(f"{pred} {order}")):
1322
  # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
1323
  # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
 
1174
 
1175
  #: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
1176
  _SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
1177
+ _ORDINAL_ORDER = re.compile(
1178
+ r"(?:\(\s*)*[+-]?\d[\d_]*(?:\s*\))*\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
1179
+ _ALL_ORDER = re.compile(r"ALL\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
1180
+
1181
+
1182
+ def _sql_terms(sql):
1183
+ """Split trusted SQL at unquoted, top-level commas, or decline unfamiliar syntax."""
1184
+ depth, quote, start, index = 0, None, 0, 0
1185
+ terms = []
1186
+ while index < len(sql):
1187
+ char = sql[index]
1188
+ if quote:
1189
+ if char == "\\":
1190
+ return None
1191
+ if char == quote:
1192
+ if index + 1 < len(sql) and sql[index + 1] == quote:
1193
+ index += 2
1194
+ continue
1195
+ quote = None
1196
+ elif char in "'\"":
1197
+ quote = char
1198
+ elif (not (char.isalnum() or char.isspace() or char in "_.,()+-*/%=<>!|:&~^")
1199
+ or sql[index:index + 2] in ("--", "/*", "*/", "->", "=>", ":=")):
1200
+ return None
1201
+ elif char == "(":
1202
+ depth += 1
1203
+ elif char == ")":
1204
+ depth -= 1
1205
+ if depth < 0:
1206
+ return None
1207
+ elif char == "," and depth == 0:
1208
+ terms.append(sql[start:index].strip())
1209
+ start = index + 1
1210
+ index += 1
1211
+ if quote or depth:
1212
+ return None
1213
+ terms.append(sql[start:].strip())
1214
+ return terms if all(terms) else None
1215
+
1216
+
1217
+ def _sql_tokens(sql):
1218
+ """Use DuckDB's lexer for identifier quoting and expression comparisons."""
1219
+ tokens = duckdb.tokenize(sql)
1220
+ encoded = sql.encode("utf-8")
1221
+ result = [(encoded[start:tokens[i + 1][0] if i + 1 < len(tokens) else len(encoded)].decode("utf-8").strip(), kind)
1222
+ for i, (start, kind) in enumerate(tokens)]
1223
+ if any(text.upper() in ("COLUMNS", "EXCLUDE", "REPLACE", "USING") for text, _kind in result):
1224
+ raise ValueError("Unrecognized page ordering syntax")
1225
+ return result
1226
+
1227
+
1228
+ def _order_needs_projection(order, projection=""):
1229
+ """Keep positional, ALL, alias-dependent and unfamiliar orders after projection."""
1230
+ terms = _sql_terms(order)
1231
+ if terms is None or any(_ORDINAL_ORDER.fullmatch(t) or _ALL_ORDER.fullmatch(t) for t in terms):
1232
+ return True
1233
+ selected = _sql_terms(projection) if projection else []
1234
+ if selected is None:
1235
+ return True
1236
+ try:
1237
+ fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
1238
+ def normalized(tokens):
1239
+ return [(text[1:-1].replace('""', '"').translate(fold) if text.startswith('"')
1240
+ else text if text.startswith("'") else text.translate(fold), kind) for text, kind in tokens]
1241
+ ordered = normalized(_sql_tokens(order))
1242
+ aliases = {}
1243
+ for term in selected:
1244
+ tokens = _sql_tokens(term)
1245
+ depth, top = 0, []
1246
+ for i, (text, _kind) in enumerate(tokens):
1247
+ if text == ")":
1248
+ depth -= 1
1249
+ if depth == 0:
1250
+ top.append(i)
1251
+ if text == "(":
1252
+ depth += 1
1253
+ if not top or any(tokens[i][0] in ("*", ":", ":=") for i in top):
1254
+ return True
1255
+ last = top[-1]
1256
+ if len(top) >= 2 and tokens[top[-2]][0].upper() == "AS":
1257
+ expression = tokens[:top[-2]]
1258
+ elif (last > 0 and tokens[last][1] == duckdb.token_type.identifier
1259
+ and tokens[last - 1][0] != "."):
1260
+ expression = tokens[:last]
1261
+ else:
1262
+ continue
1263
+ alias = normalized([tokens[last]])[0][0]
1264
+ if tokens[last][0].startswith("'"):
1265
+ alias = tokens[last][0][1:-1].replace("''", "'").translate(fold)
1266
+ aliases.setdefault(alias, []).append(normalized(expression))
1267
+ for alias, expressions in aliases.items():
1268
+ covered = set()
1269
+ # A repeated source expression is safe, including generated casts and passthrough aliases.
1270
+ if all(expr == expressions[0] for expr in expressions):
1271
+ expr = expressions[0]
1272
+ for i in range(len(ordered) - len(expr) + 1):
1273
+ if expr and ordered[i:i + len(expr)] == expr:
1274
+ covered.update(range(i, i + len(expr)))
1275
+ for i, token in enumerate(ordered):
1276
+ if (token[0] == alias and token[1] in (duckdb.token_type.identifier, duckdb.token_type.keyword)
1277
+ and i not in covered
1278
+ and (i == 0 or ordered[i - 1][0] != ".")
1279
+ and (i + 1 == len(ordered) or ordered[i + 1][0] not in (".", "("))):
1280
+ return True
1281
+ except (duckdb.Error, AttributeError, ValueError, UnicodeError):
1282
+ return True
1283
+ return False
1284
+
1285
+
1286
+ def _physical_page_order(order, page_key):
1287
+ """Make physical pages total without changing ALL or unfamiliar SQL ordering semantics."""
1288
+ terms = _sql_terms(order)
1289
+ if terms is None or any(_ALL_ORDER.fullmatch(term) for term in terms):
1290
+ return order
1291
+ try:
1292
+ _sql_tokens(order)
1293
+ except (duckdb.Error, AttributeError, ValueError):
1294
+ return order
1295
+ return f"{order}, {page_key} ASC"
1296
+
1297
+
1298
+ def _page_is_row_local(projection, order, cur):
1299
+ """Prove outer expressions scalar with DuckDB's parser and bounded catalogue lookups."""
1300
+ metadata, verified, expanding, bound = {}, set(), set(), set()
1301
+
1302
+ def parse(expressions, ordering=""):
1303
+ sql = "SELECT " + expressions + (" ORDER BY " + ordering if ordering else "")
1304
+ parsed = json.loads(cur.execute("SELECT json_serialize_sql(?)", [sql]).fetchone()[0])
1305
+ if parsed.get("error") or len(parsed["statements"]) != 1:
1306
+ raise ValueError("Unrecognized page expression")
1307
+ node = parsed["statements"][0]["node"]
1308
+ if (node["type"] != "SELECT_NODE" or node["from_table"]["type"] != "EMPTY"
1309
+ or node["cte_map"]["map"] or node["group_expressions"] or node["group_sets"]
1310
+ or node["aggregate_handling"] != "STANDARD_HANDLING"
1311
+ or any(node[key] is not None for key in ("where_clause", "having", "sample", "qualify"))
1312
+ or any(m["type"] != "ORDER_MODIFIER" for m in node["modifiers"])):
1313
+ raise ValueError("Projection changes the outer population")
1314
+ return [node["select_list"], node["modifiers"]]
1315
+
1316
+ def aggregate_is_local(value, scope):
1317
+ columns = set()
1318
+
1319
+ def collect(node):
1320
+ if isinstance(node, list):
1321
+ return all(collect(v) for v in node)
1322
+ if not isinstance(node, dict):
1323
+ return True
1324
+ kind = node.get("class")
1325
+ if kind == "COLUMN_REF":
1326
+ columns.add(tuple(node["column_names"]))
1327
+ elif kind is not None and kind not in {
1328
+ "CONSTANT", "CAST", "CASE", "COMPARISON", "CONJUNCTION",
1329
+ "OPERATOR", "FUNCTION", "PARAMETER"}:
1330
+ return False
1331
+ return all(collect(v) for v in node.values())
1332
+
1333
+ if not collect(value):
1334
+ return False
1335
+ if not columns:
1336
+ return True
1337
+ if scope is None or scope[1] or scope[0]["type"] != "SELECT_NODE":
1338
+ return False
1339
+ source = scope[0]["from_table"]
1340
+ if (source["type"] != "BASE_TABLE" or source.get("column_name_alias")
1341
+ or source.get("at_clause") or source.get("sample")):
1342
+ return False
1343
+ alias = source.get("alias") or source["table_name"]
1344
+ fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
1345
+ if any(len(c) != 2 or c[0].translate(fold) != alias.translate(fold) for c in columns):
1346
+ return False
1347
+ quote = lambda name: '"' + name.replace('"', '""') + '"'
1348
+ table = '.'.join(quote(source[key]) for key in ("catalog_name", "schema_name", "table_name")
1349
+ if source.get(key))
1350
+ selected = ','.join('.'.join(map(quote, c)) for c in sorted(columns))
1351
+ sql = f"DESCRIBE SELECT {selected} FROM {table} AS {quote(alias)}"
1352
+ if sql not in bound:
1353
+ # A matching alias alone cannot prove that a missing column will not bind outward.
1354
+ cur.execute(sql).fetchall()
1355
+ bound.add(sql)
1356
+ return True
1357
+
1358
+ def references(value, required, nested, scalar=True, scope=None):
1359
+ if isinstance(value, list):
1360
+ return all(references(v, required, nested, scalar, scope) for v in value)
1361
+ if not isinstance(value, dict):
1362
+ return True
1363
+ if "cte_map" in value:
1364
+ scope = (value, bool(value["cte_map"]["map"]) or bool(scope and scope[1]))
1365
+ kind = value.get("class")
1366
+ if scalar and kind == "SUBQUERY":
1367
+ # Nested aggregates must bind locally even inside a syntactic scalar subquery.
1368
+ return (value["subquery_type"] == "SCALAR" and value.get("child") is None
1369
+ and references(value["subquery"], required, nested, False, scope))
1370
+ if scalar and kind is not None:
1371
+ if kind not in {"COLUMN_REF", "CONSTANT", "CAST", "CASE", "COMPARISON",
1372
+ "CONJUNCTION", "OPERATOR", "FUNCTION", "PARAMETER"}:
1373
+ return False
1374
+ if kind == "OPERATOR" and value["type"] not in {
1375
+ "OPERATOR_COALESCE", "OPERATOR_NOT", "OPERATOR_IS_NULL",
1376
+ "OPERATOR_IS_NOT_NULL", "COMPARE_IN", "COMPARE_NOT_IN"}:
1377
+ return False
1378
+ if "function_name" in value:
1379
+ if value.get("catalog") or value.get("schema", "") not in ("", "main"):
1380
+ return False
1381
+ if scalar and (value.get("distinct") or value.get("filter") is not None
1382
+ or value.get("export_state") or value.get("order_bys", {}).get("orders")):
1383
+ return False
1384
+ name = value["function_name"].lower()
1385
+ required[name] = required.get(name, False) or scalar
1386
+ if not scalar:
1387
+ nested.append((name, value, scope))
1388
+ return all(references(v, required, nested, scalar, scope) for v in value.values())
1389
+
1390
+ def prove(expressions, ordering=""):
1391
+ required, nested = {}, []
1392
+ if not references(parse(expressions, ordering), required, nested):
1393
+ return False
1394
+ missing = sorted(set(required) - metadata.keys())
1395
+ if missing:
1396
+ rows = cur.execute(
1397
+ "SELECT lower(function_name), function_type, internal, macro_definition "
1398
+ "FROM duckdb_functions() WHERE lower(function_name) = ANY(?)", [missing]).fetchall()
1399
+ metadata.update((name, []) for name in missing)
1400
+ for name, function_type, internal, definition in rows:
1401
+ metadata[name].append((function_type, internal, definition))
1402
+ for name, scalar in required.items():
1403
+ if (name, scalar) in verified:
1404
+ continue
1405
+ if not metadata[name] or name in expanding or len(expanding) >= 16:
1406
+ return False
1407
+ for function_type, internal, definition in metadata[name]:
1408
+ if function_type == "macro":
1409
+ # Only engine-owned macros with independently scalar bodies are admitted.
1410
+ if not internal or not definition:
1411
+ return False
1412
+ expanding.add(name)
1413
+ try:
1414
+ if not prove(definition):
1415
+ return False
1416
+ finally:
1417
+ expanding.remove(name)
1418
+ elif function_type not in ({"scalar"} if scalar else {"scalar", "aggregate", "table"}):
1419
+ return False
1420
+ verified.add((name, scalar))
1421
+ for name, value, scope in nested:
1422
+ if (value.get("class") != "WINDOW"
1423
+ and any(kind == "aggregate" for kind, _internal, _definition in metadata[name])
1424
+ and not aggregate_is_local(value, scope)):
1425
+ return False
1426
+ return True
1427
+
1428
+ try:
1429
+ return prove(projection, order)
1430
+ except Exception as exc:
1431
+ # A parser/catalogue refusal never authorizes moving the original projection.
1432
+ _request_timing.raise_if_resource_error(exc)
1433
+ return False
1434
 
1435
 
1436
  def mirror_file_marks(cur):
 
1531
  the page's rows before the projection runs, when only the projection holds a subquery.
1532
  `page_key` opts a physical table into a narrow, materialised key page before projection.
1533
  It must uniquely identify source rows, independently of the displayed record identity.
1534
+ Proven row-local pages end with this physical key ascending. Fallback SQL retains its order.
1535
  """
1536
  if bool(table) == bool(from_sql):
1537
  raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
 
1567
  total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
1568
  grp = str(group_by or "").strip()
1569
  fold = f" GROUP BY {grp}" if grp else ""
1570
+ needs_projection = _order_needs_projection(order, proj) if page_first else True
1571
+ row_local = (page_first and table and not grp and not needs_projection
1572
+ and _page_is_row_local(proj, order, cur))
1573
+ if row_local and page_key:
1574
+ order = _physical_page_order(order, page_key)
1575
+ if row_local and page_key:
1576
  # Only keys cross the materialisation boundary; identity order still uses the full population.
1577
  got = cur.execute(
1578
  f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
1579
  f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
1580
  f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
1581
  f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
1582
+ elif (row_local and not page_key and _SUBQUERY.search(proj)
1583
  and not _SUBQUERY.search(f"{pred} {order}")):
1584
  # A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
1585
  # of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
requirements.txt CHANGED
@@ -21,7 +21,7 @@ python-dotenv>=1.2.2
21
  pandas>=2.0
22
  requests>=2.28
23
  huggingface_hub>=0.20
24
- duckdb>=1.5
25
  pyyaml>=6.0
26
  pillow>=10.0
27
  beautifulsoup4>=4.12
 
21
  pandas>=2.0
22
  requests>=2.28
23
  huggingface_hub>=0.20
24
+ duckdb==1.5.4
25
  pyyaml>=6.0
26
  pillow>=10.0
27
  beautifulsoup4>=4.12
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 285bbf5c91b135a7238573fe2243bf1b895d71df04bb4ad9c04f4d49c8b31fc3
 
1
+ 4a7e48c0f21990c3c89f863bcfd80f31584284e596c9fe0b8f9a9c0757214665