Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +75 -13
- api/routes_odoo_tables.py +146 -152
- api/routes_tables.py +6 -1
- api/sheet_warm.py +147 -2
- platform/core/refresh_proof.py +295 -0
- platform/harness/datastore.py +265 -3
- requirements.txt +1 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "52ab0bfe7b",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
52ab0bfe7b
|
api/amazon_relational.py
CHANGED
|
@@ -238,8 +238,14 @@ def _format(col, duck_type):
|
|
| 238 |
return None
|
| 239 |
|
| 240 |
|
| 241 |
-
def mirror_columns(cur, table):
|
| 242 |
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
try:
|
| 244 |
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
|
| 245 |
except Exception: # noqa: BLE001
|
|
@@ -369,7 +375,7 @@ def _parent_lookup_sql(cur):
|
|
| 369 |
return resolved
|
| 370 |
|
| 371 |
|
| 372 |
-
def lookups(cur, key, names=None):
|
| 373 |
"""`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
|
| 374 |
|
| 375 |
`expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
|
|
@@ -385,6 +391,23 @@ def lookups(cur, key, names=None):
|
|
| 385 |
sku = sku if sku in names else None
|
| 386 |
if not asin and not sku:
|
| 387 |
return []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
|
| 389 |
parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
|
| 390 |
latest = _latest_listings_sql(listing_names)
|
|
@@ -529,14 +552,15 @@ def _quoted(name):
|
|
| 529 |
AMAZON_STORED_BYTES = 4 * 1024 * 1024
|
| 530 |
|
| 531 |
|
| 532 |
-
def _read_through(cur, key, count):
|
| 533 |
"""Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
|
| 534 |
ut = _odoo._ut()
|
| 535 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 536 |
return True
|
| 537 |
if not count:
|
| 538 |
return False
|
| 539 |
-
names = [name for name, _dtype in
|
|
|
|
| 540 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
| 541 |
# Include the stored id, refreshed stamp and row-object punctuation, with the same
|
| 542 |
# ASCII escaping as the document writer. The largest sampled row is conservative.
|
|
@@ -590,9 +614,9 @@ def _integer(value):
|
|
| 590 |
return None if value in (None, "") else int(value)
|
| 591 |
|
| 592 |
|
| 593 |
-
def grid_source(cur, key):
|
| 594 |
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
|
| 595 |
-
columns = mirror_columns(cur, SOURCE[key])
|
| 596 |
if not columns:
|
| 597 |
return missing_source(key)
|
| 598 |
names = {name for name, _dtype in columns}
|
|
@@ -645,7 +669,7 @@ def grid_source(cur, key):
|
|
| 645 |
cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
|
| 646 |
# S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
|
| 647 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 648 |
-
for lookup in lookups(cur, key, names):
|
| 649 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 650 |
spec = {"table": SOURCE[key], "where": "", "id": ident,
|
| 651 |
"needs_excluded": False, "cols": cols}
|
|
@@ -654,6 +678,19 @@ def grid_source(cur, key):
|
|
| 654 |
return spec
|
| 655 |
|
| 656 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 657 |
def grid_sources(cur=None):
|
| 658 |
"""Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
|
| 659 |
own_cursor = cur is None
|
|
@@ -773,7 +810,7 @@ def wall_link_expression(source, predicate, identity):
|
|
| 773 |
f"WHERE {predicate}) AS _wall_links)")
|
| 774 |
|
| 775 |
|
| 776 |
-
def fill_link_cells(rows, fields, read_targets):
|
| 777 |
"""Populate derived Amazon links with one read per distinct target on this page.
|
| 778 |
|
| 779 |
The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
|
|
@@ -782,17 +819,42 @@ def fill_link_cells(rows, fields, read_targets):
|
|
| 782 |
Neither path stores history rows or caps the matching target set.
|
| 783 |
"""
|
| 784 |
edges = link_edges(fields)
|
| 785 |
-
requests = {}
|
| 786 |
for _field, target, frm, on in edges:
|
| 787 |
values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
|
| 788 |
requests.setdefault(target, {}).setdefault(on, set()).update(values)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
indexes = {}
|
|
|
|
| 790 |
for target, matches in requests.items():
|
| 791 |
indexes[target] = {column: {} for column in matches}
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 796 |
for field, target, frm, on in edges:
|
| 797 |
for row in rows:
|
| 798 |
row[field["key"]] = ",".join(str(pid) for pid in sorted(
|
|
|
|
| 238 |
return None
|
| 239 |
|
| 240 |
|
| 241 |
+
def mirror_columns(cur, table, *, strict=False):
|
| 242 |
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
| 243 |
+
if strict:
|
| 244 |
+
# A successful empty metadata query proves absence on this cursor only.
|
| 245 |
+
return cur.execute(
|
| 246 |
+
"SELECT column_name, data_type FROM information_schema.columns "
|
| 247 |
+
"WHERE table_catalog = current_database() AND table_schema = current_schema() "
|
| 248 |
+
"AND table_name = ? ORDER BY ordinal_position", [table]).fetchall()
|
| 249 |
try:
|
| 250 |
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
|
| 251 |
except Exception: # noqa: BLE001
|
|
|
|
| 375 |
return resolved
|
| 376 |
|
| 377 |
|
| 378 |
+
def lookups(cur, key, names=None, *, allowed=True):
|
| 379 |
"""`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
|
| 380 |
|
| 381 |
`expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
|
|
|
|
| 391 |
sku = sku if sku in names else None
|
| 392 |
if not asin and not sku:
|
| 393 |
return []
|
| 394 |
+
if not allowed:
|
| 395 |
+
# Keep declared cells blank without inspecting a denied lookup dependency.
|
| 396 |
+
keys = []
|
| 397 |
+
if asin:
|
| 398 |
+
if "parent_asin" not in names:
|
| 399 |
+
keys.append((LOOKUP_PARENT, asin, asin_via))
|
| 400 |
+
if not sku:
|
| 401 |
+
keys.append((LOOKUP_SKU, asin, asin_via))
|
| 402 |
+
elif sku:
|
| 403 |
+
if not names & _NATIVE_ASIN:
|
| 404 |
+
keys.append((LOOKUP_ASIN, sku, sku_via))
|
| 405 |
+
if "parent_asin" not in names:
|
| 406 |
+
keys.append((LOOKUP_PARENT, sku, sku_via))
|
| 407 |
+
return [{"key": name, "label": _LOOKUP_LABELS[name], "after": after,
|
| 408 |
+
"expr": "CAST(NULL AS VARCHAR)", "opens": {"via": via} if via else None,
|
| 409 |
+
"description": _LOOKUP_DESCRIPTIONS[name]}
|
| 410 |
+
for name, after, via in keys if name not in names]
|
| 411 |
listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
|
| 412 |
parents = None if "parent_asin" in names else _parent_lookup_sql(cur)
|
| 413 |
latest = _latest_listings_sql(listing_names)
|
|
|
|
| 552 |
AMAZON_STORED_BYTES = 4 * 1024 * 1024
|
| 553 |
|
| 554 |
|
| 555 |
+
def _read_through(cur, key, count, *, columns=None):
|
| 556 |
"""Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
|
| 557 |
ut = _odoo._ut()
|
| 558 |
if key in READ_THROUGH or count > ut.MAX_ROWS:
|
| 559 |
return True
|
| 560 |
if not count:
|
| 561 |
return False
|
| 562 |
+
names = [name for name, _dtype in (columns if columns is not None
|
| 563 |
+
else mirror_columns(cur, SOURCE[key]))]
|
| 564 |
sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
|
| 565 |
# Include the stored id, refreshed stamp and row-object punctuation, with the same
|
| 566 |
# ASCII escaping as the document writer. The largest sampled row is conservative.
|
|
|
|
| 614 |
return None if value in (None, "") else int(value)
|
| 615 |
|
| 616 |
|
| 617 |
+
def grid_source(cur, key, *, columns=None, allow_lookups=True):
|
| 618 |
"""One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
|
| 619 |
+
columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
|
| 620 |
if not columns:
|
| 621 |
return missing_source(key)
|
| 622 |
names = {name for name, _dtype in columns}
|
|
|
|
| 669 |
cols[name] = (expr, _number if fractional else {"int": _integer}.get(kind, _text))
|
| 670 |
# S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
|
| 671 |
# group (count by parent) and a sort compile against them in SQL with no per-row Python.
|
| 672 |
+
for lookup in lookups(cur, key, names, allowed=allow_lookups):
|
| 673 |
cols[lookup["key"]] = (lookup["expr"], _text)
|
| 674 |
spec = {"table": SOURCE[key], "where": "", "id": ident,
|
| 675 |
"needs_excluded": False, "cols": cols}
|
|
|
|
| 678 |
return spec
|
| 679 |
|
| 680 |
|
| 681 |
+
def page_source(cur, key, definition=None, *, allow_lookups=True):
|
| 682 |
+
"""Resolve one target's storage policy on the admitted cursor without provider discovery."""
|
| 683 |
+
columns = mirror_columns(cur, SOURCE[key], strict=True)
|
| 684 |
+
mandatory = key in READ_THROUGH or bool((definition or {}).get("readThrough"))
|
| 685 |
+
if not columns:
|
| 686 |
+
return missing_source(key) if mandatory else None
|
| 687 |
+
if not mandatory:
|
| 688 |
+
count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 689 |
+
if not _read_through(cur, key, count, columns=columns):
|
| 690 |
+
return None
|
| 691 |
+
return grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
def grid_sources(cur=None):
|
| 695 |
"""Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
|
| 696 |
own_cursor = cur is None
|
|
|
|
| 810 |
f"WHERE {predicate}) AS _wall_links)")
|
| 811 |
|
| 812 |
|
| 813 |
+
def fill_link_cells(rows, fields, read_targets, *, max_bytes=None, capacity_refusal=None):
|
| 814 |
"""Populate derived Amazon links with one read per distinct target on this page.
|
| 815 |
|
| 816 |
The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
|
|
|
|
| 819 |
Neither path stores history rows or caps the matching target set.
|
| 820 |
"""
|
| 821 |
edges = link_edges(fields)
|
| 822 |
+
requests, weights = {}, {}
|
| 823 |
for _field, target, frm, on in edges:
|
| 824 |
values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
|
| 825 |
requests.setdefault(target, {}).setdefault(on, set()).update(values)
|
| 826 |
+
for row in rows:
|
| 827 |
+
if row.get(frm) not in (None, ""):
|
| 828 |
+
slot = (target, on, str(row[frm]))
|
| 829 |
+
weights[slot] = weights.get(slot, 0) + 1
|
| 830 |
indexes = {}
|
| 831 |
+
remaining = max_bytes
|
| 832 |
for target, matches in requests.items():
|
| 833 |
indexes[target] = {column: {} for column in matches}
|
| 834 |
+
iterator = iter(read_targets(target, matches) if any(matches.values()) else ())
|
| 835 |
+
try:
|
| 836 |
+
for row in iterator:
|
| 837 |
+
pid = int(row["pid"])
|
| 838 |
+
for column, wanted in matches.items():
|
| 839 |
+
value = str(row[column]) if row.get(column) not in (None, "") else None
|
| 840 |
+
if value not in wanted:
|
| 841 |
+
continue
|
| 842 |
+
bucket = indexes[target][column].get(value)
|
| 843 |
+
if bucket is not None and pid in bucket:
|
| 844 |
+
continue
|
| 845 |
+
if remaining is not None:
|
| 846 |
+
# Charge every output occurrence before allocating its distinct ID.
|
| 847 |
+
cost = (len(str(pid)) + (1 if bucket else 2)) * weights[target, column, value]
|
| 848 |
+
if cost > remaining:
|
| 849 |
+
raise capacity_refusal()
|
| 850 |
+
remaining -= cost
|
| 851 |
+
if bucket is None:
|
| 852 |
+
bucket = indexes[target][column][value] = set()
|
| 853 |
+
bucket.add(pid)
|
| 854 |
+
finally:
|
| 855 |
+
close = getattr(iterator, "close", None)
|
| 856 |
+
if close is not None:
|
| 857 |
+
close()
|
| 858 |
for field, target, frm, on in edges:
|
| 859 |
for row in rows:
|
| 860 |
row[field["key"]] = ",".join(str(pid) for pid in sorted(
|
api/routes_odoo_tables.py
CHANGED
|
@@ -778,25 +778,19 @@ GRID_SOURCES = {}
|
|
| 778 |
_SOURCE_PROVIDERS = []
|
| 779 |
_SOURCE_ANALYSES = {}
|
| 780 |
|
| 781 |
-
#
|
| 782 |
-
#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
|
| 783 |
-
#: keys to the shared dict as it went, and returned the dict to any caller that found it
|
| 784 |
-
#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
|
| 785 |
-
#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
|
| 786 |
-
#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
|
| 787 |
-
#: this lock and publishes it in one `dict.update`; a reader waits on the lock until
|
| 788 |
-
#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
|
| 789 |
-
#: Amazon fallback) happens only after a build has been published, so it can never stand in
|
| 790 |
-
#: for one either.
|
| 791 |
_SOURCES_LOCK = threading.RLock()
|
| 792 |
_SOURCES_BUILT = False
|
| 793 |
-
_SOURCES_BUILDER = None #
|
|
|
|
|
|
|
| 794 |
|
| 795 |
|
| 796 |
def invalidate_sources():
|
| 797 |
"""Forget measured bindings after a connector publishes a new mirror schema."""
|
| 798 |
-
global _SOURCES_BUILT
|
| 799 |
with _SOURCES_LOCK:
|
|
|
|
| 800 |
_SOURCES_BUILT = False
|
| 801 |
GRID_SOURCES.clear()
|
| 802 |
|
|
@@ -848,54 +842,54 @@ def _sources():
|
|
| 848 |
importing that module at file-import time would drag the Odoo layer into every process that
|
| 849 |
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
|
| 850 |
|
| 851 |
-
|
| 852 |
-
lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
|
| 853 |
-
and so does one that holds only admissions made since the last invalidation.
|
| 854 |
"""
|
| 855 |
-
global _SOURCES_BUILT
|
| 856 |
if _SOURCES_BUILT and GRID_SOURCES:
|
| 857 |
return GRID_SOURCES
|
| 858 |
with _SOURCES_LOCK:
|
| 859 |
-
if (_SOURCES_BUILT and GRID_SOURCES) or
|
| 860 |
return GRID_SOURCES
|
| 861 |
-
|
| 862 |
-
|
| 863 |
-
|
| 864 |
-
|
| 865 |
-
|
| 866 |
-
|
| 867 |
-
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
# this build, so it stands; everything else comes from the build. ONE update, so a
|
| 877 |
-
# lock-free reader of the dict sees the old entry or the new one, never a gap.
|
| 878 |
GRID_SOURCES.update({**built, **GRID_SOURCES})
|
| 879 |
_SOURCES_BUILT = True
|
| 880 |
-
|
| 881 |
-
|
|
|
|
|
|
|
| 882 |
return GRID_SOURCES
|
| 883 |
|
| 884 |
|
| 885 |
-
def _learn_amazon_sources(cur=None):
|
| 886 |
-
"""
|
| 887 |
-
|
|
|
|
|
|
|
|
|
|
| 888 |
if cur is None:
|
| 889 |
extra = _amazon_sources()
|
| 890 |
else:
|
| 891 |
import amazon_relational
|
| 892 |
extra = amazon_relational.grid_sources(cur)
|
| 893 |
-
if not extra:
|
| 894 |
-
return
|
| 895 |
with _SOURCES_LOCK:
|
| 896 |
-
|
| 897 |
-
|
| 898 |
GRID_SOURCES.update(extra)
|
|
|
|
| 899 |
_ut().register_connected(*extra)
|
| 900 |
|
| 901 |
|
|
@@ -1298,13 +1292,7 @@ def _amazon_sources():
|
|
| 1298 |
|
| 1299 |
|
| 1300 |
def require_amazon_mirror(table_key, session, definition=None, st=None):
|
| 1301 |
-
"""
|
| 1302 |
-
|
| 1303 |
-
The provider contributes no keys on an Odoo-only mirror. A persisted read-through
|
| 1304 |
-
definition still must refuse there instead of falling through to its empty stored rows.
|
| 1305 |
-
S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
|
| 1306 |
-
window route stops paying a second definitions projection for this one check.
|
| 1307 |
-
"""
|
| 1308 |
if not str(table_key).startswith("ut_amzn_"):
|
| 1309 |
return
|
| 1310 |
import amazon_relational as amazon
|
|
@@ -1322,28 +1310,7 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
|
|
| 1322 |
if mismatch:
|
| 1323 |
raise err(409, "cross_tenant_store", mismatch)
|
| 1324 |
try:
|
| 1325 |
-
|
| 1326 |
-
if (spec is not None or definition.get("readThrough")
|
| 1327 |
-
or table_key in amazon.READ_THROUGH):
|
| 1328 |
-
# Check current table presence even if a binding was cached before a restart.
|
| 1329 |
-
current = _page_source_for(cur, table_key, st)
|
| 1330 |
-
if current is None:
|
| 1331 |
-
current = amazon.grid_source(cur, table_key)
|
| 1332 |
-
if current.get("missing"):
|
| 1333 |
-
raise err(503, "amazon_mirror_not_loaded", current["missing"])
|
| 1334 |
-
# A dynamically oversized snapshot may have a durable stub before this process
|
| 1335 |
-
# has measured it. Admit its binding without changing another source's registry.
|
| 1336 |
-
# ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
|
| 1337 |
-
# key. Admitted into an empty or Amazon-less registry, this one key used to make
|
| 1338 |
-
# `_sources()` skip its build, so the grid resolved while its link targets
|
| 1339 |
-
# (orders, read-through) were dispatched as stored tables: a first-page 500.
|
| 1340 |
-
if spec is None or spec.get("missing"):
|
| 1341 |
-
_learn_amazon_sources(cur)
|
| 1342 |
-
with _SOURCES_LOCK:
|
| 1343 |
-
_sources()
|
| 1344 |
-
held = GRID_SOURCES.get(table_key)
|
| 1345 |
-
if held is None or held.get("missing"):
|
| 1346 |
-
GRID_SOURCES[table_key] = current
|
| 1347 |
finally:
|
| 1348 |
cur.close()
|
| 1349 |
|
|
@@ -1559,7 +1526,27 @@ def _degrade(expr, have_by_alias, default_alias=""):
|
|
| 1559 |
return expr if bare.lower() in have else "NULL"
|
| 1560 |
|
| 1561 |
|
| 1562 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1563 |
"""The resolved spec for one connected grid, or None when this grid is not read-through yet.
|
| 1564 |
|
| 1565 |
⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
|
|
@@ -1586,25 +1573,15 @@ def _source_for(cur, table_key):
|
|
| 1586 |
# table and turns Products ordered into the wrong 404.
|
| 1587 |
if module_cache_source(key) is not None:
|
| 1588 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1589 |
spec = _sources().get(key)
|
| 1590 |
-
if (not spec or spec.get("missing")) and key.startswith("ut_amzn_"):
|
| 1591 |
-
# HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
|
| 1592 |
-
# Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
|
| 1593 |
-
# runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
|
| 1594 |
-
# read-through Amazon grid answered "no read-through binding". S70-T12 re-asks the
|
| 1595 |
-
# complete Amazon provider on this restored cursor, including cached missing entries.
|
| 1596 |
-
_learn_amazon_sources(cur)
|
| 1597 |
-
spec = GRID_SOURCES.get(key)
|
| 1598 |
if not spec:
|
| 1599 |
return None
|
| 1600 |
if spec.get("missing"):
|
| 1601 |
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1602 |
-
if key.startswith("ut_amzn_"):
|
| 1603 |
-
import amazon_relational as amazon
|
| 1604 |
-
if key in amazon.SOURCE:
|
| 1605 |
-
spec = amazon.grid_source(cur, key)
|
| 1606 |
-
if spec.get("missing"):
|
| 1607 |
-
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1608 |
from harness import datastore
|
| 1609 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1610 |
try:
|
|
@@ -1662,19 +1639,14 @@ def _source_for(cur, table_key):
|
|
| 1662 |
return {**spec, "cols": cols}
|
| 1663 |
|
| 1664 |
|
| 1665 |
-
def _page_source_for(cur, table_key, st=None):
|
| 1666 |
-
"""
|
| 1667 |
-
|
| 1668 |
-
|
| 1669 |
-
|
| 1670 |
-
|
| 1671 |
-
|
| 1672 |
-
|
| 1673 |
-
return plans[key]
|
| 1674 |
-
spec = _source_for(cur, table_key)
|
| 1675 |
-
if plans is not None:
|
| 1676 |
-
plans[key] = spec
|
| 1677 |
-
return spec
|
| 1678 |
|
| 1679 |
|
| 1680 |
#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
|
|
@@ -1912,13 +1884,14 @@ def _read_wall_columns(table_key, session, cur, spec, fields, nodes, st, trail=(
|
|
| 1912 |
targets[key] = None
|
| 1913 |
return None
|
| 1914 |
definition = _ut().get(key, st=handle)
|
| 1915 |
-
target_spec = _page_source_for(cur, key, handle)
|
| 1916 |
if target_spec is None and str(key).startswith("ut_amzn_"):
|
| 1917 |
# Small Amazon targets (notably listings) are stored snapshots, but
|
| 1918 |
# their raw mirror still answers the same automatic link in SQL.
|
| 1919 |
import amazon_relational
|
| 1920 |
if key in amazon_relational.SOURCE:
|
| 1921 |
-
target_spec = amazon_relational.grid_source(cur, key
|
|
|
|
| 1922 |
if not definition or target_spec is None or target_spec.get("missing"):
|
| 1923 |
refuse(field)
|
| 1924 |
target_fields = definition.get("fields") or []
|
|
@@ -2408,21 +2381,25 @@ def _related_source_value(session, entry, record):
|
|
| 2408 |
return None
|
| 2409 |
|
| 2410 |
|
| 2411 |
-
def _materialised_table(table_key):
|
| 2412 |
-
"""
|
| 2413 |
-
|
| 2414 |
-
⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
|
| 2415 |
-
(`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
|
| 2416 |
-
the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
|
| 2417 |
-
"""
|
| 2418 |
if module_cache_source(table_key) is not None:
|
| 2419 |
return True
|
| 2420 |
-
if table_key in _sources():
|
| 2421 |
-
return False
|
| 2422 |
if str(table_key).startswith("ut_amzn_"):
|
| 2423 |
-
|
| 2424 |
-
|
| 2425 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2426 |
|
| 2427 |
|
| 2428 |
#: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
|
|
@@ -2520,7 +2497,7 @@ def _page_defs(session):
|
|
| 2520 |
return lent, token
|
| 2521 |
|
| 2522 |
|
| 2523 |
-
def _stored_link_rows(session, table, token=_UNREAD):
|
| 2524 |
"""`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
|
| 2525 |
|
| 2526 |
⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
|
|
@@ -2534,7 +2511,8 @@ def _stored_link_rows(session, table, token=_UNREAD):
|
|
| 2534 |
|
| 2535 |
rt = session.runtime
|
| 2536 |
if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
|
| 2537 |
-
or module_cache_source(table) is not None
|
|
|
|
| 2538 |
return perm_scope._read(table, session.user, rt)
|
| 2539 |
return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
|
| 2540 |
keep=lambda v: bool(v[1]),
|
|
@@ -2567,7 +2545,7 @@ class _RelationRows:
|
|
| 2567 |
|
| 2568 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2569 |
traversal_of=None, st=None, rev=..., normalize_match=False,
|
| 2570 |
-
compiled=None, stream=False): # `...` IS `_UNREAD`
|
| 2571 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2572 |
|
| 2573 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
@@ -2585,7 +2563,10 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2585 |
import core.relations as relations
|
| 2586 |
|
| 2587 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 2588 |
-
if not
|
|
|
|
|
|
|
|
|
|
| 2589 |
read = compiled if compiled is not None else (
|
| 2590 |
_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 2591 |
if traversal_of is not None else _compiled_read(table, session, st=st))
|
|
@@ -2636,21 +2617,22 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2636 |
else:
|
| 2637 |
sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
|
| 2638 |
def batches():
|
| 2639 |
-
|
| 2640 |
-
|
| 2641 |
-
|
| 2642 |
-
|
| 2643 |
-
|
| 2644 |
-
|
| 2645 |
-
|
| 2646 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2647 |
return _RelationRows(batches) if stream else list(batches())
|
| 2648 |
|
| 2649 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 2650 |
# complete contract before taking any join value, including a join on a hidden column.
|
| 2651 |
-
if traversal_of is None and not perm_scope.may_read(
|
| 2652 |
-
session.user, table, st=st if st is not None else session.runtime):
|
| 2653 |
-
raise err(403, "forbidden_table", "This account cannot open the linked database.")
|
| 2654 |
if module_cache_source(table) is not None:
|
| 2655 |
session.require(table)
|
| 2656 |
# The revision memo serves a caller that handed this request's token (the page's link fills,
|
|
@@ -2661,7 +2643,7 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, matches=Non
|
|
| 2661 |
fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
|
| 2662 |
if definition is not None else perm_scope._read(table, session.user, session.runtime))
|
| 2663 |
else:
|
| 2664 |
-
fields, rows = _stored_link_rows(session, table, token=rev)
|
| 2665 |
if ids is not None:
|
| 2666 |
wanted = {str(value) for value in ids}
|
| 2667 |
rows = [row for row in rows if str(row.get("pid")) in wanted]
|
|
@@ -3701,7 +3683,12 @@ def resolved_fields(table_key, defn):
|
|
| 3701 |
fields = rel.namespaced_fields(mk_fields(), connection)
|
| 3702 |
break
|
| 3703 |
if fields is None:
|
| 3704 |
-
if table_key
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3705 |
raise err(404, "not_connected", "that database is not a connected grid")
|
| 3706 |
fields = (defn or {}).get("fields") or []
|
| 3707 |
if not fields:
|
|
@@ -3806,7 +3793,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 3806 |
if mismatch:
|
| 3807 |
raise err(409, "cross_tenant_store", mismatch)
|
| 3808 |
|
| 3809 |
-
spec = _page_source_for(cur, table_key, st)
|
| 3810 |
if spec is None:
|
| 3811 |
from core import perm_scope
|
| 3812 |
rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
|
|
@@ -4413,13 +4400,10 @@ def odoo_table_rows(table_key: str,
|
|
| 4413 |
"limit, filters, sorts and search in the query string.")
|
| 4414 |
ids = _record_ids(body)
|
| 4415 |
|
| 4416 |
-
#
|
| 4417 |
-
# asks it instead of reading its own definitions projection (one store round trip fewer on
|
| 4418 |
-
# every Amazon page). Not reordered past it: the check may register this grid's binding,
|
| 4419 |
-
# which is what `_materialised_table` below then reads.
|
| 4420 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4421 |
-
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4422 |
-
if _materialised_table(table_key):
|
| 4423 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 4424 |
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 4425 |
|
|
@@ -4496,12 +4480,15 @@ def odoo_table_rows(table_key: str,
|
|
| 4496 |
if not _may_open(session, target, st=page_st):
|
| 4497 |
return []
|
| 4498 |
return _relation_rows(session, target, list(matches), matches=matches,
|
| 4499 |
-
st=page_st, rev=page_rev
|
|
|
|
|
|
|
| 4500 |
|
| 4501 |
# Fields are the current reader's contract; target rows cross the same wall
|
| 4502 |
# as /related pins. No unscoped SQL subquery can leak a hidden target's id.
|
| 4503 |
-
amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets
|
| 4504 |
-
|
|
|
|
| 4505 |
|
| 4506 |
# ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
| 4507 |
#
|
|
@@ -4526,8 +4513,6 @@ def odoo_table_rows(table_key: str,
|
|
| 4526 |
for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
|
| 4527 |
overlays.setdefault(pid, {}).update(cells)
|
| 4528 |
if_keys = {field["key"] for field in fields if field.get("type") == "if"}
|
| 4529 |
-
for row in rows_src:
|
| 4530 |
-
overlays[str(row["pid"])].update({key: row.get(key, "") for key in if_keys})
|
| 4531 |
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
|
| 4532 |
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
|
| 4533 |
# in an overlay column that reads a hidden mirror column was outside its reach - and a formula
|
|
@@ -4544,7 +4529,13 @@ def odoo_table_rows(table_key: str,
|
|
| 4544 |
rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
|
| 4545 |
overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
|
| 4546 |
for pid, cells in overlays.items()}
|
| 4547 |
-
rows =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4548 |
|
| 4549 |
unfiltered = win["total"] if where == base_where else datastore.count_rows(
|
| 4550 |
where=base_where, params=tuple(wall_params), cur=cur, **frm)
|
|
@@ -4581,6 +4572,13 @@ MAX_AGGREGATES = 200
|
|
| 4581 |
LINK_VALUE_MAX_BYTES = 1_048_576
|
| 4582 |
|
| 4583 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4584 |
def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
|
| 4585 |
"""Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
|
| 4586 |
read = read if read is not None else _compiled_read(target, session, st=st)
|
|
@@ -4601,9 +4599,7 @@ def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=
|
|
| 4601 |
required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
|
| 4602 |
f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
|
| 4603 |
if required > remaining:
|
| 4604 |
-
raise
|
| 4605 |
-
f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
|
| 4606 |
-
'Reduce the page size or open the related database with filters.')
|
| 4607 |
# The byte check bounds both the aggregate and its Python result. IDs are never a
|
| 4608 |
# Python population; DISTINCT and numeric ordering preserve the existing wire value.
|
| 4609 |
cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
|
|
@@ -4653,9 +4649,7 @@ def _fill_odoo_link_cells(session, rows, fields, st=None, rev=...):
|
|
| 4653 |
# Count repeated cells as well: several page rows may share one target key.
|
| 4654 |
remaining -= len(value) + (2 if value else 0)
|
| 4655 |
if remaining < 0:
|
| 4656 |
-
raise
|
| 4657 |
-
f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
|
| 4658 |
-
'Reduce the page size or open the related database with filters.')
|
| 4659 |
row[field['key']] = value
|
| 4660 |
|
| 4661 |
|
|
@@ -4912,9 +4906,9 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
|
|
| 4912 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 4913 |
|
| 4914 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4915 |
-
require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4916 |
|
| 4917 |
-
if _materialised_table(table_key):
|
| 4918 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
| 4919 |
import odoo_default_scope
|
| 4920 |
if (odoo_default_scope.applies(table_key)
|
|
|
|
| 778 |
_SOURCE_PROVIDERS = []
|
| 779 |
_SOURCE_ANALYSES = {}
|
| 780 |
|
| 781 |
+
# Provider SQL runs outside the publication lock. Invalidation rejects an older build.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 782 |
_SOURCES_LOCK = threading.RLock()
|
| 783 |
_SOURCES_BUILT = False
|
| 784 |
+
_SOURCES_BUILDER = None # compatibility for callers that reset legacy build state
|
| 785 |
+
_SOURCES_BUILDING = threading.local()
|
| 786 |
+
_SOURCES_EPOCH = 0
|
| 787 |
|
| 788 |
|
| 789 |
def invalidate_sources():
|
| 790 |
"""Forget measured bindings after a connector publishes a new mirror schema."""
|
| 791 |
+
global _SOURCES_BUILT, _SOURCES_EPOCH
|
| 792 |
with _SOURCES_LOCK:
|
| 793 |
+
_SOURCES_EPOCH += 1
|
| 794 |
_SOURCES_BUILT = False
|
| 795 |
GRID_SOURCES.clear()
|
| 796 |
|
|
|
|
| 842 |
importing that module at file-import time would drag the Odoo layer into every process that
|
| 843 |
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
|
| 844 |
|
| 845 |
+
Providers build a local dictionary outside the publication lock.
|
|
|
|
|
|
|
| 846 |
"""
|
| 847 |
+
global _SOURCES_BUILT
|
| 848 |
if _SOURCES_BUILT and GRID_SOURCES:
|
| 849 |
return GRID_SOURCES
|
| 850 |
with _SOURCES_LOCK:
|
| 851 |
+
if (_SOURCES_BUILT and GRID_SOURCES) or getattr(_SOURCES_BUILDING, "active", False):
|
| 852 |
return GRID_SOURCES
|
| 853 |
+
epoch = _SOURCES_EPOCH
|
| 854 |
+
from harness import datastore
|
| 855 |
+
generation = datastore.mirror_generation()
|
| 856 |
+
_SOURCES_BUILDING.active = True
|
| 857 |
+
try:
|
| 858 |
+
built = {}
|
| 859 |
+
for build in list(_SOURCE_PROVIDERS):
|
| 860 |
+
try:
|
| 861 |
+
extra = build() or {}
|
| 862 |
+
except Exception: # noqa: BLE001
|
| 863 |
+
continue
|
| 864 |
+
built.update(extra)
|
| 865 |
+
with _SOURCES_LOCK:
|
| 866 |
+
if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
|
| 867 |
+
return GRID_SOURCES
|
|
|
|
|
|
|
| 868 |
GRID_SOURCES.update({**built, **GRID_SOURCES})
|
| 869 |
_SOURCES_BUILT = True
|
| 870 |
+
if built:
|
| 871 |
+
_ut().register_connected(*built)
|
| 872 |
+
finally:
|
| 873 |
+
_SOURCES_BUILDING.active = False
|
| 874 |
return GRID_SOURCES
|
| 875 |
|
| 876 |
|
| 877 |
+
def _learn_amazon_sources(cur=None, *, tenant=None):
|
| 878 |
+
"""Refresh discovery for non-page callers without retaining negative results."""
|
| 879 |
+
with _SOURCES_LOCK:
|
| 880 |
+
epoch = _SOURCES_EPOCH
|
| 881 |
+
from harness import datastore
|
| 882 |
+
generation = datastore.mirror_generation()
|
| 883 |
if cur is None:
|
| 884 |
extra = _amazon_sources()
|
| 885 |
else:
|
| 886 |
import amazon_relational
|
| 887 |
extra = amazon_relational.grid_sources(cur)
|
|
|
|
|
|
|
| 888 |
with _SOURCES_LOCK:
|
| 889 |
+
if epoch != _SOURCES_EPOCH or generation != datastore.mirror_generation():
|
| 890 |
+
return
|
| 891 |
GRID_SOURCES.update(extra)
|
| 892 |
+
if extra:
|
| 893 |
_ut().register_connected(*extra)
|
| 894 |
|
| 895 |
|
|
|
|
| 1292 |
|
| 1293 |
|
| 1294 |
def require_amazon_mirror(table_key, session, definition=None, st=None):
|
| 1295 |
+
"""Admit one Amazon grid and return its binding, or None for a stored snapshot."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1296 |
if not str(table_key).startswith("ut_amzn_"):
|
| 1297 |
return
|
| 1298 |
import amazon_relational as amazon
|
|
|
|
| 1310 |
if mismatch:
|
| 1311 |
raise err(409, "cross_tenant_store", mismatch)
|
| 1312 |
try:
|
| 1313 |
+
return _amazon_page_source(cur, table_key, definition, allow_lookups=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1314 |
finally:
|
| 1315 |
cur.close()
|
| 1316 |
|
|
|
|
| 1526 |
return expr if bare.lower() in have else "NULL"
|
| 1527 |
|
| 1528 |
|
| 1529 |
+
def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_lookups=True):
|
| 1530 |
+
"""Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
|
| 1531 |
+
import amazon_relational as amazon
|
| 1532 |
+
|
| 1533 |
+
try:
|
| 1534 |
+
if binding:
|
| 1535 |
+
columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
|
| 1536 |
+
spec = amazon.grid_source(cur, key, columns=columns, allow_lookups=allow_lookups)
|
| 1537 |
+
else:
|
| 1538 |
+
spec = amazon.page_source(cur, key, definition, allow_lookups=allow_lookups)
|
| 1539 |
+
except Exception as exc: # noqa: BLE001
|
| 1540 |
+
from core.request_timing import raise_if_resource_error
|
| 1541 |
+
raise_if_resource_error(exc)
|
| 1542 |
+
raise err(503, "amazon_source_unavailable",
|
| 1543 |
+
"This Amazon database could not be resolved. Retry the request.") from exc
|
| 1544 |
+
if spec is not None and spec.get("missing"):
|
| 1545 |
+
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
| 1546 |
+
return spec
|
| 1547 |
+
|
| 1548 |
+
|
| 1549 |
+
def _source_for(cur, table_key, *, allow_lookups=True):
|
| 1550 |
"""The resolved spec for one connected grid, or None when this grid is not read-through yet.
|
| 1551 |
|
| 1552 |
⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
|
|
|
|
| 1573 |
# table and turns Products ordered into the wrong 404.
|
| 1574 |
if module_cache_source(key) is not None:
|
| 1575 |
return None
|
| 1576 |
+
if key.startswith("ut_amzn_"):
|
| 1577 |
+
import amazon_relational as amazon
|
| 1578 |
+
if key in amazon.SOURCE:
|
| 1579 |
+
return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
|
| 1580 |
spec = _sources().get(key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1581 |
if not spec:
|
| 1582 |
return None
|
| 1583 |
if spec.get("missing"):
|
| 1584 |
raise err(503, "amazon_mirror_not_loaded", spec["missing"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1585 |
from harness import datastore
|
| 1586 |
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
|
| 1587 |
try:
|
|
|
|
| 1639 |
return {**spec, "cols": cols}
|
| 1640 |
|
| 1641 |
|
| 1642 |
+
def _page_source_for(cur, table_key, st=None, *, session=None):
|
| 1643 |
+
"""Plan admitted lookups afresh because an autocommit cursor can change snapshots."""
|
| 1644 |
+
if str(table_key).startswith("ut_amzn_") and session is not None:
|
| 1645 |
+
import amazon_relational as amazon
|
| 1646 |
+
from routes_grid import _may_open
|
| 1647 |
+
allowed = _may_open(session, amazon.LISTINGS, st=st)
|
| 1648 |
+
return _source_for(cur, table_key, allow_lookups=allowed)
|
| 1649 |
+
return _source_for(cur, table_key)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1650 |
|
| 1651 |
|
| 1652 |
#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in
|
|
|
|
| 1884 |
targets[key] = None
|
| 1885 |
return None
|
| 1886 |
definition = _ut().get(key, st=handle)
|
| 1887 |
+
target_spec = _page_source_for(cur, key, handle, session=session)
|
| 1888 |
if target_spec is None and str(key).startswith("ut_amzn_"):
|
| 1889 |
# Small Amazon targets (notably listings) are stored snapshots, but
|
| 1890 |
# their raw mirror still answers the same automatic link in SQL.
|
| 1891 |
import amazon_relational
|
| 1892 |
if key in amazon_relational.SOURCE:
|
| 1893 |
+
target_spec = amazon_relational.grid_source(cur, key,
|
| 1894 |
+
allow_lookups=_may_open(session, amazon_relational.LISTINGS, st=handle))
|
| 1895 |
if not definition or target_spec is None or target_spec.get("missing"):
|
| 1896 |
refuse(field)
|
| 1897 |
target_fields = definition.get("fields") or []
|
|
|
|
| 2381 |
return None
|
| 2382 |
|
| 2383 |
|
| 2384 |
+
def _materialised_table(table_key, *, cur=None, st=None):
|
| 2385 |
+
"""Dispatch one target against its cursor without treating registry misses as evidence."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2386 |
if module_cache_source(table_key) is not None:
|
| 2387 |
return True
|
|
|
|
|
|
|
| 2388 |
if str(table_key).startswith("ut_amzn_"):
|
| 2389 |
+
from harness import datastore
|
| 2390 |
+
import amazon_relational as amazon
|
| 2391 |
+
if table_key not in amazon.SOURCE:
|
| 2392 |
+
return True
|
| 2393 |
+
own_cursor = cur is None
|
| 2394 |
+
cur = datastore.ro_cursor() if own_cursor else cur
|
| 2395 |
+
try:
|
| 2396 |
+
definition = _ut().get(table_key, st=st) if st is not None else None
|
| 2397 |
+
spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
|
| 2398 |
+
return spec is None
|
| 2399 |
+
finally:
|
| 2400 |
+
if own_cursor:
|
| 2401 |
+
cur.close()
|
| 2402 |
+
return table_key not in _sources()
|
| 2403 |
|
| 2404 |
|
| 2405 |
#: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
|
|
|
|
| 2497 |
return lent, token
|
| 2498 |
|
| 2499 |
|
| 2500 |
+
def _stored_link_rows(session, table, token=_UNREAD, *, materialised=None):
|
| 2501 |
"""`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
|
| 2502 |
|
| 2503 |
⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
|
|
|
|
| 2511 |
|
| 2512 |
rt = session.runtime
|
| 2513 |
if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
|
| 2514 |
+
or module_cache_source(table) is not None
|
| 2515 |
+
or not (materialised if materialised is not None else _materialised_table(table))):
|
| 2516 |
return perm_scope._read(table, session.user, rt)
|
| 2517 |
return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
|
| 2518 |
keep=lambda v: bool(v[1]),
|
|
|
|
| 2545 |
|
| 2546 |
def _relation_rows(session, table, columns, *, ids=None, match=None, matches=None, where=None,
|
| 2547 |
traversal_of=None, st=None, rev=..., normalize_match=False,
|
| 2548 |
+
compiled=None, stream=False, materialised=None): # `...` IS `_UNREAD`
|
| 2549 |
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
|
| 2550 |
|
| 2551 |
Join columns may be hidden on the wire. Only identities leave the resolver; the native
|
|
|
|
| 2563 |
import core.relations as relations
|
| 2564 |
|
| 2565 |
columns = set(columns) | {relations.IDENTITY_COL}
|
| 2566 |
+
if traversal_of is None and not perm_scope.may_read(
|
| 2567 |
+
session.user, table, st=st if st is not None else session.runtime):
|
| 2568 |
+
raise err(403, "forbidden_table", "This account cannot open the linked database.")
|
| 2569 |
+
if not (materialised if materialised is not None else _materialised_table(table)):
|
| 2570 |
read = compiled if compiled is not None else (
|
| 2571 |
_compiled_read(table, session, traversal_intermediate=True, st=st)
|
| 2572 |
if traversal_of is not None else _compiled_read(table, session, st=st))
|
|
|
|
| 2617 |
else:
|
| 2618 |
sql = f"SELECT {projection} FROM {source} WHERE {predicate}"
|
| 2619 |
def batches():
|
| 2620 |
+
try:
|
| 2621 |
+
cursor = read["cur"].execute(sql, params)
|
| 2622 |
+
while True:
|
| 2623 |
+
batch = cursor.fetchmany(2048)
|
| 2624 |
+
if not batch:
|
| 2625 |
+
return
|
| 2626 |
+
for row in batch:
|
| 2627 |
+
yield dict(zip(names, row))
|
| 2628 |
+
batch.clear()
|
| 2629 |
+
finally:
|
| 2630 |
+
if stream and compiled is None:
|
| 2631 |
+
read["cur"].close()
|
| 2632 |
return _RelationRows(batches) if stream else list(batches())
|
| 2633 |
|
| 2634 |
# This is an internal join, never a cell response. Apply the same permanent wall on the
|
| 2635 |
# complete contract before taking any join value, including a join on a hidden column.
|
|
|
|
|
|
|
|
|
|
| 2636 |
if module_cache_source(table) is not None:
|
| 2637 |
session.require(table)
|
| 2638 |
# The revision memo serves a caller that handed this request's token (the page's link fills,
|
|
|
|
| 2643 |
fields, rows = (perm_scope._read(table, session.user, session.runtime, defn=definition)
|
| 2644 |
if definition is not None else perm_scope._read(table, session.user, session.runtime))
|
| 2645 |
else:
|
| 2646 |
+
fields, rows = _stored_link_rows(session, table, token=rev, materialised=True)
|
| 2647 |
if ids is not None:
|
| 2648 |
wanted = {str(value) for value in ids}
|
| 2649 |
rows = [row for row in rows if str(row.get("pid")) in wanted]
|
|
|
|
| 3683 |
fields = rel.namespaced_fields(mk_fields(), connection)
|
| 3684 |
break
|
| 3685 |
if fields is None:
|
| 3686 |
+
if str(table_key).startswith("ut_amzn_"):
|
| 3687 |
+
import amazon_relational as amazon
|
| 3688 |
+
connected = table_key in amazon.SOURCE
|
| 3689 |
+
else:
|
| 3690 |
+
connected = table_key in _sources()
|
| 3691 |
+
if not connected:
|
| 3692 |
raise err(404, "not_connected", "that database is not a connected grid")
|
| 3693 |
fields = (defn or {}).get("fields") or []
|
| 3694 |
if not fields:
|
|
|
|
| 3793 |
if mismatch:
|
| 3794 |
raise err(409, "cross_tenant_store", mismatch)
|
| 3795 |
|
| 3796 |
+
spec = _page_source_for(cur, table_key, st, session=session)
|
| 3797 |
if spec is None:
|
| 3798 |
from core import perm_scope
|
| 3799 |
rule = ((perm_scope.entry(session.user, table_key) or {}).get("filter")
|
|
|
|
| 4400 |
"limit, filters, sorts and search in the query string.")
|
| 4401 |
ids = _record_ids(body)
|
| 4402 |
|
| 4403 |
+
# Admission resolves this Amazon database without publishing a process-wide binding.
|
|
|
|
|
|
|
|
|
|
| 4404 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4405 |
+
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4406 |
+
if (admitted is None if page is not None else _materialised_table(table_key)):
|
| 4407 |
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
| 4408 |
sorts, search, related, groupBy, groupKey, ids=ids)
|
| 4409 |
|
|
|
|
| 4480 |
if not _may_open(session, target, st=page_st):
|
| 4481 |
return []
|
| 4482 |
return _relation_rows(session, target, list(matches), matches=matches,
|
| 4483 |
+
st=page_st, rev=page_rev,
|
| 4484 |
+
stream=True,
|
| 4485 |
+
materialised=_materialised_table(target, cur=cur, st=page_st))
|
| 4486 |
|
| 4487 |
# Fields are the current reader's contract; target rows cross the same wall
|
| 4488 |
# as /related pins. No unscoped SQL subquery can leak a hidden target's id.
|
| 4489 |
+
amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets,
|
| 4490 |
+
max_bytes=LINK_VALUE_MAX_BYTES, capacity_refusal=_link_capacity_refusal)
|
| 4491 |
+
overlays = {}
|
| 4492 |
|
| 4493 |
# ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
|
| 4494 |
#
|
|
|
|
| 4513 |
for pid, cells in page_cells(session.runtime, table_key, [r["pid"] for r in rows_src]).items():
|
| 4514 |
overlays.setdefault(pid, {}).update(cells)
|
| 4515 |
if_keys = {field["key"] for field in fields if field.get("type") == "if"}
|
|
|
|
|
|
|
| 4516 |
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
|
| 4517 |
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
|
| 4518 |
# in an overlay column that reads a hidden mirror column was outside its reach - and a formula
|
|
|
|
| 4529 |
rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src]
|
| 4530 |
overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden}
|
| 4531 |
for pid, cells in overlays.items()}
|
| 4532 |
+
rows = []
|
| 4533 |
+
for row in rows_src:
|
| 4534 |
+
# Physical payloads stay separate even when business identities share a pid.
|
| 4535 |
+
cells = {key: value for key, value in row.items() if key != "pid"}
|
| 4536 |
+
cells.update(overlays.get(str(row["pid"]), {}))
|
| 4537 |
+
cells.update({key: row.get(key, "") for key in if_keys if key not in _hidden})
|
| 4538 |
+
rows.extend(aios_grid.rows_from_pool([row], fields, {str(row["pid"]): cells}))
|
| 4539 |
|
| 4540 |
unfiltered = win["total"] if where == base_where else datastore.count_rows(
|
| 4541 |
where=base_where, params=tuple(wall_params), cur=cur, **frm)
|
|
|
|
| 4572 |
LINK_VALUE_MAX_BYTES = 1_048_576
|
| 4573 |
|
| 4574 |
|
| 4575 |
+
def _link_capacity_refusal():
|
| 4576 |
+
"""Name the complete-value capacity and the action that reduces it."""
|
| 4577 |
+
return err(503, 'relation_value_capacity',
|
| 4578 |
+
f'Complete linked values exceed the {LINK_VALUE_MAX_BYTES:,}-byte response capacity. '
|
| 4579 |
+
'Reduce the page size or open the related database with filters.')
|
| 4580 |
+
|
| 4581 |
+
|
| 4582 |
def _sql_link_cells(session, target, on, values, *, st, single, remaining, read=None):
|
| 4583 |
"""Reduce scoped target identities in SQL; refuse bytes before string aggregation."""
|
| 4584 |
read = read if read is not None else _compiled_read(target, session, st=st)
|
|
|
|
| 4599 |
required = int(cur.execute(f'SELECT coalesce(sum(n), 0) FROM (SELECT '
|
| 4600 |
f'sum(length(CAST(pid AS VARCHAR))) + count(*) + 1 AS n FROM {population} GROUP BY k)').fetchone()[0])
|
| 4601 |
if required > remaining:
|
| 4602 |
+
raise _link_capacity_refusal()
|
|
|
|
|
|
|
| 4603 |
# The byte check bounds both the aggregate and its Python result. IDs are never a
|
| 4604 |
# Python population; DISTINCT and numeric ordering preserve the existing wire value.
|
| 4605 |
cells = dict(cur.execute(f"SELECT k, string_agg(CAST(pid AS VARCHAR), ',' ORDER BY pid) "
|
|
|
|
| 4649 |
# Count repeated cells as well: several page rows may share one target key.
|
| 4650 |
remaining -= len(value) + (2 if value else 0)
|
| 4651 |
if remaining < 0:
|
| 4652 |
+
raise _link_capacity_refusal()
|
|
|
|
|
|
|
| 4653 |
row[field['key']] = value
|
| 4654 |
|
| 4655 |
|
|
|
|
| 4906 |
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
|
| 4907 |
|
| 4908 |
page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
|
| 4909 |
+
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
| 4910 |
|
| 4911 |
+
if (admitted is None if page is not None else _materialised_table(table_key)):
|
| 4912 |
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
|
| 4913 |
import odoo_default_scope
|
| 4914 |
if (odoo_default_scope.applies(table_key)
|
api/routes_tables.py
CHANGED
|
@@ -584,7 +584,8 @@ def _query_activity_page(session: Session, *, object=None, by=None, since=None,
|
|
| 584 |
context_store = _ut().lend(context_store, **{
|
| 585 |
name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
|
| 586 |
from routes_customers import _AssemblyStore
|
| 587 |
-
|
|
|
|
| 588 |
from copy import copy
|
| 589 |
field_session = copy(session)
|
| 590 |
field_session.runtime = context_store
|
|
@@ -1853,6 +1854,10 @@ def _compute_ut_assembly(session: Session, table_key: str, storage_key: str = ""
|
|
| 1853 |
metadata_membership = False
|
| 1854 |
if fields_only:
|
| 1855 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1856 |
fields_base = list(defn.get("fields") or [])
|
| 1857 |
pids, rows_src = frozenset(), []
|
| 1858 |
elif with_rows:
|
|
|
|
| 584 |
context_store = _ut().lend(context_store, **{
|
| 585 |
name: documents[name] for name in metadata if isinstance(documents.get(name), dict)})
|
| 586 |
from routes_customers import _AssemblyStore
|
| 587 |
+
# Lend the projected catalogue outside the assembly reader's per-read copy boundary.
|
| 588 |
+
context_store = _ut().lend_defs(_AssemblyStore(context_store, schema_only=True), schema_only=True)
|
| 589 |
from copy import copy
|
| 590 |
field_session = copy(session)
|
| 591 |
field_session.runtime = context_store
|
|
|
|
| 1854 |
metadata_membership = False
|
| 1855 |
if fields_only:
|
| 1856 |
defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
|
| 1857 |
+
if getattr(st, 'schema_only', False):
|
| 1858 |
+
from copy import deepcopy
|
| 1859 |
+
# Detach selected definitions while the request lends its projected catalogue.
|
| 1860 |
+
defn = deepcopy(defn)
|
| 1861 |
fields_base = list(defn.get("fields") or [])
|
| 1862 |
pids, rows_src = frozenset(), []
|
| 1863 |
elif with_rows:
|
api/sheet_warm.py
CHANGED
|
@@ -11,13 +11,15 @@ from __future__ import annotations
|
|
| 11 |
|
| 12 |
import atexit
|
| 13 |
import os
|
|
|
|
| 14 |
import threading
|
| 15 |
import time
|
|
|
|
| 16 |
from datetime import datetime, time as dtime, timedelta, timezone
|
| 17 |
from time import perf_counter
|
| 18 |
|
| 19 |
import routes_sheet
|
| 20 |
-
from core import external_access, perms, shares, sheets, users
|
| 21 |
from deps import Session
|
| 22 |
|
| 23 |
#: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
|
|
@@ -357,6 +359,146 @@ def _desk_skip(rt, tenant):
|
|
| 357 |
return str(text or "").startswith("pg:") and token == seen
|
| 358 |
|
| 359 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
def desk_run_one(rt, tenant):
|
| 361 |
"""Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
|
| 362 |
row = routes_sheet._refresh_desk_claim(rt)
|
|
@@ -364,6 +506,7 @@ def desk_run_one(rt, tenant):
|
|
| 364 |
return False
|
| 365 |
started = perf_counter()
|
| 366 |
error = None
|
|
|
|
| 367 |
try:
|
| 368 |
book = sheets.get_workbook(rt, row["workbook"])
|
| 369 |
rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
|
|
@@ -371,9 +514,11 @@ def desk_run_one(rt, tenant):
|
|
| 371 |
if session is None:
|
| 372 |
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 373 |
else:
|
| 374 |
-
|
|
|
|
| 375 |
except Exception as exc: # noqa: BLE001
|
| 376 |
error = routes_sheet._refresh_desk_error(exc)
|
|
|
|
| 377 |
routes_sheet._refresh_desk_finish(rt, row["id"], error)
|
| 378 |
print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
|
| 379 |
f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
|
|
|
|
| 11 |
|
| 12 |
import atexit
|
| 13 |
import os
|
| 14 |
+
import re
|
| 15 |
import threading
|
| 16 |
import time
|
| 17 |
+
import uuid
|
| 18 |
from datetime import datetime, time as dtime, timedelta, timezone
|
| 19 |
from time import perf_counter
|
| 20 |
|
| 21 |
import routes_sheet
|
| 22 |
+
from core import external_access, perms, refresh_proof, shares, sheets, users
|
| 23 |
from deps import Session
|
| 24 |
|
| 25 |
#: The rollover warm starts this long after 00:00 UTC, so the day clock has moved for every reader.
|
|
|
|
| 359 |
return str(text or "").startswith("pg:") and token == seen
|
| 360 |
|
| 361 |
|
| 362 |
+
DESK_RESULT_PREFIX = "[sheet-desk-result] "
|
| 363 |
+
_DESK_PROOF_REASONS = frozenset((
|
| 364 |
+
"workbook_incomplete", "sheet_incomplete", "sheet_partial", "cell_malformed", "refusal_malformed",
|
| 365 |
+
"identity_mismatch", "original_workbook_missing", "workbook_mismatch", "result_identity_mismatch",
|
| 366 |
+
"inventory_changed", "execution_build_unknown", "attempt_malformed", "attempt_not_committed",
|
| 367 |
+
"viewer_only", "partial_scope", "held_or_unresolved", "editor_commit_unproven",
|
| 368 |
+
"workbook_empty", "authored_definitions_changed", "proof_capacity_exceeded",
|
| 369 |
+
"generated_manifest_invalid", "generated_output_missing", "generated_output_invalid",
|
| 370 |
+
))
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def desk_result_census(book):
|
| 374 |
+
"""Validate a complete workbook and count its cells without retaining their values."""
|
| 375 |
+
identity = refresh_proof.snapshot(book)
|
| 376 |
+
return identity["census"], identity["inventory"], identity["unresolved"]
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def _desk_input_identity(book):
|
| 380 |
+
try:
|
| 381 |
+
return refresh_proof.snapshot(book)
|
| 382 |
+
except refresh_proof.ProofCapacityError as exc:
|
| 383 |
+
return {"error": "proof_capacity_exceeded", "capacityCause": str(exc)}
|
| 384 |
+
except ValueError as exc:
|
| 385 |
+
return {"error": str(exc) if str(exc) in _DESK_PROOF_REASONS else "cell_malformed"}
|
| 386 |
+
except Exception: # noqa: BLE001
|
| 387 |
+
return {"error": "cell_malformed"}
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def desk_result_capture(row, tenant, session, before, result, *, admitted=None):
|
| 391 |
+
"""Describe only the original resolver return and the worker's admitted pre-run inventory."""
|
| 392 |
+
out = {"schema": "sheet-desk-result/v1", "source": "_refresh_desk_run.return",
|
| 393 |
+
"executionId": uuid.uuid4().hex,
|
| 394 |
+
"requestId": row.get("id") if isinstance(row.get("id"), str) else None,
|
| 395 |
+
"tenant": tenant if isinstance(tenant, str) else None,
|
| 396 |
+
"reader": row.get("reader") if isinstance(row.get("reader"), str) else None,
|
| 397 |
+
"workbook": row.get("workbook") if isinstance(row.get("workbook"), str) else None,
|
| 398 |
+
"version": os.environ.get("AIOS_VERSION", ""),
|
| 399 |
+
"complete": False, "persisted": False, "reason": None, "revision": None,
|
| 400 |
+
"priorRevision": None, "attempt": None, "census": None, "inventory": None, "sheets": None,
|
| 401 |
+
"authored": None, "priorAuthored": None, "recommendation": None, "capacityCause": None}
|
| 402 |
+
try:
|
| 403 |
+
if (session is None or row.get("tenant") != tenant or session.tenant != tenant
|
| 404 |
+
or session.uname != row.get("reader") or before.get("id") != row.get("workbook")
|
| 405 |
+
or session.user.get("username") != row.get("reader")
|
| 406 |
+
or session.user.get("tenant", tenant) != tenant
|
| 407 |
+
or any(key and key != tenant for key in
|
| 408 |
+
(getattr(session.runtime, "key", None),
|
| 409 |
+
getattr(getattr(session.runtime, "tenant", None), "key", None)))):
|
| 410 |
+
raise ValueError("identity_mismatch")
|
| 411 |
+
if not isinstance(result, dict) or not isinstance(result.get("workbook"), dict):
|
| 412 |
+
raise ValueError("original_workbook_missing")
|
| 413 |
+
book = result["workbook"]
|
| 414 |
+
if book.get("id") != row.get("workbook"):
|
| 415 |
+
raise ValueError("workbook_mismatch")
|
| 416 |
+
if (any(item.get("tenant", tenant) != tenant for item in (result, book))
|
| 417 |
+
or result.get("reader", session.uname) != session.uname):
|
| 418 |
+
raise ValueError("result_identity_mismatch")
|
| 419 |
+
prior = admitted if admitted is not None else _desk_input_identity(before)
|
| 420 |
+
if prior.get("error"):
|
| 421 |
+
out["capacityCause"] = prior.get("capacityCause")
|
| 422 |
+
raise ValueError(prior["error"])
|
| 423 |
+
returned = refresh_proof.snapshot(book)
|
| 424 |
+
out["census"], unresolved = returned["census"], returned["unresolved"]
|
| 425 |
+
out["inventory"] = refresh_proof.digest(returned["inventory"])
|
| 426 |
+
out["sheets"] = returned["sheets"]
|
| 427 |
+
out["authored"], out["priorAuthored"] = returned["authored"], prior["authored"]
|
| 428 |
+
out["revision"] = book.get("rev") if type(book.get("rev")) is int else None
|
| 429 |
+
out["priorRevision"] = prior["revision"] if type(prior["revision"]) is int else None
|
| 430 |
+
if (returned["inventory"] != prior["inventory"] or returned["sheets"] != prior["sheets"]):
|
| 431 |
+
raise ValueError("inventory_changed")
|
| 432 |
+
if returned["authored"] != prior["authored"]:
|
| 433 |
+
raise ValueError("authored_definitions_changed")
|
| 434 |
+
if not re.search(r"\b[0-9a-f]{7,40}\b", out["version"]):
|
| 435 |
+
raise ValueError("execution_build_unknown")
|
| 436 |
+
if result.get("attempt") is not None and not isinstance(result["attempt"], dict):
|
| 437 |
+
raise ValueError("attempt_malformed")
|
| 438 |
+
if isinstance(result.get("attempt"), dict):
|
| 439 |
+
attempt = result["attempt"]
|
| 440 |
+
if any(attempt.get(key) is not None and not isinstance(attempt[key], str) for key in ("id", "state")):
|
| 441 |
+
raise ValueError("attempt_malformed")
|
| 442 |
+
out["attempt"] = {key: attempt.get(key) for key in ("id", "state")}
|
| 443 |
+
if attempt.get("state") != "committed":
|
| 444 |
+
raise ValueError("attempt_not_committed")
|
| 445 |
+
role = routes_sheet._role_on_workbook(session, before)
|
| 446 |
+
if role not in ("owner", "edit") or "refreshedFor" in book:
|
| 447 |
+
raise ValueError("viewer_only")
|
| 448 |
+
args = row.get("args") or {}
|
| 449 |
+
if (args != {"kind": "workbook"} and args != {"kind": "scope", "scope": {"workbook": True}}):
|
| 450 |
+
raise ValueError("partial_scope")
|
| 451 |
+
report = result.get("binding_refresh")
|
| 452 |
+
statuses = report.get("databases") if isinstance(report, dict) else None
|
| 453 |
+
if (not isinstance(statuses, dict) or any(state not in ("resolved", "skipped")
|
| 454 |
+
for state in statuses.values()) or unresolved or result.get("reused")
|
| 455 |
+
or result.get("held") or book.get("held") or result.get("partial") or book.get("partial")
|
| 456 |
+
or result.get("viewer") or result.get("viewer_only")
|
| 457 |
+
or result.get("state") not in (None, "committed")
|
| 458 |
+
or result.get("outcome") not in (None, "committed")
|
| 459 |
+
or result.get("persisted") is False or book.get("persisted") is False
|
| 460 |
+
or result.get("ok") is False):
|
| 461 |
+
raise ValueError("held_or_unresolved")
|
| 462 |
+
if (type(book.get("rev")) is not int or type(prior["revision"]) is not int
|
| 463 |
+
or prior["revision"] < 0 or book["rev"] != prior["revision"] + 1
|
| 464 |
+
or not isinstance(book.get("refreshed_at"), str) or not book["refreshed_at"]
|
| 465 |
+
or (any(_is_bound(cell) for tab in book["sheets"] for cell in tab["cells"].values())
|
| 466 |
+
and not statuses)):
|
| 467 |
+
raise ValueError("editor_commit_unproven")
|
| 468 |
+
empty_noop = prior["census"]["cells"] == returned["census"]["cells"] == 0
|
| 469 |
+
out.update(complete=True, persisted=True,
|
| 470 |
+
reason="empty_noop" if empty_noop else "editor_revision_advanced")
|
| 471 |
+
refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
|
| 472 |
+
except refresh_proof.ProofCapacityError as exc:
|
| 473 |
+
out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
|
| 474 |
+
recommendation=refresh_proof.RECOMMENDATION, capacityCause=str(exc))
|
| 475 |
+
except (ValueError, TypeError, AttributeError) as exc:
|
| 476 |
+
reason = str(exc) if isinstance(exc, ValueError) else "capture_malformed"
|
| 477 |
+
out.update(complete=False, persisted=False)
|
| 478 |
+
out["reason"] = reason if reason in _DESK_PROOF_REASONS else "capture_malformed"
|
| 479 |
+
if out["reason"] == "proof_capacity_exceeded":
|
| 480 |
+
out["recommendation"] = refresh_proof.RECOMMENDATION
|
| 481 |
+
try:
|
| 482 |
+
refresh_proof.bounded_json(out, refresh_proof.MAX_LINE_BYTES)
|
| 483 |
+
except (ValueError, TypeError):
|
| 484 |
+
out.update(complete=False, persisted=False, reason="proof_capacity_exceeded", census=None,
|
| 485 |
+
sheets=None, attempt=None, recommendation=refresh_proof.RECOMMENDATION,
|
| 486 |
+
capacityCause="log_line_capacity")
|
| 487 |
+
for key in ("requestId", "tenant", "reader", "workbook", "version"):
|
| 488 |
+
if isinstance(out[key], str) and len(out[key]) > 1024:
|
| 489 |
+
out[key] = None
|
| 490 |
+
return out
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def _desk_result_log(row, tenant, session, before, result, *, admitted=None):
|
| 494 |
+
"""A logging failure supplies no proof and never changes the desk's business outcome."""
|
| 495 |
+
try:
|
| 496 |
+
capture = desk_result_capture(row, tenant, session, before, result, admitted=admitted)
|
| 497 |
+
print(DESK_RESULT_PREFIX + refresh_proof.bounded_json(capture, refresh_proof.MAX_LINE_BYTES), flush=True)
|
| 498 |
+
except Exception: # noqa: BLE001
|
| 499 |
+
pass
|
| 500 |
+
|
| 501 |
+
|
| 502 |
def desk_run_one(rt, tenant):
|
| 503 |
"""Claim and run the oldest queued refresh of one tenant. Returns True when one ran."""
|
| 504 |
row = routes_sheet._refresh_desk_claim(rt)
|
|
|
|
| 506 |
return False
|
| 507 |
started = perf_counter()
|
| 508 |
error = None
|
| 509 |
+
book, session, result, admitted = None, None, None, None
|
| 510 |
try:
|
| 511 |
book = sheets.get_workbook(rt, row["workbook"])
|
| 512 |
rec = shares.grants("workbook", book.get("id") or row["workbook"], st=rt)
|
|
|
|
| 514 |
if session is None:
|
| 515 |
error = {"code": "workbook_missing", "message": "That workbook no longer exists."}
|
| 516 |
else:
|
| 517 |
+
admitted = _desk_input_identity(book)
|
| 518 |
+
result = routes_sheet._refresh_desk_run(session, row)
|
| 519 |
except Exception as exc: # noqa: BLE001
|
| 520 |
error = routes_sheet._refresh_desk_error(exc)
|
| 521 |
+
_desk_result_log(row, tenant, session, book, result, admitted=admitted)
|
| 522 |
routes_sheet._refresh_desk_finish(rt, row["id"], error)
|
| 523 |
print(f"[sheet-desk] wb={row['workbook']} reader={row.get('reader')} "
|
| 524 |
f"state={'failed' if error else 'done'} ms={int((perf_counter() - started) * 1000)}"
|
platform/core/refresh_proof.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure, bounded identities for authored Sheet inputs and complete returned outputs."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import re
|
| 8 |
+
|
| 9 |
+
MAX_CELLS = 100_000
|
| 10 |
+
MAX_TABS = 128
|
| 11 |
+
MAX_DOCUMENT_BYTES = 32 * 1024 * 1024
|
| 12 |
+
MAX_LINE_BYTES = 256 * 1024
|
| 13 |
+
MAX_LOG_BYTES = 8 * 1024 * 1024
|
| 14 |
+
MAX_LOG_ROWS = 4096
|
| 15 |
+
MAX_STATE_BYTES = 16 * 1024 * 1024
|
| 16 |
+
MAX_STATE_ROWS = 16_384
|
| 17 |
+
RECOMMENDATION = "Increase the named proof budget and collect a fresh complete proof."
|
| 18 |
+
_UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ProofCapacityError(ValueError):
|
| 22 |
+
"""A complete proof cannot fit the named finite budget."""
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def bounded_json(value, limit=MAX_DOCUMENT_BYTES):
|
| 26 |
+
"""Check depth, nodes and scalar sizes before allocating encoded JSON."""
|
| 27 |
+
remaining, nodes = [limit], [0]
|
| 28 |
+
|
| 29 |
+
def check(item, depth=0):
|
| 30 |
+
nodes[0] += 1
|
| 31 |
+
if depth > 40 or nodes[0] > 1_000_000:
|
| 32 |
+
raise ProofCapacityError("json_structure_capacity")
|
| 33 |
+
if isinstance(item, str):
|
| 34 |
+
if len(item) > remaining[0]:
|
| 35 |
+
raise ProofCapacityError("json_byte_capacity")
|
| 36 |
+
cost = len(item.encode("utf-8")) + 2
|
| 37 |
+
elif isinstance(item, (dict, list, tuple)):
|
| 38 |
+
if len(item) > limit:
|
| 39 |
+
raise ProofCapacityError("json_structure_capacity")
|
| 40 |
+
cost = 2 + 2 * len(item)
|
| 41 |
+
elif item is None or type(item) in (bool, int, float):
|
| 42 |
+
if type(item) is int and item.bit_length() > 256:
|
| 43 |
+
raise ProofCapacityError("json_scalar_capacity")
|
| 44 |
+
if type(item) is float and not math.isfinite(item):
|
| 45 |
+
raise ValueError("json_nonfinite")
|
| 46 |
+
cost = 32
|
| 47 |
+
else:
|
| 48 |
+
raise ValueError("json_malformed")
|
| 49 |
+
remaining[0] -= cost
|
| 50 |
+
if remaining[0] < 0:
|
| 51 |
+
raise ProofCapacityError("json_byte_capacity")
|
| 52 |
+
if isinstance(item, dict):
|
| 53 |
+
for key, child in item.items():
|
| 54 |
+
if not isinstance(key, str):
|
| 55 |
+
raise ValueError("json_key_malformed")
|
| 56 |
+
check(key, depth + 1)
|
| 57 |
+
check(child, depth + 1)
|
| 58 |
+
elif isinstance(item, (list, tuple)):
|
| 59 |
+
for child in item:
|
| 60 |
+
check(child, depth + 1)
|
| 61 |
+
|
| 62 |
+
check(value)
|
| 63 |
+
chunks, size = [], 0
|
| 64 |
+
for chunk in json.JSONEncoder(sort_keys=True, separators=(",", ":"),
|
| 65 |
+
ensure_ascii=False, allow_nan=False).iterencode(value):
|
| 66 |
+
size += len(chunk.encode("utf-8"))
|
| 67 |
+
if size > limit:
|
| 68 |
+
raise ProofCapacityError("json_byte_capacity")
|
| 69 |
+
chunks.append(chunk)
|
| 70 |
+
return "".join(chunks)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def digest(value):
|
| 74 |
+
return hashlib.sha256(bounded_json(value).encode("utf-8")).hexdigest()
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _address(addr):
|
| 78 |
+
if not isinstance(addr, str) or not re.fullmatch(r"[A-Z]{1,3}[1-9][0-9]{0,6}", addr):
|
| 79 |
+
return False
|
| 80 |
+
col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
|
| 81 |
+
n = 0
|
| 82 |
+
for char in col:
|
| 83 |
+
n = n * 26 + ord(char) - 64
|
| 84 |
+
return n <= 16384 and int(row) <= 1048576
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _definition(cell):
|
| 88 |
+
kind = cell["kind"]
|
| 89 |
+
if kind == "literal":
|
| 90 |
+
return {key: cell[key] for key in ("kind", "text", "value") if key in cell}
|
| 91 |
+
if kind == "formula":
|
| 92 |
+
meta = cell.get("formulaMeta") or {}
|
| 93 |
+
return {"kind": kind, "formula": cell["formula"],
|
| 94 |
+
"formulaMeta": {k: v for k, v in meta.items()
|
| 95 |
+
if k not in ("spill", "inferredFormat")}}
|
| 96 |
+
if kind == "bound":
|
| 97 |
+
return {"kind": kind, "binding": cell["binding"]}
|
| 98 |
+
return {"kind": kind}
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _position(addr):
|
| 102 |
+
if not _address(addr):
|
| 103 |
+
raise ValueError("generated_manifest_invalid")
|
| 104 |
+
col, row = re.fullmatch(r"([A-Z]+)([0-9]+)", addr).groups()
|
| 105 |
+
number = 0
|
| 106 |
+
for char in col:
|
| 107 |
+
number = number * 26 + ord(char) - 64
|
| 108 |
+
return number, int(row)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _cell_address(col, row):
|
| 112 |
+
letters = ""
|
| 113 |
+
while col:
|
| 114 |
+
col, digit = divmod(col - 1, 26)
|
| 115 |
+
letters = chr(65 + digit) + letters
|
| 116 |
+
return letters + str(row)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _generated_manifest(tab, scope, remaining):
|
| 120 |
+
"""Expand production spill extents and placed Grid blocks before accepting their children."""
|
| 121 |
+
expected = {}
|
| 122 |
+
for addr, root in tab["cells"].items():
|
| 123 |
+
if not isinstance(root, dict):
|
| 124 |
+
raise ValueError("cell_malformed")
|
| 125 |
+
meta = root.get("formulaMeta") or {}
|
| 126 |
+
if not isinstance(meta, dict):
|
| 127 |
+
raise ValueError("cell_malformed")
|
| 128 |
+
has_spill = "spill" in root or "spill" in meta
|
| 129 |
+
spill = root.get("spill", meta.get("spill"))
|
| 130 |
+
binding = root.get("binding") or {}
|
| 131 |
+
view = (root.get("kind") == "bound" and isinstance(binding, dict)
|
| 132 |
+
and binding.get("mode", binding.get("kind")) == "view")
|
| 133 |
+
block = binding.get("block") if view and binding.get("viewType") == "grid" else None
|
| 134 |
+
if not has_spill and block is None:
|
| 135 |
+
continue
|
| 136 |
+
col, row = _position(addr)
|
| 137 |
+
extent = spill if has_spill else block
|
| 138 |
+
if (not isinstance(extent, dict) or any(type(extent.get(k)) is not int or extent[k] < 1
|
| 139 |
+
for k in ("rows", "cols"))):
|
| 140 |
+
raise ValueError("generated_manifest_invalid")
|
| 141 |
+
rows, cols = extent["rows"], extent["cols"]
|
| 142 |
+
if col + cols - 1 > 16384 or row + rows - 1 > 1048576:
|
| 143 |
+
raise ValueError("generated_manifest_invalid")
|
| 144 |
+
if rows * cols > remaining - len(expected):
|
| 145 |
+
raise ProofCapacityError("generated_cell_capacity")
|
| 146 |
+
if has_spill:
|
| 147 |
+
if (root.get("kind") != "formula" or set(spill) != {"version", "origin", "rows", "cols", "ref"}
|
| 148 |
+
or type(spill.get("version")) is not int or spill["version"] != 1
|
| 149 |
+
or spill.get("origin") != scope + "!" + addr
|
| 150 |
+
or spill.get("ref") != addr + ":" + _cell_address(col + cols - 1, row + rows - 1)
|
| 151 |
+
or any(key in holder and holder[key] != spill for holder, key in
|
| 152 |
+
((root, "spill"), (meta, "spill")))):
|
| 153 |
+
raise ValueError("generated_manifest_invalid")
|
| 154 |
+
else:
|
| 155 |
+
fields = binding.get("fields")
|
| 156 |
+
if (rows < 2 or not isinstance(fields, list) or len(fields) != cols
|
| 157 |
+
or any(not isinstance(field, str) or not field for field in fields)
|
| 158 |
+
or len(set(fields)) != len(fields)):
|
| 159 |
+
raise ValueError("generated_manifest_invalid")
|
| 160 |
+
for dy in range(rows):
|
| 161 |
+
for dx in range(cols):
|
| 162 |
+
child_addr = _cell_address(col + dx, row + dy)
|
| 163 |
+
if child_addr in expected:
|
| 164 |
+
raise ValueError("generated_manifest_invalid")
|
| 165 |
+
expected[child_addr] = (("spill", scope + "!" + addr) if has_spill else
|
| 166 |
+
("view", addr, fields[dx], -1 if dy == 0 else -2 if dy == rows - 1 else dy - 1))
|
| 167 |
+
if child_addr not in tab["cells"]:
|
| 168 |
+
raise ValueError("generated_output_missing")
|
| 169 |
+
return expected
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def snapshot(book):
|
| 173 |
+
"""Bind authored definitions and revision while censusing every validated output cell."""
|
| 174 |
+
from core import sheets
|
| 175 |
+
|
| 176 |
+
if not isinstance(book, dict) or not isinstance(book.get("sheets"), list) or not book["sheets"]:
|
| 177 |
+
raise ValueError("workbook_incomplete")
|
| 178 |
+
if len(book["sheets"]) > MAX_TABS:
|
| 179 |
+
raise ProofCapacityError("sheet_capacity")
|
| 180 |
+
total = 0
|
| 181 |
+
for tab in book["sheets"]:
|
| 182 |
+
if isinstance(tab, dict) and isinstance(tab.get("cells"), dict):
|
| 183 |
+
total += len(tab["cells"])
|
| 184 |
+
if total > MAX_CELLS:
|
| 185 |
+
raise ProofCapacityError("cell_capacity")
|
| 186 |
+
# Bound the existing document before canonical cleaners copy any nested cell object.
|
| 187 |
+
bounded_json(book)
|
| 188 |
+
census = {"cells": 0, "bound": 0, "formulas": 0, "refusals": []}
|
| 189 |
+
rows, definitions, tabs, names, unresolved = [], [], set(), set(), False
|
| 190 |
+
generated_cells = 0
|
| 191 |
+
for index, tab in enumerate(book["sheets"]):
|
| 192 |
+
if (not isinstance(tab, dict) or not isinstance(tab.get("id"), str) or not tab["id"]
|
| 193 |
+
or tab["id"] in tabs or not isinstance(tab.get("cells"), dict)
|
| 194 |
+
or (tab.get("name") is not None and not isinstance(tab["name"], str))):
|
| 195 |
+
raise ValueError("sheet_incomplete")
|
| 196 |
+
tabs.add(tab["id"])
|
| 197 |
+
if "cellCount" in tab and (type(tab["cellCount"]) is not int or tab["cellCount"] != len(tab["cells"])):
|
| 198 |
+
raise ValueError("sheet_partial")
|
| 199 |
+
if census["cells"] + len(tab["cells"]) > MAX_CELLS:
|
| 200 |
+
raise ProofCapacityError("cell_capacity")
|
| 201 |
+
name = tab.get("name") or ""
|
| 202 |
+
scope = name.upper() if name and name.upper() not in names else (_UNADDRESSABLE_SHEET + str(index)).upper()
|
| 203 |
+
names.add(scope)
|
| 204 |
+
generated = _generated_manifest(tab, scope, MAX_CELLS - generated_cells)
|
| 205 |
+
generated_cells += len(generated)
|
| 206 |
+
for addr, cell in tab["cells"].items():
|
| 207 |
+
if (not _address(addr) or not isinstance(cell, dict)
|
| 208 |
+
or cell.get("kind") not in sheets.CELL_KINDS
|
| 209 |
+
or cell.get("state") not in sheets.CELL_STATES):
|
| 210 |
+
raise ValueError("cell_malformed")
|
| 211 |
+
try:
|
| 212 |
+
sheets.clean_cell(cell, addr)
|
| 213 |
+
except sheets.SheetRefusal as exc:
|
| 214 |
+
raise ValueError("cell_malformed") from exc
|
| 215 |
+
kind = cell["kind"]
|
| 216 |
+
if ((kind == "formula" and (not isinstance(cell.get("formula"), str)
|
| 217 |
+
or not cell["formula"].startswith("=") or len(cell["formula"]) < 2))
|
| 218 |
+
or (kind == "bound" and (not isinstance(cell.get("binding"), dict) or not cell["binding"]))
|
| 219 |
+
or (kind == "literal" and "value" not in cell and "text" not in cell)):
|
| 220 |
+
raise ValueError("cell_malformed")
|
| 221 |
+
census["cells"] += 1
|
| 222 |
+
census["bound"] += kind == "bound"
|
| 223 |
+
census["formulas"] += kind == "formula"
|
| 224 |
+
unresolved |= kind in ("bound", "formula") and cell["state"] not in ("fresh", "refused")
|
| 225 |
+
unresolved |= cell.get("staleReason") == "refresh_held"
|
| 226 |
+
if cell["state"] == "refused":
|
| 227 |
+
refusal, error = cell.get("refusal") or {}, cell.get("error") or {}
|
| 228 |
+
if not isinstance(refusal, dict) or not isinstance(error, dict):
|
| 229 |
+
raise ValueError("refusal_malformed")
|
| 230 |
+
code = refusal.get("code") or error.get("code") or "refused"
|
| 231 |
+
if not isinstance(code, str) or not code:
|
| 232 |
+
raise ValueError("refusal_malformed")
|
| 233 |
+
census["refusals"].append({"cell": "%s!%s" % (name or tab["id"], addr),
|
| 234 |
+
"kind": kind, "code": code})
|
| 235 |
+
authored = cell
|
| 236 |
+
meta = cell.get("formulaMeta") or {}
|
| 237 |
+
marker = meta.get("spillChild")
|
| 238 |
+
declared = generated.get(addr)
|
| 239 |
+
if marker is not None:
|
| 240 |
+
key = scope + "!" + addr
|
| 241 |
+
origin = marker.get("origin") if isinstance(marker, dict) else None
|
| 242 |
+
root_addr = origin.rpartition("!")[2] if isinstance(origin, str) else ""
|
| 243 |
+
root = tab["cells"].get(root_addr, {})
|
| 244 |
+
if (not isinstance(marker, dict) or type(marker.get("version")) is not int
|
| 245 |
+
or marker["version"] != 1 or marker.get("address") != key
|
| 246 |
+
or origin != scope + "!" + root_addr or origin == key
|
| 247 |
+
or cell.get("spillParent") != origin or root.get("kind") != "formula"
|
| 248 |
+
or set(marker) - {"version", "origin", "address", "base"}
|
| 249 |
+
or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
|
| 250 |
+
raise ValueError("cell_malformed")
|
| 251 |
+
base = marker.get("base")
|
| 252 |
+
if base and (not isinstance(base, dict) or base.get("kind") != "empty"
|
| 253 |
+
or any(base.get(k) is not None for k in ("value", "text", "formula", "binding"))):
|
| 254 |
+
raise ValueError("cell_malformed")
|
| 255 |
+
if meta.get("spillClear") is True:
|
| 256 |
+
if (declared is not None or cell.get("value") is not None or cell.get("display") != ""
|
| 257 |
+
or cell["state"] != "typed" or cell.get("refusal") or cell.get("error")):
|
| 258 |
+
raise ValueError("generated_output_invalid")
|
| 259 |
+
elif ("spillClear" in meta or declared != ("spill", origin)
|
| 260 |
+
or cell["state"] not in ("fresh", "refused")):
|
| 261 |
+
raise ValueError("generated_output_invalid")
|
| 262 |
+
authored = base if base else None
|
| 263 |
+
elif "spillParent" in cell or "spillClear" in meta:
|
| 264 |
+
raise ValueError("cell_malformed")
|
| 265 |
+
view = cell.get("viewChild")
|
| 266 |
+
if view is not None:
|
| 267 |
+
root = tab["cells"].get(view.get("origin"), {}) if isinstance(view, dict) else {}
|
| 268 |
+
binding = root.get("binding") or {}
|
| 269 |
+
if (marker is not None or not isinstance(view, dict) or not _address(view.get("origin"))
|
| 270 |
+
or view["origin"] == addr or not isinstance(view.get("field"), str) or not view["field"]
|
| 271 |
+
or type(view.get("line")) is not int or view["line"] < -2
|
| 272 |
+
or root.get("kind") != "bound" or not isinstance(binding, dict)
|
| 273 |
+
or (binding.get("mode") != "view" and binding.get("kind") != "view")
|
| 274 |
+
or kind != "literal" or "text" in cell or "formula" in cell or "binding" in cell):
|
| 275 |
+
raise ValueError("cell_malformed")
|
| 276 |
+
if declared != ("view", view["origin"], view["field"], view["line"]):
|
| 277 |
+
raise ValueError("generated_output_invalid")
|
| 278 |
+
authored = None
|
| 279 |
+
if declared is not None:
|
| 280 |
+
is_root = (declared[1] == (scope + "!" + addr if declared[0] == "spill" else addr))
|
| 281 |
+
if not is_root and ((declared[0] == "spill" and marker is None)
|
| 282 |
+
or (declared[0] == "view" and view is None)):
|
| 283 |
+
raise ValueError("generated_output_invalid")
|
| 284 |
+
if authored is not None:
|
| 285 |
+
rows.append({"sheet": tab["id"], "address": addr, "kind": authored["kind"]})
|
| 286 |
+
definitions.append({"sheet": tab["id"], "address": addr, "definition": _definition(authored)})
|
| 287 |
+
if census["cells"] and not rows:
|
| 288 |
+
raise ValueError("workbook_empty")
|
| 289 |
+
rows.sort(key=lambda row: (row["sheet"], row["address"]))
|
| 290 |
+
definitions.sort(key=lambda row: (row["sheet"], row["address"]))
|
| 291 |
+
authored = digest({"sheets": [(tab["id"], tab.get("name")) for tab in book["sheets"]],
|
| 292 |
+
"cells": definitions, "namedRanges": book.get("namedRanges"),
|
| 293 |
+
"dateSystem": book.get("dateSystem")})
|
| 294 |
+
return {"revision": book.get("rev"), "inventory": rows, "authored": authored,
|
| 295 |
+
"sheets": sorted(tabs), "census": census, "unresolved": bool(unresolved)}
|
platform/harness/datastore.py
CHANGED
|
@@ -1174,7 +1174,263 @@ _IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
| 1174 |
|
| 1175 |
#: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
|
| 1176 |
_SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
|
| 1177 |
-
_ORDINAL_ORDER = re.compile(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1178 |
|
| 1179 |
|
| 1180 |
def mirror_file_marks(cur):
|
|
@@ -1275,6 +1531,7 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 1275 |
the page's rows before the projection runs, when only the projection holds a subquery.
|
| 1276 |
`page_key` opts a physical table into a narrow, materialised key page before projection.
|
| 1277 |
It must uniquely identify source rows, independently of the displayed record identity.
|
|
|
|
| 1278 |
"""
|
| 1279 |
if bool(table) == bool(from_sql):
|
| 1280 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
@@ -1310,14 +1567,19 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 1310 |
total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
|
| 1311 |
grp = str(group_by or "").strip()
|
| 1312 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 1313 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1314 |
# Only keys cross the materialisation boundary; identity order still uses the full population.
|
| 1315 |
got = cur.execute(
|
| 1316 |
f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
|
| 1317 |
f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
|
| 1318 |
f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
|
| 1319 |
f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
|
| 1320 |
-
elif (
|
| 1321 |
and not _SUBQUERY.search(f"{pred} {order}")):
|
| 1322 |
# A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
|
| 1323 |
# of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
|
|
|
|
| 1174 |
|
| 1175 |
#: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
|
| 1176 |
_SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
|
| 1177 |
+
_ORDINAL_ORDER = re.compile(
|
| 1178 |
+
r"(?:\(\s*)*[+-]?\d[\d_]*(?:\s*\))*\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
|
| 1179 |
+
_ALL_ORDER = re.compile(r"ALL\s*(?:ASC|DESC)?\s*(?:NULLS\s+(?:FIRST|LAST))?", re.I)
|
| 1180 |
+
|
| 1181 |
+
|
| 1182 |
+
def _sql_terms(sql):
|
| 1183 |
+
"""Split trusted SQL at unquoted, top-level commas, or decline unfamiliar syntax."""
|
| 1184 |
+
depth, quote, start, index = 0, None, 0, 0
|
| 1185 |
+
terms = []
|
| 1186 |
+
while index < len(sql):
|
| 1187 |
+
char = sql[index]
|
| 1188 |
+
if quote:
|
| 1189 |
+
if char == "\\":
|
| 1190 |
+
return None
|
| 1191 |
+
if char == quote:
|
| 1192 |
+
if index + 1 < len(sql) and sql[index + 1] == quote:
|
| 1193 |
+
index += 2
|
| 1194 |
+
continue
|
| 1195 |
+
quote = None
|
| 1196 |
+
elif char in "'\"":
|
| 1197 |
+
quote = char
|
| 1198 |
+
elif (not (char.isalnum() or char.isspace() or char in "_.,()+-*/%=<>!|:&~^")
|
| 1199 |
+
or sql[index:index + 2] in ("--", "/*", "*/", "->", "=>", ":=")):
|
| 1200 |
+
return None
|
| 1201 |
+
elif char == "(":
|
| 1202 |
+
depth += 1
|
| 1203 |
+
elif char == ")":
|
| 1204 |
+
depth -= 1
|
| 1205 |
+
if depth < 0:
|
| 1206 |
+
return None
|
| 1207 |
+
elif char == "," and depth == 0:
|
| 1208 |
+
terms.append(sql[start:index].strip())
|
| 1209 |
+
start = index + 1
|
| 1210 |
+
index += 1
|
| 1211 |
+
if quote or depth:
|
| 1212 |
+
return None
|
| 1213 |
+
terms.append(sql[start:].strip())
|
| 1214 |
+
return terms if all(terms) else None
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
+
def _sql_tokens(sql):
|
| 1218 |
+
"""Use DuckDB's lexer for identifier quoting and expression comparisons."""
|
| 1219 |
+
tokens = duckdb.tokenize(sql)
|
| 1220 |
+
encoded = sql.encode("utf-8")
|
| 1221 |
+
result = [(encoded[start:tokens[i + 1][0] if i + 1 < len(tokens) else len(encoded)].decode("utf-8").strip(), kind)
|
| 1222 |
+
for i, (start, kind) in enumerate(tokens)]
|
| 1223 |
+
if any(text.upper() in ("COLUMNS", "EXCLUDE", "REPLACE", "USING") for text, _kind in result):
|
| 1224 |
+
raise ValueError("Unrecognized page ordering syntax")
|
| 1225 |
+
return result
|
| 1226 |
+
|
| 1227 |
+
|
| 1228 |
+
def _order_needs_projection(order, projection=""):
|
| 1229 |
+
"""Keep positional, ALL, alias-dependent and unfamiliar orders after projection."""
|
| 1230 |
+
terms = _sql_terms(order)
|
| 1231 |
+
if terms is None or any(_ORDINAL_ORDER.fullmatch(t) or _ALL_ORDER.fullmatch(t) for t in terms):
|
| 1232 |
+
return True
|
| 1233 |
+
selected = _sql_terms(projection) if projection else []
|
| 1234 |
+
if selected is None:
|
| 1235 |
+
return True
|
| 1236 |
+
try:
|
| 1237 |
+
fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
| 1238 |
+
def normalized(tokens):
|
| 1239 |
+
return [(text[1:-1].replace('""', '"').translate(fold) if text.startswith('"')
|
| 1240 |
+
else text if text.startswith("'") else text.translate(fold), kind) for text, kind in tokens]
|
| 1241 |
+
ordered = normalized(_sql_tokens(order))
|
| 1242 |
+
aliases = {}
|
| 1243 |
+
for term in selected:
|
| 1244 |
+
tokens = _sql_tokens(term)
|
| 1245 |
+
depth, top = 0, []
|
| 1246 |
+
for i, (text, _kind) in enumerate(tokens):
|
| 1247 |
+
if text == ")":
|
| 1248 |
+
depth -= 1
|
| 1249 |
+
if depth == 0:
|
| 1250 |
+
top.append(i)
|
| 1251 |
+
if text == "(":
|
| 1252 |
+
depth += 1
|
| 1253 |
+
if not top or any(tokens[i][0] in ("*", ":", ":=") for i in top):
|
| 1254 |
+
return True
|
| 1255 |
+
last = top[-1]
|
| 1256 |
+
if len(top) >= 2 and tokens[top[-2]][0].upper() == "AS":
|
| 1257 |
+
expression = tokens[:top[-2]]
|
| 1258 |
+
elif (last > 0 and tokens[last][1] == duckdb.token_type.identifier
|
| 1259 |
+
and tokens[last - 1][0] != "."):
|
| 1260 |
+
expression = tokens[:last]
|
| 1261 |
+
else:
|
| 1262 |
+
continue
|
| 1263 |
+
alias = normalized([tokens[last]])[0][0]
|
| 1264 |
+
if tokens[last][0].startswith("'"):
|
| 1265 |
+
alias = tokens[last][0][1:-1].replace("''", "'").translate(fold)
|
| 1266 |
+
aliases.setdefault(alias, []).append(normalized(expression))
|
| 1267 |
+
for alias, expressions in aliases.items():
|
| 1268 |
+
covered = set()
|
| 1269 |
+
# A repeated source expression is safe, including generated casts and passthrough aliases.
|
| 1270 |
+
if all(expr == expressions[0] for expr in expressions):
|
| 1271 |
+
expr = expressions[0]
|
| 1272 |
+
for i in range(len(ordered) - len(expr) + 1):
|
| 1273 |
+
if expr and ordered[i:i + len(expr)] == expr:
|
| 1274 |
+
covered.update(range(i, i + len(expr)))
|
| 1275 |
+
for i, token in enumerate(ordered):
|
| 1276 |
+
if (token[0] == alias and token[1] in (duckdb.token_type.identifier, duckdb.token_type.keyword)
|
| 1277 |
+
and i not in covered
|
| 1278 |
+
and (i == 0 or ordered[i - 1][0] != ".")
|
| 1279 |
+
and (i + 1 == len(ordered) or ordered[i + 1][0] not in (".", "("))):
|
| 1280 |
+
return True
|
| 1281 |
+
except (duckdb.Error, AttributeError, ValueError, UnicodeError):
|
| 1282 |
+
return True
|
| 1283 |
+
return False
|
| 1284 |
+
|
| 1285 |
+
|
| 1286 |
+
def _physical_page_order(order, page_key):
|
| 1287 |
+
"""Make physical pages total without changing ALL or unfamiliar SQL ordering semantics."""
|
| 1288 |
+
terms = _sql_terms(order)
|
| 1289 |
+
if terms is None or any(_ALL_ORDER.fullmatch(term) for term in terms):
|
| 1290 |
+
return order
|
| 1291 |
+
try:
|
| 1292 |
+
_sql_tokens(order)
|
| 1293 |
+
except (duckdb.Error, AttributeError, ValueError):
|
| 1294 |
+
return order
|
| 1295 |
+
return f"{order}, {page_key} ASC"
|
| 1296 |
+
|
| 1297 |
+
|
| 1298 |
+
def _page_is_row_local(projection, order, cur):
|
| 1299 |
+
"""Prove outer expressions scalar with DuckDB's parser and bounded catalogue lookups."""
|
| 1300 |
+
metadata, verified, expanding, bound = {}, set(), set(), set()
|
| 1301 |
+
|
| 1302 |
+
def parse(expressions, ordering=""):
|
| 1303 |
+
sql = "SELECT " + expressions + (" ORDER BY " + ordering if ordering else "")
|
| 1304 |
+
parsed = json.loads(cur.execute("SELECT json_serialize_sql(?)", [sql]).fetchone()[0])
|
| 1305 |
+
if parsed.get("error") or len(parsed["statements"]) != 1:
|
| 1306 |
+
raise ValueError("Unrecognized page expression")
|
| 1307 |
+
node = parsed["statements"][0]["node"]
|
| 1308 |
+
if (node["type"] != "SELECT_NODE" or node["from_table"]["type"] != "EMPTY"
|
| 1309 |
+
or node["cte_map"]["map"] or node["group_expressions"] or node["group_sets"]
|
| 1310 |
+
or node["aggregate_handling"] != "STANDARD_HANDLING"
|
| 1311 |
+
or any(node[key] is not None for key in ("where_clause", "having", "sample", "qualify"))
|
| 1312 |
+
or any(m["type"] != "ORDER_MODIFIER" for m in node["modifiers"])):
|
| 1313 |
+
raise ValueError("Projection changes the outer population")
|
| 1314 |
+
return [node["select_list"], node["modifiers"]]
|
| 1315 |
+
|
| 1316 |
+
def aggregate_is_local(value, scope):
|
| 1317 |
+
columns = set()
|
| 1318 |
+
|
| 1319 |
+
def collect(node):
|
| 1320 |
+
if isinstance(node, list):
|
| 1321 |
+
return all(collect(v) for v in node)
|
| 1322 |
+
if not isinstance(node, dict):
|
| 1323 |
+
return True
|
| 1324 |
+
kind = node.get("class")
|
| 1325 |
+
if kind == "COLUMN_REF":
|
| 1326 |
+
columns.add(tuple(node["column_names"]))
|
| 1327 |
+
elif kind is not None and kind not in {
|
| 1328 |
+
"CONSTANT", "CAST", "CASE", "COMPARISON", "CONJUNCTION",
|
| 1329 |
+
"OPERATOR", "FUNCTION", "PARAMETER"}:
|
| 1330 |
+
return False
|
| 1331 |
+
return all(collect(v) for v in node.values())
|
| 1332 |
+
|
| 1333 |
+
if not collect(value):
|
| 1334 |
+
return False
|
| 1335 |
+
if not columns:
|
| 1336 |
+
return True
|
| 1337 |
+
if scope is None or scope[1] or scope[0]["type"] != "SELECT_NODE":
|
| 1338 |
+
return False
|
| 1339 |
+
source = scope[0]["from_table"]
|
| 1340 |
+
if (source["type"] != "BASE_TABLE" or source.get("column_name_alias")
|
| 1341 |
+
or source.get("at_clause") or source.get("sample")):
|
| 1342 |
+
return False
|
| 1343 |
+
alias = source.get("alias") or source["table_name"]
|
| 1344 |
+
fold = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
| 1345 |
+
if any(len(c) != 2 or c[0].translate(fold) != alias.translate(fold) for c in columns):
|
| 1346 |
+
return False
|
| 1347 |
+
quote = lambda name: '"' + name.replace('"', '""') + '"'
|
| 1348 |
+
table = '.'.join(quote(source[key]) for key in ("catalog_name", "schema_name", "table_name")
|
| 1349 |
+
if source.get(key))
|
| 1350 |
+
selected = ','.join('.'.join(map(quote, c)) for c in sorted(columns))
|
| 1351 |
+
sql = f"DESCRIBE SELECT {selected} FROM {table} AS {quote(alias)}"
|
| 1352 |
+
if sql not in bound:
|
| 1353 |
+
# A matching alias alone cannot prove that a missing column will not bind outward.
|
| 1354 |
+
cur.execute(sql).fetchall()
|
| 1355 |
+
bound.add(sql)
|
| 1356 |
+
return True
|
| 1357 |
+
|
| 1358 |
+
def references(value, required, nested, scalar=True, scope=None):
|
| 1359 |
+
if isinstance(value, list):
|
| 1360 |
+
return all(references(v, required, nested, scalar, scope) for v in value)
|
| 1361 |
+
if not isinstance(value, dict):
|
| 1362 |
+
return True
|
| 1363 |
+
if "cte_map" in value:
|
| 1364 |
+
scope = (value, bool(value["cte_map"]["map"]) or bool(scope and scope[1]))
|
| 1365 |
+
kind = value.get("class")
|
| 1366 |
+
if scalar and kind == "SUBQUERY":
|
| 1367 |
+
# Nested aggregates must bind locally even inside a syntactic scalar subquery.
|
| 1368 |
+
return (value["subquery_type"] == "SCALAR" and value.get("child") is None
|
| 1369 |
+
and references(value["subquery"], required, nested, False, scope))
|
| 1370 |
+
if scalar and kind is not None:
|
| 1371 |
+
if kind not in {"COLUMN_REF", "CONSTANT", "CAST", "CASE", "COMPARISON",
|
| 1372 |
+
"CONJUNCTION", "OPERATOR", "FUNCTION", "PARAMETER"}:
|
| 1373 |
+
return False
|
| 1374 |
+
if kind == "OPERATOR" and value["type"] not in {
|
| 1375 |
+
"OPERATOR_COALESCE", "OPERATOR_NOT", "OPERATOR_IS_NULL",
|
| 1376 |
+
"OPERATOR_IS_NOT_NULL", "COMPARE_IN", "COMPARE_NOT_IN"}:
|
| 1377 |
+
return False
|
| 1378 |
+
if "function_name" in value:
|
| 1379 |
+
if value.get("catalog") or value.get("schema", "") not in ("", "main"):
|
| 1380 |
+
return False
|
| 1381 |
+
if scalar and (value.get("distinct") or value.get("filter") is not None
|
| 1382 |
+
or value.get("export_state") or value.get("order_bys", {}).get("orders")):
|
| 1383 |
+
return False
|
| 1384 |
+
name = value["function_name"].lower()
|
| 1385 |
+
required[name] = required.get(name, False) or scalar
|
| 1386 |
+
if not scalar:
|
| 1387 |
+
nested.append((name, value, scope))
|
| 1388 |
+
return all(references(v, required, nested, scalar, scope) for v in value.values())
|
| 1389 |
+
|
| 1390 |
+
def prove(expressions, ordering=""):
|
| 1391 |
+
required, nested = {}, []
|
| 1392 |
+
if not references(parse(expressions, ordering), required, nested):
|
| 1393 |
+
return False
|
| 1394 |
+
missing = sorted(set(required) - metadata.keys())
|
| 1395 |
+
if missing:
|
| 1396 |
+
rows = cur.execute(
|
| 1397 |
+
"SELECT lower(function_name), function_type, internal, macro_definition "
|
| 1398 |
+
"FROM duckdb_functions() WHERE lower(function_name) = ANY(?)", [missing]).fetchall()
|
| 1399 |
+
metadata.update((name, []) for name in missing)
|
| 1400 |
+
for name, function_type, internal, definition in rows:
|
| 1401 |
+
metadata[name].append((function_type, internal, definition))
|
| 1402 |
+
for name, scalar in required.items():
|
| 1403 |
+
if (name, scalar) in verified:
|
| 1404 |
+
continue
|
| 1405 |
+
if not metadata[name] or name in expanding or len(expanding) >= 16:
|
| 1406 |
+
return False
|
| 1407 |
+
for function_type, internal, definition in metadata[name]:
|
| 1408 |
+
if function_type == "macro":
|
| 1409 |
+
# Only engine-owned macros with independently scalar bodies are admitted.
|
| 1410 |
+
if not internal or not definition:
|
| 1411 |
+
return False
|
| 1412 |
+
expanding.add(name)
|
| 1413 |
+
try:
|
| 1414 |
+
if not prove(definition):
|
| 1415 |
+
return False
|
| 1416 |
+
finally:
|
| 1417 |
+
expanding.remove(name)
|
| 1418 |
+
elif function_type not in ({"scalar"} if scalar else {"scalar", "aggregate", "table"}):
|
| 1419 |
+
return False
|
| 1420 |
+
verified.add((name, scalar))
|
| 1421 |
+
for name, value, scope in nested:
|
| 1422 |
+
if (value.get("class") != "WINDOW"
|
| 1423 |
+
and any(kind == "aggregate" for kind, _internal, _definition in metadata[name])
|
| 1424 |
+
and not aggregate_is_local(value, scope)):
|
| 1425 |
+
return False
|
| 1426 |
+
return True
|
| 1427 |
+
|
| 1428 |
+
try:
|
| 1429 |
+
return prove(projection, order)
|
| 1430 |
+
except Exception as exc:
|
| 1431 |
+
# A parser/catalogue refusal never authorizes moving the original projection.
|
| 1432 |
+
_request_timing.raise_if_resource_error(exc)
|
| 1433 |
+
return False
|
| 1434 |
|
| 1435 |
|
| 1436 |
def mirror_file_marks(cur):
|
|
|
|
| 1531 |
the page's rows before the projection runs, when only the projection holds a subquery.
|
| 1532 |
`page_key` opts a physical table into a narrow, materialised key page before projection.
|
| 1533 |
It must uniquely identify source rows, independently of the displayed record identity.
|
| 1534 |
+
Proven row-local pages end with this physical key ascending. Fallback SQL retains its order.
|
| 1535 |
"""
|
| 1536 |
if bool(table) == bool(from_sql):
|
| 1537 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
|
|
| 1567 |
total = count_rows(table=table, from_sql=from_sql, where=pred, params=params, cur=cur)
|
| 1568 |
grp = str(group_by or "").strip()
|
| 1569 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 1570 |
+
needs_projection = _order_needs_projection(order, proj) if page_first else True
|
| 1571 |
+
row_local = (page_first and table and not grp and not needs_projection
|
| 1572 |
+
and _page_is_row_local(proj, order, cur))
|
| 1573 |
+
if row_local and page_key:
|
| 1574 |
+
order = _physical_page_order(order, page_key)
|
| 1575 |
+
if row_local and page_key:
|
| 1576 |
# Only keys cross the materialisation boundary; identity order still uses the full population.
|
| 1577 |
got = cur.execute(
|
| 1578 |
f'WITH _mirror_page AS MATERIALIZED (SELECT {page_key} AS "_page_key" '
|
| 1579 |
f'FROM {name}{tail} ORDER BY {order} LIMIT {limit} OFFSET {offset}) '
|
| 1580 |
f'SELECT {proj} FROM {name} SEMI JOIN _mirror_page '
|
| 1581 |
f'ON {page_key} = _mirror_page."_page_key" ORDER BY {order}', params).fetchall()
|
| 1582 |
+
elif (row_local and not page_key and _SUBQUERY.search(proj)
|
| 1583 |
and not _SUBQUERY.search(f"{pred} {order}")):
|
| 1584 |
# A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
|
| 1585 |
# of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
|
requirements.txt
CHANGED
|
@@ -21,7 +21,7 @@ python-dotenv>=1.2.2
|
|
| 21 |
pandas>=2.0
|
| 22 |
requests>=2.28
|
| 23 |
huggingface_hub>=0.20
|
| 24 |
-
duckdb
|
| 25 |
pyyaml>=6.0
|
| 26 |
pillow>=10.0
|
| 27 |
beautifulsoup4>=4.12
|
|
|
|
| 21 |
pandas>=2.0
|
| 22 |
requests>=2.28
|
| 23 |
huggingface_hub>=0.20
|
| 24 |
+
duckdb==1.5.4
|
| 25 |
pyyaml>=6.0
|
| 26 |
pillow>=10.0
|
| 27 |
beautifulsoup4>=4.12
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
4a7e48c0f21990c3c89f863bcfd80f31584284e596c9fe0b8f9a9c0757214665
|