Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +11 -4
- api/assistant_mirror.py +31 -3
- api/deps.py +12 -7
- api/main.py +69 -0
- api/routes_odoo_tables.py +1 -1
- platform/harness/datastore.py +22 -5
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "fe5f25c0f",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
fe5f25c0f
|
api/amazon_relational.py
CHANGED
|
@@ -470,11 +470,18 @@ def _read_through(cur, key, count):
|
|
| 470 |
return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
|
| 471 |
|
| 472 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
def missing_source(key):
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
"
|
|
|
|
|
|
|
|
|
|
| 478 |
|
| 479 |
|
| 480 |
def _text(value):
|
|
|
|
| 470 |
return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
|
| 471 |
|
| 472 |
|
| 473 |
+
#: Set by the boot's resync loop until its Amazon restore ends (`main._restore_amazon_mirror`).
|
| 474 |
+
RESTORE_PENDING = {"since": None}
|
| 475 |
+
|
| 476 |
+
|
| 477 |
def missing_source(key):
|
| 478 |
+
"""The ONE sentence every door gives for an Amazon table this server's mirror does not hold yet."""
|
| 479 |
+
label = key.removeprefix(PREFIX).replace("_", " ")
|
| 480 |
+
if RESTORE_PENDING.get("since") is not None:
|
| 481 |
+
return {"missing": f"Amazon {label} is still loading after a server restart, "
|
| 482 |
+
"so ask again in 5 minutes."}
|
| 483 |
+
return {"missing": f"Amazon {label} is still loading from Amazon, "
|
| 484 |
+
"so ask again after the next Amazon sync finishes."}
|
| 485 |
|
| 486 |
|
| 487 |
def _text(value):
|
api/assistant_mirror.py
CHANGED
|
@@ -93,6 +93,33 @@ def open_cursor(session, table_key=None):
|
|
| 93 |
return cur, None
|
| 94 |
|
| 95 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
def binding(cur, table_key):
|
| 97 |
"""The resolved read-through spec for one connected grid, or None when it has no binding.
|
| 98 |
|
|
@@ -340,7 +367,7 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
|
|
| 340 |
# ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
|
| 341 |
first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 342 |
where=where, params=tuple(params), order_by=spec["id"],
|
| 343 |
-
limit=datastore.WINDOW_MAX, cur=cur, **frm)
|
| 344 |
total = int(first["total"])
|
| 345 |
if total > ASSISTANT_READ_MAX:
|
| 346 |
raise MirrorRefusal(
|
|
@@ -358,7 +385,8 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
|
|
| 358 |
more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
|
| 359 |
nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 360 |
where=more_where, params=tuple(params) + (last,),
|
| 361 |
-
order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur,
|
|
|
|
| 362 |
page = list(nxt["rows"])
|
| 363 |
if not page:
|
| 364 |
break
|
|
@@ -481,7 +509,7 @@ def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree
|
|
| 481 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 482 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
|
| 483 |
params=tuple(params), order_by=spec["id"], offset=int(offset),
|
| 484 |
-
limit=int(limit), cur=cur, **frm)
|
| 485 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 486 |
defn=defn if isinstance(defn, dict) else None)
|
| 487 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
|
|
|
| 93 |
return cur, None
|
| 94 |
|
| 95 |
|
| 96 |
+
#: The one sentence for a mirror-served table this deployment has no binding for at all.
|
| 97 |
+
NO_BINDING = ("this database is read live from its connector mirror, and this deployment has no "
|
| 98 |
+
"read-through binding for it yet, so it holds no rows the assistant can read")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def unavailable_reason(table_key, definition=None):
|
| 102 |
+
"""The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows.
|
| 103 |
+
|
| 104 |
+
A bound table whose mirror tables are absent is loading (a restart starts from the boot seed);
|
| 105 |
+
a table with no binding at all gets `NO_BINDING`.
|
| 106 |
+
"""
|
| 107 |
+
key = str(table_key or "")
|
| 108 |
+
try:
|
| 109 |
+
if key.startswith("ut_amzn_"):
|
| 110 |
+
import amazon_relational as amazon
|
| 111 |
+
if key in amazon.SOURCE:
|
| 112 |
+
return amazon.missing_source(key)["missing"]
|
| 113 |
+
spec = _rot()._sources().get(key)
|
| 114 |
+
if spec and not spec.get("missing"):
|
| 115 |
+
label = str((definition or {}).get("label") or key)
|
| 116 |
+
return (f"{label} is still loading into this server after a restart, "
|
| 117 |
+
"so ask again in an hour.")
|
| 118 |
+
except Exception: # noqa: BLE001
|
| 119 |
+
pass
|
| 120 |
+
return NO_BINDING
|
| 121 |
+
|
| 122 |
+
|
| 123 |
def binding(cur, table_key):
|
| 124 |
"""The resolved read-through spec for one connected grid, or None when it has no binding.
|
| 125 |
|
|
|
|
| 367 |
# ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
|
| 368 |
first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 369 |
where=where, params=tuple(params), order_by=spec["id"],
|
| 370 |
+
limit=datastore.WINDOW_MAX, cur=cur, page_first=True, **frm)
|
| 371 |
total = int(first["total"])
|
| 372 |
if total > ASSISTANT_READ_MAX:
|
| 373 |
raise MirrorRefusal(
|
|
|
|
| 385 |
more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
|
| 386 |
nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 387 |
where=more_where, params=tuple(params) + (last,),
|
| 388 |
+
order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur,
|
| 389 |
+
page_first=True, **frm)
|
| 390 |
page = list(nxt["rows"])
|
| 391 |
if not page:
|
| 392 |
break
|
|
|
|
| 509 |
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
|
| 510 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
|
| 511 |
params=tuple(params), order_by=spec["id"], offset=int(offset),
|
| 512 |
+
limit=int(limit), cur=cur, page_first=True, **frm)
|
| 513 |
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
|
| 514 |
defn=defn if isinstance(defn, dict) else None)
|
| 515 |
rows = rot._pool_from_window(spec, win, stamp=stamp)
|
api/deps.py
CHANGED
|
@@ -492,8 +492,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
|
|
| 492 |
spec = assistant_mirror.binding(cur, database)
|
| 493 |
if spec is None:
|
| 494 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 495 |
-
|
| 496 |
-
"has no read-through binding for it yet, so it holds no rows to read")
|
| 497 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 498 |
if not fields:
|
| 499 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
|
@@ -1028,11 +1027,15 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 1028 |
import assistant_mirror
|
| 1029 |
cur, refusal = assistant_mirror.open_cursor(session)
|
| 1030 |
mirror_cur = None if refusal is not None else cur
|
|
|
|
|
|
|
| 1031 |
except Exception: # noqa: BLE001
|
| 1032 |
mirror_cur = None
|
| 1033 |
return mirror_cur
|
| 1034 |
|
| 1035 |
amazon_cur = {}
|
|
|
|
|
|
|
| 1036 |
|
| 1037 |
def _amazon_mirror(key):
|
| 1038 |
"""ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
|
|
@@ -1041,6 +1044,8 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 1041 |
import assistant_mirror
|
| 1042 |
cur, refusal = assistant_mirror.open_cursor(session, key)
|
| 1043 |
amazon_cur["cur"] = None if refusal is not None else cur
|
|
|
|
|
|
|
| 1044 |
except Exception: # noqa: BLE001
|
| 1045 |
amazon_cur["cur"] = None
|
| 1046 |
return amazon_cur["cur"]
|
|
@@ -1165,8 +1170,9 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 1165 |
# first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
|
| 1166 |
cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
|
| 1167 |
if cur is None:
|
| 1168 |
-
row["reason"] = ("
|
| 1169 |
-
"
|
|
|
|
| 1170 |
continue
|
| 1171 |
# ⛔ `is_bound`, NOT `binding` - THE AVAILABILITY QUESTION, NOT THE READ ONE, and the
|
| 1172 |
# difference is 300 ms PER KEY here. Resolving a full spec runs `excluded_ids`, a scan of
|
|
@@ -1180,9 +1186,8 @@ def assistant_source_status(session: Session, databases=None):
|
|
| 1180 |
if bound:
|
| 1181 |
row["answerable"] = True
|
| 1182 |
else:
|
| 1183 |
-
|
| 1184 |
-
|
| 1185 |
-
"rows the assistant can read")
|
| 1186 |
return out
|
| 1187 |
|
| 1188 |
|
|
|
|
| 492 |
spec = assistant_mirror.binding(cur, database)
|
| 493 |
if spec is None:
|
| 494 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
| 495 |
+
assistant_mirror.unavailable_reason(database, definition))
|
|
|
|
| 496 |
fields = assistant_mirror.answerable_fields(database, spec, declared)
|
| 497 |
if not fields:
|
| 498 |
_assistant_refuse(409, "assistant_snapshot_unavailable",
|
|
|
|
| 1027 |
import assistant_mirror
|
| 1028 |
cur, refusal = assistant_mirror.open_cursor(session)
|
| 1029 |
mirror_cur = None if refusal is not None else cur
|
| 1030 |
+
if refusal is not None:
|
| 1031 |
+
closed_reason["odoo"] = refusal.message
|
| 1032 |
except Exception: # noqa: BLE001
|
| 1033 |
mirror_cur = None
|
| 1034 |
return mirror_cur
|
| 1035 |
|
| 1036 |
amazon_cur = {}
|
| 1037 |
+
#: The cursor refusal's own message, so this list and `query_database` say one sentence.
|
| 1038 |
+
closed_reason = {}
|
| 1039 |
|
| 1040 |
def _amazon_mirror(key):
|
| 1041 |
"""ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
|
|
|
|
| 1044 |
import assistant_mirror
|
| 1045 |
cur, refusal = assistant_mirror.open_cursor(session, key)
|
| 1046 |
amazon_cur["cur"] = None if refusal is not None else cur
|
| 1047 |
+
if refusal is not None:
|
| 1048 |
+
closed_reason["amazon"] = refusal.message
|
| 1049 |
except Exception: # noqa: BLE001
|
| 1050 |
amazon_cur["cur"] = None
|
| 1051 |
return amazon_cur["cur"]
|
|
|
|
| 1170 |
# first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
|
| 1171 |
cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
|
| 1172 |
if cur is None:
|
| 1173 |
+
row["reason"] = (closed_reason.get("amazon" if key.startswith("ut_amzn_") else "odoo")
|
| 1174 |
+
or "this database is read live from its connector mirror, and that "
|
| 1175 |
+
"mirror is not open on this deployment right now; try again shortly")
|
| 1176 |
continue
|
| 1177 |
# ⛔ `is_bound`, NOT `binding` - THE AVAILABILITY QUESTION, NOT THE READ ONE, and the
|
| 1178 |
# difference is 300 ms PER KEY here. Resolving a full spec runs `excluded_ids`, a scan of
|
|
|
|
| 1186 |
if bound:
|
| 1187 |
row["answerable"] = True
|
| 1188 |
else:
|
| 1189 |
+
import assistant_mirror
|
| 1190 |
+
row["reason"] = assistant_mirror.unavailable_reason(key, definition)
|
|
|
|
| 1191 |
return out
|
| 1192 |
|
| 1193 |
|
api/main.py
CHANGED
|
@@ -898,6 +898,40 @@ def _pull_amazon(why, vendor=True):
|
|
| 898 |
return False
|
| 899 |
|
| 900 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 901 |
class _SyncNowRefused(Exception):
|
| 902 |
"""An application-authored, credential-free sentence for a Sync now receipt."""
|
| 903 |
|
|
@@ -1926,7 +1960,21 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 1926 |
# relational rebuild, which `manual` says nothing about.
|
| 1927 |
if manual:
|
| 1928 |
return None, None, None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1929 |
sync_report = _ds.sync_all(entities=entities, log=lambda *a, **k: None)
|
|
|
|
|
|
|
|
|
|
| 1930 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 1931 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 1932 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
@@ -1941,6 +1989,19 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 1941 |
return None, None, _classify_resync_error(e)
|
| 1942 |
|
| 1943 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1944 |
def _classify_resync_error(exc):
|
| 1945 |
"""D-314, the half that fits inside this loop: a failed pass leaves a MARK, not just a line.
|
| 1946 |
|
|
@@ -2183,7 +2244,15 @@ def _store_resync_loop():
|
|
| 2183 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2184 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2185 |
if not passes:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2186 |
_await_boot_seed()
|
|
|
|
|
|
|
|
|
|
| 2187 |
_boot_catch_up = ((not passes or catch_up_retries)
|
| 2188 |
and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
|
| 2189 |
# ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
|
|
|
|
| 898 |
return False
|
| 899 |
|
| 900 |
|
| 901 |
+
def _restore_amazon_mirror(why):
|
| 902 |
+
"""Restore Amazon's saved mirror and republish its registry: store reads only, no vendor call."""
|
| 903 |
+
try:
|
| 904 |
+
return _restore_amazon_mirror_once(why)
|
| 905 |
+
finally:
|
| 906 |
+
try:
|
| 907 |
+
import amazon_relational as _ar
|
| 908 |
+
_ar.RESTORE_PENDING["since"] = None
|
| 909 |
+
except Exception: # noqa: BLE001
|
| 910 |
+
pass
|
| 911 |
+
|
| 912 |
+
|
| 913 |
+
def _restore_amazon_mirror_once(why):
|
| 914 |
+
try:
|
| 915 |
+
import time as _t
|
| 916 |
+
from harness import amazon_store as _amazon
|
| 917 |
+
from harness import runtime as _runtime
|
| 918 |
+
import routes_odoo_tables as _rot
|
| 919 |
+
tenant_key = "royal-imports"
|
| 920 |
+
rt = _runtime.get_runtime(tenant_key)
|
| 921 |
+
if not _amazon.credential(tenant_key, rt=rt):
|
| 922 |
+
return False
|
| 923 |
+
started = _t.time()
|
| 924 |
+
result = _amazon.restore_mirror(tenant_key, rt)
|
| 925 |
+
_rot.invalidate_sources()
|
| 926 |
+
_rot._sources()
|
| 927 |
+
print(f"[aios-api] amazon mirror {result.get('state')} ({why}) in "
|
| 928 |
+
f"{_t.time() - started:.0f}s, before the Odoo catch-up")
|
| 929 |
+
return True
|
| 930 |
+
except Exception as exc: # noqa: BLE001
|
| 931 |
+
print(f"[aios-api] amazon mirror restore FAILED ({why}): {type(exc).__name__}: {exc}")
|
| 932 |
+
return False
|
| 933 |
+
|
| 934 |
+
|
| 935 |
class _SyncNowRefused(Exception):
|
| 936 |
"""An application-authored, credential-free sentence for a Sync now receipt."""
|
| 937 |
|
|
|
|
| 1960 |
# relational rebuild, which `manual` says nothing about.
|
| 1961 |
if manual:
|
| 1962 |
return None, None, None
|
| 1963 |
+
absent = _absent_mirror_entities(_ds, entities)
|
| 1964 |
+
first = {}
|
| 1965 |
+
if absent:
|
| 1966 |
+
# Tables the boot seed lacks first: minutes, where the whole catch-up runs an hour.
|
| 1967 |
+
try:
|
| 1968 |
+
first = _ds.sync_all(entities=absent, log=lambda *a, **k: None)
|
| 1969 |
+
print(f"[aios-api] store resync: synced {', '.join(absent)} first, "
|
| 1970 |
+
f"absent from the boot seed")
|
| 1971 |
+
except Exception as exc: # noqa: BLE001
|
| 1972 |
+
print(f"[aios-api] store resync: the absent tables first failed "
|
| 1973 |
+
f"({type(exc).__name__}); the full pass below still syncs them")
|
| 1974 |
sync_report = _ds.sync_all(entities=entities, log=lambda *a, **k: None)
|
| 1975 |
+
for key, row in first.items():
|
| 1976 |
+
if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
|
| 1977 |
+
sync_report[key]["pulled"] = sync_report[key].get("pulled", 0) + row.get("pulled", 0)
|
| 1978 |
# Wave 21: purge hard-deleted rows the cursor sync cannot see (the boot pass's comment has
|
| 1979 |
# the measured case); without it deleted Odoo lines inflate every sum on the mirror FOREVER.
|
| 1980 |
# ⚠ INSIDE the non-manual branch: a reconcile is a pass over the mirror this loop was
|
|
|
|
| 1989 |
return None, None, _classify_resync_error(e)
|
| 1990 |
|
| 1991 |
|
| 1992 |
+
def _absent_mirror_entities(ds, entities=None):
|
| 1993 |
+
"""The mirror entities this pass syncs that have no table in the DuckDB file yet, in sync order."""
|
| 1994 |
+
try:
|
| 1995 |
+
cur = ds.connect().cursor()
|
| 1996 |
+
try:
|
| 1997 |
+
return [key for key in (list(ds.ENTITIES) if entities is None else list(entities))
|
| 1998 |
+
if key in ds.ENTITIES and not ds.columns_of(key, cur=cur)]
|
| 1999 |
+
finally:
|
| 2000 |
+
cur.close()
|
| 2001 |
+
except Exception: # noqa: BLE001
|
| 2002 |
+
return []
|
| 2003 |
+
|
| 2004 |
+
|
| 2005 |
def _classify_resync_error(exc):
|
| 2006 |
"""D-314, the half that fits inside this loop: a failed pass leaves a MARK, not just a line.
|
| 2007 |
|
|
|
|
| 2244 |
# restores Amazon's saved mirror. The store-touching half (Meta and Amazon vendor pulls,
|
| 2245 |
# the relational rebuild) waits for the pass that is actually due.
|
| 2246 |
if not passes:
|
| 2247 |
+
try:
|
| 2248 |
+
import amazon_relational as _ar
|
| 2249 |
+
_ar.RESTORE_PENDING["since"] = _t.time()
|
| 2250 |
+
except Exception: # noqa: BLE001
|
| 2251 |
+
pass
|
| 2252 |
_await_boot_seed()
|
| 2253 |
+
# Amazon's saved mirror first: minutes of store reads, where the Odoo catch-up below
|
| 2254 |
+
# runs about an hour after a restart and Amazon answered nothing until it ended.
|
| 2255 |
+
_restore_amazon_mirror("boot")
|
| 2256 |
_boot_catch_up = ((not passes or catch_up_retries)
|
| 2257 |
and _resync_boot_recent(last_good_pass_at, _t.time(), _every))
|
| 2258 |
# ⛔⛔ HOTFIX 2026-09-24 — A DESK WAKE IS NOT A SCHEDULED PASS.
|
api/routes_odoo_tables.py
CHANGED
|
@@ -3657,7 +3657,7 @@ def odoo_table_rows(table_key: str,
|
|
| 3657 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 3658 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 3659 |
where=where, params=tuple(params), order_by=order,
|
| 3660 |
-
offset=offset, limit=(limit or None), cur=cur, **frm)
|
| 3661 |
if win["clamped"]:
|
| 3662 |
limits.append({
|
| 3663 |
"subject": "limit", "effect": "window_clamped",
|
|
|
|
| 3657 |
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
|
| 3658 |
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
|
| 3659 |
where=where, params=tuple(params), order_by=order,
|
| 3660 |
+
offset=offset, limit=(limit or None), cur=cur, page_first=True, **frm)
|
| 3661 |
if win["clamped"]:
|
| 3662 |
limits.append({
|
| 3663 |
"subject": "limit", "effect": "window_clamped",
|
platform/harness/datastore.py
CHANGED
|
@@ -601,8 +601,13 @@ WINDOW_DEFAULT = 200
|
|
| 601 |
_IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
| 602 |
|
| 603 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
def window(table=None, select=None, where=None, params=(), order_by=None,
|
| 605 |
-
offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None
|
|
|
|
| 606 |
"""One page of a mirror table PLUS the true count of everything the same predicate matches.
|
| 607 |
|
| 608 |
Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
|
|
@@ -626,6 +631,9 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 626 |
`count(DISTINCT ...)`, which is a projection, not a second meaning for this field.
|
| 627 |
⚠ It defaults to None and emits nothing then, so every existing caller's two statements are
|
| 628 |
byte-identical to what they were before this argument existed.
|
|
|
|
|
|
|
|
|
|
| 629 |
"""
|
| 630 |
if bool(table) == bool(from_sql):
|
| 631 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
@@ -661,10 +669,19 @@ def window(table=None, select=None, where=None, params=(), order_by=None,
|
|
| 661 |
total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0)
|
| 662 |
grp = str(group_by or "").strip()
|
| 663 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 668 |
cols = [d[0] for d in (cur.description or [])]
|
| 669 |
return {"rows": [tuple(r) for r in got], "columns": cols,
|
| 670 |
"total": total, "offset": offset, "limit": limit, "clamped": clamped}
|
|
|
|
| 601 |
_IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
| 602 |
|
| 603 |
|
| 604 |
+
#: A scalar subquery inside SQL text: `window(page_first=True)` defers it until the page is chosen.
|
| 605 |
+
_SUBQUERY = re.compile(r"\(\s*SELECT\b", re.IGNORECASE)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
def window(table=None, select=None, where=None, params=(), order_by=None,
|
| 609 |
+
offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None, group_by=None,
|
| 610 |
+
page_first=False):
|
| 611 |
"""One page of a mirror table PLUS the true count of everything the same predicate matches.
|
| 612 |
|
| 613 |
Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int,
|
|
|
|
| 631 |
`count(DISTINCT ...)`, which is a projection, not a second meaning for this field.
|
| 632 |
⚠ It defaults to None and emits nothing then, so every existing caller's two statements are
|
| 633 |
byte-identical to what they were before this argument existed.
|
| 634 |
+
|
| 635 |
+
`page_first` (a row page whose `order_by` is an expression over the table's own columns) picks
|
| 636 |
+
the page's rows before the projection runs, when only the projection holds a subquery.
|
| 637 |
"""
|
| 638 |
if bool(table) == bool(from_sql):
|
| 639 |
raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`")
|
|
|
|
| 669 |
total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0)
|
| 670 |
grp = str(group_by or "").strip()
|
| 671 |
fold = f" GROUP BY {grp}" if grp else ""
|
| 672 |
+
if (page_first and table and not grp and _SUBQUERY.search(proj)
|
| 673 |
+
and not _SUBQUERY.search(f"{pred} {order}")):
|
| 674 |
+
# A correlated lookup in the projection joins EVERY row before the LIMIT: one 50-row page
|
| 675 |
+
# of the 952k-row Amazon transactions held 2.8 GB and restarted staging (hotfix sdlc71).
|
| 676 |
+
got = cur.execute(
|
| 677 |
+
f"SELECT {proj} FROM (SELECT * FROM {name}{tail} ORDER BY {order} "
|
| 678 |
+
f"LIMIT {int(limit)} OFFSET {int(offset)}) AS {name} ORDER BY {order}",
|
| 679 |
+
params).fetchall()
|
| 680 |
+
else:
|
| 681 |
+
got = cur.execute(
|
| 682 |
+
f"SELECT {proj} FROM {name}{tail}{fold} "
|
| 683 |
+
f"ORDER BY {order} LIMIT {int(limit)} OFFSET {int(offset)}",
|
| 684 |
+
params).fetchall()
|
| 685 |
cols = [d[0] for d in (cur.description or [])]
|
| 686 |
return {"rows": [tuple(r) for r in got], "columns": cols,
|
| 687 |
"total": total, "offset": offset, "limit": limit, "clamped": clamped}
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
4073c7bb3f0821ae686c33783c33d634088e1ecb973bedefd18728b446704658
|