loopable / api /assistant_mirror.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
85f794d verified
Raw History Blame
38.8 kB
"""assistant_mirror.py - the assistant's read of a MIRROR-SERVED connector database (R11 / C7).
⭐⭐ WHY THIS FILE EXISTS. `deps._assistant_user_table` reads a user table out of the tenant
document and refuses, 409, when the table does not materialise: *"this database is mirror-served
and needs its materialised reader"*. Its own comment named the missing half - *"E must consume the
mirror through its own `mirror_cursor`-guarded seam"* - and this is that seam. Owner R11: *"Why
can't it be queried by the AI if user has access to it? just query it using our own Filter work. I
want the AI to be able to use our tools."*
β›” C7 IS THE WHOLE CONSTRAINT, AND IT IS A CONSTRAINT ON WHAT THIS FILE MAY CONTAIN, not only on
what it must do: *"the assistant reads a mirror-served database through the SAME filter compiler
the grid uses, under `mirror_cursor` and the assistant's existing permission validation. No second
filter dialect, and no widening of what a scoped user may reach."* So every predicate below is
compiled by `harness.filter_sql.compile_filter_tree`, every column comes from
`routes_odoo_tables._filter_columns`, and the cursor comes from `routes_odoo_tables._mirror_cur`.
There is no hand-built SQL for a condition anywhere in this module. If you find yourself writing a
string with an operator in it, you are writing the second dialect this contract forbids.
β›” AND NO PERMISSION DECISION IS TAKEN HERE. This module is handed an already-resolved wall tree
and an already-resolved list of columns the caller's own filter may name; `deps.py` owns both,
through `_assistant_preflight_request`, which is the assistant's existing permission validation
that C7 says to reuse rather than restate. A second permission wall would be a second thing to
forget.
⚠ IMPORT DIRECTION. `routes_odoo_tables` imports `deps` at module level, so BOTH the import from
`deps` into here and the import from here into `routes_odoo_tables` are lazy, inside the functions
that need them. A module-level import either way is a cycle at startup.
"""
#: ⭐ S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what
#: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window):
#: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an
#: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is
#: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's
#: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631).
ASSISTANT_READ_MAX = 200_000
class MirrorRefusal(Exception):
"""A named, fail-closed refusal with the status and code `deps` should raise it as.
β›” AN EXCEPTION RATHER THAN A RETURN, so a caller cannot read "refused" as "no rows". This
module never answers a question it could not fully answer; the caller turns this into the
ordinary assistant error shape.
"""
def __init__(self, status, code, message):
super().__init__(message)
self.status = int(status)
self.code = str(code)
self.message = str(message)
def _rot():
"""The grid's own read-through module, imported lazily to break the `deps` cycle."""
import routes_odoo_tables
return routes_odoo_tables
def _cause(cause):
"""The closed-set refusal sentence `deps` owns, imported lazily to break the `deps` cycle."""
import deps
return deps._assistant_source_message(cause)
def open_cursor(session, table_key=None):
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
⭐ S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's
`_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its
own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so
the assistant no longer answers 503 for Amazon while Odoo is still backfilling.
β›” THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
and retrying works: 503. A mis-pinned process serving another tenant's file never becomes
un-mis-pinned by waiting: 409. Collapsing them would render the loudest refusal in the system
as a spinner.
"""
rot = _rot()
try:
cur, mismatch = rot._mirror_cur(session, table_key)
except RuntimeError as exc:
if "could not be restored" in str(exc):
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("restore_blocked"))
if "has not been attached" in str(exc):
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("loading"))
return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("syncing"))
except Exception as exc: # noqa: BLE001
# ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
# `deps`, which every fixture and gate in the repo imports and drives with runtimes that
# are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
# whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
# which reads to a user as the product being broken rather than the mirror being shut.
# Named and fail-closed beats a stack trace. The class goes to the server log, never the reply.
import deps
deps._assistant_log_cause("assistant mirror open", exc)
return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("mirror_open"))
if mismatch:
return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
return cur, None
#: The one sentence for a mirror-served table this deployment has no binding for at all.
NO_BINDING = ("this database is read live from its connector mirror, and this deployment has no "
"read-through binding for it yet, so it holds no rows the assistant can read")
class ReadinessSnapshot:
"""Reuse mirror metadata within one source listing, without caching permissions or row access."""
def __init__(self):
self._sources = None
self._columns = {}
self._phases = {}
self._loading = {}
self._amazon_learned = False
self._amazon_pending = None
def amazon_pending(self):
"""`amazon_relational.restore_pending()`, read once per listing: a data fact, never a permission."""
if self._amazon_pending is None:
import amazon_relational
self._amazon_pending = bool(amazon_relational.restore_pending())
return self._amazon_pending
def source(self, key):
if self._sources is None:
self._sources = dict(_rot()._sources())
return self._sources.get(key)
def learn_amazon(self, cur):
if not self._amazon_learned:
self._amazon_learned = True
rot = _rot()
rot._learn_amazon_sources(cur)
self._sources = dict(rot.GRID_SOURCES)
def columns(self, cur, name):
from harness import datastore
key = (id(cur), name)
if key not in self._columns:
self._columns[key] = datastore.columns_of(name, cur=cur)
return self._columns[key]
def loading(self, cur, name):
key = (id(cur), name)
if key not in self._loading:
cursor_id = id(cur)
if cursor_id not in self._phases:
try:
phases = dict(cur.execute("SELECT entity, phase FROM _sync_state").fetchall())
except Exception: # noqa: BLE001
phases = None
self._phases[cursor_id] = phases
phases = self._phases[cursor_id]
if phases is None:
return False # Mirrors without sync bookkeeping retain standalone semantics.
if name in phases:
self._loading[key] = phases[name] != "live"
else:
self._loading[key] = not cur.execute(
f"SELECT count(*) FROM (SELECT 1 FROM {name} LIMIT 1)").fetchone()[0]
return self._loading[key]
def amazon_empty_reason(status):
"""The one-line reason in an Amazon `sourceStatus`, or "" when it names no code and reason."""
if not isinstance(status, dict) or not status.get("empty") or not status.get("code"):
return ""
reason = status.get("reason")
return " ".join(reason.split()) if isinstance(reason, str) else ""
def unavailable_reason(table_key, definition=None, *, snapshot=None):
"""The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows.
A bound table whose mirror tables are absent is loading (a restart starts from the boot seed);
a table with no binding at all gets `NO_BINDING`.
"""
key = str(table_key or "")
try:
if key.startswith("ut_amzn_"):
import amazon_relational as amazon
if key in amazon.SOURCE:
# D-2643: the stored definition's empty reason, from the one Amazon producer.
status = (definition or {}).get("sourceStatus") if isinstance(definition, dict) else None
reason = amazon_empty_reason(status)
if reason:
return reason
pending = snapshot.amazon_pending() if snapshot is not None else None
return amazon.missing_source(key, pending=pending)["missing"]
spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
if spec and not spec.get("missing"):
return _cause("loading")
except Exception: # noqa: BLE001
pass
return _cause("no_binding")
def still_loading(table_key, *, cur=None, snapshot=None):
"""True while an Odoo table behind this binding is in its first backfill, so its rows are partial.
Evidence only: a `_sync_state` phase other than live, or an empty table with no state row yet
(created, first batch not written). A mirror with no sync bookkeeping is not loading.
"""
from harness import datastore
owned_cur = None
try:
snapshot = snapshot if snapshot is not None else ReadinessSnapshot()
spec = snapshot.source(str(table_key or "")) or {}
names = set((spec.get("tables") or {}).values()) | ({spec["table"]} if spec.get("table") else set())
names = sorted(n for n in names if n in datastore.ENTITIES)
if not names:
return False
if cur is None:
owned_cur = cur = datastore.ro_cursor()
return any(snapshot.loading(cur, name) for name in names)
except Exception: # noqa: BLE001
return False
finally:
if owned_cur is not None:
try:
owned_cur.close()
except Exception: # noqa: BLE001
pass
def binding(cur, table_key):
"""The resolved read-through spec for one connected grid, or None when it has no binding.
`None` is a real, reportable answer: this deployment's mirror carries no table for this grid,
which is not the same as the grid being empty.
"""
try:
spec = _rot()._source_for(cur, str(table_key or ""))
except Exception: # noqa: BLE001
# A spec that cannot be resolved is indistinguishable, to every caller, from a grid with
# no binding: neither can be read. Fail closed to the reportable answer.
return None
return amazon_lag(cur, str(table_key or ""), spec)
def amazon_lag(cur, table_key, spec):
"""D-2642: a dated Amazon binding with rows also carries its newest row date, cadence and lag line.
The keys come from `amazon_store.table_empty_status`, the one per-table producer. A trailing table
also gets a `source_behind` limit. A status that cannot be read carries the unknown lag line.
"""
if not spec or spec.get("missing") or not table_key.startswith("ut_amzn_"):
return spec
from harness import amazon_store
table = table_key.removeprefix("ut_amzn_")
if table not in amazon_store.LAG_DATE_COLUMNS:
return spec
try:
if cur.execute(f'SELECT 1 FROM "{amazon_store.table_name(table)}" LIMIT 1').fetchone() is None:
return spec
status = amazon_store.table_empty_status(cur, table, rows=1, lag=True)
dated = {name: status[name] for name in amazon_store.LAG_KEYS if name in status}
except Exception: # noqa: BLE001
dated = amazon_store.unknown_lag(table)
spec = {**spec, "sourceStatus": {"empty": False, **(spec.get("sourceStatus") or {}), **dated}}
if dated.get("lag"):
spec["limits"] = [*(spec.get("limits") or []),
{"effect": "source_behind", "code": dated.get("lagCode"), "cause": dated["lag"]}]
return spec
def is_bound(cur, table_key, *, snapshot=None):
"""CAN this deployment read `table_key` through the mirror at all? The availability question.
⭐⭐ THE CHEAP PREFIX OF `_source_for`, AND IT IS HERE BECAUSE THE FULL CALL COSTS 300 ms.
MEASURED on this worktree's mirror, warm: `_source_for('ut_odoo_accounts')` 2.9 ms, but
`ut_odoo_order_lines`, `ut_odoo_gl_lines` and `ut_odoo_orders` carry `needs_excluded`, and
that branch runs `odoo_relational.excluded_ids`, a scan of all 5,334 `res_partner` rows, at
44 ms to 297 ms EACH. `assistant_source_status` is called on mount by BOTH the Assistant and
Query and used to answer in 0.4 ms warm; resolving every read-through grid's full spec there
measured **+605 ms median** on the whole batch. A picker that now tells the truth about ten
databases must not cost two thirds of a second to say it.
β›” IT IS A PREFIX, NOT A SECOND ANSWER, and that distinction is what keeps it honest.
`_source_for` returns `None` in EXACTLY two cases: this deployment registers no spec for the
key, or the mirror carries no table behind one of the spec's aliases. Both are checked here,
with the same calls, in the same order. Everything after them in `_source_for` builds the
COLUMN EXPRESSIONS a read needs and cannot change whether the answer is `None`. The expensive
`excluded_ids` lookup is part of that second half.
⚠ SO IF `_source_for` EVER GROWS A THIRD `return None`, THIS MUST FOLLOW IT. The failure that
would cause is bounded and visible rather than silent: a source reported answerable that then
answers a named refusal when actually read. It can never produce a wrong NUMBER, which is the
direction that matters.
"""
from harness import datastore
try:
rot = _rot()
key = str(table_key or "")
spec = snapshot.source(key) if snapshot is not None else rot._sources().get(key)
if not spec and key.startswith("ut_amzn_"):
# S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key.
# `_source_for` and `_materialised_table` both learn them on demand; so must this.
if snapshot is not None:
snapshot.learn_amazon(cur)
spec = snapshot.source(key)
else:
rot._learn_amazon_sources()
spec = rot.GRID_SOURCES.get(key)
if not spec:
# A dynamically sized Amazon snapshot is admitted per key by the grid door, not by
# the provider build, so the registry can lack a key the grid serves (measured on
# staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the
# provider's own per-key resolver, exactly as `require_amazon_mirror` does, without
# admitting it (the read path admits it through that same door).
import amazon_relational as amazon
if key not in amazon.SOURCE:
return False
# A yes/no: no columns is unbound. `grid_source` would read the keychain to word why.
if not amazon.mirror_columns(cur, amazon.SOURCE[key]):
return False
current = amazon.grid_source(cur, key) or {}
return bool(current) and not current.get("missing")
if not spec or spec.get("missing"):
return False
aliases = dict(spec.get("tables") or {})
if spec.get("table"):
aliases.setdefault("", spec["table"])
return all(snapshot.columns(cur, name) if snapshot is not None
else datastore.columns_of(name, cur=cur) for name in aliases.values())
except Exception: # noqa: BLE001
return False
def declared_fields(table_key, definition=None):
"""`mk_fields()` for one connected grid, or None when this key is not a connected grid.
⭐ S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that
is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid,
a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before
this, those databases were listed as answerable and then refused 409 on every read.
⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
each table's stored `fields` from this same function, so reading it directly is reading the
source both agree on, and it is why `odoo_table_rows` uses it too. A stored definition can
also carry user-added columns that have no mirror expression at all.
"""
import odoo_relational
for _bucket, key, _label, mk_fields in odoo_relational.TABLES:
if key == str(table_key or ""):
return [dict(field) for field in (mk_fields() or [])
if isinstance(field, dict) and field.get("key")]
key = str(table_key or "")
if not isinstance(definition, dict) or not key.startswith("ut_"):
return None
try:
if _rot()._materialised_table(key):
return None
except Exception: # noqa: BLE001
return None
fields = [dict(field) for field in (definition.get("fields") or [])
if isinstance(field, dict) and field.get("key")]
return fields or None
def answerable_fields(table_key, spec, fields=None):
"""The declared field contract INTERSECTED with the columns the mirror can actually express.
β›” THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
with no expression in `spec["cols"]` (a link, a rollup, a column this deployment has not
backfilled) cannot be answered from SQL at all. Offering it anyway would put it in the
assistant's field list, where a request naming it comes back present-but-empty rather than
refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
for it with a loud `assistant_unknown_field`.
"""
if fields is None:
fields = declared_fields(table_key)
if fields is None:
return None
cols = set((spec or {}).get("cols") or ())
return [field for field in fields if field["key"] in cols]
def _compiled(nodes, conj, columns, *, code, subject, rot, read_wall=False):
"""Compile ONE tree through the grid's compiler, or refuse. Never returns a partial predicate.
β›” THREE WAYS TO FAIL AND ALL THREE FAIL CLOSED, which is the same policy `odoo_table_rows`
carries for the administrator's wall - a column with no mirror expression, an operator with no
SQL form, an aggregate. The grid can afford to report an unpushable USER condition in its
`limits` array and let the client see it; the assistant's snapshot envelope has no such
channel, and an ignored condition does not narrow, it WIDENS. So on this path both trees
refuse.
"""
unknown = sorted(rot._leaf_cols(nodes) - set(columns))
if unknown:
raise MirrorRefusal(
409, code,
f"{subject} names {', '.join(unknown)}, and this database's live connection cannot "
f"read that column, so the condition cannot be applied here. Applying the rest would "
f"answer a wider question than the one asked")
try:
compiler = rot._compile_read_wall if read_wall else rot._fs().compile_filter_tree
pred = compiler(nodes, conj=conj, columns=columns, today=rot._today())
except ValueError as exc:
raise MirrorRefusal(
409, code,
f"{subject} uses a condition this database's live connection cannot evaluate "
f"({exc}), so it is refused rather than ignored")
if pred is None:
return None
if pred.uses_aggregate:
raise MirrorRefusal(
409, code,
f"{subject} is a condition on an aggregate column, and this database's live "
f"connection cannot evaluate one of those a row at a time")
return pred
def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, *, table_key=None):
"""`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34.
Extracted verbatim from `read_rows` so the row walk, the page read and the server-side
aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler.
Three readers with one predicate cannot disagree about which rows the caller may see.
"""
fe = rot._fe()
cols = rot._filter_columns(spec, fields)
# ── the administrator's wall, compiled FIRST and pushed down ──────────────────────────────
# β›” PUSHED DOWN RATHER THAN APPLIED AFTERWARDS, for one reason that is specific to this door:
# the row TOTAL below decides whether this read is refused as too large. A wall applied only
# in Python would count rows the reader may not see, and refuse a scoped user whose own slice
# of a million-row table is two hundred rows.
#
# ⚠ AND IT IS THE ASSISTANT'S WALL, NOT `row_scope_applies`. The grid door uses
# `perm_scope.row_scope_applies`, which is False for an admin. `assistant_apply_row_scope`
# has no admin break-glass: it applies the grant's filter to every caller. The caller hands
# the tree it will apply in Python, so the two passes are the SAME predicate by construction
# rather than by two spellings that agree today.
#
# READ THROUGH `tree_parts`, WHICH IS THE ONE READER OF THIS SHAPE. A stored wall is
# `{conj, nodes}` and both consumers need a bare node list plus a separate conj. Handing
# either the dict is silently EMPTY rather than an error, i.e. no wall at all, on the one path
# where that means serving the whole database.
where = spec.get("where") or ""
params = []
wall_nodes, wall_conj = fe.tree_parts(wall_tree)
if wall_nodes:
wall = _compiled(wall_nodes, wall_conj, cols, code="assistant_scope_not_applicable",
subject="the restriction an administrator set on your view of this "
"database", rot=rot, read_wall=True)
if wall is not None:
where = f"({where}) AND {wall.sql}" if where else wall.sql
params.extend(wall.params)
# ── the caller's own filter ───────────────────────────────────────────────────────────────
# β›” AGAINST THE VISIBLE COLUMNS ONLY. `filter_keys` is the assistant's visible field list,
# already resolved by the caller's `_assistant_preflight_request`. Compiling against the full
# contract instead would let a scoped reader infer a hidden column's values by filtering on
# it, which is exactly the widening C7 forbids. The wall above is compiled against the FULL
# contract on purpose, precisely as `perm_scope._scoped` evaluates it against unstripped
# fields.
user_cols = ({key: value for key, value in cols.items() if key in set(filter_keys)}
if filter_keys is not None else dict(cols))
nodes, conj = fe.tree_parts(filter_tree)
if nodes:
pred = _compiled(nodes, conj, user_cols, code="assistant_filter_not_answerable",
subject="the filter on this request", rot=rot)
if pred is not None:
where = f"({where}) AND {pred.sql}" if where else pred.sql
params.extend(pred.params)
# The trusted link pin intersects every SQL count, aggregate and page for its source.
from core import relations
pin = relations.current_query_link_pin()
if pin is not None and (table_key is None or pin.source == table_key):
link_cols = dict(user_cols)
if pin.column == "pid":
link_cols["pid"] = {"sql": spec["id"]}
try:
link_sql, link_params = relations.query_link_predicate(link_cols)
except ValueError as exc:
raise MirrorRefusal(409, "query_link_unavailable", str(exc)) from exc
if link_sql:
where = f"({where}) AND ({link_sql})" if where else link_sql
params.extend(link_params)
return where, params, cols
def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
filter_keys=None, defn=None, serialize=True):
"""Every permitted row of a mirror-served grid, or a refusal that names its cause.
Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
(`{pid, **cells}`), sorted by pid.
β›” NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
every aggregate from those same records, so a short set does not produce a smaller answer, it
produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
already settled this policy in the same words: REFUSE, NEVER TRIM.
β›” FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
the field projection in Python AFTER this returns. Hand it a row missing a column the wall
names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
pass would quietly disagree with the pushdown it is meant to double-check.
"""
from harness import datastore
rot = _rot()
where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys,
table_key=table_key)
# ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
# spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
# ⭐⭐ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX`
# pass this replaced refused any connected database past 5,000 matching rows, which is a cap
# on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages,
# KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs
# one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a
# user condition: every condition above still comes from the grid's compiler.
# β›” STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that
# ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
where=where, params=tuple(params), order_by=spec["id"],
limit=datastore.WINDOW_MAX, cur=cur, page_first=True, **frm)
total = int(first["total"])
if total > ASSISTANT_READ_MAX:
raise MirrorRefusal(
409, "assistant_source_too_large",
f"this database matches {total:,} rows for that question, and one assistant answer "
f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question "
f"cannot exhaust the server's memory. Answering over part of them would report a "
f"wrong total, so it is refused instead. Narrow it with a filter (for example a date "
f"range) and ask again, or ask for totals as aggregates, which the server computes "
f"over every row")
got = list(first["rows"])
page = list(first["rows"])
while len(got) < total and len(page) >= datastore.WINDOW_MAX:
last = page[-1][0]
more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
where=more_where, params=tuple(params) + (last,),
order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur,
page_first=True, **frm)
page = list(nxt["rows"])
if not page:
break
got.extend(page)
if len(got) != total:
raise MirrorRefusal(
409, "assistant_source_changed",
f"this database changed while it was being read ({len(got):,} rows read against "
f"{total:,} counted), so the answer would not describe one consistent set of rows. "
f"Ask again once its connector sync has finished")
win = {**first, "rows": got, "total": total}
# S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does
# not re-read the whole tenant document (measured about 3 s a question on staging).
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
defn=defn if isinstance(defn, dict) else None)
rows = rot._pool_from_window(spec, win, stamp=stamp)
if serialize:
rows = _typed_rows(rot, fields, rows)
rows.sort(key=lambda row: row["pid"])
return rows, stamp, int(win["total"])
#: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`).
_GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"}
def _json_value(rot, value):
"""A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a
date or timestamp as its ISO text, anything else as is. Never a substituted 0."""
if value is None:
return None
if isinstance(value, (int, float)) and not isinstance(value, bool):
return rot._agg_number(value)
iso = getattr(value, "isoformat", None)
if callable(iso):
return iso()
try:
from decimal import Decimal
if isinstance(value, Decimal):
return rot._agg_number(value)
except Exception: # noqa: BLE001
pass
return value
def field_value(rot, field, value):
"""Serialize a native checkbox as a Boolean while preserving other JSON values."""
if field.get("type") == "checkbox" and value is not None:
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "t", "yes", "y", "on"}
return bool(value)
return _json_value(rot, value)
def _typed_rows(rot, fields, rows, *, preserve_blank=False):
"""Apply the declared field codecs without changing source records."""
flags = [field for field in fields if field.get("type") == "checkbox"]
if not flags:
return rows
def encode(field, value):
if preserve_blank and (value is None or isinstance(value, str) and value.strip() in {'', '(none)'}):
return value
return field_value(rot, field, value)
return [{**row, **{field["key"]: encode(field, row[field["key"]])
for field in flags if field["key"] in row}} for row in rows]
def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
filter_tree=None, filter_keys=None):
"""Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the
count of rows the permitted predicate matches, from `datastore.window`'s own count statement
over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract).
β›”β›” THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall
and the caller's filter with the grid's compiler, so a sum over a million-row table runs over
exactly the rows this caller may read and no row is pulled into Python to compute it. There
is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size.
β›” AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and
a field outside it has no expression here at all, so a sum over a hidden column cannot be
built (the caller's `_normalise_aggregates` has already refused it by name).
"""
from harness import datastore
rot = _rot()
where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys,
table_key=table_key)
visible = set(filter_keys) if filter_keys is not None else set(cols)
fs = rot._fs()
proj, slots = [], []
for item in specs or ():
fn, field = item.get("fn"), item.get("field")
if field is None:
slots.append(None) # count(*): the window's own total
continue
col = cols.get(field) if field in visible else None
if col is None:
from deps import assistant_unknown_field_message
raise MirrorRefusal(400, "assistant_unknown_field",
assistant_unknown_field_message(field, visible & set(cols)))
if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES:
raise MirrorRefusal(400, "query_aggregate_type",
"sum and avg require a readable numeric field.")
if fn in _GRID_AGG:
expr = fs.aggregate_sql(col, _GRID_AGG[fn])
elif fn in {"min", "max"}:
expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES
else f"{fn}({col['sql']})")
elif fn == "count":
expr = f"count({col['sql']})"
else:
raise MirrorRefusal(400, "query_aggregates_invalid",
"That aggregate is not available.")
slots.append(len(proj))
proj.append(f'{expr} AS "a{len(proj)}"')
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params),
order_by="1", limit=1, cur=cur, **frm)
row = (win["rows"] or [()])[0]
total = int(win["total"])
out = []
for item, slot in zip(specs or (), slots):
value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row)
else None)
out.append({**item, "value": value})
return out, total
def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
filter_tree=None, filter_keys=None, defn=None, sorts=None, serialize=True):
"""Return one SQL-ordered permitted page, its source stamp and its full population count."""
from harness import datastore
rot = _rot()
where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys,
table_key=table_key)
visible = set(filter_keys) if filter_keys is not None else set(cols)
order = []
for rule in sorts or ():
if (not isinstance(rule, dict) or set(rule) != {"field", "direction"}
or not isinstance(rule.get("field"), str)
or rule["field"] not in visible or rule["field"] not in cols
or rule.get("direction") not in ("asc", "desc")):
from deps import assistant_unknown_field_message
raise MirrorRefusal(400, "assistant_unknown_field",
assistant_unknown_field_message(
rule.get("field") if isinstance(rule, dict) else None,
visible & set(cols)))
column_sql = cols[rule["field"]]["sql"]
direction = "ASC" if rule["direction"] == "asc" else "DESC"
# SQL expressions come only from the resolved schema, directions from fixed literals.
if cols[rule["field"]]["type"] == "text":
order.append(f"lower({column_sql}) {direction} NULLS LAST")
order.append(f"{column_sql} {direction} NULLS LAST")
order.append(f'{spec["id"]} ASC')
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
params=tuple(params), order_by=", ".join(order), offset=int(offset),
limit=int(limit), cur=cur, page_first=True, **frm)
stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
defn=defn if isinstance(defn, dict) else None)
rows = rot._pool_from_window(spec, win, stamp=stamp)
if serialize:
rows = _typed_rows(rot, fields, rows)
return rows, stamp, int(win["total"])