Download api/assistant_mirror.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 38.8 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_mirror.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable@9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_mirror.py
-
curl -L -o assistant_mirror.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_mirror.py
38.8 kB
| """assistant_mirror.py - the assistant's read of a MIRROR-SERVED connector database (R11 / C7). | |
| ββ WHY THIS FILE EXISTS. `deps._assistant_user_table` reads a user table out of the tenant | |
| document and refuses, 409, when the table does not materialise: *"this database is mirror-served | |
| and needs its materialised reader"*. Its own comment named the missing half - *"E must consume the | |
| mirror through its own `mirror_cursor`-guarded seam"* - and this is that seam. Owner R11: *"Why | |
| can't it be queried by the AI if user has access to it? just query it using our own Filter work. I | |
| want the AI to be able to use our tools."* | |
| β C7 IS THE WHOLE CONSTRAINT, AND IT IS A CONSTRAINT ON WHAT THIS FILE MAY CONTAIN, not only on | |
| what it must do: *"the assistant reads a mirror-served database through the SAME filter compiler | |
| the grid uses, under `mirror_cursor` and the assistant's existing permission validation. No second | |
| filter dialect, and no widening of what a scoped user may reach."* So every predicate below is | |
| compiled by `harness.filter_sql.compile_filter_tree`, every column comes from | |
| `routes_odoo_tables._filter_columns`, and the cursor comes from `routes_odoo_tables._mirror_cur`. | |
| There is no hand-built SQL for a condition anywhere in this module. If you find yourself writing a | |
| string with an operator in it, you are writing the second dialect this contract forbids. | |
| β AND NO PERMISSION DECISION IS TAKEN HERE. This module is handed an already-resolved wall tree | |
| and an already-resolved list of columns the caller's own filter may name; `deps.py` owns both, | |
| through `_assistant_preflight_request`, which is the assistant's existing permission validation | |
| that C7 says to reuse rather than restate. A second permission wall would be a second thing to | |
| forget. | |
| β IMPORT DIRECTION. `routes_odoo_tables` imports `deps` at module level, so BOTH the import from | |
| `deps` into here and the import from here into `routes_odoo_tables` are lazy, inside the functions | |
| that need them. A module-level import either way is a cycle at startup. | |
| """ | |
| #: β S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what | |
| #: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window): | |
| #: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an | |
| #: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is | |
| #: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's | |
| #: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631). | |
| ASSISTANT_READ_MAX = 200_000 | |
| class MirrorRefusal(Exception): | |
| """A named, fail-closed refusal with the status and code `deps` should raise it as. | |
| β AN EXCEPTION RATHER THAN A RETURN, so a caller cannot read "refused" as "no rows". This | |
| module never answers a question it could not fully answer; the caller turns this into the | |
| ordinary assistant error shape. | |
| """ | |
| def __init__(self, status, code, message): | |
| super().__init__(message) | |
| self.status = int(status) | |
| self.code = str(code) | |
| self.message = str(message) | |
| def _rot(): | |
| """The grid's own read-through module, imported lazily to break the `deps` cycle.""" | |
| import routes_odoo_tables | |
| return routes_odoo_tables | |
| def _cause(cause): | |
| """The closed-set refusal sentence `deps` owns, imported lazily to break the `deps` cycle.""" | |
| import deps | |
| return deps._assistant_source_message(cause) | |
| def open_cursor(session, table_key=None): | |
| """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None. | |
| β S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's | |
| `_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its | |
| own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so | |
| the assistant no longer answers 503 for Amazon while Odoo is still backfilling. | |
| β THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on | |
| the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a | |
| sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync | |
| and retrying works: 503. A mis-pinned process serving another tenant's file never becomes | |
| un-mis-pinned by waiting: 409. Collapsing them would render the loudest refusal in the system | |
| as a spinner. | |
| """ | |
| rot = _rot() | |
| try: | |
| cur, mismatch = rot._mirror_cur(session, table_key) | |
| except RuntimeError as exc: | |
| if "could not be restored" in str(exc): | |
| return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("restore_blocked")) | |
| if "has not been attached" in str(exc): | |
| return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("loading")) | |
| return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("syncing")) | |
| except Exception as exc: # noqa: BLE001 | |
| # β A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from | |
| # `deps`, which every fixture and gate in the repo imports and drives with runtimes that | |
| # are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on | |
| # whatever it is handed. An unexpected failure there is a 500 on an assistant answer, | |
| # which reads to a user as the product being broken rather than the mirror being shut. | |
| # Named and fail-closed beats a stack trace. The class goes to the server log, never the reply. | |
| import deps | |
| deps._assistant_log_cause("assistant mirror open", exc) | |
| return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("mirror_open")) | |
| if mismatch: | |
| return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch)) | |
| return cur, None | |
| #: The one sentence for a mirror-served table this deployment has no binding for at all. | |
| NO_BINDING = ("this database is read live from its connector mirror, and this deployment has no " | |
| "read-through binding for it yet, so it holds no rows the assistant can read") | |
| class ReadinessSnapshot: | |
| """Reuse mirror metadata within one source listing, without caching permissions or row access.""" | |
| def __init__(self): | |
| self._sources = None | |
| self._columns = {} | |
| self._phases = {} | |
| self._loading = {} | |
| self._amazon_learned = False | |
| self._amazon_pending = None | |
| def amazon_pending(self): | |
| """`amazon_relational.restore_pending()`, read once per listing: a data fact, never a permission.""" | |
| if self._amazon_pending is None: | |
| import amazon_relational | |
| self._amazon_pending = bool(amazon_relational.restore_pending()) | |
| return self._amazon_pending | |
| def source(self, key): | |
| if self._sources is None: | |
| self._sources = dict(_rot()._sources()) | |
| return self._sources.get(key) | |
| def learn_amazon(self, cur): | |
| if not self._amazon_learned: | |
| self._amazon_learned = True | |
| rot = _rot() | |
| rot._learn_amazon_sources(cur) | |
| self._sources = dict(rot.GRID_SOURCES) | |
| def columns(self, cur, name): | |
| from harness import datastore | |
| key = (id(cur), name) | |
| if key not in self._columns: | |
| self._columns[key] = datastore.columns_of(name, cur=cur) | |
| return self._columns[key] | |
| def loading(self, cur, name): | |
| key = (id(cur), name) | |
| if key not in self._loading: | |
| cursor_id = id(cur) | |
| if cursor_id not in self._phases: | |
| try: | |
| phases = dict(cur.execute("SELECT entity, phase FROM _sync_state").fetchall()) | |
| except Exception: # noqa: BLE001 | |
| phases = None | |
| self._phases[cursor_id] = phases | |
| phases = self._phases[cursor_id] | |
| if phases is None: | |
| return False # Mirrors without sync bookkeeping retain standalone semantics. | |
| if name in phases: | |
| self._loading[key] = phases[name] != "live" | |
| else: | |
| self._loading[key] = not cur.execute( | |
| f"SELECT count(*) FROM (SELECT 1 FROM {name} LIMIT 1)").fetchone()[0] | |
| return self._loading[key] | |
| def amazon_empty_reason(status): | |
| """The one-line reason in an Amazon `sourceStatus`, or "" when it names no code and reason.""" | |
| if not isinstance(status, dict) or not status.get("empty") or not status.get("code"): | |
| return "" | |
| reason = status.get("reason") | |
| return " ".join(reason.split()) if isinstance(reason, str) else "" | |
| def unavailable_reason(table_key, definition=None, *, snapshot=None): | |
| """The ONE sentence `list_databases` and `query_database` both give when the mirror holds no rows. | |
| A bound table whose mirror tables are absent is loading (a restart starts from the boot seed); | |
| a table with no binding at all gets `NO_BINDING`. | |
| """ | |
| key = str(table_key or "") | |
| try: | |
| if key.startswith("ut_amzn_"): | |
| import amazon_relational as amazon | |
| if key in amazon.SOURCE: | |
| # D-2643: the stored definition's empty reason, from the one Amazon producer. | |
| status = (definition or {}).get("sourceStatus") if isinstance(definition, dict) else None | |
| reason = amazon_empty_reason(status) | |
| if reason: | |
| return reason | |
| pending = snapshot.amazon_pending() if snapshot is not None else None | |
| return amazon.missing_source(key, pending=pending)["missing"] | |
| spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key) | |
| if spec and not spec.get("missing"): | |
| return _cause("loading") | |
| except Exception: # noqa: BLE001 | |
| pass | |
| return _cause("no_binding") | |
| def still_loading(table_key, *, cur=None, snapshot=None): | |
| """True while an Odoo table behind this binding is in its first backfill, so its rows are partial. | |
| Evidence only: a `_sync_state` phase other than live, or an empty table with no state row yet | |
| (created, first batch not written). A mirror with no sync bookkeeping is not loading. | |
| """ | |
| from harness import datastore | |
| owned_cur = None | |
| try: | |
| snapshot = snapshot if snapshot is not None else ReadinessSnapshot() | |
| spec = snapshot.source(str(table_key or "")) or {} | |
| names = set((spec.get("tables") or {}).values()) | ({spec["table"]} if spec.get("table") else set()) | |
| names = sorted(n for n in names if n in datastore.ENTITIES) | |
| if not names: | |
| return False | |
| if cur is None: | |
| owned_cur = cur = datastore.ro_cursor() | |
| return any(snapshot.loading(cur, name) for name in names) | |
| except Exception: # noqa: BLE001 | |
| return False | |
| finally: | |
| if owned_cur is not None: | |
| try: | |
| owned_cur.close() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| def binding(cur, table_key): | |
| """The resolved read-through spec for one connected grid, or None when it has no binding. | |
| `None` is a real, reportable answer: this deployment's mirror carries no table for this grid, | |
| which is not the same as the grid being empty. | |
| """ | |
| try: | |
| spec = _rot()._source_for(cur, str(table_key or "")) | |
| except Exception: # noqa: BLE001 | |
| # A spec that cannot be resolved is indistinguishable, to every caller, from a grid with | |
| # no binding: neither can be read. Fail closed to the reportable answer. | |
| return None | |
| return amazon_lag(cur, str(table_key or ""), spec) | |
| def amazon_lag(cur, table_key, spec): | |
| """D-2642: a dated Amazon binding with rows also carries its newest row date, cadence and lag line. | |
| The keys come from `amazon_store.table_empty_status`, the one per-table producer. A trailing table | |
| also gets a `source_behind` limit. A status that cannot be read carries the unknown lag line. | |
| """ | |
| if not spec or spec.get("missing") or not table_key.startswith("ut_amzn_"): | |
| return spec | |
| from harness import amazon_store | |
| table = table_key.removeprefix("ut_amzn_") | |
| if table not in amazon_store.LAG_DATE_COLUMNS: | |
| return spec | |
| try: | |
| if cur.execute(f'SELECT 1 FROM "{amazon_store.table_name(table)}" LIMIT 1').fetchone() is None: | |
| return spec | |
| status = amazon_store.table_empty_status(cur, table, rows=1, lag=True) | |
| dated = {name: status[name] for name in amazon_store.LAG_KEYS if name in status} | |
| except Exception: # noqa: BLE001 | |
| dated = amazon_store.unknown_lag(table) | |
| spec = {**spec, "sourceStatus": {"empty": False, **(spec.get("sourceStatus") or {}), **dated}} | |
| if dated.get("lag"): | |
| spec["limits"] = [*(spec.get("limits") or []), | |
| {"effect": "source_behind", "code": dated.get("lagCode"), "cause": dated["lag"]}] | |
| return spec | |
| def is_bound(cur, table_key, *, snapshot=None): | |
| """CAN this deployment read `table_key` through the mirror at all? The availability question. | |
| ββ THE CHEAP PREFIX OF `_source_for`, AND IT IS HERE BECAUSE THE FULL CALL COSTS 300 ms. | |
| MEASURED on this worktree's mirror, warm: `_source_for('ut_odoo_accounts')` 2.9 ms, but | |
| `ut_odoo_order_lines`, `ut_odoo_gl_lines` and `ut_odoo_orders` carry `needs_excluded`, and | |
| that branch runs `odoo_relational.excluded_ids`, a scan of all 5,334 `res_partner` rows, at | |
| 44 ms to 297 ms EACH. `assistant_source_status` is called on mount by BOTH the Assistant and | |
| Query and used to answer in 0.4 ms warm; resolving every read-through grid's full spec there | |
| measured **+605 ms median** on the whole batch. A picker that now tells the truth about ten | |
| databases must not cost two thirds of a second to say it. | |
| β IT IS A PREFIX, NOT A SECOND ANSWER, and that distinction is what keeps it honest. | |
| `_source_for` returns `None` in EXACTLY two cases: this deployment registers no spec for the | |
| key, or the mirror carries no table behind one of the spec's aliases. Both are checked here, | |
| with the same calls, in the same order. Everything after them in `_source_for` builds the | |
| COLUMN EXPRESSIONS a read needs and cannot change whether the answer is `None`. The expensive | |
| `excluded_ids` lookup is part of that second half. | |
| β SO IF `_source_for` EVER GROWS A THIRD `return None`, THIS MUST FOLLOW IT. The failure that | |
| would cause is bounded and visible rather than silent: a source reported answerable that then | |
| answers a named refusal when actually read. It can never produce a wrong NUMBER, which is the | |
| direction that matters. | |
| """ | |
| from harness import datastore | |
| try: | |
| rot = _rot() | |
| key = str(table_key or "") | |
| spec = snapshot.source(key) if snapshot is not None else rot._sources().get(key) | |
| if not spec and key.startswith("ut_amzn_"): | |
| # S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key. | |
| # `_source_for` and `_materialised_table` both learn them on demand; so must this. | |
| if snapshot is not None: | |
| snapshot.learn_amazon(cur) | |
| spec = snapshot.source(key) | |
| else: | |
| rot._learn_amazon_sources() | |
| spec = rot.GRID_SOURCES.get(key) | |
| if not spec: | |
| # A dynamically sized Amazon snapshot is admitted per key by the grid door, not by | |
| # the provider build, so the registry can lack a key the grid serves (measured on | |
| # staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the | |
| # provider's own per-key resolver, exactly as `require_amazon_mirror` does, without | |
| # admitting it (the read path admits it through that same door). | |
| import amazon_relational as amazon | |
| if key not in amazon.SOURCE: | |
| return False | |
| # A yes/no: no columns is unbound. `grid_source` would read the keychain to word why. | |
| if not amazon.mirror_columns(cur, amazon.SOURCE[key]): | |
| return False | |
| current = amazon.grid_source(cur, key) or {} | |
| return bool(current) and not current.get("missing") | |
| if not spec or spec.get("missing"): | |
| return False | |
| aliases = dict(spec.get("tables") or {}) | |
| if spec.get("table"): | |
| aliases.setdefault("", spec["table"]) | |
| return all(snapshot.columns(cur, name) if snapshot is not None | |
| else datastore.columns_of(name, cur=cur) for name in aliases.values()) | |
| except Exception: # noqa: BLE001 | |
| return False | |
| def declared_fields(table_key, definition=None): | |
| """`mk_fields()` for one connected grid, or None when this key is not a connected grid. | |
| β S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that | |
| is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid, | |
| a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before | |
| this, those databases were listed as answerable and then refused 409 on every read. | |
| β THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built | |
| each table's stored `fields` from this same function, so reading it directly is reading the | |
| source both agree on, and it is why `odoo_table_rows` uses it too. A stored definition can | |
| also carry user-added columns that have no mirror expression at all. | |
| """ | |
| import odoo_relational | |
| for _bucket, key, _label, mk_fields in odoo_relational.TABLES: | |
| if key == str(table_key or ""): | |
| return [dict(field) for field in (mk_fields() or []) | |
| if isinstance(field, dict) and field.get("key")] | |
| key = str(table_key or "") | |
| if not isinstance(definition, dict) or not key.startswith("ut_"): | |
| return None | |
| try: | |
| if _rot()._materialised_table(key): | |
| return None | |
| except Exception: # noqa: BLE001 | |
| return None | |
| fields = [dict(field) for field in (definition.get("fields") or []) | |
| if isinstance(field, dict) and field.get("key")] | |
| return fields or None | |
| def answerable_fields(table_key, spec, fields=None): | |
| """The declared field contract INTERSECTED with the columns the mirror can actually express. | |
| β THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field | |
| with no expression in `spec["cols"]` (a link, a rollup, a column this deployment has not | |
| backfilled) cannot be answered from SQL at all. Offering it anyway would put it in the | |
| assistant's field list, where a request naming it comes back present-but-empty rather than | |
| refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request | |
| for it with a loud `assistant_unknown_field`. | |
| """ | |
| if fields is None: | |
| fields = declared_fields(table_key) | |
| if fields is None: | |
| return None | |
| cols = set((spec or {}).get("cols") or ()) | |
| return [field for field in fields if field["key"] in cols] | |
| def _compiled(nodes, conj, columns, *, code, subject, rot, read_wall=False): | |
| """Compile ONE tree through the grid's compiler, or refuse. Never returns a partial predicate. | |
| β THREE WAYS TO FAIL AND ALL THREE FAIL CLOSED, which is the same policy `odoo_table_rows` | |
| carries for the administrator's wall - a column with no mirror expression, an operator with no | |
| SQL form, an aggregate. The grid can afford to report an unpushable USER condition in its | |
| `limits` array and let the client see it; the assistant's snapshot envelope has no such | |
| channel, and an ignored condition does not narrow, it WIDENS. So on this path both trees | |
| refuse. | |
| """ | |
| unknown = sorted(rot._leaf_cols(nodes) - set(columns)) | |
| if unknown: | |
| raise MirrorRefusal( | |
| 409, code, | |
| f"{subject} names {', '.join(unknown)}, and this database's live connection cannot " | |
| f"read that column, so the condition cannot be applied here. Applying the rest would " | |
| f"answer a wider question than the one asked") | |
| try: | |
| compiler = rot._compile_read_wall if read_wall else rot._fs().compile_filter_tree | |
| pred = compiler(nodes, conj=conj, columns=columns, today=rot._today()) | |
| except ValueError as exc: | |
| raise MirrorRefusal( | |
| 409, code, | |
| f"{subject} uses a condition this database's live connection cannot evaluate " | |
| f"({exc}), so it is refused rather than ignored") | |
| if pred is None: | |
| return None | |
| if pred.uses_aggregate: | |
| raise MirrorRefusal( | |
| 409, code, | |
| f"{subject} is a condition on an aggregate column, and this database's live " | |
| f"connection cannot evaluate one of those a row at a time") | |
| return pred | |
| def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, *, table_key=None): | |
| """`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34. | |
| Extracted verbatim from `read_rows` so the row walk, the page read and the server-side | |
| aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler. | |
| Three readers with one predicate cannot disagree about which rows the caller may see. | |
| """ | |
| fe = rot._fe() | |
| cols = rot._filter_columns(spec, fields) | |
| # ββ the administrator's wall, compiled FIRST and pushed down ββββββββββββββββββββββββββββββ | |
| # β PUSHED DOWN RATHER THAN APPLIED AFTERWARDS, for one reason that is specific to this door: | |
| # the row TOTAL below decides whether this read is refused as too large. A wall applied only | |
| # in Python would count rows the reader may not see, and refuse a scoped user whose own slice | |
| # of a million-row table is two hundred rows. | |
| # | |
| # β AND IT IS THE ASSISTANT'S WALL, NOT `row_scope_applies`. The grid door uses | |
| # `perm_scope.row_scope_applies`, which is False for an admin. `assistant_apply_row_scope` | |
| # has no admin break-glass: it applies the grant's filter to every caller. The caller hands | |
| # the tree it will apply in Python, so the two passes are the SAME predicate by construction | |
| # rather than by two spellings that agree today. | |
| # | |
| # READ THROUGH `tree_parts`, WHICH IS THE ONE READER OF THIS SHAPE. A stored wall is | |
| # `{conj, nodes}` and both consumers need a bare node list plus a separate conj. Handing | |
| # either the dict is silently EMPTY rather than an error, i.e. no wall at all, on the one path | |
| # where that means serving the whole database. | |
| where = spec.get("where") or "" | |
| params = [] | |
| wall_nodes, wall_conj = fe.tree_parts(wall_tree) | |
| if wall_nodes: | |
| wall = _compiled(wall_nodes, wall_conj, cols, code="assistant_scope_not_applicable", | |
| subject="the restriction an administrator set on your view of this " | |
| "database", rot=rot, read_wall=True) | |
| if wall is not None: | |
| where = f"({where}) AND {wall.sql}" if where else wall.sql | |
| params.extend(wall.params) | |
| # ββ the caller's own filter βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β AGAINST THE VISIBLE COLUMNS ONLY. `filter_keys` is the assistant's visible field list, | |
| # already resolved by the caller's `_assistant_preflight_request`. Compiling against the full | |
| # contract instead would let a scoped reader infer a hidden column's values by filtering on | |
| # it, which is exactly the widening C7 forbids. The wall above is compiled against the FULL | |
| # contract on purpose, precisely as `perm_scope._scoped` evaluates it against unstripped | |
| # fields. | |
| user_cols = ({key: value for key, value in cols.items() if key in set(filter_keys)} | |
| if filter_keys is not None else dict(cols)) | |
| nodes, conj = fe.tree_parts(filter_tree) | |
| if nodes: | |
| pred = _compiled(nodes, conj, user_cols, code="assistant_filter_not_answerable", | |
| subject="the filter on this request", rot=rot) | |
| if pred is not None: | |
| where = f"({where}) AND {pred.sql}" if where else pred.sql | |
| params.extend(pred.params) | |
| # The trusted link pin intersects every SQL count, aggregate and page for its source. | |
| from core import relations | |
| pin = relations.current_query_link_pin() | |
| if pin is not None and (table_key is None or pin.source == table_key): | |
| link_cols = dict(user_cols) | |
| if pin.column == "pid": | |
| link_cols["pid"] = {"sql": spec["id"]} | |
| try: | |
| link_sql, link_params = relations.query_link_predicate(link_cols) | |
| except ValueError as exc: | |
| raise MirrorRefusal(409, "query_link_unavailable", str(exc)) from exc | |
| if link_sql: | |
| where = f"({where}) AND ({link_sql})" if where else link_sql | |
| params.extend(link_params) | |
| return where, params, cols | |
| def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None, | |
| filter_keys=None, defn=None, serialize=True): | |
| """Every permitted row of a mirror-served grid, or a refusal that names its cause. | |
| Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape | |
| (`{pid, **cells}`), sorted by pid. | |
| β NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a | |
| connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED | |
| with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt` | |
| states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes | |
| every aggregate from those same records, so a short set does not produce a smaller answer, it | |
| produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over | |
| already settled this policy in the same words: REFUSE, NEVER TRIM. | |
| β FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and | |
| the field projection in Python AFTER this returns. Hand it a row missing a column the wall | |
| names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces | |
| pass would quietly disagree with the pushdown it is meant to double-check. | |
| """ | |
| from harness import datastore | |
| rot = _rot() | |
| where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, | |
| table_key=table_key) | |
| # β EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the | |
| # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. | |
| select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) | |
| frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} | |
| # ββ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX` | |
| # pass this replaced refused any connected database past 5,000 matching rows, which is a cap | |
| # on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages, | |
| # KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs | |
| # one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a | |
| # user condition: every condition above still comes from the grid's compiler. | |
| # β STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that | |
| # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short. | |
| first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', | |
| where=where, params=tuple(params), order_by=spec["id"], | |
| limit=datastore.WINDOW_MAX, cur=cur, page_first=True, **frm) | |
| total = int(first["total"]) | |
| if total > ASSISTANT_READ_MAX: | |
| raise MirrorRefusal( | |
| 409, "assistant_source_too_large", | |
| f"this database matches {total:,} rows for that question, and one assistant answer " | |
| f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question " | |
| f"cannot exhaust the server's memory. Answering over part of them would report a " | |
| f"wrong total, so it is refused instead. Narrow it with a filter (for example a date " | |
| f"range) and ask again, or ask for totals as aggregates, which the server computes " | |
| f"over every row") | |
| got = list(first["rows"]) | |
| page = list(first["rows"]) | |
| while len(got) < total and len(page) >= datastore.WINDOW_MAX: | |
| last = page[-1][0] | |
| more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?" | |
| nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', | |
| where=more_where, params=tuple(params) + (last,), | |
| order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, | |
| page_first=True, **frm) | |
| page = list(nxt["rows"]) | |
| if not page: | |
| break | |
| got.extend(page) | |
| if len(got) != total: | |
| raise MirrorRefusal( | |
| 409, "assistant_source_changed", | |
| f"this database changed while it was being read ({len(got):,} rows read against " | |
| f"{total:,} counted), so the answer would not describe one consistent set of rows. " | |
| f"Ask again once its connector sync has finished") | |
| win = {**first, "rows": got, "total": total} | |
| # S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does | |
| # not re-read the whole tenant document (measured about 3 s a question on staging). | |
| stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), | |
| defn=defn if isinstance(defn, dict) else None) | |
| rows = rot._pool_from_window(spec, win, stamp=stamp) | |
| if serialize: | |
| rows = _typed_rows(rot, fields, rows) | |
| rows.sort(key=lambda row: row["pid"]) | |
| return rows, stamp, int(win["total"]) | |
| #: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`). | |
| _GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"} | |
| def _json_value(rot, value): | |
| """A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a | |
| date or timestamp as its ISO text, anything else as is. Never a substituted 0.""" | |
| if value is None: | |
| return None | |
| if isinstance(value, (int, float)) and not isinstance(value, bool): | |
| return rot._agg_number(value) | |
| iso = getattr(value, "isoformat", None) | |
| if callable(iso): | |
| return iso() | |
| try: | |
| from decimal import Decimal | |
| if isinstance(value, Decimal): | |
| return rot._agg_number(value) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| return value | |
| def field_value(rot, field, value): | |
| """Serialize a native checkbox as a Boolean while preserving other JSON values.""" | |
| if field.get("type") == "checkbox" and value is not None: | |
| if isinstance(value, str): | |
| return value.strip().lower() in {"1", "true", "t", "yes", "y", "on"} | |
| return bool(value) | |
| return _json_value(rot, value) | |
| def _typed_rows(rot, fields, rows, *, preserve_blank=False): | |
| """Apply the declared field codecs without changing source records.""" | |
| flags = [field for field in fields if field.get("type") == "checkbox"] | |
| if not flags: | |
| return rows | |
| def encode(field, value): | |
| if preserve_blank and (value is None or isinstance(value, str) and value.strip() in {'', '(none)'}): | |
| return value | |
| return field_value(rot, field, value) | |
| return [{**row, **{field["key"]: encode(field, row[field["key"]]) | |
| for field in flags if field["key"] in row}} for row in rows] | |
| def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None, | |
| filter_tree=None, filter_keys=None): | |
| """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631). | |
| Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the | |
| count of rows the permitted predicate matches, from `datastore.window`'s own count statement | |
| over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract). | |
| ββ THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall | |
| and the caller's filter with the grid's compiler, so a sum over a million-row table runs over | |
| exactly the rows this caller may read and no row is pulled into Python to compute it. There | |
| is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size. | |
| β AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and | |
| a field outside it has no expression here at all, so a sum over a hidden column cannot be | |
| built (the caller's `_normalise_aggregates` has already refused it by name). | |
| """ | |
| from harness import datastore | |
| rot = _rot() | |
| where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, | |
| table_key=table_key) | |
| visible = set(filter_keys) if filter_keys is not None else set(cols) | |
| fs = rot._fs() | |
| proj, slots = [], [] | |
| for item in specs or (): | |
| fn, field = item.get("fn"), item.get("field") | |
| if field is None: | |
| slots.append(None) # count(*): the window's own total | |
| continue | |
| col = cols.get(field) if field in visible else None | |
| if col is None: | |
| from deps import assistant_unknown_field_message | |
| raise MirrorRefusal(400, "assistant_unknown_field", | |
| assistant_unknown_field_message(field, visible & set(cols))) | |
| if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES: | |
| raise MirrorRefusal(400, "query_aggregate_type", | |
| "sum and avg require a readable numeric field.") | |
| if fn in _GRID_AGG: | |
| expr = fs.aggregate_sql(col, _GRID_AGG[fn]) | |
| elif fn in {"min", "max"}: | |
| expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES | |
| else f"{fn}({col['sql']})") | |
| elif fn == "count": | |
| expr = f"count({col['sql']})" | |
| else: | |
| raise MirrorRefusal(400, "query_aggregates_invalid", | |
| "That aggregate is not available.") | |
| slots.append(len(proj)) | |
| proj.append(f'{expr} AS "a{len(proj)}"') | |
| frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} | |
| win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params), | |
| order_by="1", limit=1, cur=cur, **frm) | |
| row = (win["rows"] or [()])[0] | |
| total = int(win["total"]) | |
| out = [] | |
| for item, slot in zip(specs or (), slots): | |
| value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row) | |
| else None) | |
| out.append({**item, "value": value}) | |
| return out, total | |
| def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None, | |
| filter_tree=None, filter_keys=None, defn=None, sorts=None, serialize=True): | |
| """Return one SQL-ordered permitted page, its source stamp and its full population count.""" | |
| from harness import datastore | |
| rot = _rot() | |
| where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys, | |
| table_key=table_key) | |
| visible = set(filter_keys) if filter_keys is not None else set(cols) | |
| order = [] | |
| for rule in sorts or (): | |
| if (not isinstance(rule, dict) or set(rule) != {"field", "direction"} | |
| or not isinstance(rule.get("field"), str) | |
| or rule["field"] not in visible or rule["field"] not in cols | |
| or rule.get("direction") not in ("asc", "desc")): | |
| from deps import assistant_unknown_field_message | |
| raise MirrorRefusal(400, "assistant_unknown_field", | |
| assistant_unknown_field_message( | |
| rule.get("field") if isinstance(rule, dict) else None, | |
| visible & set(cols))) | |
| column_sql = cols[rule["field"]]["sql"] | |
| direction = "ASC" if rule["direction"] == "asc" else "DESC" | |
| # SQL expressions come only from the resolved schema, directions from fixed literals. | |
| if cols[rule["field"]]["type"] == "text": | |
| order.append(f"lower({column_sql}) {direction} NULLS LAST") | |
| order.append(f"{column_sql} {direction} NULLS LAST") | |
| order.append(f'{spec["id"]} ASC') | |
| select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) | |
| frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} | |
| win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where, | |
| params=tuple(params), order_by=", ".join(order), offset=int(offset), | |
| limit=int(limit), cur=cur, page_first=True, **frm) | |
| stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), | |
| defn=defn if isinstance(defn, dict) else None) | |
| rows = rot._pool_from_window(spec, win, stamp=stamp) | |
| if serialize: | |
| rows = _typed_rows(rot, fields, rows) | |
| return rows, stamp, int(win["total"]) | |