diff --git a/Dockerfile b/Dockerfile index f8cff4b4cdbb754a18680ca59ab17d0725cf2102..0558431db11754088ad3954f9d7d3f0df67f99b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,12 +20,24 @@ # ---- stage 1: build the React bundle ---- FROM node:22-slim AS web WORKDIR /web -# Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash -# 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts). -COPY web/package.json ./ -RUN npm install --no-audit --no-fund --no-package-lock +# The lockfile is the audited supply-chain boundary. It now resolves entirely from the public +# registry; discarding it here made every image an unreviewed dependency upgrade. +# +# ⛔⛔ `npm ci` WAS TRIED HERE AND CANNOT WORK FROM A WINDOWS-RESOLVED LOCKFILE. Measured +# 2026-09-13: the build died with `Missing: @emnapi/runtime@1.11.3 from lock file`. npm on Windows +# prunes the linux-x64 rolldown binding and its transitive `@emnapi/runtime`, and `npm ci` treats +# any absent entry as a fatal lockfile mismatch rather than a platform gap. Regenerating with +# `--os=linux --cpu=x64` only inverts the failure: the lockfile then builds on Linux and breaks +# `npm ci` on every developer's Windows box. +# +# `npm install` WITH the lockfile present keeps the audited versions as the boundary — it honours +# every pin it can resolve — while repairing a platform-only gap instead of aborting. This is NOT +# the pre-83bfed7 `--no-package-lock`, which discarded the audit entirely. +# The real fix is a lockfile generated on Linux; booked as W-170. +COPY web/package.json web/package-lock.json ./ +RUN npm install --no-audit --no-fund --ignore-scripts COPY web/ ./ -RUN rm -f package-lock.json && npm run build # -> /web/dist +RUN npm run build # -> /web/dist # ---- stage 2: python API serving the bundle ---- FROM python:3.12-slim diff --git a/RELEASES.json b/RELEASES.json index b5a698ec6a6b98db135e999261f4401294d6edba..e731c9ab24b0a24ec174d258239663d0f0431dca 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "v71 (96143dc)", + "current": "v73 (bb94de5)", "releases": [ + { + "version": "v73", + "sha": "bb94de5", + "date": "2026-09-13", + "subject": "release v73" + }, { "version": "v71", "sha": "96143dc", diff --git a/VERSION b/VERSION index 2448460301d6666ef814efc9db01e7f4ad907e8b..1e15e29d9ad5fcc8920c4f4747bfa4fd135d93f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v71 (96143dc) +v73 (bb94de5) diff --git a/api/amazon_relational.py b/api/amazon_relational.py new file mode 100644 index 0000000000000000000000000000000000000000..fd47fcd38d081ead163082978398eb80a37da345 --- /dev/null +++ b/api/amazon_relational.py @@ -0,0 +1,421 @@ +"""amazon_relational.py — the Amazon mirror becomes LOCKED DATABASES under Data (wave 52, R14). + +`harness/amazon_store.py` pulls SP-API report documents into the tenant's DuckDB mirror; this reads +that mirror and spawns the `ut_amzn_*` grids a person opens. It is `meta_relational.py`'s shape +with one source swapped, which is itself `odoo_relational.py`'s shape. + + plan(cur) the rows that WOULD be written — no store write at all + apply_plan(rt, built) create-or-merge every table and its rows, in ONE sync write + refresh(rt, why) both, with the mirror opened for this tenant + +⭐⭐ **THE COLUMNS ARE DERIVED FROM THE MIRROR, NEVER TYPED OUT HERE**, exactly as Meta's are. For +Amazon that is doubly load-bearing: five of the seven tables have their columns MEASURED from the +report document's own header row at sync time (`amazon_schema.PROVENANCE`), so a hand-written field +list here would not merely rot, it would contradict the only measurement that exists. Add a column +to the report and it appears in the grid with no edit in this file. + +⛔⛔ **R14 — THIS FILE IS WHERE THE CONNECTOR ENDS.** *"this is just database from Amazon, all its +fields mapped into our app so it can be seen under Data module and later be pulled into Sheet. All +database connector has this exact flow."* No dashboard, no module, no metric page. `Data` lists +these tables because they are `ut_*` tables; `sheet/BindingEditor.tsx:61` binds them on the `ut_` +prefix. Both are prefix-driven, so **neither needs a line of work here** — step 5 of `/connector` is +a verification, not a build. + +⛔ **R12 — AMAZON IS A STANDALONE DATABASE SET.** Owner: *"Its all different. Assume that its a +whole new databases which they are."* `LINKS` below therefore joins Amazon tables only to OTHER +AMAZON TABLES: order lines to their order, pricing to its listing. **Nothing here joins to +`product_data`, to `ut_odoo_*`, or to the hand-maintained ASIN mastersheet.** ⭐ That is what +removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever +converted to an Amazon unit, because the two never meet. + +⚠ MATERIALISED, WITH THE CEILING REPORTED RATHER THAN ENFORCED. Same posture as Meta's: a table +over `MAX_ROWS` is REFUSED with its cause and its fix named, never truncated. For Amazon that is +standing rule 1's second half, and it is the likeliest table to hit it — an order-item history is +one row per line, not per order. +""" +import hashlib +import json +import os +import sys +from pathlib import Path + +_API = Path(__file__).resolve().parent +if str(_API.parent.parent / "platform") not in sys.path: + sys.path.insert(0, str(_API.parent.parent / "platform")) + +import odoo_relational as _odoo # noqa: E402 +from core import connector_spec # noqa: E402 +from harness import amazon_schema, amazon_store, datastore # noqa: E402 + +Refused = _odoo.Refused + +#: ⛔ THE PREFIX IS `ut_amzn_`, AND IT MUST ALSO BE IN `routes_connectors.CONNECTED_PREFIXES` — +#: otherwise `is_connected` falls through and every Amazon table is silently re-capped at +#: `MAX_ROWS`, breaking standing rule 1 with nothing raising. `verify_amazon` asserts the pair. +PREFIX = "ut_amzn_" + +#: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot +#: exist on one side and not the other. +SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.TABLES} + +#: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE** +#: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level +#: names because `verify_amazon`, `verify_meta.section_connector_spec` and the connector routes all +#: read them BY NAME — but each is a DERIVED VIEW over ONE declaration now, so the shape can no +#: longer be stated twice and say two things. Every comment that used to justify a literal here +#: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values +#: into `connector_spec.AMAZON_TABLES`; read it there. +_SPEC = {s.key: s for s in connector_spec.tables_for("amazon")} + +#: ⛔ A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR +#: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal +#: `SOURCE`'s, so the drift reddens a gate rather than raising at import inside a live Space, and +#: it names the missing table instead of a KeyError deep in `apply_plan`. +LABELS = {k: _SPEC[k].label for k in SOURCE if k in _SPEC} + +#: The column each grid shows first. ⚠ Every one is a name Amazon's own reports use, so a table +#: whose header does not carry it simply has no pinned column rather than a broken one. +PINNED = {k: _SPEC[k].pinned for k in SOURCE if k in _SPEC} + +#: ⭐ THE ROW IDENTITY PER TABLE, IN FALLING PREFERENCE — a flat-file report has no `id` column the +#: way Graph does. When none is present the row is keyed by a hash of its own content (`_row_id`), +#: which still gives a STABLE id across re-syncs and is what makes the delete-then-insert merge +#: idempotent instead of appending a second copy of every row each time. +IDENTITY = {k: tuple(_SPEC[k].identity) for k in SOURCE if k in _SPEC} + +#: `{ut_ key: [(this table's column, target ut_ key, the column it matches there), ...]}`. +#: ⚠ THE CARDINALITY IS DROPPED HERE AND KEPT IN THE SPEC. This view stays a 3-tuple because that +#: is the shape `verify_amazon.section_relational` unpacks; `connector_spec.edges_for` is what +#: `_fields_for` actually renders from, and it carries all four. +#: ⛔ AMAZON TO AMAZON ONLY (R12), and a link is DECLARED in the spec but VERIFIED AT SPAWN: five of +#: seven tables are `measured_at_sync`, so every entry is a hypothesis about a report column name +#: until `_fields_for` confirms BOTH sides carry it. A wrong guess here is cheap; a wrong guess in a +#: JOIN would not be. A table with no links is absent rather than present-and-empty, matching the +#: dict this replaced. +LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links] + for k in SOURCE if k in _SPEC and _SPEC[k].links} + + +def source_attribution(key): + """Durable origin metadata for one materialised Amazon database. + + DPP 1.8 permits a database separation strategy or a mechanism that tags and identifies the + origin of data. The DuckDB mirror uses the reserved ``amzn_*`` namespace already; this stamp + is the corresponding mechanism in the shared ``user_tables`` document. It is written on + every refresh so a user-facing label change cannot erase the Amazon origin or the exact raw + table/report that supplied the rows. + """ + if key not in SOURCE or key not in _SPEC: + raise KeyError(f"unknown Amazon database {key!r}") + return { + "key": "amazon_sp", + "origin": "Amazon Services API", + "rawTable": SOURCE[key], + "native": _SPEC[key].native, + "readOnly": True, + } + + +#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601. +_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date", + "settlement_end_date", "deposit_date", "posted_date", "open_date", "date", + "report_date", "start_date", "end_date", "_time") + +#: ⛔ MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace, +#: and an int column would round every row of a settlement. The grid shows what the report said and +#: converts nothing. +_MONEY = {"item_price", "item_tax", "shipping_price", "shipping_tax", "gift_wrap_price", + "gift_wrap_tax", "item_promotion_discount", "ship_promotion_discount", "price", + "amount", "total_amount", "principal", "shipping", "tax", "promotion", "fee", + "your_price", "landed_price", "listing_price", "open_date_price", + "sales_by_asin_ordered_product_sales_amount", + "sales_by_asin_ordered_product_sales_b2_b_amount"} + +#: Percentages and rates are fractional too, but they are not money. `currency` is the only +#: fractional field type the grid offers, so they take it and the LABEL carries the meaning. +_RATE_SUFFIX = ("_percentage", "_percentage_b2_b", "_rate", "_ctr") + + +def _label(col): + return col.replace("_", " ").strip().capitalize() + + +def _ftype(col, duck_type): + """The grid field type for one mirror column. Name first, storage second. + + ⚠ NAME BEATS STORAGE FOR DATES ON PURPOSE. Every flat-file column arrives as text, so a + purchase date is a VARCHAR in the mirror and would render as a string in the grid unless the + name is allowed to override. The store's `_coltype` already refuses to guess a type for text + it cannot parse; this is the presentation half of the same judgement. + """ + if any(col == s or col.endswith(s) for s in _DATEISH): + return "date" + if col in _MONEY or any(col.endswith(s) for s in _RATE_SUFFIX): + return "currency" + base = str(duck_type or "").upper().split("(")[0] + if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"): + return "int" + if base in ("DOUBLE", "FLOAT"): + return "currency" + return "text" + + +def mirror_columns(cur, table): + """[(name, type)] for a mirror table, or [] when it has never been synced.""" + try: + return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()] + except Exception: # noqa: BLE001 + return [] + + +def _fields_for(cur, key): + """One preset field per mirror column — the whole schema, derived. See the header.""" + cols = mirror_columns(cur, SOURCE[key]) + if not cols: + return [] + names = {c[0] for c in cols} + pinned = PINNED.get(key) if PINNED.get(key) in names else None + out = [] + for name, dtype in cols: + out.append(_odoo._preset({ + "key": name, "label": _label(name), "type": _ftype(name, dtype), + "source": "overlay", + "default": name == pinned or name in _DEFAULT_COLUMNS, + "pinned": name == pinned, + }, flow="amazon_relational")) + # ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not + # known until it syncs, so a link to a table that has never run would name a column that does + # not exist. **Absent is a normal state here, not a failure** — which is exactly what lets the + # map above declare hypotheses about report column names before any report has been seen. + # ⚠ A TABLE MAY NOW CARRY SEVERAL LINKS. The first version allowed one, which could express the + # transaction spine or the product spine but never both, and Amazon's data needs both. + # ⭐⭐ THE EDGE'S KEY AND LABEL COME FROM `connector_spec.edges_for`, NOT FROM AN f-STRING HERE + # (W53-T37). `f"{target}_link__{from_col}"` and `f"{label} by {_label(from_col)}"` are ONE + # convention shared by every connector: the source column is in the key because two links to + # the same target (orders by id and by sku) would otherwise collide on one key and the second + # would silently overwrite the first. Meta carried the older `f"{target}_link"` and therefore + # carried that collision, latent; a shared generator is what retires it for the next connector + # too, instead of each module re-typing the rule and one of them getting it wrong. + for edge in connector_spec.edges_for(key): + if edge["from_col"] not in names: + continue + if not any(c[0] == edge["to_col"] + for c in mirror_columns(cur, SOURCE.get(edge["target"], ""))): + continue + out.append(_odoo._preset({ + "key": edge["key"], "label": edge["label"], + "type": "link", "source": "overlay", "default": True, + "link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"]}, + }, flow="amazon_relational")) + out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational")) + return out + + +#: Columns worth showing before a person picks their own. Deliberately short: these tables are +#: wide, and a default of "everything" is the same as no default at all. +_DEFAULT_COLUMNS = { + "amazon_order_id", "purchase_date", "order_status", "sku", "seller_sku", "asin", "asin1", + "quantity", "quantity_purchased", "item_price", "item_status", "product_name", "item_name", + "settlement_id", "amount", "amount_type", "amount_description", "posted_date", + "parent_asin", "child_asin", "search_term", "search_frequency_rank", + "traffic_by_asin_buy_box_percentage", "sales_by_asin_units_ordered", +} + + +def _row_id(row, key): + """A STABLE id for one mirror row. + + ⭐ STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace` + merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and + re-insert of the grid — losing any per-row state the product hangs off that id. + + Falls back to a content hash rather than a counter for exactly that reason: a settlement line + with no id column still hashes to the same value next week if its content has not changed. + """ + for col in IDENTITY.get(key, ()): + val = row.get(col) + if val not in (None, ""): + return str(val) + payload = json.dumps({k: row.get(k) for k in sorted(row)}, sort_keys=True, + ensure_ascii=True, separators=(",", ":"), default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32] + + +def _read(cur, key): + """[(row dict)] straight off the mirror, each carrying a stable `_id`.""" + cols = [c[0] for c in mirror_columns(cur, SOURCE[key])] + if not cols: + return [] + quoted = ", ".join(f'"{c}"' for c in cols) + rows = cur.execute(f"SELECT {quoted} FROM {SOURCE[key]}").fetchall() + out = [] + for r in rows: + row = {} + for name, val in zip(cols, r): + row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val)) + row["_id"] = _row_id(row, key) + out.append(row) + return out + + +def available(tenant_key="royal-imports"): + """Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches.""" + try: + counts = amazon_store.status(tenant_key) + except Exception: # noqa: BLE001 + return False + return any(bool((v or {}).get("rows")) for v in counts.values()) + + +def plan(cur, rt=None): + """{bucket_key: rows} plus `problems`. No store write; every cap checked BEFORE any commit.""" + built = {"problems": []} + ut = _odoo._ut() + cap = getattr(ut, "MAX_ROWS", 60_000) + for key in SOURCE: + # ⛔ COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking + # the cap afterwards means the one table too big to keep is the one built in memory first. + try: + n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0] + except Exception: # noqa: BLE001 + continue # absent = never synced + if n > cap: + # ⛔ REPORTED WITH ITS CAUSE AND ITS FIX, NEVER TRUNCATED. Standing rule 1's second + # half, and `ut_amzn_order_items` is the table most likely to reach it: an order + # history is one row per LINE, so a mid-size seller passes 60,000 in a year. + built["problems"].append( + f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was " + f"truncated and nothing was written for this table. Cause: this seller's history " + f"is larger than a materialised ut_* table can hold. Fix: bind it read-through " + f"off the mirror as `ut_odoo_gl_lines` is " + f"(odoo_relational.READ_THROUGH_KEYS), which `routes_connected_tables` " + f"already serves.") + continue + rows = _read(cur, key) + if rows: + built[key] = rows + return built + + +def plan_fingerprint(cur, built): + """Digest the exact derived Amazon content that would be published. + + `amazon_store.sync()` counts fetched rows, not changed rows, so it is not equality evidence. + Row order out of DuckDB is arbitrary and therefore non-semantic, so rows are canonicalised + before hashing. Callers fail safe to a refresh if this raises. + """ + tables = [] + for key in SOURCE: + rows = built.get(key) or [] + canonical = sorted(rows, key=lambda row: json.dumps( + row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str)) + tables.append({"key": key, "fields": _fields_for(cur, key), "rows": canonical}) + payload = json.dumps(tables, sort_keys=True, ensure_ascii=True, + separators=(",", ":"), default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def apply_plan(rt, built, username="amazon", today=None): + """Create-or-merge every table in `built`, in ONE `flush="sync"` write. + + ⚠ ONE write for all seven, not seven. `Store.update` is a full download plus a full upload of + the tenant document, so seven would be seven round trips and seven commits per resync. It is + also ATOMIC: a `Refused` at MAX_TABLES aborts before anything persists, rather than leaving the + half-spawn `plan()` exists to prevent. + """ + if built.get("problems"): + raise Refused("; ".join(built["problems"])) + stamp = today or _odoo._iso_today() + written = {} + cur_mirror = datastore.ro_con() + plans = [(key, LABELS[key], _fields_for(cur_mirror, key), built[key]) + for key in SOURCE if key in built] + + def _apply_all(doc): + doc = doc if isinstance(doc, dict) else {} + for key, label, fields, rows in plans: + written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows, + username, stamp) + # DPP 1.8 attribution is explicit in the stored definition, not inferred from a + # mutable display label or from the process having imported this module. + doc[key]["connector"] = source_attribution(key) + return doc + + rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync") + return written + + +def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print): + """Open, plan and conditionally apply the Amazon mirror. + + Returns `{}` and says why when the mirror holds no Amazon tables — a tenant that never + connected Amazon is a normal state, not a failure. + """ + state = {"present": False, "known": False, "fingerprint": None, + "changed": False, "applied": False, "written": {}} + tenant = tenant_key or getattr(getattr(rt, "tenant", None), "key", None) or "royal-imports" + path = datastore.path_for(tenant) + if Path(datastore.DB_PATH) != Path(path): + datastore.use_path(path) + cur = datastore.ro_con() + if not any(mirror_columns(cur, t) for t in SOURCE.values()): + log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn " + f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)") + return state + state["present"] = True + built = plan(cur, rt) + problems = built.get("problems") or [] + for p in problems: + log(f"[amazon_relational] PROBLEM {p}") + applicable = {k: v for k, v in built.items() if k != "problems"} + if not problems: + try: + state["fingerprint"] = plan_fingerprint(cur, applicable) + state["known"] = True + except Exception as e: # noqa: BLE001 + log(f"[amazon_relational] {why}: plan fingerprint unavailable; forcing refresh " + f"({type(e).__name__}: {e})") + if state["known"] and state["fingerprint"] == previous_fingerprint: + log(f"[amazon_relational] {why}: Amazon plan unchanged; skipped full user_tables write") + return state + state["changed"] = True + state["written"] = apply_plan(rt, applicable) + state["applied"] = True + for key, counts in state["written"].items(): + log(f"[amazon_relational] {why}: {key:<28} {counts}") + return state + + +def refresh(rt, why="boot", tenant_key=None, log=print): + """Backward-compatible facade. Use `refresh_state` for egress-aware state.""" + return refresh_state(rt, why, tenant_key, log=log)["written"] + + +def main(argv=None): + """`python aios-web/api/amazon_relational.py --refresh [--tenant X]` — the manual door.""" + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--refresh", action="store_true") + ap.add_argument("--plan", action="store_true") + ap.add_argument("--tenant", default="royal-imports") + a = ap.parse_args(argv) + os.environ.setdefault("AIOS_PREWARM", "0") + from harness import runtime as _rt + rt = _rt.get_runtime(a.tenant) + if a.plan: + path = datastore.path_for(a.tenant) + if Path(datastore.DB_PATH) != Path(path): + datastore.use_path(path) + built = plan(datastore.ro_con(), rt) + for k, v in built.items(): + print(f" {k:<28} {len(v) if isinstance(v, list) else v}") + return 0 + if a.refresh: + refresh(rt, why="cli", tenant_key=a.tenant) + return 0 + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/api/application_registry.py b/api/application_registry.py index ee2cc6dc423a87f734bdc64d7e6413e32cad54a7..91b6a24cb7cafadad6adfc04366c8ae4608aad5b 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -7,14 +7,59 @@ import uuid APPLICATION_TYPES = ("chart", "route", "map", "timeseries") +APPLICATION_TEMPLATE_KINDS = ("route", "catalog") _KEY = "applications_v1" _MAX = 500 +# Routing and Catalog are the first authored App templates. These are wire contracts, not +# executable module declarations: the client renders a trusted platform surface from the stored +# definition. ``required`` is consumed by the creation flow and enforced again below, so a +# future client cannot weaken a template by skipping a wizard step. +_TEMPLATE_CONTRACTS = { + "route": { + "kind": "route", + "label": "Routing", + "applicationType": "route", + "displayMode": "map", + "fieldRoles": [ + {"key": "title", "label": "Record title", "required": True, "types": []}, + {"key": "latitude", "label": "Latitude", "required": False, + "types": ["text", "int", "currency", "pct", "rating", "number", "float", "decimal"]}, + {"key": "longitude", "label": "Longitude", "required": False, + "types": ["text", "int", "currency", "pct", "rating", "number", "float", "decimal"]}, + {"key": "address", "label": "Address", "required": False, + "types": ["text"]}, + ], + "defaultPlacement": {"section": "app", "order": 0}, + }, + "catalog": { + "kind": "catalog", + "label": "Catalog", + # Catalog is a template discriminator over an existing trusted renderer contract. The + # governed Application type vocabulary stays four-way; the client launches the Catalog + # surface from ``templateKind`` rather than teaching every shared type mirror a fifth word. + "applicationType": "map", + "displayMode": "map", + "fieldRoles": [ + {"key": "key", "label": "Record key", "required": True, "types": []}, + {"key": "title", "label": "Record title", "required": True, "types": []}, + {"key": "image", "label": "Image", "required": False, + "types": ["image", "attachment", "url", "text"]}, + ], + "defaultPlacement": {"section": "app", "order": 0}, + }, +} + class ApplicationError(ValueError): pass +def template_contracts(): + """The trusted renderer contracts offered by the Application creation flow.""" + return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_TEMPLATE_KINDS] + + def blank_doc(): return {"schema": 1, "applications": {}, "defaults": {}} @@ -125,6 +170,129 @@ def _allowed(compatible, allowed_databases): else set(compatible or ()) & set(allowed_databases)) +def _contract_fields(database, semantic_records=None, field_contracts=None): + """Return ``{field key: type}`` from a caller's already permission-scoped contract. + + The nav creation door supplies its served schema fields. The older Application door supplies + governed semantic records. Supporting both here keeps one validator while never making this + registry reach around either caller's database/field wall. + """ + fields = {} + raw = ((field_contracts or {}).get(database) + if isinstance(field_contracts, dict) else None) + if isinstance(raw, dict): + raw = list(raw.values()) + for row in raw if isinstance(raw, (list, tuple)) else (): + if not isinstance(row, dict): + continue + key = str(row.get("key") or row.get("field") or "").strip() + if key: + fields[key] = str(row.get("type") or row.get("dataType") or "").strip().lower() + records = ((semantic_records or {}).get(database) + if isinstance(semantic_records, dict) else None) + for row in (records or {}).values() if isinstance(records, dict) else (): + if not isinstance(row, dict): + continue + key = str(row.get("field") or row.get("key") or "").strip() + if key and key not in fields: + fields[key] = str(row.get("dataType") or row.get("type") or "").strip().lower() + return fields + + +def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None, + allowed_databases=None): + contract = _TEMPLATE_CONTRACTS[kind] + template_kind = str(payload.get("templateKind") or "").strip().lower() + if template_kind != kind: + raise ApplicationError(f"{kind} applications require templateKind {kind!r}") + source = str(payload.get("sourceDatabase") or "").strip()[:120] + if not source: + raise ApplicationError("sourceDatabase is required") + if source not in set(allowed_databases or ()): + raise PermissionError("an Application cannot bind a database you cannot open") + if "compatibleDatabases" in payload: + compatible = _compat(payload.get("compatibleDatabases")) + if compatible != [source]: + raise ApplicationError("a template Application has exactly one sourceDatabase") + + raw_map = payload.get("fieldMap") + if not isinstance(raw_map, dict): + raise ApplicationError("fieldMap must be an object") + roles = {row["key"]: row for row in contract["fieldRoles"]} + extras = sorted(set(raw_map) - set(roles)) + if extras: + raise ApplicationError("fieldMap contains roles outside the template contract: " + + ", ".join(extras)) + missing = sorted(key for key, role in roles.items() + if role["required"] and not str(raw_map.get(key) or "").strip()) + if missing: + raise ApplicationError("fieldMap is missing required roles: " + ", ".join(missing)) + available = _contract_fields(source, semantic_records, field_contracts) + if not available: + raise ApplicationError("the source database has no readable field contract") + field_map = {} + for role, raw_field in raw_map.items(): + field = str(raw_field or "").strip() + if not field: + continue + if len(field) > 160 or field not in available: + raise ApplicationError(f"fieldMap role {role!r} does not name a source field") + allowed_types = set(roles[role].get("types") or ()) + if allowed_types and available[field] not in allowed_types: + raise ApplicationError( + f"fieldMap role {role!r} cannot use field type {available[field]!r}") + field_map[role] = field + latitude, longitude = field_map.get("latitude"), field_map.get("longitude") + if bool(latitude) != bool(longitude): + raise ApplicationError("Routing needs both coordinate fields or neither") + if latitude and latitude == longitude: + raise ApplicationError("Routing latitude and longitude must use different fields") + # A new Route must name at least one placement source. Stored definitions are not revalidated + # on read, so this create-time refusal does not retire older Applications with an empty map. + if kind == "route" and not (latitude and longitude) and not field_map.get("address"): + raise ApplicationError("Routing needs both coordinate fields or an address field") + + raw_placement = payload.get("navPlacement") + if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}: + raise ApplicationError("navPlacement must contain section and order") + section = str(raw_placement.get("section") or "").strip().lower() + order = raw_placement.get("order") + if section != "app": + raise ApplicationError("template Applications must be placed in App") + if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999: + raise ApplicationError("navPlacement order must be an integer from 0 to 999") + return {"templateKind": kind, "sourceDatabase": source, "fieldMap": field_map, + "navPlacement": {"section": "app", "order": order}} + + +def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None, + allowed_databases=None): + """Validate the server-authored identity/tenant envelope and any template definition.""" + payload = payload if isinstance(payload, dict) else {} + if payload.get("id") not in (None, ""): + raise ApplicationError("application id is server assigned") + if "tenant" in payload and str(payload.get("tenant") or "") != str(tenant): + raise PermissionError("an Application cannot be created for another tenant") + template_kind = str(payload.get("templateKind") or "").strip().lower() + if template_kind and template_kind not in APPLICATION_TEMPLATE_KINDS: + raise ApplicationError("application template kind is not recognized") + expected_type = (_TEMPLATE_CONTRACTS[template_kind]["applicationType"] + if template_kind else "") + kind = str(payload.get("type") or expected_type).strip().lower() + if kind not in APPLICATION_TYPES: + raise ApplicationError("application type is not recognized") + if template_kind: + if kind != expected_type: + raise ApplicationError( + f"{template_kind} template Applications require type {expected_type!r}") + return _template_definition( + template_kind, payload, semantic_records=semantic_records, + field_contracts=field_contracts, allowed_databases=allowed_databases) + if any(key in payload for key in ("sourceDatabase", "fieldMap", "navPlacement")): + raise ApplicationError("only a template Application may carry template fields") + return {} + + def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, is_default=False): row=copy.deepcopy(app) @@ -160,23 +328,34 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True) -def create(runtime, tenant, user, payload, *, semantic_records=None, allowed_databases=None): +def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None, + allowed_databases=None): payload=payload if isinstance(payload,dict) else {} - kind=str(payload.get("type") or "").lower() - if kind not in APPLICATION_TYPES: raise ApplicationError("application type is not recognized") + template_kind=str(payload.get("templateKind") or "").strip().lower() + expected_type = (_TEMPLATE_CONTRACTS.get(template_kind) or {}).get("applicationType") + kind=str(payload.get("type") or expected_type or "").strip().lower() + definition = validate_application( + payload, tenant, semantic_records=semantic_records, field_contracts=field_contracts, + allowed_databases=allowed_databases) name=" ".join(str(payload.get("name") or "").split())[:120] if not name: raise ApplicationError("application name is required") app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat() - compatible = _compat(payload.get("compatibleDatabases")) + compatible = ([definition["sourceDatabase"]] if definition + else _compat(payload.get("compatibleDatabases"))) if _allowed(compatible, allowed_databases) != set(compatible): raise PermissionError("an Application cannot bind a database you cannot open") + raw_config = payload.get("config") + if definition and raw_config is None: + raw_config = {"schemaVersion": 1, + "display": {"mode": _TEMPLATE_CONTRACTS[template_kind]["displayMode"]}} app={"id":app_id,"tenant":str(tenant),"type":kind,"name":name, "compatibleDatabases":compatible, "semanticRoleBindings":_governed_bindings( payload.get("semanticRoleBindings"), compatible, semantic_records or {}), - "config":_typed_config(kind, payload.get("config")), + "config":_typed_config(kind, raw_config), "share":{"mode":"private","users":[]},"starredBy":[],"createdBy":user, "createdAt":now,"updatedAt":now,"revision":1} + app.update(definition) def apply(raw): doc=_clean_doc(raw) if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached") @@ -188,11 +367,19 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, allowed_dat def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, allowed_databases=None, target_database=None): patch=patch if isinstance(patch,dict) else {}; landed={} + immutable = sorted(set(patch) & { + "id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement" + }) + if immutable: + raise ApplicationError("Application identity and template definition are immutable: " + + ", ".join(immutable)) def apply(raw): doc=_clean_doc(raw); app=doc["applications"].get(app_id) if not app: raise ApplicationError("application does not exist") if not _allowed(app.get("compatibleDatabases"), allowed_databases): raise ApplicationError("application does not exist") + if app.get("templateKind") in APPLICATION_TEMPLATE_KINDS and "compatibleDatabases" in patch: + raise ApplicationError("a template Application's sourceDatabase is immutable") star_only = set(patch) <= {"starred"} if star_only: if not may_read(app,user,is_admin): raise PermissionError("application is not available") @@ -230,15 +417,19 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records=None, - allowed_databases=None): + field_contracts=None, allowed_databases=None): app=load(runtime)["applications"].get(app_id) if not may_read(app,user,is_admin): raise ApplicationError("application does not exist") if not _allowed(app.get("compatibleDatabases"), allowed_databases): raise ApplicationError("application does not exist") - return create(runtime,tenant,user,{"type":app["type"],"name":f"{app['name']} copy", + payload={"type":app["type"],"name":f"{app['name']} copy", "compatibleDatabases":app.get("compatibleDatabases"), - "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}, - semantic_records=semantic_records, allowed_databases=allowed_databases) + "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")} + for key in ("templateKind", "sourceDatabase", "fieldMap", "navPlacement"): + if key in app: payload[key]=copy.deepcopy(app[key]) + return create(runtime,tenant,user,payload, + semantic_records=semantic_records, field_contracts=field_contracts, + allowed_databases=allowed_databases) def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None): diff --git a/api/automation_engine.py b/api/automation_engine.py index 3afe591486c012aaf645401f2b0e7dc77f2aa752..12e563752eefb872c876b18ee09e70ef3ab7040d 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -4097,7 +4097,7 @@ POST_SNAPSHOT_FIELDS = [ # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, # and `ut_ig_posts` already performs the first of them. See the verdict in - # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for + # `AutoWiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for # moving the window here. field_def("posted_at", "Posted at", "date"), field_def("type", "Type", "select", options=["image", "video", "carousel"]), @@ -13770,7 +13770,7 @@ def _rollup_fold(fn, values, ftype="text"): LINKABLE_MODULES = ("customer_data", "product_data") -def _module_limit(subject, cause, recommendation, effect="unresolvable"): +def _module_limit(subject, cause, recommendation, effect="unresolvable", limit=None): """A refusal in `core.user_tables::limit_report`'s vocabulary — `{subject, effect, cause, recommendation}`, the same four keys `perm_scope.Unresolvable.as_limit()` puts on the wire. @@ -13778,9 +13778,20 @@ def _module_limit(subject, cause, recommendation, effect="unresolvable"): this engine cannot reach used to resolve to `{}` and blank the column, which reads exactly like "there are no linked records". A reason a caller can render is the difference between a measurement and a lie. + + ⭐ W53-T40 — `limit` IS OPTIONAL AND IS OMITTED UNLESS A NUMBER IS THE POINT, which keeps + every refusal already shipped byte-identical. `core.user_tables.limit_report` carries the key + on exactly the reports where a CEILING is the subject (`{'subject': 'rows', 'limit': 0, + 'effect': 'read_through', ...}`), and the two W53-T40 reports below are that shape: a + read-through grain whose ceiling here is 0, and a second-hop fan-out whose ceiling is + `LINK_THROUGH_MAX_IDS`. Naming the number in prose alone would leave a caller parsing English + to find out how much it did not get. """ - return {"subject": str(subject), "effect": effect, - "cause": str(cause), "recommendation": str(recommendation)} + out = {"subject": str(subject), "effect": effect, + "cause": str(cause), "recommendation": str(recommendation)} + if limit is not None: + out["limit"] = int(limit) + return out def _module_link_fields(key): @@ -13901,6 +13912,112 @@ def _module_table(rt, key, memo, want_rows=False): return table, None +def _as_field_limit(lk_key, base, context): + """⭐ W53-T40 — ONE REFUSAL, RE-SUBJECTED FROM THE DATABASE IT IS ABOUT TO THE COLUMN IT BLANKS. + + `_resolve_table` refuses in terms of the DATABASE it could not read, which is the right + subject for its own caller and the WRONG one for a link: what a client renders is a COLUMN, + and "which field is blank, and why" is the entire question `relation_cells`' second return + value exists to answer. Every other refusal that function raises is already keyed on the field + key; this keeps a refusal that arrives from one layer down in the same vocabulary. + + ⛔ THE CAUSE AND THE RECOMMENDATION ARE CARRIED VERBATIM, never restated. The layer that + refused knows why it refused; paraphrasing it here would be a second account of one fact, and + the two would drift the first time the lower one is corrected. `context` is PREPENDED, so the + route is stated and then the reason follows in the words of whoever had it. + """ + base = base or {} + return _module_limit( + lk_key, (str(context or "").strip() + " " + str(base.get("cause") or "")).strip(), + str(base.get("recommendation") or ""), + effect=str(base.get("effect") or "unresolvable"), limit=base.get("limit")) + + +def _mirror_table(rt, key, defn): + """⭐⭐ W53-T40 (d)+(b) — A READ-THROUGH DATABASE'S ROWS, FROM THE CONNECTOR MIRROR, or a + REFUSAL that names the field, the cause and the fix. `(table_or_None, limit_or_None)`. + + ⛔ THIS IS TWO ARMS AND THEY ARE THE TICKET'S (d) AND (b), IN THAT ORDER. + + (d) **NO READER REGISTERED ⇒ REFUSE, LOUDLY.** A read-through grid's stored document is a + real definition with `rows: {}` — never `None` — so the old `if linked is None` refusal + in `relation_cells` could not see it: the link fell through to the ordinary join arm, + `_linked_rows_by_join` indexed an empty dict, every cell resolved to `[]`, and the + column was permanently blank with NO `limit_report`. That is the one case + `relation_cells`' own docstring says cannot happen, and it happened because the guard + was keyed on the wrong sentinel. Re-keying it means making THIS function answer the + real question — "are this database's rows resolvable here?" — so the existing `is None` + test becomes true for the structurally-rows-absent case rather than being duplicated. + + (b) **A READER IS REGISTERED ⇒ WALK IT.** The rows already exist and are already served: + `routes_odoo_tables.GRID_SOURCES` renders them through `routes_tables._read_through_rows`, + published as `core.perm_scope._MIRROR_READER` by `routes_tables._register_mirror`. What + was missing was a CALLER. This is the caller. + + ⭐⭐ THE SENTINEL IS `core.user_tables.materialises`, NOT A HAND-READ `readThrough` KEY, and + the difference is a cold process. `materialises` is three-tier — the app-layer + `register_read_through` set, then the definition's own `readThrough` STAMP written by + `odoo_relational._ensure_table_inplace`, then the default — so a worker that never served a + read-through grid still gets the right answer. A hand-read of the stamp alone would be one + of those tiers pretending to be all three ([[one-evaluator-per-question]]). + + ⛔ THE SHAPE IS COPIED FROM `core.perm_scope._read_user_table`, NOT INVENTED HERE. That + function has carried exactly this branch since W38 — refuse when `_MIRROR_READER is None`, + otherwise project the declared field keys and hand back the reader's rows — and a second + spelling of it would be a second opinion about what a read-through read is. + + ⚠ THE DOCUMENT IS COPIED, NEVER MUTATED. `store` is the live `user_tables` bucket; writing + mirror rows into it would materialise the very grain R6 forbids materialising, in memory, + silently, and then hand it to whatever reads the store next. + + ⚠ `rt` RIDES INTO THE READER AS ITS THIRD ARGUMENT. `routes_tables._c1_mirror_rows` names + that parameter `st` and passes it straight through to `_read_through_rows(..., rt=st)`, whose + docstring says why it must be the SESSION'S runtime: one Space process serves every tenant + and `datastore.ro_con()` reads a process-global path, so the tenant handle is the only thing + standing between a relation pass and another tenant's mirror. + """ + from core import perm_scope # lazy: the cycle rule `_ut()` documents + key = str(key or "") + reader = getattr(perm_scope, "_MIRROR_READER", None) + if reader is None: + # (d). ⛔ ASCII ONLY: this rides out in `limits` and can reach a client (standing rule 2). + return None, _module_limit( + key, + f"'{key}' is served READ-THROUGH from the connector mirror, so it stores no rows in " + f"this workspace document (row_limit 0), and no mirror reader is registered in this " + f"process. A link resolved against its stored rows would find none and render a " + f"permanently blank column that looks configured", + "call `perm_scope.register_mirror(...)` from the app layer before resolving a " + "relation at this grain (importing `routes_tables` does it), or read the grain " + "directly through the API, where every total is a SQL count over the whole table", + effect="read_through", limit=0) + fields = list((defn or {}).get("fields") or []) + wanted = {str(f.get("key")) for f in fields if f.get("key")} + try: + raw = list(reader(key, wanted, rt) or ()) + except Exception as e: # noqa: BLE001 + # ⛔ A MIRROR THAT CANNOT ANSWER IS A REASON, NOT AN OUTAGE, and `perm_scope.Unresolvable` + # already carries the four keys this vocabulary wants: reuse them rather than flattening + # a stated cause into `str(e)`. + cause = str(getattr(e, "cause", "") or "") or f"{type(e).__name__}: {e}" + return None, _module_limit( + key, + f"'{key}' is served READ-THROUGH from the connector mirror and the mirror could not " + f"answer for it: {cause}", + str(getattr(e, "recommendation", "") or "") + or ("retry once the connector mirror has finished opening on this process, or read " + "the grain a window at a time through the API"), + effect=str(getattr(e, "effect", "") or "read_through"), limit=0) + # `pid` IS THE ROW ID on every read-through grid — `routes_tables` stamps it onto each row it + # serves and `_read_through_rows` keeps it through the projection. A row without one cannot be + # addressed by a link cell at all, so it is dropped rather than given an invented id. + table = dict(defn or {}) + table["key"] = key + table["rows"] = {str(r.get("pid")): dict(r) for r in raw if (r or {}).get("pid") is not None} + table["_rows_built"] = True + return table, None + + def _resolve_table(rt, store, key, memo, want_rows=False): """A database by key, from EITHER namespace — `(table_or_None, limit_or_None)`. @@ -13909,9 +14026,31 @@ def _resolve_table(rt, store, key, memo, want_rows=False): the TARGET table were two separate `store.get(...)` calls, C5 named only the target, and a target-only fix leaves a link DECLARED ON `customer_data` returning 0 rows touched — which reads exactly like "nothing needed doing". + + ⭐⭐ W53-T40 — AND THAT IS ALSO WHY THE READ-THROUGH BRANCH LANDS HERE RATHER THAN IN EITHER + CALLER. `_PRODUCT_LINK_BLOCKER` named this symbol as "the door BOTH of them go through", so a + third namespace — the connector MIRROR — is one branch here instead of two branches that must + be kept in step. A stored document is never `None`, so before this branch existed a + read-through target came back as a real definition with `rows: {}` and every caller read that + as "no linked records". + + ⚠ ONLY WHEN `want_rows`. The field contract of a read-through grid is in the stored document + like everybody else's, and `relation_cells` resolves the source table fields-only precisely so + a table with no links pays nothing. Reaching the mirror for a fields-only resolve would put a + 256,593-row read behind a dict scan. + + ⚠ MEMOISED ON A TUPLE KEY. `_module_table` memoises on the bare key; a mirror table for the + same key is a different object with a different cost, and both can be wanted in one pass (the + bridge of a two-hop is resolved once per link field). """ - hit = (store or {}).get(str(key or "")) + key = str(key or "") + hit = (store or {}).get(key) if hit is not None: + if want_rows and not _ut().materialises(key, defn=hit): + cached = memo.get(("mirror", key)) + if cached is None: + cached = memo[("mirror", key)] = _mirror_table(rt, key, hit) + return cached return hit, None return _module_table(rt, key, memo, want_rows=want_rows) @@ -14083,9 +14222,18 @@ def relation_cells(rt, table_key, tables=None): if not links and not rollups: return {}, limits rows = (t or {}).get("rows") or {} - if not rows and t is not None and not t.get("_rows_built", True): + if not rows and t is not None and (not t.get("_rows_built", True) + or not _ut().materialises(table_key, defn=t)): # A registry source: NOW the pool is worth its cost, because this table really does # declare relations. One fetch, memoised, shared with every link and rollup below. + # ⭐⭐ W53-T40 (d) — AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason + # that is not symmetry. `_rows_built` is the REGISTRY sentinel; a read-through `ut_*` + # document carries no such key, so `.get("_rows_built", True)` answered True, `rows` stayed + # `{}`, and a link DECLARED ON a read-through grain returned `{}` cells with no limit — + # the same silent blank on the source side that (d) ends on the target side. The predicate + # is `materialises`, the one evaluator of "are this database's rows stored here", and the + # cheap-by-construction guarantee is untouched because this sits BELOW the declaration + # check: a table with no links and no rollups has already returned. t, limit = _resolve_table(rt, store, table_key, memo, want_rows=True) if limit is not None: limits.append(limit) @@ -14108,12 +14256,22 @@ def relation_cells(rt, table_key, tables=None): want_rows=True) if linked is None: refused.add(lk_key) - limits.append(limit if limit is not None else _module_limit( - lk_key, - f"this link points at '{bag.get('table')}', which is not a database this " - f"workspace holds", - "re-point the link at an existing database, or delete the column; a link whose " - "target is gone is not a relation")) + if str((limit or {}).get("effect")) == "read_through": + # ⭐ W53-T40 (d) — THE NEW REFUSAL IS RE-SUBJECTED ONTO THE FIELD, and ONLY the new + # one. `_module_table`'s pre-existing refusals are passed through in their own + # words exactly as they always were; re-keying those would churn a report other + # callers already read. This arm did not exist before this ticket, so nothing + # downstream has an older reading of it to protect. + limits.append(_as_field_limit( + lk_key, limit, + f"this link points at '{bag.get('table')}' and its rows could not be read.")) + else: + limits.append(limit if limit is not None else _module_limit( + lk_key, + f"this link points at '{bag.get('table')}', which is not a database this " + f"workspace holds", + "re-point the link at an existing database, or delete the column; a link " + "whose target is gone is not a relation")) continue linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") for lf in (linked.get("fields") or [])} @@ -14130,6 +14288,108 @@ def relation_cells(rt, table_key, tables=None): inverse_index.setdefault(target_id, []).append( (str(source_id), source_row or {})) resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} + elif bag.get("through"): + # ⭐⭐ W53-T40 (b)+(c) — THE TWO-HOP TRAVERSAL, AND IT IS TRAVERSAL, NOT AGGREGATION. + # R12 keeps aggregation at depth 1; this walks `source -> bridge -> target` and hands + # the TARGET ROWS back in the same `resolved[link][row_id] = [(id, row), ...]` shape + # every one-hop arm above produces, so every rollup, cap and cell below is unchanged. + # + # ⛔ THE INDEX IS `_linked_rows_by_join` ON BOTH HOPS AND THE NORMALISER IS + # `_join_norm` ON BOTH SIDES OF BOTH. `_join_norm`'s own docstring says why: a join is + # a comparison, and a comparison normalised in two places is one edit from silently + # reporting zero. A second spelling here would be four half-comparisons instead of two. + # + # ⛔ THE ROUTE THIS SHIPS FOR IS `customer_data -> ut_odoo_order_lines -> + # product_data`, and it is the ONLY route Odoo declares between a partner and a + # product: `sale.order.line` carries `order_partner_id` and `product_id`, both stored. + # ⛔ THERE IS NO INVOICE ROUTE AND THERE MUST NOT BE ONE THROUGH THIS GRAIN. Invoice to + # line to product is THREE hops and the first of them, `invoice_origin`, is FREE TEXT + # filled on 34.1% of posted invoices — a depth-3 path over a weak hop returns a + # confidently wrong number rather than an error. + through_key = str(bag.get("through") or "") + on_key = str(bag.get("on") or "") + hop_in = str(bag.get("throughOn") or "") + hop_out = str(bag.get("throughTo") or "") + from_key = _link_from_key(fields, bag) + if not (on_key and hop_in and hop_out and from_key): + # ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with + # a missing leg has no second hop to make; resolving it to `[]` would render a + # permanently blank column that looks configured. `_clean_link` refuses this shape + # at the definition door, so reaching here means a bag that never went through it. + refused.add(lk_key) + limits.append(_module_limit( + lk_key, + f"this link travels through '{through_key}' but the route is incomplete. A " + f"two-hop join names four columns: `from` on '{table_key}', `throughOn` and " + f"`throughTo` on the bridge, and `on` on '{bag.get('table')}'. Present here: " + f"from={from_key or '(none)'}, throughOn={hop_in or '(none)'}, " + f"throughTo={hop_out or '(none)'}, on={on_key or '(none)'}", + "declare every leg of the route on the link, or remove `through` and let it " + "be the one-hop join it is actually configured as")) + continue + # THE BRIDGE, THROUGH THE SAME DOOR AS THE SOURCE AND THE TARGET. This is where the + # read-through refusal lands for the shipped route: `ut_odoo_order_lines` stores no + # rows in this document on purpose, so `_resolve_table` reaches the mirror for it, or + # refuses with a cause and a fix when no reader is registered in this process. + bridge, limit = _resolve_table(rt, store, through_key, memo, want_rows=True) + if bridge is None: + refused.add(lk_key) + limits.append(_as_field_limit( + lk_key, limit, + f"this link travels {table_key} -> {through_key} -> {bag.get('table')} and " + f"the bridge could not be read.") if limit is not None else _module_limit( + lk_key, + f"this link travels through '{through_key}', which is not a database this " + f"workspace holds", + "re-point the link's `through` at an existing database, or delete the " + "column; a route across a database that is gone is not a relation")) + continue + bridge_idx = _linked_rows_by_join(bridge, hop_in) + target_idx = _linked_rows_by_join(linked, on_key) + cap = int(_ut().LINK_THROUGH_MAX_IDS) + + def _walk(row, _bidx=bridge_idx, _tidx=target_idx, _cap=cap): + """One source row's DISTINCT target records, and whether the ceiling stopped it. + + ⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines; + twenty copies of one product in the cell would be a count of LINES wearing the + label of a set of PRODUCTS. + """ + seen, out = set(), [] + for _bid, brow in _bidx.get(_join_norm((row or {}).get(from_key)), ()): + for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()): + if tid in seen: + continue + if len(out) >= _cap: + return out, True + seen.add(tid) + out.append((tid, trow)) + return out, False + + two_hop, capped = {}, 0 + for rid, row in rows.items(): + hits_2, hit_cap = _walk(row) + two_hop[str(rid)] = hits_2 + capped += 1 if hit_cap else 0 + resolved[lk_key] = two_hop + if capped: + # ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND IT IS THE HALF THAT GETS DROPPED: a + # limit that genuinely cannot be removed is REPORTED, with cause and a recommended + # fix, never silently enforced. ⛔ AND THIS ONE IS LOUDER THAN THE `LINK_MAX_IDS` + # display cap, because it is a different quantity: the display cap slices the CELL + # and leaves `resolved` whole for the rollups, while this bounds `resolved` + # ITSELF — so a rollup folded over a capped row describes a subset. + # ⛔ ASCII ONLY: this rides out in `limits` and can reach a client. + limits.append(_module_limit( + lk_key, + f"this link travels {table_key} -> {through_key} -> {bag.get('table')}, and " + f"{capped} of {len(rows)} rows reached the {cap}-record fan-out ceiling on " + f"the second hop. Those cells, and any rollup folded over them, describe the " + f"first {cap} distinct records rather than the whole set", + "narrow the route with a `filter` on the link, or read the bridge grain " + "directly through the API, where every total is a SQL count over the whole " + "table rather than over a resolved window", + effect="capped", limit=cap)) elif bag.get("on"): idx = _linked_rows_by_join(linked, str(bag["on"])) from_key = _link_from_key(fields, bag) @@ -14293,6 +14553,80 @@ def relation_cells(rt, table_key, tables=None): return changes, limits +def related_product_metric_values(rt, customer_id, products, fields, offer, *, + team_id=None, agent=None, today=None): + """Read-only customer × real product metrics; the caller supplies walled server rows. + + Product grid pids are CRCs and never enter the semantic product dimension. Keep the + BU in flat filters too: an entity binding's scope_variant suppresses team_id upstream. + No memo survives this request, and an unsupported metric/window stays blank. + """ + from harness import semantic as sem, windows + import time + + rt.assert_datastore_matches() + today = today or time.strftime("%Y-%m-%d") + by_key = {m["key"]: m for m in offer} + ids = {int(row["pid"]): int(row["product_id"]) for row in products} + filters = {"order_partner": [customer_id], "product": sorted(set(ids.values()))} + if team_id is not None: + filters["team"] = [team_id] + if agent is not None: + filters["agent"] = [agent] + out = {pid: {f["key"]: None for f in fields} for pid in ids} + if not ids: + return out, [] + + # Membership comes from confirmed order lines, never the client link cell or SKU code. + membership = sem.store_query("sales_lines", ["revenue"], group_by=["product"], + filters=filters, team_id=team_id, limit=sem.MAX_GROUPS, + scope_variant="all_channels") + if membership.get("truncated"): + raise ValueError("The related product set is incomplete.") + linked = {int(r["product_id"]) for r in membership["rows"]} + if set(ids.values()) - linked: + raise ValueError("A product is not within this customer's permitted ordered products.") + + groups, reasons = {}, [] + for field in fields: + spec = field.get("measure") or {} + measure = by_key.get(spec.get("key")) + if not measure or measure.get("topic") != "sales_lines": + reasons.append("Some Metrics cannot be evaluated for a customer and product together.") + continue + window = windows.normalize(spec.get("window")) + bounds = windows.resolve(spec.get("window"), today) if window is not None else None + if bounds is None: + reasons.append("Some Metric date windows could not be resolved.") + continue + groups.setdefault((*bounds, measure.get("scope_variant")), []).append((field, measure)) + + for (start, end, variant), group in groups.items(): + keys = sorted({m.get("base_key") or m["key"] for _, m in group}) + result = sem.store_query("sales_lines", keys, group_by=["product"], + date_from=start, date_to=end, filters=filters, + team_id=team_id, exclude_services=False, + limit=sem.MAX_GROUPS, scope_variant=variant) + if result.get("truncated"): + raise ValueError("The scoped Metric result is incomplete.") + values = {int(r["product_id"]): r for r in result["rows"]} + for pid, real_id in ids.items(): + row = values.get(real_id, {}) + for field, measure in group: + guard = measure.get("guard") + if guard is not None and not row.get(guard): + continue + value = row.get(measure.get("base_key") or measure["key"]) + if value is None: + value = 0 if measure.get("empty") == "zero" else None + if value is not None: + value *= measure.get("scale", 1) + if measure.get("round") is not None: + value = round(value, measure["round"]) + out[pid][field["key"]] = value + return out, list(dict.fromkeys(reasons)) + + def compute_relation_cells(rt, table_key, tables=None): """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py new file mode 100644 index 0000000000000000000000000000000000000000..9dfa5d00a5b354f31aba524728bf4c4a119e393c --- /dev/null +++ b/api/connectors_amazon.py @@ -0,0 +1,307 @@ +"""connectors_amazon.py — the API-layer FACE of the Amazon connector: the liveness probe. + +⛔⛔ **THE TRANSPORT IS NOT HERE. IT IS `platform/harness/amazon_client.py`, AND THAT IS A LAYER +RULE, NOT A PREFERENCE.** `harness/amazon_store.py` needs the same client this probe needs, and +nothing under `platform/` may import an API-layer module -- layers point one way. The first version +of this connector put the transport here and had the store reach up for it; `verify_meta`'s +architecture leg caught it in one line. Everything the client exports is re-exported below, so a +caller that imports it from here still works. + +⛔ **THE PROBER IS REGISTERED, NEVER IMPORTED BY `core`.** `core` never imports up +(ARCHITECTURE.md), and the tempting shortcut -- a `sys.path.insert` plus an `import` from inside +`core/keychain.py` -- is an architecture inversion no gate was watching for until wave 31. +`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer +down. + +⛔ **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers. +""" +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + +_PLATFORM = Path(__file__).resolve().parents[2] / "platform" +if str(_PLATFORM) not in sys.path: + sys.path.insert(0, str(_PLATFORM)) + +# ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report +# map and the transport; this module adds the probe and the registration and nothing else. +from harness.amazon_client import ( # noqa: F401,E402 + AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE, + CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS, + REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox, + ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report, + _competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH, + _COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH, + _request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory, + competitive_summaries, marketplace_participations, report_document, +) + +#: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data +#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the +#: comments in `keychain.py` warn about it in as many words. +ENTRY_TYPE = "amazon_sp" + +#: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's +#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and +#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo +#: models. The document lives on the request's tenant runtime, so two tenants can make opposite +#: choices without a process-global map ever seeing either answer. +ENABLEMENT_KEY = "amazon_connector_enablement" +ENABLEMENT_ROUTE = "/connectors/amazon_sp/databases/{databaseKey}/enablement" + + +class EnablementConfigError(ValueError): + """The tenant's stored Amazon enablement document is malformed.""" + + +def database_keys(): + """The authored Amazon database keys, derived from the connector's one report roster.""" + return tuple("ut_amzn_" + str(name) for name in REPORT_TYPES) + + +def read_database_enablement(rt): + """Return this tenant's fail-closed Amazon database enablement state. + + An absent document preserves the connector's shipped behaviour: all seven databases are on. + Once a document exists, every value must be a real boolean and every key must belong to the + authored Amazon roster. A malformed or unreadable document disables every row in the detail + response and marks the capability unavailable; it never guesses through corrupt tenant state. + """ + keys = database_keys() + try: + current = rt.get(ENABLEMENT_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + return {"available": False, "databases": {key: False for key in keys}} + # Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are + # the migration default rather than malformed state. + if current is None or current == {}: + return {"available": True, "databases": {key: True for key in keys}} + if not isinstance(current, dict): + return {"available": False, "databases": {key: False for key in keys}} + stored = current.get("databases") + if not isinstance(stored, dict): + return {"available": False, "databases": {key: False for key in keys}} + if (set(stored) - set(keys) + or any(type(value) is not bool for value in stored.values())): + return {"available": False, "databases": {key: False for key in keys}} + return {"available": True, + # Absent keys stay on so a later report addition does not silently disappear for + # every tenant that saved an older version of this document. + "databases": {key: stored.get(key, True) for key in keys}} + + +def set_database_enabled(rt, database_key, enabled): + """Persist one tenant's Amazon database choice without touching its materialised rows.""" + key = str(database_key or "").strip() + known = set(database_keys()) + if key not in known: + raise KeyError(key) + if type(enabled) is not bool: + raise ValueError("enabled must be true or false") + + def _set(current): + if current is None: + current = {} + if not isinstance(current, dict): + raise EnablementConfigError("the stored Amazon enablement document is malformed") + stored = current.get("databases", {}) + if not isinstance(stored, dict): + raise EnablementConfigError("the stored Amazon database choices are malformed") + if (set(stored) - known + or any(type(value) is not bool for value in stored.values())): + raise EnablementConfigError("the stored Amazon database choices are malformed") + # A fresh document contains the whole current roster. That makes the stored answer + # inspectable while retaining the migration rule above for future database additions. + choices = {name: stored.get(name, True) for name in database_keys()} + choices[key] = enabled + return {**current, "databases": choices} + + rt.update(ENABLEMENT_KEY, _set, flush="sync") + return read_database_enablement(rt) + + +def enablement_capability(rt, is_admin): + """The provider-aware capability served by Amazon's connector detail response.""" + state = read_database_enablement(rt) + can_manage = bool(is_admin) and state["available"] + return {"provider": ENTRY_TYPE, + "mode": "database", + "canManage": can_manage, + # A member receives no write URL. The route still enforces administrator status; this + # is the matching affordance wall, not the permission boundary. + "updatePath": ENABLEMENT_ROUTE if can_manage else "", + # Disabling changes only this config document. No mirror or ``ut_amzn_*`` row is + # removed, so re-enabling resumes against the same materialised database. + "preservesData": True, + "available": state["available"]} + + +def probe_credential(fields, timeout=8): + """`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe. + + ⛔ IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED + BY IT. `core` never imports up (ARCHITECTURE.md); `keychain.register_prober` is the slot, and + `connectors_meta` established the idiom after a first draft did `sys.path.insert` plus an + `import` from inside `core`. + + ⚠ ZERO marketplace participations is reported as NOT ok, deliberately, for the same reason Meta + reports zero ad accounts that way: a credential that authenticates but reaches nothing is an + authorization answer, and `ok: true` would hide the one thing the admin has to fix. + """ + f = fields or {} + missing = [k for k in ("client_id", "client_secret", "refresh_token") if not f.get(k)] + if missing: + return {"ok": False, + "message": "store " + ", ".join(f"`{m}`" for m in missing) + + " alongside `seller_id` and `marketplace_id`"} + try: + parts = marketplace_participations(f, timeout=timeout) + except AmazonError as e: + return {"ok": False, "message": str(e)[:200]} + live = [p for p in parts + if ((p or {}).get("participation") or {}).get("isParticipating") is not False] + n = len(live) + if not n: + return {"ok": False, + "message": "the credential authenticates but participates in NO marketplace: " + "that is an authorization answer, not an empty account"} + names = ", ".join(sorted({str(((p or {}).get("marketplace") or {}).get("countryCode") or "?") + for p in live})) + return {"ok": True, "message": f"Amazon answered. {n} marketplace(s) reachable: {names}"} + + +def register(): + """Fill the keychain's probe slot for this connector. + + Called at import by the API layer's connector registry (`routes_connectors._register_probers`), + so any process that can serve the connector directory has also registered the prober. Never + raises: a probe slot that could not be filled costs a liveness CHECK, never the ability to + store a credential. + """ + try: + import core.keychain as kc + return kc.register_prober(ENTRY_TYPE, probe_credential) + except Exception: # noqa: BLE001 + return False + + +def _env_credential(): + """A credential assembled from the environment, for `--check` only. + + ⛔ NOT a runtime path. The product reads Amazon credentials from the per-tenant keychain and + from nowhere else; this exists so an operator can prove a freshly minted refresh token works + before pasting it into the app. + """ + # ⚠ `OPTIONAL_FIELDS` joins the read so `AMAZON_SP_SANDBOX=true` reaches `is_sandbox` and the + # probe talks to the sandbox host. Absent still means PRODUCTION, which is the safe default. + f = {k: (os.environ.get("AMAZON_SP_" + k.upper()) or "").strip() + for k in tuple(CREDENTIAL_FIELDS) + OPTIONAL_FIELDS} + return {k: v for k, v in f.items() if v} + + +def probe_report(fields, report_type=None, days=30, log=print): + """Drive the WHOLE report chain once and print the MEASURED header. Writes to no store. + + `createReport -> poll -> getReportDocument -> download -> decompress -> parse`, then print the + columns the document actually carried. **This is the only way to learn a flat-file report's + schema short of a production sync**, and it is exactly what the connector cannot know from + Amazon's published artifacts: five of our seven tables have none. + + ⛔ NOTHING IS PERSISTED — no mirror table, no `ut_amzn_*`, no `_sync_state` row. That is what + makes it safe to point at the SANDBOX. `amazon_store.sync` refuses a sandbox credential so mock + rows can never reach a grid; this path sidesteps that question entirely by writing nothing. + + ⚠⚠ SANDBOX REALITY, MEASURED 2026-09-04, AND IT IS NOT WHAT "the sandbox mirrors production" + SUGGESTS: the static sandbox's `getReportDocument` **always returns + `GET_MERCHANT_LISTINGS_ALL_DATA`, whatever report type you asked for** (amzn/selling-partner- + api-models#231). So this probe proves the CHAIN for any report type, and yields a genuine header + for exactly ONE of our seven tables — `listings`. The other six still need production. + + ⚠ And the parse goes through `amazon_store.parse_flat_file`, the SAME function `sync` uses. + A second reader here would prove something production does not do. + """ + from datetime import datetime, timedelta, timezone + from harness.amazon_store import parse_flat_file + + rt = report_type or REPORT_TYPES["listings"]["report"] + sbx = is_sandbox(fields) + end = datetime.now(timezone.utc) + start = end - timedelta(days=days) + log(f"host : {'SANDBOX' if sbx else 'PRODUCTION'} {base_for(fields)}") + log(f"report : {rt}") + if sbx and rt != "GET_MERCHANT_LISTINGS_ALL_DATA": + log("WARNING: the static sandbox returns GET_MERCHANT_LISTINGS_ALL_DATA whatever you ask " + "for, so the document below is NOT this report type. The chain is still proven.") + t0 = time.time() + text, meta = report_document( + fields, rt, + start=start.strftime("%Y-%m-%dT%H:%M:%SZ"), + end=end.strftime("%Y-%m-%dT%H:%M:%SZ"), log=log) + log(f"document : {len(text):,} chars, compression={meta.get('compression')} " + f"({time.time() - t0:.1f}s)") + + cols, rows = parse_flat_file(text) + log("") + log(f"MEASURED HEADER: {len(cols)} columns, {len(rows)} data rows") + for i, c in enumerate(cols): + log(f" {i + 1:>3}. {c}") + if rows: + # ⚠ TRUNCATED, and only the first row. A report document is the SELLER'S data; a probe + # prints enough to prove the shape and no more. + log("") + log("first row, values truncated to 40 chars:") + for c in cols[:12]: + log(f" {c:<34} {str(rows[0].get(c))[:40]}") + return cols, rows + + +def main(argv): + """`--check` proves a credential; `--probe-report` proves the whole report chain.""" + if "--check" not in argv and "--probe-report" not in argv: + print("usage: connectors_amazon.py --check prove the credential") + print(" connectors_amazon.py --probe-report [--report TYPE]") + print(" drive the full report chain and print") + print(" the MEASURED header. Writes nothing.") + print(" reads AMAZON_SP_CLIENT_ID / _CLIENT_SECRET / _REFRESH_TOKEN / _SELLER_ID /") + print(" _MARKETPLACE_ID, plus AMAZON_SP_SANDBOX=true for the sandbox host") + return 2 + f = _env_credential() + have = [k for k in CREDENTIAL_FIELDS if f.get(k)] + print("credential fields present: " + (", ".join(have) if have else "NONE") + + " (values are never printed)") + if not {"client_id", "client_secret", "refresh_token"} <= set(have): + print("REFUSED: client_id, client_secret and refresh_token are all required") + return 2 + t0 = time.time() + res = probe_credential(f, timeout=_TIMEOUT) + print(f"{'OK' if res.get('ok') else 'REFUSED'}: {res.get('message')} " + f"({time.time() - t0:.1f}s)") + if not res.get("ok"): + return 1 + print("\nreport types this connector will pull (owner ruling R4):") + for table, spec in REPORT_TYPES.items(): + rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)" + print(f" {table:<16} {rep}") + + if "--probe-report" in argv: + which = None + for i, a in enumerate(argv): + if a == "--report" and i + 1 < len(argv): + which = argv[i + 1] + print("") + try: + probe_report(f, report_type=which) + except Exception as e: # noqa: BLE001 + # ⚠ REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator + # needs to read (a role they lack, a throttle, an unsupported sandbox report type), not + # a traceback that buries it. + print(f"REPORT PROBE REFUSED: {type(e).__name__}: {e}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/api/connectors_bd.py b/api/connectors_bd.py index 82297bf993d6f1c589ad53d62f7529ab534fa9c4..b0b67c66eaf1cc55d88bc7b8ad9501c68f119187 100644 --- a/api/connectors_bd.py +++ b/api/connectors_bd.py @@ -182,7 +182,7 @@ def bd_call(path, params=None, body=None): if not key: # ⛔ THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends # somebody to read Instagram's status page instead of setting an env var. - return None, "AIOS_BRIGHTDATA_KEY is not configured — the paid rung is closed" + return None, "AIOS_BRIGHTDATA_KEY is not configured; the paid rung is closed" qs = "&".join(f"{k}={requests.utils.quote(str(v))}" for k, v in (params or {}).items() if v not in (None, "")) url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "") @@ -220,7 +220,7 @@ def bd_call(path, params=None, body=None): # paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow. if len(raw) >= BD_MAX_KB * 1024: return None, (f"the profile source's answer exceeded this client's {BD_MAX_KB // 1024} MB read " - f"ceiling and was TRUNCATED — ask for fewer records; nothing was parsed " + f"ceiling and was TRUNCATED; ask for fewer records; nothing was parsed " f"from a partial document") text = raw.decode("utf-8", "replace").strip() if not text: @@ -336,7 +336,7 @@ def bd_snapshot_progress(sid): if isinstance(codes, dict) and codes else "") if errors or named or status in ("failed", "error"): return "failed", 0, ("the source finished this request and collected nothing" - + (f" — it reported {named}" if named else "") + + (f"; it reported {named}" if named else "") + "; the target is unreachable, private, or no longer exists") return "done", 0, "the source finished this request and found no records for it" @@ -388,7 +388,7 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None): deferred.append({"snapshotId": sid, "datasetId": str(dataset_id), "urls": requested_urls}) return [], (f"the profile source deferred this batch to {sid} and it was not ready within " - f"{int(budget)}s — the records are collected, not lost; ask for a smaller batch " + f"{int(budget)}s; the records are collected, not lost; ask for a smaller batch " f"or collect {sid} from the control panel") @@ -575,7 +575,7 @@ def bd_filter_start(predicates, operator="and", records_limit=5, dataset_id="", if not str(dataset_id or "").strip(): # Fail-closed, and it names what is missing. A transport that guesses a corpus is a # transport that bills the wrong one. - return "", "the search was not started — no corpus was named for this platform" + return "", "the search was not started; no corpus was named for this platform" nodes = bd_group(expand_predicates(predicates), operator) filt = {"operator": operator, "filters": nodes} handles = [str(h).strip().lstrip("@").lower() for h in (exclude_handles or [])] @@ -627,7 +627,7 @@ def bd_filter_status(snapshot_id): return "empty", 0, "" if status == "failed" and err: # NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name. - extra = (" — the matched set is too large for this account's balance; narrow the " + extra = ("; the matched set is too large for this account's balance; narrow the " "conditions or lower the limit" if "FUNDS" in err.upper() else "") return status, 0, f"the search failed: {err}{extra}" return status, int(d.get("dataset_size") or 0), "" @@ -646,7 +646,7 @@ def bd_filter_rows(snapshot_id): return [], note rows = _bd_rows(payload) if not rows: - return [], "the search is ready but its rows are not downloadable yet — the vendor's " \ + return [], "the search is ready but its rows are not downloadable yet; the vendor's " \ "delivery lags its own `ready` status by minutes" return rows, "" diff --git a/api/connectors_ig.py b/api/connectors_ig.py index 19887284f1f1ee9ad1ca9419959aa9fab8f287fc..adf95fd225694aa975c32b745a8e91995d818519 100644 --- a/api/connectors_ig.py +++ b/api/connectors_ig.py @@ -658,7 +658,7 @@ def bd_profile_posts(handle, count, deferred=None, wait=None, post_type="", raw_ deferred.append({"snapshotId": sid, "datasetId": str(BD_DS_POSTS), "urls": [f"https://www.instagram.com/{handle}/"], "kind": "posts"}) return [], (f"the post source deferred this request to {sid} and it was not ready within " - f"{int(budget)}s — the records are collected, not lost") + f"{int(budget)}s; the records are collected, not lost") def group_types(groups): @@ -815,7 +815,7 @@ APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240) #: a rung that returns this into `res["gone"]`, which is what lets a run stop re-buying a dead #: handle every morning ([[flag-shipped-without-its-writer]] — the flag needs a writer AND a #: reader, so both are in this file's call graph). -ACCOUNT_GONE_NOTE = ("Instagram has no account with that handle — the source reports it does " +ACCOUNT_GONE_NOTE = ("Instagram has no account with that handle; the source reports it does " "not exist (it may have been deleted, renamed, or suspended)") @@ -841,7 +841,7 @@ def apify_posts(urls): return [], "" key = apify_key() if not key: - return [], "AIOS_APIFY_KEY is not configured — the Apify rung is closed" + return [], "AIOS_APIFY_KEY is not configured; the Apify rung is closed" url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items" f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}") body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)), @@ -895,7 +895,7 @@ def apify_reels(handle, limit): return [], "no handle to read reels for" key = apify_key() if not key: - return [], "AIOS_APIFY_KEY is not configured — the Apify rung is closed" + return [], "AIOS_APIFY_KEY is not configured; the Apify rung is closed" try: n = max(1, min(int(limit), MAX_POST_WINDOW)) except (TypeError, ValueError): @@ -949,7 +949,7 @@ def apify_profile(handle): return None, "no handle" key = apify_key() if not key: - return None, "AIOS_APIFY_KEY is not configured — the Apify rung is closed" + return None, "AIOS_APIFY_KEY is not configured; the Apify rung is closed" url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items" f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}") body = {"directUrls": [f"https://www.instagram.com/{h}/"], @@ -1065,7 +1065,7 @@ def bd_corpus_profile(handle, log=print): time.sleep(BD_CORPUS_POLL) break return None, (f"the profile corpus did not deliver {h!r} within " - f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building — it is not lost)") + f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building; it is not lost)") def _tag_metric_deferrals(items, start, kind, influencer_key): @@ -1266,7 +1266,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False, sid = str(cached.get(DEFERRED_MARK)) return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", "deferredProfile": [sid], - "note": (f"the profile source deferred this batch to {sid} — the record is " + "note": (f"the profile source deferred this batch to {sid}; the record is " f"collected, not lost; the next run picks it up")} if isinstance(cached, dict) and cached: rows, note = [cached], "" @@ -1292,7 +1292,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False, # against the schema. Losing it behind the corpus fallback would make a mapping # regression look like a vendor outage. scrape_note = note or ("the scrape answered, but no follower/following counts were " - "readable in it (the field names may have moved — see " + "readable in it (the field names may have moved; see " "instagram-capture.md)") cnode, cnote = bd_corpus_profile(handle, log=log) if cnode: @@ -1303,7 +1303,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False, # a read taken now. Said out loud on the attempt, because "when was this true" is the # one question a stored measurement must always be able to answer. note = (f"the live scrape did not answer ({_s(scrape_note, 120)}), so this came from " - f"the vendor's pre-collected profile corpus — the values are its last " + f"the vendor's pre-collected profile corpus; the values are its last " f"capture, not a read taken just now") else: # ⭐ THE DEFERRAL IS REGISTERED ONLY WHEN THE PAID RUNG GIVES UP. If the corpus @@ -1323,7 +1323,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False, # is discarded, so a row where it is the only "readable" field is a row we cannot read. return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", "note": "the profile source answered, but no follower/following counts were readable in " - "it (the field names may have moved — see instagram-capture.md)"} + "it (the field names may have moved; see instagram-capture.md)"} posts = [] for p in (node.get("posts") or [])[:max_posts]: got = _bd_post_identity(p) @@ -1652,4 +1652,3 @@ def pull_profile(url, max_posts=None, log=print, "no provider could read this profile " f"({'; '.join(attempts) or 'blocked'})")} - diff --git a/api/connectors_meta.py b/api/connectors_meta.py index 11bfd80dfed0bab72ec191fa627db58d25786161..b4f78c10cb7c7aca0d72a8c2b09bee5a769d1b41 100644 --- a/api/connectors_meta.py +++ b/api/connectors_meta.py @@ -325,7 +325,7 @@ def probe(): report["errors"].append(f"reaching /me/adaccounts: {e}") return report if not report["accounts"]: - report["errors"].append("the token authenticates but reaches ZERO ad accounts — a " + report["errors"].append("the token authenticates but reaches ZERO ad accounts; a " "permissions/scope answer, not an empty business") return report act = report["accounts"][0] @@ -373,7 +373,7 @@ def probe(): def as_markdown(report): """The wave asset. ⛔ Field NAMES and TYPES only — no row values, no account identifiers beyond the ad account name the owner already knows, and no query strings.""" - L = ["# W31-T40 — the Meta Ads schema, MEASURED", "", + L = ["# W31-T40: the Meta Ads schema, MEASURED", "", f"Graph version: `{report['graphVersion']}` · token present: " f"**{'yes' if report['token'] else 'NO'}** · probe calls: **{len(CALLS)}**", ""] if report["errors"]: @@ -392,25 +392,25 @@ def as_markdown(report): f"{a.get('account_status','')} | {a.get('currency','')} | " f"{a.get('timezone_name','')} |") L.append("") - L += ["## Method — and why it is not `?metadata=1`", "", + L += ["## Method: and why it is not `?metadata=1`", "", "⛔ **Graph v21.0 publishes no introspection table.** `GET /act_…?metadata=1` returns " - "keys `['account_id','id']` and `GET /?metadata=1` returns `['id']` — " + "keys `['account_id','id']` and `GET /?metadata=1` returns `['id']`; " "`metadata.fields` is EMPTY. The first run of this probe reported *0 fields* for all " "five levels and looked like a broken probe; it was a withdrawn API.", "", "⭐ So every level below is measured by **ACCEPTANCE**: the names in " "`connectors_meta.ENTITY_CANDIDATES` are offered, and what Graph takes is the catalog. " "A `(#100)` refusal NAMES the offending field, so a batch converges in one retry; a " "`(#200)` permission refusal names nothing, so the batch is **bisected** rather than " - "dropped — otherwise the Ad Account level reports a confident, wrong `0 fields`.", "", + "dropped; otherwise the Ad Account level reports a confident, wrong `0 fields`.", "", "⛔⛔ **ACCEPTED IS NOT RETURNED.** A 20-field batch on a live campaign answers HTTP 200 " "with 14 keys, because Graph omits nulls. Every list below is what was **asked and " - "accepted** — the response's own keys are never inspected. Building columns from " + "accepted**; the response's own keys are never inspected. Building columns from " "returned keys drops roughly a third of the schema, which is R2's *do not drop any " "column* broken with nothing going red.", ""] for key, label, _edge in LEVELS: lv = report["levels"].get(key) or {} fields = lv.get("accepted") or [] - L += [f"## {label} — **{len(fields)} accepted** of {lv.get('asked', 0)} asked", ""] + L += [f"## {label}: **{len(fields)} accepted** of {lv.get('asked', 0)} asked", ""] if lv.get("note"): L += [f"⚠ {lv['note']}", ""] if fields: @@ -420,12 +420,12 @@ def as_markdown(report): "guessed at):", ""] L += [f"- `{k}`" for k in sorted(lv["rejected"])] + [""] if lv.get("gated"): - L += ["⚠ **Permission-gated — a REPORTED limit, not a silent omission** (R6's second " + L += ["⚠ **Permission-gated: a REPORTED limit, not a silent omission** (R6's second " "sentence):", ""] - L += [f"- `{k}` — {v}" for k, v in sorted(lv["gated"].items())] + [""] + L += [f"- `{k}`: {v}" for k, v in sorted(lv["gated"].items())] + [""] ins = report.get("insights") or {} if ins: - L += [f"## Insights at `time_increment=1` — {len(ins.get('accepted') or [])} accepted of " + L += [f"## Insights at `time_increment=1`: {len(ins.get('accepted') or [])} accepted of " f"{ins.get('candidatesAsked', 0)} asked", "", "⚠ Insights publishes no `metadata=1` field table, so this catalog is measured by " "ACCEPTANCE: every name below was offered to the API and taken. A name absent from " @@ -433,7 +433,7 @@ def as_markdown(report): "```", ", ".join(ins.get("accepted") or []), "```", ""] if ins.get("rejected"): L += ["### Refused by name", ""] - L += [f"- `{k}` — {v}" for k, v in sorted((ins.get("rejected") or {}).items())] + [""] + L += [f"- `{k}`: {v}" for k, v in sorted((ins.get("rejected") or {}).items())] + [""] return "\n".join(L) @@ -461,7 +461,7 @@ def probe_credential(fields, timeout=8): return {"ok": False, "message": str(e)[:200]} n = len(got.get("data") or []) return {"ok": n > 0, - "message": (f"Meta answered — {n} ad account(s) reachable" if n else + "message": (f"Meta answered; {n} ad account(s) reachable" if n else "the token authenticates but reaches NO ad account: that is a " "scope/permission answer, not an empty business")} @@ -494,7 +494,7 @@ def main(argv): print(f"REFUSED: {e}") return 1 report = probe() - out = (Path(__file__).resolve().parents[2] / ".claude" / "wiki" / "research" / "waves" + out = (Path(__file__).resolve().parents[2] / "AutoWiki" / "research" / "waves" / "wave31" / "proto" / "meta-ads-schema.md") out.parent.mkdir(parents=True, exist_ok=True) out.write_text(as_markdown(report), encoding="utf-8") diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 0603f4f23f7075252fdcef06ce19371ac1620523..3a692a7f2505cee8db8e0f3cf6950c4d7045e52b 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -7,7 +7,7 @@ exists at all rather than another thousand lines inside the engine. ⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +`AutoWiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a name below reads through a candidate list it is because the vendor has two names for one fact (`biography`/`signature`, `region`/`country`), never because the name is uncertain. diff --git a/api/deps.py b/api/deps.py index 72fc24821c6bbf83881ac09cfc2beea102b5ff06..e72935107ebf55076734ad9fe7c6cd8a32d1f7b8 100644 --- a/api/deps.py +++ b/api/deps.py @@ -103,10 +103,40 @@ class Session: # the Customer/Product pool builders, connector routes, provider routes, Odoo, or an environment # lookup from this boundary. A cache miss is a named refusal, never permission to refresh from the # source of truth. That is what "data the app already stores or serves" means in executable form. -_ASSISTANT_BUILT_INS = { +# These two keys name registry modules backed by runtime pools. They are DATABASES the product +# may open, but they are not materialised `ut_*` documents and they are not DuckDB read-through +# mirrors. Keeping that distinction in the request boundary prevents a route resolver from +# treating `product_data` as though `_sources()` could bind it to a mirror table. +MODULE_CACHE_DATABASES = { "customer_data": {"label": "Odoo customers", "cache_prefix": "pool"}, "product_data": {"label": "Odoo products", "cache_prefix": "product_pool"}, } +# The Assistant predates the public taxonomy above. Its private name remains an alias so its +# snapshot-only contract and every existing caller stay byte-for-byte unchanged. +_ASSISTANT_BUILT_INS = MODULE_CACHE_DATABASES + + +def module_cache_source(database: str): + """Metadata for a runtime-cache database, or None for every materialised/mirror key.""" + meta = MODULE_CACHE_DATABASES.get(str(database or "")) + return dict(meta) if meta else None + + +def module_cache_rows(session: Session, database: str): + """The current viewer's scoped rows for one known runtime-cache database. + + This is deliberately the interactive grid boundary, not the Assistant snapshot reader: + `scoped_table` applies the same team/agent and row grants as the module page, while + `Session.require` keeps the account and tenant-catalogue checks in their one chokepoint. + Unknown keys fail before either check, so an absent relation target cannot be mislabeled as + a permission denial. + """ + key = str(database or "") + if module_cache_source(key) is None: + raise KeyError(key) + session.require(key) + import core.perm_scope as perm_scope + return list(perm_scope.scoped_table(session.user, key, st=session.runtime) or []) def _assistant_refuse(status, code, message): @@ -746,24 +776,16 @@ def _user_for(claims): """The account named by a verified cookie, or None — with the epoch check that makes a stateless cookie revocable. - ⛔ THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities `users.verify` + ⛔ THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities the login route issues.** Any divergence produces the worst failure shape there is — a login that returns 200 - with a cookie and then 401s every request after it. This function got that wrong once (the - emergency-master branch below ran only when NO record existed, while `verify` grants the master - identity even when one does), so it is now written as a deliberate mirror of `verify`'s - structure: try the record, else the master. + with a cookie and then 401s every request after it. - TWO CASES, in `verify`'s own order: + TWO CASES: * THE RECORD, if it is readable and satisfies everything a session adds on top of a login: the account is active, and its CURRENT epoch equals the cookie's — otherwise the session is revoked (a password change or a deactivation bumped it). - * THE EMERGENCY MASTER — username 'admin' with APP_PASSWORD configured — exactly as - `verify`'s last branch grants it, whether or not a record exists. This is what stops a - store outage, a self-deactivation or a bumped epoch locking the owner out of the product. - ⚠ AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That - is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's - epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it. - Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`. + * THE QA RUNNER, only when both non-production deployment gates remain enabled and its + password-derived epoch still matches. Live deployments cannot enable it. * anything else → None. An unknown username is NOT admitted just because its signature was valid: a signature proves the cookie is ours, not that the account still exists. """ @@ -801,17 +823,6 @@ def _user_for(claims): if claim_tenant != pub.get("tenant"): return None return pub - if (uname == "admin" and os.environ.get("APP_PASSWORD", "") - and claim_tenant == "royal-imports"): - # The emergency master is TENANT #0's break-glass, not the platform's: a synthetic - # all-access identity minted into another tenant's session would be the exact - # cross-tenant widening the rule above closes. - # ⚠ WAVE 40 (R2): the same mark `users.verify`'s master branch carries. The two dicts are - # documented mirrors of each other, so the key rides on both or they are not mirrors. - return {"username": "admin", "name": "Administrator", "role": "admin", - "bus": "all", "modules": "all", "tenant": "royal-imports", - "emergency_master": True, - "epoch": int(claims.get("e") or 0)} return None @@ -832,7 +843,7 @@ def _user_for(claims): # HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each # write once per window (still bounded); a restart re-arms every user's first request; and the # resolution of `last_active` is therefore "within the last hour", which is exactly what the plane -# renders it as. Bounded like `routes_auth._FAILS` so it cannot grow into a leak on a deployment +# renders it as. Bounded independently so it cannot grow into a leak on a deployment # with many accounts. _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict() _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600) diff --git a/api/formula_runtime.py b/api/formula_runtime.py index 6443b91ccf02ac25c63383b46c9f1dca055d135c..84ba9960a8fb91b0d9dc32414b3e2c17833eff12 100644 --- a/api/formula_runtime.py +++ b/api/formula_runtime.py @@ -11,8 +11,14 @@ import datetime as dt import hashlib import json import math +from functools import lru_cache +import random import re import warnings +import unicodedata +from bisect import bisect_left, bisect_right, insort +from collections import OrderedDict, deque +from threading import RLock from decimal import Decimal, ROUND_FLOOR, ROUND_HALF_UP @@ -31,6 +37,14 @@ AUTHORITY = "server" #: republished verbatim in `contract_dto()`, which is what the browser reads. CONSUMERS = ("preview", "grid", "filter", "export", "automation", "query", "timeseries", "sheet") FN_ARITY = { + "FV": (3, 5), + "DATEDIF": (3, 3), + "IRR": (1, 2), + "XIRR": (2, 3), + "NORM.DIST": (4, 4), + "BAHTTEXT": (1, 1), + "ADD": (2, 2), + "MULTIPLY": (2, 2), "ABS": (1, 1), "ROUND": (1, 2), "ROUNDUP": (1, 2), "ROUNDDOWN": (1, 2), "INT": (1, 1), "MOD": (2, 2), "SQRT": (1, 1), "POWER": (2, 2), "EXP": (1, 1), "LN": (1, 1), "SUM": (1, MANY), "AVERAGE": (1, MANY), "COUNT": (1, MANY), @@ -67,6 +81,8 @@ FN_ARITY = { # Text and Information. "TEXTJOIN": (3, MANY), "REPT": (2, 2), "EXACT": (2, 2), "ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1), + "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0), + "FILTER": (2, 3), "SORT": (1, 4), } ALIASES = {"CONCAT": "CONCATENATE"} @@ -85,8 +101,8 @@ ALIASES = {"CONCAT": "CONCATENATE"} #: ⛔ AND THIS IS NOT `ALIASES`. `catalogue_rows()` publishes `ALIASES` to consumers as the #: alternative spellings a PERSON may type; `STDEV_P` is a transport form of this parser's own #: making and offering it to a user would advertise a spelling Excel does not have. -DOTTED_SPELLINGS = {"STDEV_P": "STDEV.P", "VAR_P": "VAR.P"} -_DOTTED_RE = re.compile(r"(STDEV|VAR)\.P(?=\s*\()", re.IGNORECASE) +DOTTED_SPELLINGS = {"STDEV_P": "STDEV.P", "VAR_P": "VAR.P", "NORM_DIST": "NORM.DIST"} +_DOTTED_RE = re.compile(r"(?:STDEV\.P|VAR\.P|NORM\.DIST)(?=\s*\()", re.IGNORECASE) #: ⭐⭐ WAVE 50 · W50-T15 (ruling R5, contract C2) — THE FORMULA CATALOGUE. #: @@ -122,6 +138,24 @@ _BOTH = ("grid", "sheet") _SHEET = ("sheet",) FN_CATALOGUE = { + "FV": {"category": "Math", "signature": "FV(rate, nper, pmt, [pv], [type])", "description": "Returns the future value of periodic payments.", "contexts": _BOTH}, + "DATEDIF": {"category": "Date", "signature": "DATEDIF(start_date, end_date, unit)", "description": "Returns elapsed calendar years, months or days.", "contexts": _BOTH}, + "IRR": {"category": "Math", "signature": "IRR(values, [guess])", "description": "Returns the periodic internal rate of return near a starting guess.", "contexts": _SHEET}, + "XIRR": {"category": "Math", "signature": "XIRR(values, dates, [guess])", "description": "Returns the annual internal rate of return for dated cash flows.", "contexts": _SHEET}, + "NORM.DIST": {"category": "Statistical", "signature": "NORM.DIST(x, mean, standard_dev, cumulative)", "description": "Returns a normal probability or density.", "contexts": _BOTH}, + "BAHTTEXT": {"category": "Text", "signature": "BAHTTEXT(number)", "description": "Spells a monetary amount in Thai baht and satang.", "contexts": _BOTH}, + "ADD": {"category": "Math", "signature": "ADD(value1, value2)", "description": "Adds two values.", "contexts": _BOTH}, + "MULTIPLY": {"category": "Math", "signature": "MULTIPLY(value1, value2)", "description": "Multiplies two values.", "contexts": _BOTH}, + "FILTER": {"category": "Lookup", "signature": "FILTER(array, include, [if_empty])", + "description": "Returns the rows or columns that meet a condition.", "contexts": _SHEET}, + "SORT": {"category": "Lookup", "signature": "SORT(array, [sort_index], [sort_order], [by_col])", + "description": "Returns an array sorted by a row or column.", "contexts": _SHEET}, + "OFFSET": {"category": "Lookup", "signature": "OFFSET(reference, rows, cols, [height], [width])", + "description": "Returns a reference shifted by a number of rows and columns.", "contexts": _SHEET}, + "INDIRECT": {"category": "Lookup", "signature": "INDIRECT(ref_text, [a1])", + "description": "Returns the reference named by a text value.", "contexts": _SHEET}, + "RAND": {"category": "Math", "signature": "RAND()", + "description": "Returns a random number between zero and one.", "contexts": _BOTH}, # ── Math ───────────────────────────────────────────────────────────────────────────────── "ABS": {"category": "Math", "signature": "ABS(number)", "description": "Returns the absolute value of a number.", "contexts": _BOTH}, @@ -491,6 +525,19 @@ EVALUATION_ERROR_FIXTURES = { # Shared contract fixture corpus. The API publishes this exact data so the browser preview gate # can evaluate the same source/row/today triples without maintaining a second handwritten grammar. PARITY_FIXTURES = { + "FV": ("FV(0,2,10,100)", -120), + "DATEDIF": ('DATEDIF("2001-06-01","2002-08-15","D")', 440), + "IRR": ("IRR(B2:B3)", 0.1), + "XIRR": ("XIRR(B2:B3,C2:C3)", 0.1), + "NORM.DIST": ("NORM.DIST(0,0,1,TRUE)", 0.5), + "BAHTTEXT": ("BAHTTEXT(21)", "ยี่สิบเอ็ดบาทถ้วน"), + "ADD": ("ADD(4,7)", 11), + "MULTIPLY": ("MULTIPLY(4,7)", 28), + "FILTER": ("SUM(FILTER(3,TRUE))", 3), + "SORT": ("SUM(SORT(3))", 3), + "OFFSET": ("OFFSET(B2, 1, 1)", 20), + "INDIRECT": ('INDIRECT("B2")', 1), + "RAND": ("RAND()>=0", "TRUE"), "ABS": ("ABS(-2)", 2.0), "ROUND": ("ROUND(2.345, 2)", 2.35), "ROUNDUP": ("ROUNDUP(2.341, 2)", 2.35), "ROUNDDOWN": ("ROUNDDOWN(2.349, 2)", 2.34), "INT": ("INT(2.9)", 2), "MOD": ("MOD(7, 4)", 3.0), "SQRT": ("SQRT(9)", 3.0), @@ -596,6 +643,9 @@ _CELLS_BC = {"B2": 1, "B3": 2, "B4": 3, "B5": 4, "B6": 5, #: `"row": {"empty": None}` for every fixture. A consumer that runs the PUBLISHED corpus therefore #: cannot reproduce a range fixture from the DTO alone and must read this table. PARITY_ROWS = { + "IRR": {"B2": -100, "B3": 110}, + "XIRR": {"B2": -100, "B3": 110, "C2": "2021-01-01", "C3": "2022-01-01"}, + "OFFSET": _CELLS_BC, "INDIRECT": _CELLS_BC, "SUMIF": _CELLS_BC, "SUMIFS": _CELLS_BC, "COUNTIF": _CELLS_BC, "COUNTIFS": _CELLS_BC, "AVERAGEIF": _CELLS_BC, "AVERAGEIFS": _CELLS_BC, "MAXIFS": _CELLS_BC, "MINIFS": _CELLS_BC, "SUMPRODUCT": _CELLS_BC, "VLOOKUP": _CELLS_BC, "XLOOKUP": _CELLS_BC, "INDEX": _CELLS_BC, @@ -839,6 +889,8 @@ ALL_ARGS = "*" #: ⛔ SUMIF PUTS THE SUM RANGE LAST AND SUMIFS PUTS IT FIRST. Reading this table backwards is the #: classic Excel bug, so both spellings carry a fixture over the same cells. RANGE_ARGS = { + "IRR": ((0,), None), "XIRR": ((0, 1), None), + "FILTER": ((0, 1), None), "SORT": ((0,), None), "SUMIF": ((0, 2), None), # SUMIF(range, criteria, [sum_range]) "AVERAGEIF": ((0, 2), None), # AVERAGEIF(range, criteria, [average_range]) "COUNTIF": ((0,), None), # COUNTIF(range, criteria) @@ -954,7 +1006,7 @@ def _rewrite(source, sheet=""): # never touched. See `DOTTED_SPELLINGS` for why the pattern names only two functions. dotted = _DOTTED_RE.match(source, i) if dotted: - out.append(dotted.group(1).upper() + "_P") + out.append(dotted.group(0).upper().replace(".", "_")) i = dotted.end() continue got = _a1_at(source, i, sheet) @@ -989,7 +1041,7 @@ def _rewrite(source, sheet=""): return "".join(out) -def _normal(node, depth=0): +def _normal(node, depth=0, compact_ranges=False): if depth > MAX_DEPTH: raise FormulaError("formula is nested too deeply") if isinstance(node, ast.Constant) and isinstance(node.value, (int, float, str, bool)): @@ -999,15 +1051,15 @@ def _normal(node, depth=0): if isinstance(node, ast.Name) and node.id.upper() in ("TRUE", "FALSE"): return {"t": "const", "v": node.id.upper() == "TRUE"} if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): - return {"t": "neg", "e": _normal(node.operand, depth + 1)} + return {"t": "neg", "e": _normal(node.operand, depth + 1, compact_ranges)} if isinstance(node, ast.BinOp): ops = {ast.Add: "+", ast.Sub: "-", ast.Mult: "*", ast.Div: "/", ast.Pow: "^", ast.BitAnd: "&"} op = ops.get(type(node.op)) if not op: raise FormulaError("that operator is not supported") - return {"t": "bin", "op": op, "l": _normal(node.left, depth + 1), - "r": _normal(node.right, depth + 1)} + return {"t": "bin", "op": op, "l": _normal(node.left, depth + 1, compact_ranges), + "r": _normal(node.right, depth + 1, compact_ranges)} if isinstance(node, ast.Compare): if len(node.ops) != 1 or len(node.comparators) != 1: raise FormulaError("comparisons cannot be chained") @@ -1016,8 +1068,8 @@ def _normal(node, depth=0): op = ops.get(type(node.ops[0])) if not op: raise FormulaError("that comparison is not supported") - return {"t": "cmp", "op": op, "l": _normal(node.left, depth + 1), - "r": _normal(node.comparators[0], depth + 1)} + return {"t": "cmp", "op": op, "l": _normal(node.left, depth + 1, compact_ranges), + "r": _normal(node.comparators[0], depth + 1, compact_ranges)} if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): fn = ALIASES.get(node.func.id.upper(), node.func.id.upper()) # The transport spelling folds back to Excel's own, so `FN_ARITY` and the catalogue are @@ -1034,6 +1086,11 @@ def _normal(node, depth=0): vals = [str(x.value) for x in node.args] if fn == "A1REF": return {"t": "a1", "k": a1_key(vals[0], vals[1])} + if compact_ranges: + bounds = _range_bounds(vals[0], vals[1], vals[2]) + return {"t": "a1range", "range": bounds, + "rows": bounds[4] - bounds[3] + 1, + "cols": bounds[2] - bounds[1] + 1} # The range is expanded HERE, at normalize time, so a malformed or oversized range is a # PARSE error the author sees immediately rather than a surprise at evaluation. # ⭐ `rows` and `cols` RIDE ON THE NODE (W50-T16) because they are known here and are @@ -1048,11 +1105,198 @@ def _normal(node, depth=0): lo, hi = FN_ARITY[fn] if not lo <= len(node.args) <= hi: raise FormulaError(f"{fn} takes {lo} to {hi} arguments") - return {"t": "call", "fn": fn, "args": [_normal(x, depth + 1) for x in node.args]} + return {"t": "call", "fn": fn, + "args": [_normal(x, depth + 1, compact_ranges) for x in node.args]} raise FormulaError("formula contains unsupported syntax") -def parse_formula(source, *, sheet=""): +_SHEET_TOKEN = re.compile( + r'(?P\s+)|(?P"(?:[^"]|"")*")|' + r"(?P(?:'[^']*\[[^']*'|\[[^\]]*\][A-Za-z0-9_ ]*)![^ ,()+*/]+)|" + r"(?P(?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!(?:\$?[A-Za-z]{1,3}\$?[1-9][0-9]*|[A-Za-z_][A-Za-z0-9_.]*))|" + r'(?P\$?[A-Za-z]{1,3}\$?[1-9][0-9]*(?![A-Za-z0-9_.]))|' + r'(?P(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?)|' + r'(?P[A-Za-z_\\][A-Za-z0-9_.\\]*)|' + r'(?P\#[A-Z0-9/]+[!?]?)|(?P<=|>=|<>|!=|[+*/^&=<>():,%#-])') +REFERENCE_ERRORS = {"null_intersection": "#NULL!", "unknown_name": "#NAME?", + "external_reference": "#REF!", "invalid_reference": "#REF!", + "spill_obstructed": "#SPILL!", "empty_array": "#CALC!"} +_ERROR_LITERALS = {"#NULL!": "null_intersection", "#REF!": "invalid_reference", + "#NAME?": "unknown_name", "#DIV/0!": "div_zero", "#VALUE!": "bad_value", + "#N/A": "no_match", "#NUM!": "bad_number"} +_VOLATILE_FUNCTIONS = frozenset(("NOW", "TODAY", "RAND", "OFFSET", "INDIRECT")) + + +_COPY_REFERENCE = re.compile( + r'"(?:[^"]|"")*"|(?(?:(?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!)?)" + r'(?P\$?[A-Za-z]{1,3}\$?[1-9][0-9]*(?![A-Za-z0-9_.]|\s*\()|' + r'\$?[A-Za-z]{1,3}:\$?[A-Za-z]{1,3}(?![A-Za-z0-9_.$])|' + r'\$?[1-9][0-9]*:\$?[1-9][0-9]*(?![A-Za-z0-9_.$]))') + + +def transform_formula_references(source, drow=0, dcol=0, mode="copy"): + """The client transformSheetFormula contract: cells/bands shift; names/text stay literal.""" + if mode == "cut" or not (drow or dcol): return source + def replace(match): + ref = match.group("ref") + if not ref: return match.group() + shifted = [] + for end in ref.split(":"): + cell = re.fullmatch(r"(\$?)([A-Za-z]+)(\$?)([0-9]+)", end) + if cell: + c = col_to_index(cell[2]) + (0 if cell[1] else dcol) + r = int(cell[4]) + (0 if cell[3] else drow) + if not (1 <= c <= 16384 and 1 <= r <= 1048576): return "#REF!" + shifted.append(cell[1] + index_to_col(c) + cell[3] + str(r)) + else: + fixed = end.startswith("$"); raw = end.lstrip("$"); column = raw.isalpha() + value = (col_to_index(raw) if column else int(raw)) + (0 if fixed else dcol if column else drow) + if not 1 <= value <= (16384 if column else 1048576): return "#REF!" + shifted.append(("$" if fixed else "") + (index_to_col(value) if column else str(value))) + return match.group("prefix") + ":".join(shifted) + return _COPY_REFERENCE.sub(replace, source) + + +def _normalize_named_ranges(value): + if isinstance(value, dict): + value = [{"name": name, **(entry if isinstance(entry, dict) else {"ref": entry})} + for name, entry in value.items()] + return tuple(dict(entry) for entry in (value or ()) + if isinstance(entry, dict) and isinstance(entry.get("name"), str) + and isinstance(entry.get("ref"), str)) + + +def _sheet_syntax(source, sheet, named_ranges=(), active_names=(), address="", budget=None): + """Reference operators share one AST with arithmetic; names bind without string eval. + + The original token rides on each address/name node. Names are local then global and + remain names when a formula is copied. Dynamic text is parsed as a reference, never code. + """ + if len(source) > MAX_SOURCE: raise FormulaError("formula source is too long") + if budget is None: named_ranges = _normalize_named_ranges(named_ranges) + budget = [MAX_NODES] if budget is None else budget + tokens, pos = [], 0 + text = _expand_bands(re.sub(r'"(?:[^"]|"")*"|(?:_xlfn\.|_xlws\.|_xludf\.)+(?=[A-Za-z_][A-Za-z0-9_.]*\s*\()', + lambda match: match[0] if match[0].startswith('"') else '', source, flags=re.I)) + while pos < len(text): + match = _SHEET_TOKEN.match(text, pos) + if not match: + raise FormulaError("the formula could not be parsed") + tokens.append((match.lastgroup, match.group())) + if len(tokens) > MAX_NODES * 2: raise FormulaError("formula has too many tokens") + pos = match.end() + # Whitespace is intersection only between operands, not around punctuation/functions. + clean = [] + for index, token in enumerate(tokens): + if token[0] != "space": + clean.append(token) + continue + prev = clean[-1] if clean else None + nxt = next((t for t in tokens[index + 1:] if t[0] != "space"), None) + if (prev and nxt and (prev[0] in ("cell", "qualified", "name") or prev[1] == ")") + and (nxt[0] in ("cell", "qualified", "name") or nxt[1] == "(") + and not (prev[0] == "name" and nxt[1] == "(")): + clean.append(("op", " ")) + index = 0 + def peek(): return clean[index][1] if index < len(clean) else "" + def take(): + nonlocal index + if index >= len(clean): raise FormulaError("the formula is incomplete") + token = clean[index]; index += 1 + return token + def reference(token, qualifier=""): + own = qualifier or sheet + if re.fullmatch(_A1_CELL, token): + key = a1_key(token, own) + _, c, r = _address_parts(key) + if c > 16384 or r > 1048576: + return {"t": "error", "code": "invalid_reference"} + return {"t": "a1", "k": key, "token": token} + name = token.upper() + candidates = [n for n in named_ranges if str(n.get("name", "")).upper() == name] + local = next((n for n in candidates if str(n.get("sheet", "")).upper() == own.upper()), None) + target = local or next((n for n in candidates if not n.get("sheet")), None) + identity = (own.upper(), name) + if target is None: bound = {"t": "error", "code": "unknown_name"} + elif identity in active_names: bound = {"t": "error", "code": "circular_reference"} + elif len(active_names) >= MAX_DEPTH: raise FormulaError("names are nested too deeply") + else: + # A relative defined name is a template anchored at base (A1 for legacy names). + # Its dependency is bound at the actual caller address, including mixed-$ axes. + _, bc, br = _address_parts(str(target.get("base") or "A1").replace("$", "").upper()) + _, cc, cr = _address_parts(address or "A1") + shifted = transform_formula_references(str(target.get("ref", "")).lstrip("="), cr-br, cc-bc) + bound = _sheet_syntax(shifted, + str(target.get("sheet") or sheet), named_ranges, + (*active_names, identity), address, budget) + return {"t": "name", "name": name, "sheet": own, "token": token, "e": bound} + def atom(depth): + budget[0] -= 1 + if budget[0] < 0: raise FormulaError("formula has too many expanded nodes") + kind, token = take() + if kind == "num": return {"t": "const", "v": float(token)} + if kind == "str": return {"t": "const", "v": token[1:-1].replace('""', '"')} + if kind == "external": return {"t": "error", "code": "external_reference"} + if kind == "err": return {"t": "error", "code": _ERROR_LITERALS.get(token, "bad_value")} + if kind in ("cell", "name"): + if peek() == "(": + take(); fn = ALIASES.get(token.upper(), token.upper()); args = [] + if peek() != ")": + while True: + args.append({"t": "const", "v": None} if peek() in (",", ")") else expr(1, depth + 1)) + if peek() != ",": break + take() + if take()[1] != ")": raise FormulaError("function arguments are incomplete") + if fn not in FN_ARITY: return {"t": "error", "code": "unknown_name"} + lo, hi = FN_ARITY[fn] + if not lo <= len(args) <= hi: raise FormulaError("function arity is invalid") + return {"t": "call", "fn": fn, "args": args} + if token.upper() in ("TRUE", "FALSE"): return {"t": "const", "v": token.upper() == "TRUE"} + return reference(token) + if kind == "qualified": + qualifier, token = token.rsplit("!", 1) + bound = reference(token, qualifier.strip("'").replace("''", "'")) + bound["token"] = qualifier + "!" + token + return bound + if token == "(": + node = expr(0, depth + 1) + if take()[1] != ")": raise FormulaError("parenthesis is incomplete") + return node + if token in ("+", "-"): + node = expr(6, depth + 1) + return node if token == "+" else {"t": "neg", "e": node} + raise FormulaError("the formula could not be parsed") + precedence = {",": 0, "=": 1, "<>": 1, "!=": 1, "<": 1, ">": 1, "<=": 1, ">=": 1, + "&": 2, "+": 3, "-": 3, "*": 4, "/": 4, "^": 5, " ": 7, ":": 8} + def expr(minimum, depth): + if depth > MAX_DEPTH: raise FormulaError("formula is nested too deeply") + node = atom(depth) + if peek() == "#": + take(); node = {"t": "spill", "e": node} + while peek() in precedence and precedence[peek()] >= minimum: + budget[0] -= 1 + if budget[0] < 0: raise FormulaError("formula has too many expanded nodes") + op = take()[1]; right = expr(precedence[op] + 1, depth + 1) + if op == ":" and node["t"] == right["t"] == "a1": + a, b = node["k"].rpartition("!"), right["k"].rpartition("!") + # An unqualified end inherits the explicit start's tab. + end_sheet = b[0] if "!" in right.get("token", "") else a[0] + bounds = _range_bounds(a[2], b[2], a[0]) + node = {"t": "a1range", "range": bounds, "rows": bounds[4]-bounds[3]+1, + "cols": bounds[2]-bounds[1]+1} if end_sheet == a[0] else {"t": "error", "code": "invalid_reference"} + elif op in (":", " ", ","): + node = {"t": "reference_op", "op": op, "l": node, "r": right} + else: + node = {"t": "cmp" if precedence[op] == 1 else "bin", "op": "!=" if op == "<>" else op, + "l": node, "r": right} + return node + root = expr(0, 0) + if index != len(clean): raise FormulaError("the formula has trailing tokens") + return root + + +def parse_formula(source, *, sheet="", compact_ranges=False, named_ranges=(), address=""): """One formula's stored definition. `sheet` is the tab it lives on, or "" for a grid field. ⭐ KEYWORD ONLY, AND DEFAULTED TO "", so every caller that has no notion of a tab keeps the @@ -1066,30 +1310,38 @@ def parse_formula(source, *, sheet=""): raise FormulaError("the formula is empty") if len(source) > MAX_SOURCE: raise FormulaError(f"formula longer than {MAX_SOURCE} characters") + if compact_ranges: + root = _sheet_syntax(source, sheet, named_ranges, address=address) + else: + root = None try: # Regex patterns such as ``"\\s"`` are data for REGEX_* rather than Python escapes. # Python 3.12 warns while parsing them even though the string is valid for this grammar. with warnings.catch_warnings(): warnings.simplefilter("ignore", SyntaxWarning) - parsed = ast.parse(_rewrite(source, sheet), mode="eval") + parsed = ast.parse(_rewrite(source, sheet), mode="eval") if root is None else None except (SyntaxError, ValueError) as exc: raise FormulaError("the formula could not be parsed") from exc - if sum(1 for _ in ast.walk(parsed)) > MAX_NODES: + if parsed is not None and sum(1 for _ in ast.walk(parsed)) > MAX_NODES: raise FormulaError(f"formula has more than {MAX_NODES} nodes") - root = _normal(parsed.body) + root = root if root is not None else _normal(parsed.body, compact_ranges=compact_ranges) refs = [] # ⭐ `cells` IS A SECOND, SEPARATE LIST AND NOT A MERGE INTO `refs`. `refs` names FIELD keys and # is already persisted in every stored definition, read by `evaluate_fields`, `evaluate_batch` # and the contract DTO; folding cell addresses into it would hand every one of those consumers # a key that is not a field. `cells` is additive, so an existing definition simply carries an # empty one. - cells = [] + cells, ranges, precedents = [], [], [] def walk(row): if row["t"] == "ref" and row["k"] not in refs: refs.append(row["k"]) if row["t"] == "a1" and row["k"] not in cells: cells.append(row["k"]) + precedents.append({"cell": row["k"]}) if row["t"] == "a1range": + if "range" in row: + ranges.append(row["range"]) + precedents.append({"range": row["range"]}) for key in row.get("keys") or (): if key not in cells: cells.append(key) @@ -1097,7 +1349,14 @@ def parse_formula(source, *, sheet=""): if isinstance(row.get(key), dict): walk(row[key]) for child in row.get("args") or (): walk(child) walk(root) - return {"version": AST_VERSION, "source": source, "ast": root, "refs": refs, "cells": cells} + result = {"version": AST_VERSION, "source": source, "ast": root, "refs": refs, "cells": cells} + if compact_ranges: + def volatile(node): + return (node.get("fn") in _VOLATILE_FUNCTIONS + or any(volatile(node[k]) for k in ("e", "l", "r") if isinstance(node.get(k), dict)) + or any(volatile(child) for child in node.get("args", ()))) + result.update(ranges=ranges, precedents=precedents, volatile=volatile(root), sheet=sheet) + return result def _num(v): @@ -1266,10 +1525,12 @@ def evaluate_result(definition, row, *, today=None, strict=False): today or dt.date.today().isoformat(), diagnostics) if diagnostics: value = None - if isinstance(value, bool): + if isinstance(value, bool) and not isinstance(row, _SheetInputs): value = "TRUE" if value else "FALSE" elif isinstance(value, float) and not math.isfinite(value): value = None + elif isinstance(value, Range) and isinstance(row, _SheetInputs): + pass elif value is not None and not isinstance(value, (int, float, str)): value = None diagnostics.append("evaluation_error") @@ -1293,9 +1554,173 @@ def _diagnose(diagnostics, code): diagnostics.append(code) +def _reference_node(node): + if node.get("t") == "name": return _reference_node(node["e"]) + return (node.get("t") in ("a1", "a1range", "reference_op", "spill") + or (node.get("t") == "call" and node.get("fn") in ("OFFSET", "INDIRECT"))) + + +def _reference_areas(node, row, today, diagnostics): + """Resolve rectangles, not values. Unions retain each area and overlapping occurrences.""" + kind = node.get("t") + if kind == "error": + _diagnose(diagnostics, node["code"]); return [] + if kind == "name": return _reference_areas(node["e"], row, today, diagnostics) + if kind == "spill": + child = node["e"] + if child.get("t") != "a1" or not isinstance(row, _SheetInputs): + _diagnose(diagnostics, "invalid_reference"); return [] + key = child["k"] + row.get(key) + owned = row.engine.spills.get(key) + if not owned: _diagnose(diagnostics, "invalid_reference"); return [] + return [owned["range"]] + if kind == "a1": + sheet, col, r = _address_parts(node["k"]) + return [(sheet, col, col, r, r)] + if kind == "a1range": + if "range" in node: return [node["range"]] + keys = node["keys"] + return [_range_bounds(keys[0].rsplit("!", 1)[-1], keys[-1].rsplit("!", 1)[-1], _sheet_half(keys[0]))] + if kind == "reference_op": + left = _reference_areas(node["l"], row, today, diagnostics) + right = _reference_areas(node["r"], row, today, diagnostics) + if diagnostics: return [] + if node["op"] == ",": return left + right + if node["op"] == ":": + if len(left) != 1 or len(right) != 1 or left[0][0] != right[0][0]: + _diagnose(diagnostics, "invalid_reference"); return [] + a, b = left[0], right[0] + return [(a[0], min(a[1], b[1]), max(a[2], b[2]), min(a[3], b[3]), max(a[4], b[4]))] + result = [] + for a in left: + for b in right: + area = (a[0], max(a[1], b[1]), min(a[2], b[2]), max(a[3], b[3]), min(a[4], b[4])) + if a[0] == b[0] and area[1] <= area[2] and area[3] <= area[4]: result.append(area) + if not result: _diagnose(diagnostics, "null_intersection") + return result + if kind == "call" and node["fn"] == "OFFSET": + base = _reference_areas(node["args"][0], row, today, diagnostics) + args = [_num(_eval(arg, row, today, diagnostics)) for arg in node["args"][1:]] + if len(base) != 1 or any(v is None for v in args): + _diagnose(diagnostics, "bad_value"); return [] + s, c0, c1, r0, r1 = base[0] + dr, dc = (int(args[0]), int(args[1])) + height = int(args[2]) if len(args) > 2 else r1-r0+1 + width = int(args[3]) if len(args) > 3 else c1-c0+1 + area = (s, c0+dc, c0+dc+width-1, r0+dr, r0+dr+height-1) + if height < 1 or width < 1: + _diagnose(diagnostics, "invalid_reference"); return [] + return _checked_areas([area], row, diagnostics) + if kind == "call" and node["fn"] == "INDIRECT": + text = _eval(node["args"][0], row, today, diagnostics) + if not isinstance(text, str): _diagnose(diagnostics, "invalid_reference"); return [] + if "[" in text or "]" in text: + # R1C1 bracket offsets are allowed only under the explicit second FALSE argument. + r1c1 = len(node["args"]) == 2 and _eval(node["args"][1], row, today, diagnostics) is False + if not r1c1 or not re.fullmatch(r"R(?:\[-?\d+\]|\d*)C(?:\[-?\d+\]|\d*)", text, re.I): + _diagnose(diagnostics, "external_reference"); return [] + here = row.get(SHEET_ADDR_KEY, "") + sheet = _sheet_half(here) + if len(node["args"]) == 2 and _eval(node["args"][1], row, today, diagnostics) is False: + m = re.fullmatch(r"R(\[-?\d+\]|\d*)C(\[-?\d+\]|\d*)", text, re.I) + if not m or not here: _diagnose(diagnostics, "invalid_reference"); return [] + _, col, r = _address_parts(here) + def coord(raw, origin): return origin + int(raw[1:-1]) if raw.startswith("[") else int(raw) if raw else origin + r, col = coord(m[1], r), coord(m[2], col) + if min(r, col) < 1: _diagnose(diagnostics, "invalid_reference"); return [] + text = index_to_col(col) + str(r) + try: + target = row.engine.reference_syntax(text, sheet, here) if isinstance(row, _SheetInputs) else _sheet_syntax(text, sheet, address=here) + except FormulaError: + _diagnose(diagnostics, "invalid_reference"); return [] + if (not _reference_node(target) and target.get("t") != "error") or target.get("t") == "call": + _diagnose(diagnostics, "invalid_reference"); return [] + return _checked_areas(_reference_areas(target, row, today, diagnostics), row, diagnostics) + _diagnose(diagnostics, "invalid_reference") + return [] + + +def _checked_areas(areas, row, diagnostics): + for s, c0, c1, r0, r1 in areas: + if min(c0, r0) < 1 or c1 > 16384 or r1 > 1048576: + _diagnose(diagnostics, "invalid_reference"); return [] + if isinstance(row, _SheetInputs) and row.engine.sheets is not None and s and s not in row.engine.sheets: + _diagnose(diagnostics, "unknown_sheet"); return [] + return areas + + +def _area_values(areas, row, sparse=True): + for bounds in areas: + if sparse and isinstance(row, _SheetInputs): + yield from row.range_values(bounds) + else: + s, c0, c1, r0, r1 = bounds + if (c1-c0+1)*(r1-r0+1) > MAX_RANGE_CELLS: raise FormulaError("range is too large") + for r in range(r0, r1+1): + for c in range(c0, c1+1): yield row.get(a1_key(index_to_col(c)+str(r), s)) + + +def _excel_number(value): + return float(format(value, ".15g")) if isinstance(value, (int, float)) and not isinstance(value, bool) else value + + +def _logical(value, diagnostics): + if isinstance(value, (bool, int, float)) or value is None: return bool(value) + if isinstance(value, str) and value.upper() in ("TRUE", "FALSE"): return value.upper() == "TRUE" + _diagnose(diagnostics, "bad_value") + return False + + +def _format_rule(node, formats): + formats = [item for item in formats if item] + kind, fn = node.get("t"), node.get("fn") + if kind == "cmp" or (kind == "bin" and node["op"] in ("&", "^")): return None + if kind == "call": + if fn in ("DATE", "DATEVALUE", "TODAY", "NOW", "EDATE", "EOMONTH", "WORKDAY", "DATEADD"): + return {"kind": "date"} + if fn in ("IRR", "XIRR"): return {"kind": "percent", "decimals": 2} + if fn not in ("IF", "IFS", "IFERROR", "IFNA", "SWITCH", "CHOOSE", "SUM", "AVERAGE", "MIN", "MAX", "SUMIF", "SUMIFS", "AVERAGEIF", "AVERAGEIFS", "INDEX", "XLOOKUP", "VLOOKUP", "HLOOKUP", "OFFSET", "INDIRECT", "FILTER", "SORT", "ROUND", "ROUNDUP", "ROUNDDOWN", "ABS", "ADD", "MULTIPLY", "FV"): + return None + if kind == "bin" and len(formats) > 1: + kinds = [f.get("kind") if isinstance(f, dict) else f for f in formats] + if node["op"] == "-" and kinds[:2] == ["date", "date"]: return None + if node["op"] == "/" and kinds[0] == kinds[1]: return {"kind": "percent", "decimals": 2} + return next((f for f in formats if isinstance(f, dict) and f.get("kind") in ("currency", "date")), formats[0] if formats else None) + + def _eval(node, row, today, diagnostics=None): + if not isinstance(row, _SheetInputs): return _eval_value(node, row, today, diagnostics) + row.format_stack.append([]) + try: + value = _eval_value(node, row, today, diagnostics) + fmt = _format_rule(node, row.format_stack[-1]) + row.last_format = fmt if ((isinstance(value, (int, float, Range)) and not isinstance(value, bool)) + or (isinstance(value, str) and isinstance(fmt, dict) and fmt.get("kind") == "date")) else None + value = _excel_number(value) + finally: + row.format_stack.pop() + if row.format_stack: row.format_stack[-1].append(row.last_format) + return value + + +def _eval_value(node, row, today, diagnostics=None): kind = node.get("t") if isinstance(node, dict) else None if kind == "const": return node.get("v") + if kind == "error": + _diagnose(diagnostics, node["code"]); return None + if kind == "name" and not _reference_node(node["e"]): + return _eval(node["e"], row, today, diagnostics) + if kind in ("name", "reference_op", "spill") or (kind == "call" and node["fn"] in ("OFFSET", "INDIRECT")): + areas = _reference_areas(node, row, today, diagnostics) + if len(areas) == 1 and areas[0][1] == areas[0][2] and areas[0][3] == areas[0][4]: + area = areas[0] + return row.get(a1_key(index_to_col(area[1])+str(area[3]), area[0])) + if isinstance(row, _SheetInputs) and len(areas) == 1: + area = areas[0] + return Range(_area_values(areas, row, sparse=False), area[4]-area[3]+1, area[2]-area[1]+1) + if not diagnostics: _diagnose(diagnostics, "range_outside_function") + return None if kind == "ref": key = node.get("k") if key not in row: @@ -1310,17 +1735,26 @@ def _eval(node, row, today, diagnostics=None): _diagnose(diagnostics, "unknown_ref") return row.get(key) if kind == "a1range": + if isinstance(row, _SheetInputs): return _range_arg(node, row, today, diagnostics) # ⛔ A BARE RANGE IS NOT A VALUE. `=B4:B8` alone has no meaning; a range is only ever an # ARGUMENT to an aggregate, and the call dispatch below is what splices it in. Returning # the list here would let `=B4:B8*2` silently produce nonsense. _diagnose(diagnostics, "range_outside_function") return None if kind == "neg": - n = _num(_eval(node["e"], row, today, diagnostics)); return None if n is None else -n + value = _eval(node["e"], row, today, diagnostics) + if isinstance(value, Range): return _array_binary(value, -1, "*", diagnostics) + n = _num(value) + return None if n is None else -n if kind == "bin": left = _eval(node["l"], row, today, diagnostics) right = _eval(node["r"], row, today, diagnostics) + if isinstance(left, Range) or isinstance(right, Range): + return _array_binary(left, right, node["op"], diagnostics) if node["op"] == "&": return _text(left) + _text(right) + if isinstance(row, _SheetInputs): + left, right = 0 if left is None else left, 0 if right is None else right + left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right a, b = _num(left), _num(right) if a is None or b is None: return None if node["op"] == "/" and b == 0: @@ -1331,38 +1765,55 @@ def _eval(node, row, today, diagnostics=None): if kind == "cmp": a = _eval(node["l"], row, today, diagnostics) b = _eval(node["r"], row, today, diagnostics) - if a is None or b is None: return None - na, nb = _num(a), _num(b) - if (na is None) != (nb is None): return None - x, y = (na, nb) if na is not None else (str(a).casefold(), str(b).casefold()) + if isinstance(a, Range) or isinstance(b, Range): + return _array_binary(a, b, node["op"], diagnostics) + if isinstance(row, _SheetInputs): + def comparison(value): + if isinstance(value, bool): return (2, value) + if isinstance(value, str): return (1, value.casefold()) + return (0, _excel_number(value or 0)) + x, y = comparison(a), comparison(b) + else: + if a is None or b is None: return None + na, nb = _num(a), _num(b) + if (na is None) != (nb is None): return None + x, y = (na, nb) if na is not None else (str(a).casefold(), str(b).casefold()) return {"<": x < y, "<=": x <= y, ">": x > y, ">=": x >= y, "=": x == y, "!=": x != y}[node["op"]] if kind == "call": fn = node["fn"] + if fn in ("RAND", "NOW", "TODAY") and isinstance(row, _SheetInputs): + return row.engine.volatile_value(node, fn) + if fn == "RAND": return random.random() # IF/IFERROR are lazy, so an unselected bad branch cannot blank the result. if fn == "IF": cond = _eval(node["args"][0], row, today, diagnostics) - if not isinstance(cond, bool): return None + if isinstance(row, _SheetInputs): cond = _logical(cond, diagnostics) + elif not isinstance(cond, bool): return None pick = 1 if cond else 2 return (_eval(node["args"][pick], row, today, diagnostics) if pick < len(node["args"]) else None) if fn == "IFERROR": # A handled failure is not an error of the whole formula, so diagnostics from the # losing branch remain local. + local = [] try: - first = _eval(node["args"][0], row, today, []) + first = _eval(node["args"][0], row, today, local) except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, - FormulaError): - first = None - return first if first is not None else _eval( - node["args"][1], row, today, diagnostics) + FormulaError, _ReferenceFault): + first = None; local.append("evaluation_error") + if isinstance(row, _SheetInputs): + answer = _eval(node["args"][1], row, today, diagnostics) if local else first + return "" if answer is None and not diagnostics else answer + return first if first is not None and not local else _eval(node["args"][1], row, today, diagnostics) if fn == "IFS": # Conditions are read in order and STOP at the first TRUE one, so a later branch that # would refuse never runs. Excel answers a no-match error when none of them is TRUE. pairs = node.get("args") or () for index in range(0, len(pairs) - 1, 2): condition = _eval(pairs[index], row, today, diagnostics) - if not isinstance(condition, bool): + if isinstance(row, _SheetInputs): condition = _logical(condition, diagnostics) + elif not isinstance(condition, bool): return None if condition: return _eval(pairs[index + 1], row, today, diagnostics) @@ -1373,19 +1824,24 @@ def _eval(node, row, today, diagnostics=None): # IFERROR. A lookup that found nothing is answered; a division by zero is still a # division by zero and its diagnostic is handed back to the caller unchanged. local = [] - first = _eval(node["args"][0], row, today, local) + try: + first = _eval(node["args"][0], row, today, local) + except _ReferenceFault as exc: + if row.engine.errors[str(exc)] != "no_match": raise + first = None; local.append("no_match") if "no_match" in local: - return _eval(node["args"][1], row, today, diagnostics) + first = _eval(node["args"][1], row, today, diagnostics) + return "" if first is None and isinstance(row, _SheetInputs) and not diagnostics else first for code in local: _diagnose(diagnostics, code) - return first + return "" if first is None and isinstance(row, _SheetInputs) and not local else first if fn == "ISERROR": # The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL. local = [] try: _eval(node["args"][0], row, today, local) except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, - FormulaError): + FormulaError, _ReferenceFault): return True return bool(local) # ⭐ A POSITIONAL FUNCTION TAKES ITS RANGES BY POSITION (W50-T16). See `RANGE_ARGS` for why @@ -1415,14 +1871,31 @@ def _eval(node, row, today, diagnostics=None): # so a range must become MORE ARGUMENTS rather than one list-shaped argument. Doing it here # instead of inside `_call` is what keeps all 46 functions unchanged. args = [] + if fn in ("COUNT", "COUNTA") and isinstance(row, _SheetInputs): + row.ignore_reference_errors += 1 + try: + for child in node.get("args") or (): + if _reference_node(child): args.extend(_area_values(_reference_areas(child, row, today, diagnostics), row)) + else: args.append(_eval(child, row, today, diagnostics)) + finally: row.ignore_reference_errors -= 1 + return _call(fn, args, today, diagnostics) for x in node.get("args") or (): + if _reference_node(x) and x.get("t") not in ("a1", "a1range"): + args.extend(_area_values(_reference_areas(x, row, today, diagnostics), row)) + continue if isinstance(x, dict) and x.get("t") == "a1range": + if "range" in x and isinstance(row, _SheetInputs): + args.extend(row.range_values(x["range"])) + continue for key in x.get("keys") or (): if key not in row: _diagnose(diagnostics, "unknown_ref") args.append(row.get(key)) else: - args.append(_eval(x, row, today, diagnostics)) + value = _eval(x, row, today, diagnostics) + if isinstance(value, Range) and fn in ("SUM", "AVERAGE", "COUNT", "COUNTA", "MIN", "MAX", "SUMPRODUCT"): + args.extend(value.values) + else: args.append(value) return _call(fn, args, today, diagnostics) raise FormulaError("formula AST node is not supported") @@ -1437,7 +1910,23 @@ def _addr_arg(node, diagnostics): def _range_arg(node, row, today, diagnostics): """One argument as a shaped `Range`. A scalar standing there becomes a 1 by 1 rectangle.""" + if _reference_node(node) and node.get("t") not in ("a1", "a1range"): + areas = _reference_areas(node, row, today, diagnostics) + if len(areas) != 1: _diagnose(diagnostics, "bad_value"); return Range.scalar(None) + s, c0, c1, r0, r1 = areas[0] + return Range(_area_values(areas, row, sparse=False), r1-r0+1, c1-c0+1) if isinstance(node, dict) and node.get("t") == "a1range": + if "range" in node and isinstance(row, _SheetInputs): + bounds = node["range"] + size = node["rows"] * node["cols"] + if size > MAX_RANGE_CELLS: + # Shaped array/lookup execution is extended by the array engine. Aggregates + # already read sparse ranges without enumerating blanks. + _diagnose(diagnostics, "range_too_large") + return Range.scalar(None) + return Range([row.get(a1_key(index_to_col(c) + str(r), bounds[0])) + for r in range(bounds[3], bounds[4] + 1) + for c in range(bounds[1], bounds[2] + 1)], node["rows"], node["cols"]) keys = list(node.get("keys") or ()) rows, cols = node.get("rows"), node.get("cols") if (isinstance(rows, bool) or isinstance(cols, bool) @@ -1449,7 +1938,47 @@ def _range_arg(node, row, today, diagnostics): # always contains empty cells; that is what COUNTBLANK counts and what SUM skips. Calling # it `unknown_ref` would refuse the commonest formula on any real sheet. return Range([row.get(key) for key in keys], rows, cols) - return Range.scalar(_eval(node, row, today, diagnostics)) + value = _eval(node, row, today, diagnostics) + return value if isinstance(value, Range) else Range.scalar(value) + + +def _array_binary(left, right, op, diagnostics): + a, b = _as_range(left), _as_range(right) + rows, cols = max(a.rows, b.rows), max(a.cols, b.cols) + if any(x.rows not in (1, rows) or x.cols not in (1, cols) for x in (a, b)): + _diagnose(diagnostics, "bad_value"); return None + values = [] + for r in range(rows): + for c in range(cols): + x, y = a.cell(r % a.rows+1, c % a.cols+1), b.cell(r % b.rows+1, c % b.cols+1) + if op == "&": value = _text(x) + _text(y) + elif op in ("=", "!=", "<", ">", "<=", ">="): + nx, ny = _num(x), _num(y) + x, y = (nx, ny) if nx is not None and ny is not None else (_text(x).casefold(), _text(y).casefold()) + value = {"=": lambda: x == y, "!=": lambda: x != y, "<": lambda: x < y, + ">": lambda: x > y, "<=": lambda: x <= y, ">=": lambda: x >= y}[op]() + else: + x, y = int(x) if isinstance(x, bool) else _num(x), int(y) if isinstance(y, bool) else _num(y) + if x is None or y is None: _diagnose(diagnostics, "bad_value"); return None + if op == "/" and y == 0: _diagnose(diagnostics, "div_zero"); return None + value = {"+": lambda: x+y, "-": lambda: x-y, "*": lambda: x*y, + "/": lambda: x/y, "^": lambda: x**y}[op]() + values.append(value) + return Range(values, rows, cols) if isinstance(left, Range) or isinstance(right, Range) else values[0] + + +def _english_sort_key(value): + """English collation: letters before their accents, canonical accent ties stay stable. + + Case is ignored, matching Intl.Collator('en', {sensitivity:'accent'}). NFD makes + composed/decomposed Latin accents identical without depending on the host locale. + """ + if isinstance(value, bool): return (2, int(value), ()) + if isinstance(value, (int, float)): return (0, value, ()) + if value is None: return (3, "", ()) + text = unicodedata.normalize("NFD", str(value).casefold()) + return (1, "".join(ch for ch in text if not unicodedata.combining(ch)), + tuple((i, ord(ch)) for i, ch in enumerate(text) if unicodedata.combining(ch))) def _strict_num(v, diagnostics): @@ -1461,6 +1990,9 @@ def _strict_num(v, diagnostics): different question. `CEILING("x", 2)` is not a request to round nothing, it is a mistake, and answering a number to it would hide the mistake inside a plausible total. """ + if isinstance(v, Range): + if len(v) != 1: _diagnose(diagnostics, "bad_value"); return None + v = v.values[0] if v is None or (isinstance(v, str) and v == ""): return None n = _num(v) @@ -1590,8 +2122,127 @@ def _percentile(values, k): return float(ordered[low] + (ordered[high] - ordered[low]) * (position - low)) +def _cashflow_rate(values, times, guess, diagnostics): + if (len(values) < 2 or not any(v < 0 for v in values) or not any(v > 0 for v in values) + or guess <= -1 or any(t < 0 for t in times)): + _diagnose(diagnostics, "numeric_error"); return None + # This bounded Newton solver is a function algorithm, never workbook iteration. + scale = max(abs(v) for v in values) + rate = guess + for _ in range(100): + try: + discount = [math.exp(-t * math.log1p(rate)) for t in times] + value = math.fsum(v/scale*d for v, d in zip(values, discount)) + derivative = math.fsum(-t*v/scale*d/(1+rate) for v, t, d in zip(values, times, discount)) + except (ValueError, OverflowError): break + if abs(value) < 1e-13: return rate + if not derivative or not math.isfinite(derivative): break + candidate = rate-value/derivative + if candidate <= -1: candidate = (rate-1)/2 + if not math.isfinite(candidate): break + if abs(candidate-rate) < 1e-12 and abs(value) < 1e-9: return candidate + rate = candidate + _diagnose(diagnostics, "numeric_error") + return None + + +def _baht_text(number): + digits = ("ศูนย์", "หนึ่ง", "สอง", "สาม", "สี่", "ห้า", "หก", "เจ็ด", "แปด", "เก้า") + places = ("", "สิบ", "ร้อย", "พัน", "หมื่น", "แสน") + def spell(integer): + if integer >= 1_000_000: + high, low = divmod(integer, 1_000_000) + return spell(high) + "ล้าน" + (spell(low) if low else "") + text = "" + for position in range(5, -1, -1): + digit = integer // 10**position % 10 + if not digit: continue + word = "ยี่" if position == 1 and digit == 2 else "" if position == 1 and digit == 1 else "เอ็ด" if position == 0 and digit == 1 and integer > 10 else digits[digit] + text += word + places[position] + return text or digits[0] + cents = int(Decimal(str(abs(number))).quantize(Decimal(".01"), rounding=ROUND_HALF_UP)*100) + whole, fraction = divmod(cents, 100) + return ("ลบ" if number < 0 else "") + spell(whole) + "บาท" + (spell(fraction) + "สตางค์" if fraction else "ถ้วน") + + def _call(fn, a, today, diagnostics=None): nums = [_num(x) for x in a]; present = [x for x in nums if x is not None] + if fn in ("ADD", "MULTIPLY", "FV", "NORM.DIST", "BAHTTEXT"): + numeric = [float(v) if isinstance(v, bool) else 0.0 if v is None else _num(v) for v in a] + if any(v is None for v in numeric): _diagnose(diagnostics, "bad_value"); return None + if fn == "ADD": return numeric[0]+numeric[1] + if fn == "MULTIPLY": return numeric[0]*numeric[1] + if fn == "BAHTTEXT": return _baht_text(numeric[0]) + if fn == "FV": + rate, nper, payment = numeric[:3] + pv, when = numeric[3] if len(numeric) > 3 else 0, numeric[4] if len(numeric) > 4 else 0 + if when not in (0, 1): _diagnose(diagnostics, "bad_value"); return None + if rate == 0: return -(pv+payment*nper) + growth = (1+rate)**nper + if isinstance(growth, complex): _diagnose(diagnostics, "numeric_error"); return None + return -(pv*growth+payment*(1+rate*when)*(growth-1)/rate) + x, mean, sigma = numeric[:3] + if sigma <= 0: _diagnose(diagnostics, "numeric_error"); return None + z = (x-mean)/sigma + return (math.erfc(-z/math.sqrt(2))/2 if _logical(a[3], diagnostics) + else math.exp(-z*z/2)/(sigma*math.sqrt(2*math.pi))) + if fn == "DATEDIF": + start, end = _date(a[0]), _date(a[1]); unit = _text(a[2]).upper() + if start is None or end is None: _diagnose(diagnostics, "bad_value"); return None + if start > end or unit not in ("Y", "M", "D", "YM", "YD", "MD"): + _diagnose(diagnostics, "numeric_error"); return None + months = (end.year-start.year)*12+end.month-start.month-(end.day < start.day) + if unit == "Y": return months//12 + if unit == "M": return months + if unit == "YM": return months % 12 + if unit == "D": return (end-start).days + if unit == "MD": + # Excel's documented legacy MD algorithm can return a negative day count. + return end.day-start.day+(calendar.monthrange(end.year if end.month > 1 else end.year-1, end.month-1 if end.month > 1 else 12)[1] if end.day < start.day else 0) + year = end.year - ((end.month, end.day) < (start.month, start.day)) + anniversary = dt.date(year, start.month, 1)+dt.timedelta(days=start.day-1) + return (end-anniversary).days + if fn in ("IRR", "XIRR"): + raw = _as_range(a[0]).values + values = [float(v) for v in raw if isinstance(v, (int, float)) and not isinstance(v, bool)] + guess_at = 1 if fn == "IRR" else 2 + guess = _num(a[guess_at]) if len(a) > guess_at else .1 + if guess is None: _diagnose(diagnostics, "bad_value"); return None + if fn == "IRR": return _cashflow_rate(values, range(len(values)), guess, diagnostics) + dates = [_date(v) for v in _as_range(a[1]).values] + if len(values) != len(dates) or any(v is None for v in dates): + _diagnose(diagnostics, "numeric_error"); return None + return _cashflow_rate(values, [(v-dates[0]).days/365 for v in dates], guess, diagnostics) + if fn == "FILTER": + array, include = _as_range(a[0]), _as_range(a[1]) + vertical = include.cols == 1 and include.rows == array.rows + horizontal = include.rows == 1 and include.cols == array.cols + if not vertical and not horizontal: + _diagnose(diagnostics, "bad_value"); return None + if any(not isinstance(v, (bool, int, float)) and v is not None for v in include.values): + _diagnose(diagnostics, "bad_value"); return None + selected = [i+1 for i, value in enumerate(include.values) if value] + if not selected: + if len(a) == 3: return _as_range(a[2]) + _diagnose(diagnostics, "empty_array"); return None + rows = selected if vertical else range(1, array.rows+1) + cols = range(1, array.cols+1) if vertical else selected + return Range([array.cell(r, c) for r in rows for c in cols], len(rows), len(cols)) + if fn == "SORT": + array = _as_range(a[0]) + index = _strict_num(a[1], diagnostics) if len(a) > 1 else 1 + order = _strict_num(a[2], diagnostics) if len(a) > 2 else 1 + by_col = a[3] if len(a) > 3 else False + if not isinstance(by_col, bool) or index is None or int(index) != index or order not in (-1, 1): + _diagnose(diagnostics, "bad_value"); return None + span = array.rows if by_col else array.cols + if not 1 <= index <= span: + _diagnose(diagnostics, "bad_value"); return None + indices = sorted(range(1, (array.cols if by_col else array.rows)+1), + key=lambda i: _english_sort_key(array.cell(int(index), i) if by_col else array.cell(i, int(index))), + reverse=order == -1) + return Range([array.cell(r, c) for r in (range(1, array.rows+1) if by_col else indices) + for c in (indices if by_col else range(1, array.cols+1))], array.rows, array.cols) if fn == "ABS": return abs(nums[0]) if nums[0] is not None else None if fn == "ROUND": return _round(a[0], a[1] if len(a) > 1 else 0) if fn == "ROUNDUP": @@ -1773,18 +2424,30 @@ def _call(fn, a, today, diagnostics=None): return table.cell(hit, offset) if vertical else table.cell(offset, hit) if fn == "XLOOKUP": lookup_array, return_array = _as_range(a[1]), _as_range(a[2]) - if len(lookup_array) != len(return_array): + vertical = lookup_array.cols == 1 + if ((not vertical and lookup_array.rows != 1) + or (lookup_array.rows != return_array.rows if vertical else lookup_array.cols != return_array.cols)): _diagnose(diagnostics, "bad_value") return None + if isinstance(a[0], Range): + wanted = a[0] + pieces = [_call(fn, [value, *a[1:]], today, diagnostics) for value in wanted.values] + if diagnostics: return None + pieces = [_as_range(piece) for piece in pieces] + if any((p.rows, p.cols) != (1, 1) for p in pieces): + _diagnose(diagnostics, "bad_value"); return None + return Range([p.values[0] for p in pieces], wanted.rows, wanted.cols) mode = 0 if len(a) > 4: mode = _strict_num(a[4], diagnostics) if mode is None: return None mode = int(mode) + if mode not in (-1, 0, 1, 2): _diagnose(diagnostics, "bad_value"); return None hit = None for i, value in enumerate(lookup_array.values): - if _values_equal(value, a[0]): + if (bool(re.fullmatch(_wildcard_pattern(_text(a[0])), _text(value), re.I)) + if mode == 2 else _values_equal(value, a[0])): hit = i break if hit is None and mode in (-1, 1): @@ -1802,7 +2465,10 @@ def _call(fn, a, today, diagnostics=None): return a[3] _diagnose(diagnostics, "no_match") return None - return return_array.values[hit] + values = ([return_array.cell(hit+1, c) for c in range(1, return_array.cols+1)] if vertical + else [return_array.cell(r, hit+1) for r in range(1, return_array.rows+1)]) + return values[0] if len(values) == 1 else Range(values, 1 if vertical else return_array.rows, + return_array.cols if vertical else 1) if fn == "INDEX": array = _as_range(a[0]) first = _strict_num(a[1], diagnostics) @@ -1818,13 +2484,14 @@ def _call(fn, a, today, diagnostics=None): elif array.cols == 1: wanted_row, wanted_col = int(first), 1 else: - # A single index into a rectangle is ambiguous, and Excel answers a whole row or - # column there. This engine has no array values, so it refuses rather than guess. - _diagnose(diagnostics, "bad_value") - return None - if not (1 <= wanted_row <= array.rows and 1 <= wanted_col <= array.cols): + wanted_row, wanted_col = int(first), 0 + if not (0 <= wanted_row <= array.rows and 0 <= wanted_col <= array.cols): _diagnose(diagnostics, "bad_value") return None + if wanted_row == 0 or wanted_col == 0: + rows = [wanted_row] if wanted_row else range(1, array.rows+1) + cols = [wanted_col] if wanted_col else range(1, array.cols+1) + return Range([array.cell(r, c) for r in rows for c in cols], len(rows), len(cols)) return array.cell(wanted_row, wanted_col) if fn == "MATCH": array = _as_range(a[1]) @@ -2216,130 +2883,863 @@ def _unknown_sheets(definition, known): return out -def evaluate_sheet(cells, *, today=None, sheets=None): - """Evaluate a whole WORKBOOK of A1 cells in dependency order (wave 49 W49-T03, W51-T23). - - `cells` is `{addr_key: {"formula": "=..."} or {"value": v}}` where `addr_key` is what - `a1_key` produces: `B4` on a single unnamed tab, `SHEET2!C3` when the caller keys a workbook. - Returns `{"values": {addr: value}, "errors": {addr: code}}`. - - ⭐⭐ A SPEC ENTRY MAY CARRY `"sheet"`, AND THAT IS HOW A WORKBOOK GETS ONE MEMOIZED PASS - (W51-T23). It names the tab that cell lives on, and it is handed to `parse_formula` so this - formula's own unqualified references key as `TAB!B4` rather than `B4`. Every tab in the book - therefore goes into ONE `cells` dict with one `resolve` memo over it, so a cross-tab reference - is an ordinary lookup and a chain that crosses tabs twice is still computed once. - ⛔ QUALIFYING THE KEYS WITHOUT `"sheet"` IS THE PROTOTYPE'S FAILURE MODE 3, MEASURED: cross-tab - references resolve and every LOCAL one breaks, 3,770 blanks against a true 785. - - `sheets` is the list of tab names the workbook actually has. When it is supplied, a formula - naming a tab outside it is refused `unknown_sheet` rather than quietly reading as blank; when - it is None nothing is checked, so every wave-49 caller keeps its behaviour exactly. - - ⚠ `SHEET_ADDR_KEY` IS STILL HANDED THE FULL KEY, and `ROW()` / `COLUMN()` still answer, - because `_call` splits an address at its last `!` before `_split_addr` reads it. That split - lands the same way for `G4` and for `TAB05!G4`, so the bare and the argument spelling both - survive a workbook pass. It has its own gate leg because it does not look like it would. - - ⛔⛔ **THE REFUSAL IS `circular_reference`, AND IT IS DELIBERATELY NOT THE `"cycle"` THAT - `evaluate_fields` ALREADY USES.** Renaming that one was considered and rejected on evidence: - `"cycle"` is a SHIPPED WIRE CODE. It is published in `contract_dto()`'s `errorCodes`, so it is - on the wire to the browser; `routes_query.py` branches on it behaviourally; and - `verify_w45_semantic_apps.py` pins the string in two fixtures. Both of those files belong to - another lane's fence. - - ⭐ AND THE TWO ARE GENUINELY DIFFERENT GRAPHS, which is what makes keeping both honest rather - than a dodge. `evaluate_fields`'s cycle is over FIELD MEMBERS within one grid row: every node is - a column of the same record. A sheet cycle is over CELL ADDRESSES and spans tabs, so its nodes - are not fields, its edges come from `parse_formula`'s `cells` list rather than `refs`, and a - range edge expands to many nodes at once. Unifying them would mean one detector walking two - unrelated graphs. If you are here to merge these names, read this paragraph first: the merge - breaks `routes_query.py` and a lane C gate in a single edit. - """ - values, errors, active, resolved = {}, {}, [], set() - parsed = {} - known = None if sheets is None else {str(n).upper() for n in sheets if str(n or "")} +def _range_bounds(first, last, sheet=""): + a, ar = _split_addr(first) + b, br = _split_addr(last) + ac, bc = col_to_index(a), col_to_index(b) + return (str(sheet).upper(), min(ac, bc), max(ac, bc), min(ar, br), max(ar, br)) + + +_BAND_SCAN = re.compile( + r'"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\'(?!\!)|' + r"(?(?:(?:'[^']{1,64}'|[A-Za-z_][A-Za-z0-9_]{0,63})!)?)" + r"\$?(?P[A-Za-z]{1,3}|[1-9][0-9]{0,6}):\$?(?P[A-Za-z]{1,3}|[1-9][0-9]{0,6})" + r"(?![A-Za-z0-9_$!])") + + +def _expand_bands(source): + """Normalize whole bands to bounded Excel rectangles, without expanding any cells.""" + def replace(match): + first, last = match.group("first"), match.group("last") + if not first or first.isalpha() != last.isalpha(): + return match.group(0) + ends = (first + "1:" + last + "1048576" if first.isalpha() + else "A" + first + ":XFD" + last) + return match.group("prefix") + ends + return _BAND_SCAN.sub(replace, source) + + +def _address_parts(key): + sheet, _, addr = str(key).rpartition("!") + col, row = _split_addr(addr) + return sheet.upper(), col_to_index(col), row + + +def _segments(lo, hi): + """Canonical dyadic cover. A million-row band takes one subscription, not a million.""" + lo -= 1 + while lo < hi: + width = (lo & -lo) if lo else 1 << (hi.bit_length() - 1) + while width > hi - lo: + width >>= 1 + yield width.bit_length() - 1, lo // width + lo += width + + +class _RangeIndex: + def __init__(self): + self.buckets, self.owned = {}, {} + self.levels = {} + + def add(self, owner, bounds): + sheet, c0, c1, r0, r1 = bounds + for cl, ci in _segments(c0, c1): + for rl, ri in _segments(r0, r1): + token = (sheet, cl, ci, rl, ri) + if token not in self.buckets: + levels = self.levels.setdefault(sheet, {}) + levels[cl, rl] = levels.get((cl, rl), 0) + 1 + self.buckets.setdefault(token, set()).add(owner) + self.owned.setdefault(owner, set()).add(token) + + def remove(self, owner): + for token in self.owned.pop(owner, ()): + self.buckets[token].discard(owner) + if not self.buckets[token]: + del self.buckets[token] + sheet, cl, _, rl, _ = token + levels = self.levels[sheet] + levels[cl, rl] -= 1 + if not levels[cl, rl]: del levels[cl, rl] + if not levels: del self.levels[sheet] + + def at(self, key, stats): + sheet, col, row = _address_parts(key) + out = set() + for cl, rl in self.levels.get(sheet, ()): + stats["range_probes"] += 1 + hits = self.buckets.get((sheet, cl, (col - 1) >> cl, rl, (row - 1) >> rl), ()) + stats["range_checks"] += len(hits) + out.update(hits) + return out + + +class _SparseCells: + """Sorted populated coordinates; range queries never inspect unrelated cells or formulas.""" + def __init__(self): + self.columns, self.rows = {}, {} + + def add(self, key): + sheet, col, row = _address_parts(key) + cols = self.columns.setdefault(sheet, []) + rows = self.rows.get((sheet, col)) + if rows is None: + insort(cols, col) + rows = self.rows[(sheet, col)] = [] + at = bisect_left(rows, row) + if at == len(rows) or rows[at] != row: + rows.insert(at, row) + + def remove(self, key): + sheet, col, row = _address_parts(key) + rows = self.rows.get((sheet, col), []) + at = bisect_left(rows, row) + if at < len(rows) and rows[at] == row: + rows.pop(at) + if not rows and (sheet, col) in self.rows: + del self.rows[sheet, col] + self.columns[sheet].remove(col) + + def within(self, bounds): + sheet, c0, c1, r0, r1 = bounds + cols = self.columns.get(sheet, []) + found = [] + for col in cols[bisect_left(cols, c0):bisect_right(cols, c1)]: + rows = self.rows[sheet, col] + found.extend((row, col) for row in rows[bisect_left(rows, r0):bisect_right(rows, r1)]) + for row, col in sorted(found): + yield a1_key(index_to_col(col) + str(row), sheet) + + +class _SheetInputs(dict): + def __init__(self, engine, address): + super().__init__({SHEET_ADDR_KEY: address}) + self.engine = engine + self.ignore_reference_errors = 0 + self.format_stack = [] + self.last_format = None + + def __contains__(self, key): + return True # Missing Sheet cells are blanks, as in the existing evaluate_sheet DTO. + + def get(self, key, default=None): + if key == SHEET_ADDR_KEY: + return super().get(key, default) + self.engine._consume(super().get(SHEET_ADDR_KEY), key) + owner = self.engine.spill_parents.get(key) + if owner in self.engine._pending: raise _NeedReference(owner) + fault = key if key in self.engine.errors else None + if not _occupied(self.engine.specs.get(key)): + producers = self.engine.prospective.at(key, self.engine.stats) + pending = producers & self.engine._pending + if pending: raise _NeedReference(sorted(pending)[0]) + fault = next((ref for ref in sorted(producers) if self.engine.errors.get(ref) == "circular_reference"), fault) + if key in self.engine._pending: + raise _NeedReference(key) + if fault: + if self.ignore_reference_errors: return "#ERROR!" + raise _ReferenceFault(fault) + self.engine.stats["reads"] += 1 + if self.format_stack: self.format_stack[-1].append(self.engine.formats.get(key)) + return self.engine.values.get(key, default) + + def range_values(self, bounds): + # A sparse range must settle potential producers before deciding which cells exist. + for key in self.engine.array_positions.within(bounds): + if key in self.engine._pending: raise _NeedReference(key) + for key in self.engine.array_roots: + if key not in self.engine._pending: continue + area = self.engine.prospective_bounds.get(key) + if area and area[0] == bounds[0] and max(area[1], bounds[1]) <= min(area[2], bounds[2]) and max(area[3], bounds[3]) <= min(area[4], bounds[4]): + raise _NeedReference(key) + return [self.get(key) for key in self.engine.positions.within(bounds)] + + +def _spill_marker(spec, key): + marker = (spec.get("formulaMeta") or {}).get("spillChild") if isinstance(spec, dict) else None + return (marker if isinstance(marker, dict) and marker.get("version") == 1 + and marker.get("address") == key and isinstance(marker.get("origin"), str) + and _is_a1_key(marker["origin"]) and _sheet_half(marker["origin"]) == _sheet_half(key) + and marker["origin"] != key and spec.get("kind") != "formula" else None) + + +def _occupied(spec): + if not isinstance(spec, dict): return spec is not None + if spec.get("kind") == "empty": return False + return (spec.get("kind") in ("formula", "bound", "binding") or bool(spec.get("formula")) + or spec.get("binding") is not None or spec.get("value") is not None + or spec.get("text") is not None or bool(spec.get("refused"))) + + +@lru_cache(maxsize=8192) +def _static_reference_shape(text): + return _array_shape(_sheet_syntax(text, "")) + + +def _array_shape(node, lookup=None, seen=()): + """Conservative output extent, independent of values; scalars never reserve children.""" + kind, args = node.get("t"), node.get("args", []) + if kind == "a1range": return (node["rows"], node["cols"]) + shape = lambda child: _array_shape(child, lookup, seen) + if kind in ("name", "neg"): return shape(node["e"]) + if kind == "spill" and lookup and node["e"].get("t") == "a1": + key = node["e"]["k"] + target = lookup(key) + if target and key not in seen: return _array_shape(target, lookup, (*seen, key)) + if kind in ("bin", "cmp"): + a, b = shape(node["l"]), shape(node["r"]) + return max(a[0], b[0]), max(a[1], b[1]) + if kind == "call": + fn = node["fn"] + if fn in ("SORT", "FILTER"): return shape(args[0]) + if fn == "INDIRECT" and args[0].get("t") == "const" and isinstance(args[0].get("v"), str): + try: return _static_reference_shape(args[0]["v"]) + except FormulaError: return 1, 1 + if fn == "OFFSET": + rows, cols = shape(args[0]) + def dimension(index, default): + value = args[index].get("v") if len(args) > index else None + return max(1, int(value)) if isinstance(value, (int, float)) and math.isfinite(value) else default + return dimension(3, rows), dimension(4, cols) + if fn == "IF": + shapes = [shape(arg) for arg in args[1:]] + return max(s[0] for s in shapes), max(s[1] for s in shapes) + if fn in ("IFERROR", "IFNA"): + a, b = [shape(arg) for arg in args]; return max(a[0], b[0]), max(a[1], b[1]) + if fn == "XLOOKUP": + source, result = shape(args[1]), shape(args[2]) + wanted = shape(args[0]) + return wanted if wanted[0]*wanted[1] > 1 else (1, result[1]) if source[1] == 1 else (result[0], 1) + if fn == "INDEX": + rows, cols = shape(args[0]) + r = args[1].get("v") + c = args[2].get("v") if len(args) > 2 else 0 if rows > 1 and cols > 1 else 1 + return (rows if r == 0 else 1, cols if c == 0 else 1) + return 1, 1 + + +def _dynamic_array_shape(node): + if node.get("t") == "call": + args = node.get("args", []) + if node["fn"] == "INDIRECT" and args[0].get("t") != "const": return True + if node["fn"] == "OFFSET" and any(arg.get("t") != "const" for arg in args[3:]): return True + return any(_dynamic_array_shape(arg) for arg in args) + return any(_dynamic_array_shape(node[k]) for k in ("e", "l", "r") if isinstance(node.get(k), dict)) + + +def _calculation_stats(): + return dict(compiled=0, evaluated=0, visited=0, scanned=0, edges=0, + range_checks=0, range_probes=0, reads=0) + + +class _NeedReference(Exception): + pass - for addr, spec in (cells or {}).items(): - if not isinstance(spec, dict): - continue - # ⛔⛔ A CELL THAT ALREADY REFUSED IS CARRIED AS A REFUSAL, NOT AS AN ABSENCE (W49-T09, C2). - # This is the whole of "refusals propagate". If a refused cell were simply left out of the - # inputs, a formula over it would answer `unknown_ref` - "this points at a cell that is not - # filled in" - which is a DIFFERENT and misleading sentence: the cell IS filled in, and the - # reason it has no value is one the reader needs to see. Worse, `SUM` over an absent cell - # answers a NUMBER, so the refusal would vanish into a smaller total that looks fine. - if spec.get("refused"): - errors[addr] = str(spec["refused"]) - values[addr] = None - resolved.add(addr) - continue - raw = spec.get("formula") - if isinstance(raw, str) and raw.strip().startswith("="): - try: - definition = parse_formula(raw.strip()[1:], sheet=str(spec.get("sheet") or "")) - except FormulaError: - errors[addr] = "bad_formula" - values[addr] = None - continue - # ⛔ A MISSING TAB IS ITS OWN REFUSAL, NOT A BLANK. Left to the lookup it would read as - # `unknown_ref` at best and as a confident 0 through a range at worst, and both send - # the reader to inspect a formula that is correct except for a tab that is not there. - if known is not None and _unknown_sheets(definition, known): - errors[addr] = "unknown_sheet" - values[addr] = None + +class _ReferenceFault(Exception): + pass + + +def _cyclic_components(nodes, reads): + """Iterative Kosaraju: cycle members only, linear in the affected graph.""" + seen, finished = set(), [] + reverse = {} + for key in nodes: + for ref in reads.get(key, ()): + if ref in nodes: reverse.setdefault(ref, []).append(key) + for root in sorted(nodes): + if root in seen: continue + seen.add(root); stack = [(root, iter(sorted(set(reads.get(root, ())) & nodes)))] + while stack: + key, children = stack[-1] + ref = next(children, None) + if ref is None: finished.append(key); stack.pop() + elif ref not in seen: + seen.add(ref); stack.append((ref, iter(sorted(set(reads.get(ref, ())) & nodes)))) + seen, components = set(), [] + for root in reversed(finished): + if root in seen: continue + stack, members = [root], []; seen.add(root) + while stack: + key = stack.pop(); members.append(key) + for ref in reverse.get(key, ()): + if ref not in seen: seen.add(ref); stack.append(ref) + if len(members) > 1 or root in reads.get(root, ()): components.append(sorted(members)) + return sorted(components) + + +class SheetCalculation: + """C8 graph contract: ordered scalar/rectangle precedents, reverse subscriptions, revision. + + A reset visits the sparse snapshot once. An update visits only supplied cells and the + reachable subgraph, including range consumers of newly populated or deleted addresses. + Result dictionaries are owned by this instance; callers must not mutate them. + """ + def __init__(self, cells, *, revision=0, sheets=None, today=None, named_ranges=(), date_system="1900"): + self.revision = revision + self.sheets = None if sheets is None else {str(s).upper() for s in sheets} + self.today = today + self.named_ranges = _normalize_named_ranges(named_ranges) + self._reference_syntax_cache = OrderedDict() + self.date_system = date_system + self.volatile_roots, self._pending = set(), set() + self._volatile_values = {} + self._clock = dt.datetime.now(dt.timezone.utc) + self.stats = _calculation_stats() + self.specs, self.compiled, self.values, self.errors, self.provenance = {}, {}, {}, {}, {} + self.formats = {} + self.reads, self.read_by = {}, {} + self.consumed, self.consumed_by = {}, {} + self.ranges, self.positions, self.formula_positions = _RangeIndex(), _SparseCells(), _SparseCells() + self.spills, self.spill_parents, self.cleared_spills = {}, {}, {} + self.prospective, self.prospective_bounds = _RangeIndex(), {} + self.array_roots, self.array_positions = set(), _SparseCells() + for key, spec in (cells or {}).items(): + marker = _spill_marker(spec, str(key).upper()) + if marker: + self.cleared_spills[str(key).upper()] = marker["origin"] + base = marker.get("base") + if isinstance(base, dict) and not _occupied(base): self._put(str(key).upper(), base) continue - parsed[addr] = definition - else: - values[addr] = spec.get("value") - - def resolve(addr): - if addr in resolved: - return values.get(addr) - if addr not in parsed: - return values.get(addr) - if addr in active: - # Every member of the loop is refused, not just the one that closed it: a person - # looking at any cell in the cycle needs to be told it is part of one. - for member in active[active.index(addr):]: - errors[member] = "circular_reference" - values[member] = None - resolved.add(member) - return None - active.append(addr) - definition = parsed[addr] - # ⭐ THE CELL TELLS THE EVALUATOR WHERE IT IS (W50-T16). `ROW()` and `COLUMN()` with no - # argument have no other way to know, and this is the only caller that does. See - # `SHEET_ADDR_KEY` for why the key cannot collide with a field or a cell. - inputs = {SHEET_ADDR_KEY: addr} - inherited = None - for key in definition.get("cells") or (): - inputs[key] = resolve(key) if key in parsed else values.get(key) - # ⭐ THE UPSTREAM REFUSAL WINS, AND THE FIRST ONE FOUND IS THE ONE REPORTED. A formula - # over two refused cells has two reasons and one place to say them; naming the first is - # honest and actionable, while inventing a combined sentence is neither. The cell is - # refused either way, so no number is served. - if inherited is None and errors.get(key): - inherited = errors[key] - if errors.get(addr) == "circular_reference": - active.pop() - return None - if inherited is not None: - values[addr] = None - errors[addr] = inherited - active.pop() - resolved.add(addr) - return None - result = evaluate_result(definition, inputs, today=today) - active.pop() - values[addr] = result["value"] - if result["error"]: - errors[addr] = result["error"] - resolved.add(addr) - return values.get(addr) + self._put(str(key).upper(), spec) + for key in self.compiled: + self._connect(key) + self._calculate(set(self.compiled)) + + def _put(self, key, spec): + self.stats["scanned"] += 1 + self.errors.pop(key, None) + self.provenance.pop(key, None) + self.formats.pop(key, None) + if spec is None: + self.specs.pop(key, None) + self.values.pop(key, None) + self.positions.remove(key) + return + spec = dict(spec) if isinstance(spec, dict) else {"value": spec} + self.specs[key] = spec + fmt = (spec.get("style") or {}).get("format") + inferred = (spec.get("formulaMeta") or {}).get("inferredFormat") + if fmt and not (isinstance(inferred, dict) and inferred.get("version") == 1 and inferred.get("format") == fmt): + self.formats[key] = fmt + self.positions.add(key) + self.values[key] = spec.get("value") + if spec.get("refused"): + self._error(key, str(spec["refused"])) + return + source = spec.get("formula") + if not isinstance(source, str) or not source.strip().startswith("="): + return + self.stats["compiled"] += 1 + try: + definition = parse_formula(source.strip()[1:], sheet=spec.get("sheet") or "", compact_ranges=True, + named_ranges=self.named_ranges, address=key) + except FormulaError: + self._error(key, "bad_formula") + return + names = [_sheet_half(ref) for ref in definition["cells"]] + names.extend(bounds[0] for bounds in definition["ranges"]) + if self.sheets is not None and any(name and name not in self.sheets for name in names): + self._error(key, "unknown_sheet") + return + self.compiled[key] = definition + definition["dynamic_array_shape"] = _dynamic_array_shape(definition["ast"]) + rows, cols = _array_shape(definition["ast"]) + if rows*cols > 1: + s, c, r = _address_parts(key) + bounds = (s, c, min(16384, c+cols-1), r, min(1048576, r+rows-1)) + self.prospective.add(key, bounds); self.prospective_bounds[key] = bounds + self.array_roots.add(key); self.array_positions.add(key) + if definition["volatile"]: self.volatile_roots.add(key) + self.formula_positions.add(key) + + def reference_syntax(self, text, sheet, address): + # Relative named definitions depend on the calling cell. Ordinary A1 reference + # text depends only on its sheet; literal R1C1 offsets were resolved by the caller. + key = (text, sheet, address if self.named_ranges else "") + if key not in self._reference_syntax_cache: + self._reference_syntax_cache[key] = _sheet_syntax(text, sheet, self.named_ranges, address=address) + if len(self._reference_syntax_cache) > 8192: self._reference_syntax_cache.popitem(last=False) + else: self._reference_syntax_cache.move_to_end(key) + return self._reference_syntax_cache[key] + + def volatile_value(self, node, fn): + token = id(node) + if token not in self._volatile_values: + now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None) + epoch = dt.datetime(1904, 1, 1) if self.date_system == "1904" else dt.datetime(1899, 12, 30) + self._volatile_values[token] = (random.random() if fn == "RAND" else + (now - epoch).total_seconds() / 86400 if fn == "NOW" else (now - epoch).days) + return self._volatile_values[token] + + def _connect(self, key): + definition = self.compiled[key] + self._index_array(key) + deps = set(definition["cells"]) + for bounds in definition["ranges"]: + self.ranges.add(key, bounds) + deps.update(self.formula_positions.within(bounds)) + self.reads[key] = deps + for ref in deps: + self.read_by.setdefault(ref, set()).add(key) + self.stats["edges"] += 1 + + def _index_array(self, key): + rows, cols = _array_shape(self.compiled[key]["ast"], lambda ref: self.compiled.get(ref, {}).get("ast"), (key,)) + self.prospective.remove(key); self.prospective_bounds.pop(key, None); self.array_roots.discard(key) + if rows*cols > 1: + s, c, r = _address_parts(key) + area = (s, c, min(16384, c+cols-1), r, min(1048576, r+rows-1)) + self.prospective.add(key, area); self.prospective_bounds[key] = area; self.array_roots.add(key) + if self.compiled[key].get("dynamic_array_shape"): + self.array_roots.add(key); self.array_positions.add(key) + + def _disconnect(self, key): + self._forget_consumed(key) + self.prospective.remove(key); self.prospective_bounds.pop(key, None) + self.array_roots.discard(key); self.array_positions.remove(key) + self.volatile_roots.discard(key) + for ref in self.reads.pop(key, ()): + self.read_by[ref].discard(key) + if not self.read_by[ref]: + del self.read_by[ref] + self.ranges.remove(key) + self.compiled.pop(key, None) + self.formula_positions.remove(key) + + def _forget_consumed(self, key): + for ref in self.consumed.pop(key, ()): + owners = self.consumed_by[ref]; owners.discard(key) + if not owners: del self.consumed_by[ref] + + def _consume(self, key, ref): + refs = self.consumed.setdefault(key, set()) + if ref not in refs: + refs.add(ref); self.consumed_by.setdefault(ref, set()).add(key) + + def _spill_cycle(self, key, children): + # A newly discovered child closes a dependency through its producer. Only + # consumed references participate: an unused branch cannot create this edge. + # Walk the producer's precedents, never the workbook or all range subscribers. + if not any(self.consumed_by.get(child) for child in children): return False + targets, reachable, edges, queue = set(children), {key}, {}, [key] + for current in queue: + deps = set() + for ref in self.consumed.get(current, ()): + owner = key if ref in targets else self.spill_parents.get(ref, ref) + if owner in self.compiled: + deps.add(owner) + if owner not in reachable: reachable.add(owner); queue.append(owner) + edges[current] = deps + component = next((members for members in _cyclic_components(reachable, edges) if key in members), None) + if not component: return False + self._pass_cyclic.update(component) + for member in component: + self._error(member, "circular_reference"); self._pending.discard(member) + self.provenance[member].update(origin=component[0], via=None, path=component) + self.formats.pop(member, None) + # Earlier consumers can already hold scalars. A graph refusal invalidates + # those descendants once; they evaluate normally so IFERROR still catches. + queue, seen = [*component, *children], set(component) | set(children) + for ref in queue: + for owner in self.read_by.get(ref, set()) | self.consumed_by.get(ref, set()) | self.ranges.at(ref, self.stats): + if owner in seen or owner in self._pass_cyclic: continue + seen.add(owner); queue.append(owner) + if owner in self.compiled: + if owner not in self._pass_dirty: self.stats["visited"] += 1 + self._pass_dirty.add(owner); self._pending.add(owner) + return True - for addr in list(parsed): - resolve(addr) - return {"values": values, "errors": errors} + def _error(self, key, code, upstream=None): + self._clear_spill(key) + self.values[key] = None + self.errors[key] = code + trace = self.provenance.get(upstream) if upstream else None + self.provenance[key] = {"code": code, "origin": trace["origin"] if trace else key, + "via": upstream} + + def _clear_spill(self, key): + previous = self.spills.pop(key, None) + if not previous: return + for child in previous["children"]: + if self.spill_parents.get(child) != key: continue + self.spill_parents.pop(child, None) + self.formats.pop(child, None) + # Replacing an owned output with an authored literal retires ownership, + # but must not send a clear for that newly authored cell. + if not _occupied(self.specs.get(child)): + self.cleared_spills[child] = key + if child not in self.specs: + self.values.pop(child, None); self.positions.remove(child) + else: + self.values[child] = self.specs[child].get("value") + fmt = (self.specs[child].get("style") or {}).get("format") + if fmt: self.formats[child] = fmt + + def _publish(self, key, value): + self._clear_spill(key) + if not isinstance(value, Range): self.values[key] = value; return + s, c, r = _address_parts(key) + if r+value.rows-1 > 1048576 or c+value.cols-1 > 16384 or len(value) > MAX_RANGE_CELLS: + self._error(key, "spill_obstructed"); return + children = [a1_key(index_to_col(c+dc)+str(r+dr), s) + for dr in range(value.rows) for dc in range(value.cols)] + if any(_occupied(self.specs.get(child)) or child in self.spill_parents for child in children[1:]): + self._error(key, "spill_obstructed"); return + if self._spill_cycle(key, children[1:]): + bounds = (s, c, c+value.cols-1, r, r+value.rows-1) + self.prospective.add(key, bounds); self.prospective_bounds[key] = bounds + return + self.spills[key] = {"origin": key, "rows": value.rows, "cols": value.cols, + "range": (s, c, c+value.cols-1, r, r+value.rows-1), "children": children[1:]} + for child, item in zip(children, value.values): + self.values[child] = item + fmt = (self.specs.get(child, {}).get("style") or {}).get("format") if child != key else None + if fmt or self.formats.get(key): self.formats[child] = fmt or self.formats[key] + self.positions.add(child) + if child != key: + self.spill_parents[child] = key; self.cleared_spills.pop(child, None) + # Runtime shape discovery is a dependency event. Newly created targets can have + # subscribers that were clean when the input delta was scheduled. + queue = [] + for child in children[1:]: + if child not in self._output_notifications: + self._output_notifications.add(child); queue.append(child) + seen = set(queue) + for child in queue: + for owner in self.read_by.get(child, set()) | self.consumed_by.get(child, set()) | self.ranges.at(child, self.stats): + if owner in seen: continue + # An already evaluated formula needs the newly discovered edge only + # if it consumed that address. Static unused branches stay untouched. + if owner in self._pass_dirty and (owner in self._pending or owner not in self.consumed_by.get(child, ())): continue + seen.add(owner); queue.append(owner) + if owner in self.compiled: + if owner not in self._pass_dirty: self.stats["visited"] += 1 + self._pass_dirty.add(owner); self._pending.add(owner) + + def _calculate(self, dirty): + self._pass_dirty, self._output_notifications = dirty, set() + for key in dirty: self._forget_consumed(key) + for key in dirty: self._index_array(key) + for key in sorted(dirty): self._clear_spill(key) + indegree, dependents = {}, {} + for key in dirty: + self.stats["visited"] += 1 + deps = self.reads.get(key, set()) & dirty + indegree[key] = len(deps) + for ref in deps: + dependents.setdefault(ref, []).append(key) + self.stats["edges"] += 1 + ready = deque(sorted(k for k, degree in indegree.items() if not degree)) + ordered = [] + while ready: + key = ready.popleft() + ordered.append(key) + for dependent in sorted(dependents.get(key, ())): + indegree[dependent] -= 1 + if indegree[dependent] == 0: + ready.append(dependent) + blocked = sorted(dirty - set(ordered)) + components = _cyclic_components(set(blocked), self.reads) + cyclic = {member for component in components for member in component} + self._pass_cyclic = cyclic + for component in components: + for key in component: + self._error(key, "circular_reference") + self.provenance[key].update(origin=component[0], via=None, path=component) + self.formats.pop(key, None) + ordered += [key for key in blocked if key not in cyclic] + self._pending = set(ordered) + # Dynamic references can discover a dirty target after topological scheduling. + # Suspend the current AST, settle its target iteratively, then retry with the same + # per-pass volatile values. No recursive dependency walk and no whole-sheet scan. + def settle(key): + stack, active = [key], {key} + while stack: + current = stack[-1] + if current not in self._pending: + stack.pop(); active.discard(current); continue + needed = next((ref for ref in self.reads.get(current, ()) if ref in self._pending), None) + try: + if needed: raise _NeedReference(needed) + calculate_one(current) + except _NeedReference as exc: + ref = str(exc) + if ref in active: + component = sorted(stack[stack.index(ref):]) + for member in component: + self._error(member, "circular_reference"); self._pending.discard(member) + self.provenance[member].update(origin=component[0], via=None, path=component) + cyclic.add(member) + else: + stack.append(ref); active.add(ref) + continue + except _ReferenceFault as exc: + ref = str(exc); self._error(current, self.errors[ref], ref) + self._pending.discard(current) + stack.pop(); active.discard(current) + def calculate_one(key): + self.stats["evaluated"] += 1 + self._forget_consumed(key) + self.errors.pop(key, None) + self.provenance.pop(key, None) + inputs = _SheetInputs(self, key) + self.formats.pop(key, None) + result = evaluate_result(self.compiled[key], inputs, today=self.today) + spec = self.specs[key] + fmt = (spec.get("style") or {}).get("format") + inferred = (spec.get("formulaMeta") or {}).get("inferredFormat") + authored = fmt and not (isinstance(inferred, dict) and inferred.get("version") == 1 and inferred.get("format") == fmt) + if authored or inputs.last_format: self.formats[key] = fmt if authored else inputs.last_format + if result["error"]: + self._error(key, result["error"]) + else: self._publish(key, result["value"]) + for key in sorted(dirty & self.array_roots) + ordered: + if key in self._pending: settle(key) + while self._pending: + settle(min(self._pending)) + self.dirty, self.order, self.cyclic = dirty, ordered + sorted(dirty-set(ordered)-cyclic) + sorted(cyclic), sorted(cyclic) + + def update(self, changes, *, revision, today=None): + if revision != self.revision + 1: + raise ValueError("a calculation delta requires the next revision; rebuild from the snapshot") + self.stats = _calculation_stats() + self._volatile_values = {} + self._clock = dt.datetime.now(dt.timezone.utc) + self.today = today if today is not None else self.today + changed = {str(key).upper(): spec for key, spec in changes.items()} + self.cleared_spills = {} + spill_roots, old_children = set(), set() + for key in changed: + owner = self.spill_parents.get(key) + if owner: spill_roots.add(owner) + spill_roots.update(self.prospective.at(key, self.stats)) + old_children.update(self.spills.get(key, {}).get("children", ())) + for key in changed: self._clear_spill(key) + # Disconnect all edited formulas before reconnecting; a multi-cell paste sees one + # final graph. Existing range formulas gain/lose formula edges at the changed point. + consumers = {key: self.ranges.at(key, self.stats) for key in changed} + for key in changed: + self._disconnect(key) + for key, spec in changed.items(): + self._put(key, spec) + for key in changed: + if key in self.compiled: + self._connect(key) + for owner in consumers[key] - changed.keys(): + if key in self.compiled: + self.reads[owner].add(key) + self.read_by.setdefault(key, set()).add(owner) + elif key not in self.compiled[owner]["cells"]: + self.reads[owner].discard(key) + self.read_by.get(key, set()).discard(owner) + roots = set(changed) | self.volatile_roots | spill_roots | old_children + seen, dirty, queue = set(roots), set(), deque(roots) + while queue: + key = queue.popleft() + if key in self.compiled: + dirty.add(key) + for child in self.spills.get(key, {}).get("children", ()): + if child not in seen: seen.add(child); queue.append(child) + area = self.prospective_bounds.get(key) + if area and (area[2]-area[1]+1)*(area[4]-area[3]+1) <= MAX_RANGE_CELLS: + for r in range(area[3], area[4]+1): + for c in range(area[1], area[2]+1): + child = a1_key(index_to_col(c)+str(r), area[0]) + if child not in seen: seen.add(child); queue.append(child) + for owner in self.read_by.get(key, set()) | self.consumed_by.get(key, set()) | self.ranges.at(key, self.stats): + self.stats["edges"] += 1 + if owner not in seen: + seen.add(owner) + queue.append(owner) + self.revision = revision + self._calculate(dirty) + return self.result() + + def result(self): + return {"values": self.values, "errors": self.errors, "provenance": self.provenance, "formats": self.formats, + "spills": self.spills, "spillParents": self.spill_parents, "clearedSpills": self.cleared_spills, + "order": self.order, "cyclic": self.cyclic, "dirty": sorted(self.dirty), + "revision": self.revision, "stats": dict(self.stats)} + + +def evaluate_sheet(cells, *, today=None, sheets=None): + """Compatibility entry point over the same compiled Sheet graph as document refresh. + + Grid fields retain their separate row evaluator and legacy Boolean DTO. This wrapper + preserves its two-key response; document consumers get graph traces and formats directly. + """ + result = SheetCalculation(cells, today=today, sheets=sheets).result() + return {"values": result["values"], "errors": result["errors"]} + + +_DOCUMENT_CACHE_LIMIT = 16 +_DOCUMENT_CACHE_CELL_LIMIT = 200_000 +_document_calculations = OrderedDict() +_document_calculations_lock = RLock() +_UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00" + + +def _document_scopes(tabs): + names, seen = [], set() + for index, tab in enumerate(tabs): + name = str(tab.get("name") or "") + if not name or name.upper() in seen: + name = _UNADDRESSABLE_SHEET + str(index) + names.append(name) + seen.add(name.upper()) + return names + + +def _document_spec(cell, sheet): + # Calculation state/errors on formulas are previous OUTPUT, never trusted input. + if cell.get("kind") == "formula" and cell.get("formula"): + return {**cell, "formula": cell["formula"], "sheet": sheet} + if cell.get("state") == "refused" or cell.get("error"): + code = (cell.get("refusal") or cell.get("error") or {}).get("code") or "refused" + return {**cell, "refused": code, "sheet": sheet} + return {**cell, "sheet": sheet} + + +def _sheet_error_message(code): + from modules.sheet import REFUSALS + if code in REFUSALS: + return REFUSALS[code].format(subject="that database") + return { + "circular_reference": "This formula refers back to itself, so it has no value.", + "bad_formula": "This formula could not be read.", + "unknown_ref": "This formula points at a cell that is not filled in.", + "unknown_sheet": "This formula points at a tab that is not in this workbook.", + "div_zero": "This formula divides by zero.", + "no_match": "This lookup found nothing to match.", + "bad_value": "This function cannot use one of the values it was given.", + "null_intersection": "#NULL! These references do not intersect.", + "unknown_name": "#NAME? This name is not defined in this workbook.", + "external_reference": "#REF! External workbook references are not supported.", + "invalid_reference": "#REF! This reference is outside the worksheet.", + "spill_obstructed": "#SPILL! This array needs an empty range inside the worksheet.", + "empty_array": "#CALC! This filter has no matching values.", + }.get(code, "This formula could not be calculated.") + + +def recompute_document(book, *, cache_key=None, changes=None): + """Recalculate a C7 document in place; return C8 values/errors/provenance/work counters. + + Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU. + Consecutive revisions plus an explicit accepted delta are required. Unkeyed reads, + refreshes, imports, missing revisions, gaps, renames, names and date-system edits rebuild. + The output projection visits the cached formula index because the route returns all + formula values; it never scans a sheet's cell map on a valid delta. + """ + with _document_calculations_lock: + tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)] + names = _document_scopes(tabs) + tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)} + signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)], + book.get("namedRanges"), book.get("dateSystem"), + dt.date.today().isoformat()], sort_keys=True) + revision = book.get("rev") + valid_revision = type(revision) is int and revision >= 0 + trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3 + and all(isinstance(part, str) and part for part in cache_key)) + entry = _document_calculations.get(cache_key) if trusted else None + incremental = (entry is not None and changes is not None and valid_revision + and entry[0] == signature and entry[1].revision + 1 == revision) + if incremental: + _, engine, formula_keys = entry + delta = {} + for raw, cell in changes.items(): + key = str(raw).upper() + scope = _sheet_half(key) + if scope not in tab_by_scope: + incremental = False + break + delta[key] = _document_spec(cell, scope) if isinstance(cell, dict) else None + if isinstance(cell, dict) and cell.get("kind") == "formula": + formula_keys.add(key) + else: + formula_keys.discard(key) + if incremental: + engine.update(delta, revision=revision) + if not incremental: + specs, formula_keys = {}, set() + for tab, name in zip(tabs, names): + for addr, cell in (tab.get("cells") or {}).items(): + if not isinstance(cell, dict): + continue + key = a1_key(addr, name) + specs[key] = _document_spec(cell, name) + if cell.get("kind") == "formula": + formula_keys.add(key) + engine = SheetCalculation(specs, revision=revision if valid_revision else 0, sheets=names, + named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900")) + projected = 0 + for key in formula_keys: + scope, _, addr = key.rpartition("!") + cell = (tab_by_scope[scope].get("cells") or {}).get(addr) + if not isinstance(cell, dict): + continue + projected += 1 + code = engine.errors.get(key) + cell["value"] = engine.values.get(key) + if code: + message = _sheet_error_message(code) + cell.update(state="refused", display="", refusal={"code": code, "message": message}, + error={"code": code, "message": message, + **engine.provenance.get(key, {})}) + excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code) + if excel: cell["error"]["excelCode"] = excel + else: + cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"])) + cell.pop("refusal", None) + cell.pop("error", None) + cell.pop("spill", None) + meta = dict(cell.get("formulaMeta") or {}) + meta.pop("spill", None) + style = dict(cell.get("style") or {}) + inherited = meta.pop("inferredFormat", None) + if isinstance(inherited, dict) and inherited.get("version") == 1 and inherited.get("format") == style.get("format"): + style.pop("format", None) + if "format" not in style and engine.formats.get(key) and not code: + style["format"] = engine.formats[key] + meta["inferredFormat"] = {"version": 1, "format": engine.formats[key]} + if style or "style" in cell: cell["style"] = style + if key in engine.spills: + spill = engine.spills[key] + details = {"version": 1, "origin": key, "rows": spill["rows"], "cols": spill["cols"], + "ref": addr + ":" + index_to_col(spill["range"][2]) + str(spill["range"][4])} + cell["spill"] = details + meta["spill"] = details + if meta: cell["formulaMeta"] = meta + elif "formulaMeta" in cell: cell["formulaMeta"] = {} + # These are output records, never authored literals. The formulaMeta marker survives + # C7 migration and the existing route's spillParent predicate forwards clears as well. + for child, origin in engine.cleared_spills.items(): + scope, _, addr = child.rpartition("!") + if scope not in tab_by_scope: continue + cells = tab_by_scope[scope].setdefault("cells", {}) + existing = cells.get(addr) + if _occupied(engine.specs.get(child)) or (existing is not None and not _spill_marker(existing, child)): continue + base = engine.specs.get(child, {}) + cells[addr] = {"kind": "literal", "value": None, "display": "", "state": "typed", + **({"style": base["style"]} if base.get("style") else {}), + "spillParent": origin, "formulaMeta": {"spillClear": True, + "spillChild": {"version": 1, "origin": origin, "address": child, + **({"base": base} if base else {})}}} + for child, origin in engine.spill_parents.items(): + scope, _, addr = child.rpartition("!") + owner_scope, _, owner_addr = origin.rpartition("!") + owner = tab_by_scope[owner_scope]["cells"][owner_addr] + base = engine.specs.get(child, {}) + style = {**(owner.get("style") or {}), **(base.get("style") or {})} + record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh", + "display": "" if engine.values.get(child) is None else str(engine.values[child]), + "spillParent": origin, "style": style, + "formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child, + **({"base": base} if base else {})}}} + tab_by_scope[scope].setdefault("cells", {})[addr] = record + if trusted and valid_revision: + _document_calculations[cache_key] = (signature, engine, formula_keys) + _document_calculations.move_to_end(cache_key) + while (len(_document_calculations) > _DOCUMENT_CACHE_LIMIT + or sum(len(item[1].specs) for item in _document_calculations.values()) + > _DOCUMENT_CACHE_CELL_LIMIT): + _document_calculations.popitem(last=False) + elif trusted: + _document_calculations.pop(cache_key, None) + result = engine.result() + result["stats"].update(rebuilt=not incremental, projected=projected) + return result def evaluate_batch(definitions, rows, *, consumer, today=None, field_revisions=None): diff --git a/api/ig_master.py b/api/ig_master.py index c9da246fab8bdf844311840f147a74e176a3d1f9..e67538d7d3c47d7bd1eca921a80432144cf1add6 100644 --- a/api/ig_master.py +++ b/api/ig_master.py @@ -70,7 +70,7 @@ def append_run(tenant, snaps, posts, psnaps): own budget, and a per-row writer would spend it the same way). Empty inputs cost nothing. """ if not configured(): - return "off", "no master store on this deployment — tenant copy only" + return "off", "no master store on this deployment; tenant copy only" slug = str(tenant or "").strip() or "unknown" stamped = { SNAP_BUCKET: ([{**r, "tenant": slug} for r in snaps or []], "snapshot_key"), @@ -87,7 +87,7 @@ def append_run(tenant, snaps, posts, psnaps): existing = dict(h.get(bucket) or {}) merged, counts = upsert(existing, rows_in, key_field, cap=MASTER_MAX_ROWS) if counts.get("capped"): - capped.append(f"{bucket} is FULL ({MASTER_MAX_ROWS:,}) — " + capped.append(f"{bucket} is FULL ({MASTER_MAX_ROWS:,}); " f"{counts['capped']} rows lost") h.update(bucket, lambda cur, m=merged: m, flush="sync") except Exception as e: # noqa: BLE001 diff --git a/api/main.py b/api/main.py index c8eb7abda6b5831c2c4928aa0237c9d5e077eb3d..268093d6d988998f1062e2c5f5c3a190db4510f7 100644 --- a/api/main.py +++ b/api/main.py @@ -66,6 +66,7 @@ import routes_sheet # noqa: E402 (wave 49 instr 1 / W49-T02 / C1 — the Sheet import routes_customers # noqa: E402 import routes_grid # noqa: E402 import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) +import routes_external # noqa: E402 (wave 56 R3/C7 — per-user external AI credential) import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) import routes_nav # noqa: E402 import routes_pages # noqa: E402 @@ -461,6 +462,7 @@ app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent h # tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them # rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) +app.include_router(routes_external.router) # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -751,6 +753,37 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations "changed": True, "applied": False, "written": {}} +def _rebuild_amazon_relational(why, rt, previous_fingerprint=None): + """Spawn/refresh the `ut_amzn_*` locked databases off the SAME mirror the two passes above used. + + ⭐ CALLED IN THE SAME PLACE AND FOR THE SAME REASON AS `_rebuild_meta_relational`, which is + D-29 and not tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and the + caller has already established which tenant's file this process holds open. Spawning Amazon + here inherits that binding instead of rebinding it under live readers, which is the documented + way to serve one tenant's rows to another with nothing raised. + + ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Amazon has no `amzn_*` + tables in its mirror; `refresh_state` returns `{}` and says so once. That is the normal state + for every tenant but this one, and it must not print an error every 30 minutes for all of them. + + ⛔ ITS OWN try/except, for the reason the passes above have theirs: an Amazon refusal must not + cancel an Odoo or Meta rebuild that already succeeded. Amazon is the likeliest of the three to + refuse, because R5 ships Brand Analytics UNPROVEN and a missing role answers 403. + """ + try: + import amazon_relational as _amzn + state = _amzn.refresh_state(rt, why, previous_fingerprint=previous_fingerprint) + counts = state["written"] + if counts: + print(f"[aios-api] amazon relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + return state + except Exception as e: # noqa: BLE001 + print(f"[aios-api] amazon relational rebuild FAILED ({why}): {type(e).__name__}: {e}") + return {"present": False, "known": False, "fingerprint": None, + "changed": True, "applied": False, "written": {}} + + def _sweep_automation_schemas(why): """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. @@ -895,7 +928,21 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out _rt = _runtime.get_runtime("royal-imports") from core import user_tables as _ut _ut.record_odoo_sync_status(phase="syncing", st=_rt) - counts = _rel.refresh(_rt, "royal-imports") + # ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections` + # delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo + # connections in slot order and records a failure against the connection it belongs + # to rather than aborting the rest. The DEFAULT connection's counts stay at the top + # level of the returned dict, so the log line below is unchanged; the per-connection + # map rides beside them under `connections`. + # ⚠ A FLAG RATHER THAN CALLING THE ALL-CONNECTIONS FUNCTION BY NAME HERE, and the + # reason is a gate: `verify_odoo_relational` locates this call by its literal spelling + # and reads the code that follows it, to prove the relation cells are computed after + # the spawn. Spelling the boot call differently moves that window off the code it is + # meant to measure. ⛔ AND THE CALL IS NAMED IN WORDS ABOVE RATHER THAN WRITTEN OUT, + # because a gate that counts RAW occurrences of a call would count the one in this + # comment too. That gate happens to strip comments first; the next one may not. + # See `odoo_relational.refresh` and its `all_connections` arm. + counts = _rel.refresh(_rt, "royal-imports", all_connections=True) raw_models, schema_errors = _rel.authoritative_schema_snapshot() _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors, errors=schema_errors, st=_rt) @@ -939,6 +986,10 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True) if isinstance(meta_state_out, dict): meta_state_out.update(meta_state) + # ⚠ AFTER Meta and inside the same tenant binding — see `_rebuild_amazon_relational`. + # It carries no fingerprint here on purpose: this branch has just done a full Odoo + # rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid. + _rebuild_amazon_relational(why, _rt) # The isolated cells/rollups/meta handlers above are deliberately allowed to report and # continue. This return value answers the narrower resync-policy question: did the Odoo # table refresh itself complete, so a confirmed zero-change pass may be skipped? @@ -1223,6 +1274,11 @@ def _store_resync_loop(): last_relational_success_at = None last_meta_fingerprint = None last_meta_applied_at = None + # ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change + # on different cadences behind one variable would let a quiet Meta pass suppress an Amazon + # rebuild that was due -- one question with two answers, which is the defect class this + # codebase already names. Unknown is never carried forward as equality evidence. + last_amzn_fingerprint = None while True: # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. # @@ -1277,7 +1333,13 @@ def _store_resync_loop(): _RESYNC_NOTICE[0] = _cad["disagreement"] if _cad["disagreement"]: print(f"[aios-api] store resync cadence: {_cad['disagreement']}") - _t.sleep(_every) + # W58: the first resident pass is the boot catch-up when AIOS_BOOT_SYNC=0. Sleeping here + # unconditionally left a newly installed seed untouched for the full hourly cadence; + # stored fields such as customer_rank can change without partner write_date advancing, + # so staging served a provably stale customer population for its first hour. Subsequent + # passes still sleep at the freshly resolved cadence, and manual still skips only Odoo. + if passes: + _t.sleep(_every) passes += 1 RESYNC_STATE["passes"] = passes sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes) @@ -1328,6 +1390,7 @@ def _store_resync_loop(): _meta_reason = _meta_rebuild_force_reason( _meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now) _meta_previous = last_meta_fingerprint if _meta_reason is None else None + _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None _reason = _resync_rebuild_reason(sync_report, delete_report, last_relational_success_at, _meta_now) _meta_state = {} @@ -1335,13 +1398,22 @@ def _store_resync_loop(): print(f"[aios-api] odoo relational rebuild required ({_reason})") if _rebuild_odoo_relational("resync", _meta_previous, _meta_state): last_relational_success_at = _t.time() + else: + # A prior successful projection no longer proves the changed mirror was applied. + # Keep recovery due on the next cadence even when that sync has no new rows. + last_relational_success_at = None else: # A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides # whether this cadence may skip both the full-document write and relation pass. try: from harness import runtime as _runtime - _meta_state = _rebuild_meta_relational( - "resync", _runtime.get_runtime("royal-imports"), _meta_previous) + _rt_resync = _runtime.get_runtime("royal-imports") + _meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous) + # A zero Odoo report says nothing about Amazon either. Same reasoning, same pass, + # and it inherits the tenant binding the line above just established. + _amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous) + last_amzn_fingerprint = (_amzn_state["fingerprint"] + if _amzn_state.get("known") else None) except Exception as e: # noqa: BLE001 print(f"[aios-api] meta relational refresh failed (resync): " f"{type(e).__name__}: {e}") diff --git a/api/meta_relational.py b/api/meta_relational.py index d1d1270bd12977a5f3cfa53056d0907deac7c093..5b50b447e3acf9635b403f52e5d76b84d32f718f 100644 --- a/api/meta_relational.py +++ b/api/meta_relational.py @@ -40,10 +40,14 @@ if str(_API.parent.parent / "platform") not in sys.path: sys.path.insert(0, str(_API.parent.parent / "platform")) import odoo_relational as _odoo # noqa: E402 +from core import connector_spec # noqa: E402 from harness import datastore, meta_store # noqa: E402 Refused = _odoo.Refused +#: The six tables this module spawns, as names rather than repeated string literals. ⚠ They are the +#: same six keys `connector_spec.META_TABLES` declares; `INSIGHTS_KEY` is the one still USED below, +#: so a typo in it grows `SOURCE` a seventh entry and `verify_meta`'s coverage leg reddens. ACCOUNTS_KEY = "ut_meta_ad_accounts" CAMPAIGNS_KEY = "ut_meta_campaigns" ADSETS_KEY = "ut_meta_adsets" @@ -51,36 +55,47 @@ ADS_KEY = "ut_meta_ads" CREATIVES_KEY = "ut_meta_creatives" INSIGHTS_KEY = "ut_meta_insights" -#: ut_* key -> the mirror table `meta_store` writes. -SOURCE = { - ACCOUNTS_KEY: "meta_ad_accounts", - CAMPAIGNS_KEY: "meta_campaigns", - ADSETS_KEY: "meta_adsets", - ADS_KEY: "meta_ads", - CREATIVES_KEY: "meta_creatives", - INSIGHTS_KEY: meta_store.INSIGHTS_TABLE, -} - -LABELS = { - ACCOUNTS_KEY: "Meta ad accounts", CAMPAIGNS_KEY: "Meta campaigns", - ADSETS_KEY: "Meta ad sets", ADS_KEY: "Meta ads", - CREATIVES_KEY: "Meta creatives", INSIGHTS_KEY: "Meta insights (daily)", -} +#: ⭐⭐ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE** +#: (W53-T37, contract C2). `SOURCE`, `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their +#: module-level names because gates and routes read them BY NAME — but each is a DERIVED VIEW over +#: ONE declaration now, and Amazon's copy of the same five dicts is derived from the same file. Two +#: connectors stating one shape in two dialects is what C2 exists to end. +_SPEC = {s.key: s for s in connector_spec.tables_for("meta")} + +#: ut_* key -> the mirror table `meta_store` writes, in the spec's declaration order (which is the +#: order `meta_store` syncs them, and the order `plan` therefore walks). +SOURCE = {k: s.source for k, s in _SPEC.items()} +#: ⚠ ONE ENTRY STAYS DERIVED FROM THE LOADER RATHER THAN FROM THE SPEC, ON PURPOSE. The daily +#: insights grain is the only Meta mirror table whose name `meta_store` also owns +#: (`meta_store.INSIGHTS_TABLE`), and reading it from there keeps `verify_meta`'s Meta-source parity +#: leg a REAL cross-check for that table instead of comparing the spec with itself. The other five +#: mirror names exist only in the spec, so there is nothing to hold them against. +SOURCE[INSIGHTS_KEY] = meta_store.INSIGHTS_TABLE + +LABELS = {k: s.label for k, s in _SPEC.items()} #: The column each grid shows first. Everything else keeps the API's own order, which is the order #: the schema was measured in — a reader comparing the grid to Graph sees the same sequence. -PINNED = {ACCOUNTS_KEY: "name", CAMPAIGNS_KEY: "name", ADSETS_KEY: "name", - ADS_KEY: "name", CREATIVES_KEY: "name", INSIGHTS_KEY: "date_start"} +PINNED = {k: s.pinned for k, s in _SPEC.items()} -#: (this table's column) -> (target ut_ key, the column it matches there). Derived links, so a +#: ⭐ THE ROW IDENTITY PER TABLE, IN FALLING PREFERENCE, NEW IN W53-T37 — this module had no +#: `IDENTITY` at all before the spec declared one. ⚠ NOTHING IN THIS MODULE READS IT YET: `_read` +#: keys every row on the Graph `id` inline, so this is the spec's answer made reachable by name for +#: the callers that will ask (the relation engine, the connector detail door), not a behaviour +#: change. Wiring a consumer that did not exist is a different ticket. +IDENTITY = {k: tuple(s.identity) for k, s in _SPEC.items()} + +#: `{ut_ key: [(this table's column, target ut_ key, the column it matches there), ...]}`, so a #: person can walk account -> campaign -> ad set -> ad in the product the way they do in Ads -#: Manager. ⚠ Only emitted when BOTH sides exist in the mirror. -LINKS = { - CAMPAIGNS_KEY: ("account_id", ACCOUNTS_KEY, "account_id"), - ADSETS_KEY: ("campaign_id", CAMPAIGNS_KEY, "id"), - ADS_KEY: ("adset_id", ADSETS_KEY, "id"), - INSIGHTS_KEY: ("ad_id", ADS_KEY, "id"), -} +#: Manager. ⚠ Only emitted when this table's own mirror carries the column. +#: ⛔⛔ **A LIST PER KEY, NOT A SCALAR 3-TUPLE** (W53-T37, and it is the point of the ticket). The +#: old scalar could hold exactly one edge per table, and the key it generated was +#: `f"{target}_link"` — so a SECOND edge to the same target would have collided on both the field +#: key and the field label and silently overwritten the first. Meta's chain is linear today, so +#: nothing collided and no gate was red; it was latent, and Amazon had already fixed the same +#: defect. The shape and the generated key are now Amazon's, from one shared generator. +LINKS = {k: [(c, t, o) for c, t, o, _cardinality in s.links] + for k, s in _SPEC.items() if s.links} #: DuckDB type -> `core.user_tables.UT_FIELD_TYPES` member. ⚠ `_clean_field` returns None for an #: unknown type, which DELETES the column on the next read rather than erroring — so this map only @@ -131,7 +146,7 @@ def _fields_for(cur, key): if not cols: return [] pinned = PINNED.get(key) - link = LINKS.get(key) + names = {c[0] for c in cols} out = [] for name, dtype in cols: f = {"key": name, "label": _label(name), "type": _ftype(name, dtype), @@ -139,13 +154,18 @@ def _fields_for(cur, key): "impressions", "clicks", "date_start"), "pinned": name == pinned} out.append(_odoo._preset(f, flow="meta_relational")) - if link: - col, target, on = link - if any(c[0] == col for c in cols): + # ⭐⭐ EVERY EDGE OF THIS TABLE, FROM THE SHARED GENERATOR (W53-T37). `connector_spec.edges_for` + # mints `f"{target}_link__{from_col}"` and `f"{target label} by {source column}"` — Amazon's + # convention, now everyone's. ⛔ THE SOURCE COLUMN IS IN THE KEY ON PURPOSE: this loop used to + # read a single scalar link and key it `f"{target}_link"`, so a second edge to one target would + # have overwritten the first on both the key and the label, with nothing raising. + for edge in connector_spec.edges_for(key): + if edge["from_col"] in names: out.append(_odoo._preset( - {"key": f"{target}_link", "label": LABELS[target], "type": "link", + {"key": edge["key"], "label": edge["label"], "type": "link", "source": "overlay", "default": True, - "link": {"table": target, "on": on, "from": col}}, flow="meta_relational")) + "link": {"table": edge["target"], "on": edge["to_col"], + "from": edge["from_col"]}}, flow="meta_relational")) out.append(_odoo._preset(_odoo._refreshed_field(), flow="meta_relational")) return out @@ -277,7 +297,7 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo datastore.use_path(path) cur = datastore.ro_con() if not any(mirror_columns(cur, t) for t in SOURCE.values()): - log(f"[meta_relational] {why}: no Meta tables in {tenant}'s mirror — nothing to spawn " + log(f"[meta_relational] {why}: no Meta tables in {tenant}'s mirror; nothing to spawn " f"(run `python platform/harness/meta_store.py --sync --tenant {tenant}` first)") return state state["present"] = True diff --git a/api/oauth_connect.py b/api/oauth_connect.py index eb9f4cfab1a88cacf2ccec133424d621f7e47afd..eb53434edc69219d7958e62c9d5014ae6821f76f 100644 --- a/api/oauth_connect.py +++ b/api/oauth_connect.py @@ -164,7 +164,7 @@ def start(provider, uname, redirect_uri, next_path=""): if p is None: return "", f"{provider!r} is not a connectable provider" if not configured(provider): - return "", (f"{p['label']} OAuth is not configured on this deployment — the owner adds " + return "", (f"{p['label']} OAuth is not configured on this deployment; the owner adds " f"{p['env_id']} / {p['env_secret']} first") _prune_states() cid, _secret = client(provider) @@ -229,11 +229,11 @@ def callback(rt, uname, state, code): entry = _STATES.pop(str(state or ""), None) # single-use, whatever happens next home = safe_next((entry or {}).get("next")) if not entry: - return "", home, "that connect attempt is unknown or expired — start again" + return "", home, "that connect attempt is unknown or expired; start again" if entry.get("uname") != str(uname or ""): # A state minted for one session consumed by another is exactly the CSRF the state # exists to refuse. Fail closed; the honest user just clicks Connect again. - return "", home, "that connect attempt belongs to a different session — start again" + return "", home, "that connect attempt belongs to a different session; start again" provider = entry.get("provider") or "google" p = provider_def(provider) if p is None or not configured(provider): @@ -250,11 +250,11 @@ def callback(rt, uname, state, code): access = str(payload.get("access_token") or "") refresh = str(payload.get("refresh_token") or "") if not access: - return "", home, "the provider answered without an access token — nothing was stored" + return "", home, "the provider answered without an access token; nothing was stored" if not refresh: # Google answers access-only when consent was silently reused — an access-only slot # dies within the hour, silently, so it is refused with the way out named. - return "", home, ("no refresh token came back — disconnect the app on the provider's " + return "", home, ("no refresh token came back; disconnect the app on the provider's " "permissions page and connect again") expiry = time.time() + float(payload.get("expires_in") or 3600) - 60 email = _userinfo_email(provider, access) @@ -302,11 +302,11 @@ def creds(rt, uname, provider="google"): try: slot = kc.read_user_secret(rt, uname, slot_name(provider)) except kc.KeychainLocked as e: - return "", f"the keychain is locked ({e}) — no token can be read" + return "", f"the keychain is locked ({e}); no token can be read" if not slot: - return "", (f"no {p['label']} connection for this user — connect it in Settings") + return "", (f"no {p['label']} connection for this user; connect it in Settings") if slot.get("reconnect"): - return "", f"the {p['label']} connection expired — reconnect it in Settings" + return "", f"the {p['label']} connection expired; reconnect it in Settings" try: expiry = float(slot.get("expiry") or 0) except (TypeError, ValueError): @@ -314,7 +314,7 @@ def creds(rt, uname, provider="google"): if time.time() >= expiry: slot, err = _refresh(rt, uname, provider, slot) if err: - return "", f"the {p['label']} connection could not refresh — {err}" + return "", f"the {p['label']} connection could not refresh; {err}" return str(slot.get("access") or ""), "" diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 272525a96f3311067802999bc9af49e17f2027cd..67f9051819bf97e177b17ce069a8598028f7a628 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -48,9 +48,13 @@ wider link would count 31,418 documents and call them open invoices, so those tw the oracle's predicate as an explicit `payment_state` condition pair. """ import datetime as _dt +import json as _json +import time as _time -#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: -#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. +#: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3, +#: which admits on CONFIGURED CREDENTIALS (`available()`) so a second Odoo tenant needs no edit. +#: It stays because five modules outside this one still gate ROUTES on `is_royal`, and because +#: `is_royal("")` is deliberately True with a `verify_automation` leg pinning that. RI_SLUGS = ("", "royal-imports") INVOICES_KEY = "ut_odoo_invoices" @@ -921,20 +925,30 @@ def module_link_fields(key): `ut_odoo_invoices` answers the ALL-CHANNELS question (the Amazon partner alone is 75% of the raw balance), so the same key would carry two numbers 4x apart depending on which pass ran last. That is the second-source-for-one-fact defect this module refuses everywhere else. - * ⭐⭐ **D-492, CHECKED BEFORE ANYTHING NEW WAS ADDED (W46-T42, 2026-08-30) AND IT STILL - HOLDS.** The register says two links a prior instruction DECLARED were never built. Read off - this function at HEAD, both are confirmed absent and neither was faked to close the row: - `customer_data` returns `invoices`, `orders` and `agent_link` and no *"Products ordered"*; - `product_data` returns `[]` unconditionally (the `if str(key or "") != "customer_data"` - guard on the first line), so *"Customers who ordered"* is never declared at all. They are - the SAME two-hop blocker, not two independent misses, and `_PRODUCT_LINK_BLOCKER` below is - their single stated reason. ⛔ A link declared to close D-492 without that blocker lifted - would render permanently blank with no `limit_report`, which is worse than the open row. - - * **NO LINK ON `product_data` AT ALL** — see `_PRODUCT_LINK_BLOCKER` below for what it would - take. Declaring one anyway is the worst available outcome: `ut_odoo_order_lines` EXISTS in - the store with zero rows, so `_resolve_table` finds it, nothing is refused, no - `limit_report` is raised, and the column renders permanently blank while looking configured. + * ⭐⭐ **D-492 IS HALF CLOSED, AND THE HALF THAT CLOSED IS THE ONE WITH A MECHANISM (W53-T40, + 2026-09-05).** The register said two links a prior instruction DECLARED were never built. + W46-T42 confirmed both absent and refused to fake either; both were the SAME missing + primitive, and W53-T40 built it. So `customer_data` now returns `invoices`, `orders`, + `agent_link` AND `products_ordered` — a resolving two-hop column, not a blank one. ⛔ THE + OLD WARNING HERE READ *"a link declared to close D-492 without that blocker lifted would + render permanently blank with no `limit_report`"*, and it stays true as a WARNING: what + changed is that the blocker was lifted first, in the order its own prescription named. + ⚠ *"Customers who ordered"* on `product_data` is STILL not declared, and its reason has + changed from unresolvable to simply undeclared — see the next bullet. + + * **NO LINK ON `product_data` AT ALL, AND IT IS NO LONGER A BLOCKED ONE.** The `if str(key or + "") != "customer_data"` guard on the first line of this function is why, and that is now the + whole reason: since W53-T40 the engine walks `source -> through -> target` in either + direction, so `product_data.product_id -> ut_odoo_order_lines.product_id -> + ut_odoo_order_lines.partner_id -> customer_data.pid` is a route the same `through` arm would + resolve today. W53-T40's `done-when` named ONE direction, so one direction shipped; the + reverse is a declaration nobody has asked for, not a missing primitive. ⛔ THE HISTORICAL + HAZARD IS RECORDED BECAUSE IT IS WHAT THE FIX WAS ABOUT: declaring this link BEFORE that + ticket was the worst available outcome, because `ut_odoo_order_lines` EXISTS in the store + with zero rows, `_resolve_table` found it, nothing was refused, no `limit_report` was + raised, and the column rendered permanently blank while looking configured. That path is + closed — `_resolve_table` now reaches the mirror or REPORTS — but the shape of the mistake + is worth keeping. ⚠ THE FIELD BAG IS SHAPED FOR THE RENDER WIRE THAT DOES NOT EXIST YET. `routes_customers` / `routes_products` still build their field lists from `aios_grid` alone, so today these columns @@ -966,159 +980,141 @@ def module_link_fields(key): "description": "The agent record for this account, matched on the agent name. Blank " "when the account has no agent or carries more than one.", "link": {"table": AGENTS_KEY, "on": AGENT_NAME_KEY, "from": AGENT_NAME_KEY}}, + # ⭐⭐ W53-T40 / D-492 — "PRODUCTS ORDERED", THE TWO-HOP THAT USED TO BE REFUSED HERE. + # `_PRODUCT_LINK_BLOCKER` stood below this list and named four missing parts; all four + # landed in W53-T40 and the blocker was retired with them (see the note where it stood). + # + # ⛔ THE ROUTE IS THE ONE ODOO DECLARES, AND IT IS THE ONLY ONE. `product.product` has no + # commercial relation to `res.partner` — `message_partner_ids` is the chatter FOLLOWERS + # many2many and `seller_ids` is VENDORS — so the single declared path is `sale.order.line`, + # which carries `order_partner_id` (many2one to `res.partner`) and `product_id` (many2one + # to `product.product`), both stored. ORDER -> LINE -> PRODUCT is genuinely TWO hops. + # ⛔ AND THE INVOICE ROUTE IS STILL REFUSED. Invoice to line to product through this grain + # is THREE hops and its first is `invoice_origin`, free text filled on 34.1% of posted + # invoices (10,706 of 31,418, measured 2026-08-29) whose backfill is unfinished. A depth-3 + # path over a weak hop returns a confidently wrong number instead of an error. + # + # ⚠ THE COLUMN SPELLINGS, EACH TAKEN FROM THE GRAIN THAT OWNS IT rather than retyped: + # `from` `customer_data.pid` IS the `res.partner` id (CUSTOMER_JOIN_FROM). + # `throughOn` `order_line_fields()` spells `order_partner_id` as JOIN_KEY, `partner_id`. + # `throughTo` the line's `product.product` id, PRODUCT_JOIN_KEY. + # `on` `product_data` carries Odoo's own id under PRODUCT_JOIN_KEY BESIDE its + # hashed `pid` (W33-T43 / R2). ⛔ IT CANNOT BE `pid` HERE: `product_data.pid` + # is `crc32(default_code)`, so joining a line's product id to it would match + # nothing while looking exactly like a configured column. + # + # ⚠ `default: False`, AND THAT IS A COST STATEMENT RATHER THAN TIMIDITY. The bridge is + # read-through: resolving this column reaches the connector mirror for 256,593 order + # lines. The other two links resolve out of documents already in hand, so they may paint + # on first render; this one is OFFERED and painted when asked for. + {"key": "products_ordered", "label": "Products ordered", "type": "link", "source": "odoo", + "derived": True, "default": False, + "description": "Every product this customer has ordered on a confirmed order, resolved " + "through the order lines. Distinct products, not a count of lines.", + "link": {"table": "product_data", "on": PRODUCT_JOIN_KEY, "from": CUSTOMER_JOIN_FROM, + "through": ORDER_LINES_KEY, "throughOn": JOIN_KEY, + "throughTo": PRODUCT_JOIN_KEY}}, )] -#: ⛔⛔ "PRODUCTS ORDERED" / "CUSTOMERS WHO ORDERED" CANNOT BE DECLARED HERE YET, AND THE REASON IS -#: NOT A MISSING DECLARATION — IT IS A MISSING PRIMITIVE. Recorded as data so the next lane reads -#: a measurement rather than an empty function. +#: ⭐⭐ W53-T40, 2026-09-05 — `_PRODUCT_LINK_BLOCKER` STOOD HERE AND IS **RETIRED**, BECAUSE ALL +#: FOUR PARTS OF ITS OWN PRESCRIPTION LANDED. It is removed rather than left with a stale `effect`, +#: and this note is what stands in its place: a refusal that has expired is a lie no test notices, +#: and the tripwire in `verify_odoo_relational.py` existed to catch exactly the day it expired. #: -#: WHAT ODOO SAYS (read from `ir.model.fields`, not assumed): `product.product` declares NO -#: COMMERCIAL relation to `res.partner`. ⚠ THE SENTENCE HERE USED TO SAY "no relation to -#: `res.partner` AT ALL", and D-492 is right that this was OVERSTATED — corrected 2026-08-30 in -#: W46-T42 rather than left standing, because a claim a reader can falsify in one grep costs the -#: surviving conclusion its authority. `product.product` inherits `mail.thread`, so -#: `message_partner_ids` (the generic FOLLOWERS relation) is a genuine many2many to `res.partner` -#: on this model. It is not a customer relation: a follower is whoever subscribed to the record's -#: chatter, it answers no question about who ORDERED the product, and joining through it would -#: mint a "customers who ordered" column populated by subscriptions. The other partner-shaped -#: field is `seller_ids`, an unstored one2many to `product.supplierinfo`, which is VENDORS. So the -#: evidence changes and THE CONCLUSION SURVIVES: the one declared path between a customer -#: and a product is `sale.order.line`, which carries `order_partner_id` (many2one to `res.partner`) -#: and `product_id` (many2one to `product.product`), both stored. So the relation is real, Odoo -#: does not declare it directly, and it is exactly TWO HOPS. +#: WHAT IT SAID, AND WHAT ANSWERED IT — part by part, in the order it prescribed: +#: (d) *"re-key `relation_cells`' refusal from `linked is None` onto 'the resolved document +#: stores no rows here'"*. ANSWERED at `automation_engine._mirror_table` + `_resolve_table`: +#: the sentinel is `core.user_tables.materialises` (the three-tier predicate OVER the +#: `readThrough` stamp, not a hand-read of the stamp), and `_resolve_table` now answers +#: `(None, limit)` for a database whose rows are structurally absent, so the existing +#: `is None` test became true for that case instead of being duplicated beside it. A +#: premature link at a read-through grain now REPORTS with a cause and a fix. +#: (a) *"`_clean_link` accepts a `through` key and validates its target the way it validates +#: `table`"*. ANSWERED at `core.user_tables._clean_link`: `through` is validated by +#: `is_linkable_target`, joined by `throughOn` / `throughTo` (the bridge's two ends, which +#: are genuinely different columns), refused without an `on`, and refused when the bridge IS +#: the target. +#: (b) *"give `_resolve_table` a read-through branch that calls the registered mirror reader, +#: then teach `relation_cells` to resolve table -> through -> target"*. ANSWERED at +#: `automation_engine._mirror_table` (the door) and `relation_cells`' `through` arm (the +#: walk). The reader was never missing — `routes_odoo_tables.GRID_SOURCES` has served these +#: rows through `routes_tables._read_through_rows` all along, registered as +#: `core.perm_scope._MIRROR_READER`. What was missing was a CALLER, and that was the whole +#: correction W46-T41 made to this blocker's `cause`. +#: (c) *"a fan-out display cap that is REPORTED through limit_report"*. ANSWERED at +#: `core.user_tables.LINK_THROUGH_MAX_IDS` (5,000 distinct target records per source row) +#: and the `effect: "capped"` report `relation_cells` emits when any row reaches it. #: -#: WHY NEITHER HOP IS EXPRESSIBLE TODAY: -#: 1. `relation_cells` resolves a link with ONE index lookup — `_linked_rows_by_join(target, on)` -#: probed by the source row's single `from` value. There is no `through` leg in the bag -#: vocabulary (`core.user_tables._clean_link` accepts table/on/from/inverse/reciprocal/via/ -#: single/filter, and returns None for anything mis-shaped) and no two-hop branch in the -#: engine. ⚠ `via` IS NOT A TABLE HOP and reading it as one is the easy mistake here: -#: `_clean_via` demands topic/source_dim/target_dim/target_key/scope_key/measure and refuses -#: unless the dims belong to a semantic topic, and `automation_engine` reads a link bag's -#: `via` ZERO times. -#: 2. ⛔⛔ THE BRIDGE HAS ROWS. THIS ITEM SAID THE OPPOSITE UNTIL W46-T41 MEASURED IT, and the -#: correction changes what the missing piece IS. `routes_odoo_tables.GRID_SOURCES` already -#: serves `ut_odoo_order_lines` from the DuckDB mirror by SQL (`sale_order_line` joined to -#: `sale_order` on the confirmed scope), `routes_tables._read_through_rows` renders it, and -#: `routes_tables._register_mirror` publishes that reader as `core.perm_scope._MIRROR_READER`. -#: MEASURED 2026-08-29: 256,593 in-scope rows, 256,502 of them carrying a `product_id` -#: (`_ORDER_LINE_GRAIN` below). What is empty is the `user_tables` DOCUMENT, not the system: -#: `row_limit` answers 0 and `plan()` builds zero python rows, by design and on purpose. -#: So the absence is A CALLER, not data — `automation_engine._resolve_table` returns the -#: stored doc first (`hit = (store or {}).get(key); if hit is not None: return hit, None`) and -#: carries no mirror branch at all, while `_linked_rows_by_join` indexes only -#: `(linked or {}).get("rows")`. -#: 2b. ⛔⛔ AND THAT IS WHY DECLARING THE LINK ANYWAY IS THE WORST AVAILABLE OUTCOME, in the -#: words of the engine's own docstring. `relation_cells` says of its second return value: -#: *"A link whose target cannot be reached is SKIPPED … and a `limit_report`-shaped dict says -#: which field, why, and what to do … a link with no matching records and a link that could -#: not be resolved BOTH leave the key out of `cells`, and only this list tells them apart."* -#: The function was rewritten expressly to end the silent blank. For a read-through target it -#: still produces one, and the reason is small and exact: the refusal predicate is -#: `if linked is None`, and `_resolve_table` hands back a stored definition, which is never -#: None. So this link lands in the ONE case the docstring says cannot happen — no `cells` key -#: AND no `limits` entry, indistinguishable from "no matching records" by the very -#: discriminator that docstring names. The guard is not missing; it is KEYED ON THE WRONG -#: SENTINEL. Standing rule 1's second sentence fails on that path, which is the whole reason -#: the link is refused here rather than declared. -#: 3. `inverse` cannot carry the other end either. It reads the source row's STORED cell, and a -#: registry module stores no rows here: its grid is assembled per render from the tenant pool, -#: so the reciprocal cell a pool row would need is precisely the projection being computed. +#: ⭐ AND THE COLUMN IS DECLARED: `module_link_fields("customer_data")` above now returns +#: `products_ordered`, `customer_data -> ut_odoo_order_lines -> product_data`, which is D-492's +#: first half closed with a resolving column rather than with a blank one. #: -#: WHAT IT WOULD TAKE (four parts, one ticket, and none of them in this file): -#: (a) a `through` leg on the link bag, resolved as a two-hop join in `automation_engine`; -#: (b) bridge rows served from the DuckDB MIRROR rather than the `user_tables` document, because -#: order lines are read-through on purpose and materialising them is the thing R6 forbade. -#: ⭐ The reader for (b) ALREADY EXISTS and is already registered; what (b) needs is the door -#: into it — a read-through branch in `_resolve_table`, the one symbol the other two both -#: pass through; -#: (c) a cap-and-report story for the fan-out under standing rule 1 — the widest customer's -#: basket and the widest product's buyer list both need a display cap plus a -#: `limit_report`-shaped account of it, never a silent truncation; -#: (d) ⭐ THE CHEAPEST OF THE FOUR AND THE ONE THAT STOPS THE SILENT FAILURE ON ITS OWN: re-key -#: `relation_cells`' refusal off `linked is None` and onto "resolved a document whose rows -#: are structurally absent". The signal is already written and already durable — -#: `_ensure_table_inplace` in this module stamps `readThrough: True` on exactly those -#: definitions, and `core.user_tables.materialises` reads it. Landing (d) first converts -#: every premature link at a read-through grain from a permanently blank column into a -#: `limit_report` that names the field and the fix. +#: ⛔⛔ WHAT DID **NOT** EXPIRE, AND IS KEPT BELOW RATHER THAN SWEPT UP WITH IT. Two of the old +#: blocker's clauses were never about the engine, so landing the engine work changed nothing about +#: them, and retiring a refusal whose condition survives is worse than leaving it standing: +#: * THE INVOICE ROUTE. Still refused, still for its own reason, and it now has its own dict +#: (`_INVOICE_PRODUCT_LINK_BLOCKER` below) so it cannot be read as a leftover of this one. +#: * "CUSTOMERS WHO ORDERED" ON `product_data`. ⚠ ITS REASON CHANGED AND SO THE SENTENCE HAD TO +#: BE REWRITTEN RATHER THAN DELETED. It is no longer UNRESOLVABLE: the mechanism is symmetric +#: and the reverse route (`product_data.product_id -> ut_odoo_order_lines.product_id -> +#: ut_odoo_order_lines.partner_id -> customer_data.pid`) is walkable by the same `through` arm +#: the moment somebody declares it. It is simply NOT DECLARED — W53-T40's `done-when` names +#: one direction, so one direction is what shipped and the other is a declaration nobody has +#: asked for yet, not a missing primitive. `module_link_fields` still returns `[]` for +#: `product_data` and its guard is the first line of that function. #: -#: ⭐⭐ W46-T41 — IT IS THREE HOPS FOR AN INVOICE, NOT TWO, AND THE WAVE'S OWN DOCUMENTS SAY TWO. -#: Recorded because the correction outlives the ticket. `ut_odoo_order_lines` IS `sale.order.line`, -#: which hangs off `sale.order` and NOT off `account.move`, so the two readings differ: -#: • ORDER -> LINE -> PRODUCT is genuinely two hops, and it is what C5's own wording describes -#: ("keyed by `order_id` and `product_id`"). This grain can serve it the day (a)+(b) land. -#: • INVOICE -> LINE -> PRODUCT through THIS grain is THREE hops, and the first of them is a -#: free-text hint rather than a key: `invoice_fields`' existing `order_link` matches -#: `origin_order` (Odoo's `invoice_origin`, a NAME) to `order_no`. MEASURED 2026-08-29: -#: `invoice_origin` is filled on 10,706 of 31,418 posted customer invoices (34.1%), of which -#: 10,683 match a `sale_order.name`; `_sync_state` shows the `account_move.cols.invoice_origin` -#: backfill still in phase `running` at 35,000 of 236,811 rows since 2026-08-15. So the first -#: hop of that route is a third populated and its own backfill is unfinished. -#: ⚠ THE HONEST TWO-HOP ROUTE FOR AN INVOICE IS THE OTHER GRAIN, and it is named here rather than -#: built because it is blocked by the identical engine gap. `account_move_line` carries `move_id`, -#: `product_id` AND `sale_line_id`, so INVOICE -> `ut_odoo_gl_lines` -> PRODUCT is a real two hops -#: (measured 2026-08-29: 477,852 of 511,167 posted customer invoice lines carry a `product_id`, -#: across 30,164 of 31,418 invoices). Two things stand in the way: `ut_odoo_gl_lines` is the OTHER -#: read-through grain, so items 2 and 2b above apply to it unchanged, and `gl_line_fields` does not -#: expose `product_id` today. A future ticket takes that route; this one does not. -_PRODUCT_LINK_BLOCKER = { - "subject": "product_data / customer_data: Products ordered and Customers who ordered", +#: ⛔ THE MEASUREMENTS SURVIVE THEIR BLOCKER and are kept because standing rule 8 wants the number +#: rather than the adjective: the bridge holds 256,593 in-scope rows of which 256,502 carry a +#: `product_id` (measured 2026-08-29; the machine-readable form is `_ORDER_LINE_GRAIN` below), and +#: `product.product` still declares NO commercial relation to `res.partner` — `message_partner_ids` +#: is the `mail.thread` FOLLOWERS many2many and `seller_ids` is an unstored one2many to +#: `product.supplierinfo`, i.e. VENDORS. Neither answers "who ordered this", which is why +#: `sale.order.line` is the only route and why it is two hops. + +#: ⛔⛔ W53-T40 — THE **INVOICE** TO PRODUCT ROUTE IS STILL REFUSED, AND THIS IS THE SURVIVING HALF +#: OF `_PRODUCT_LINK_BLOCKER` RESTATED TO WHAT IS ACTUALLY TRUE TODAY. Its cause is DATA, never the +#: engine: the engine now walks two hops and would walk these too, which is precisely why the +#: refusal has to be written down instead of assumed. A route that has become MECHANICALLY possible +#: while remaining EVIDENTIALLY wrong is the one a future lane is most likely to build by accident. +#: +#: ⚠ ASCII ONLY IN THESE VALUES. They travel into `plan()`'s `limits` report and can reach a +#: client, where standing rule 2 forbids an em dash or an en dash. +_INVOICE_PRODUCT_LINK_BLOCKER = { + "subject": "ut_odoo_invoices: Products invoiced", "effect": "unresolvable", - # ⛔ THE `cause` CHANGED IN W46-T41 AND THE OLD ONE WAS WRONG IN A LOAD-BEARING WAY. It read - # "the bridge grain ut_odoo_order_lines is read-through by design and stores no rows to walk", - # which is true of the `user_tables` DOCUMENT and false of the system: the rows are measured, - # served and registered elsewhere. A blocker that blames missing DATA sends the next lane to - # materialise 256,593 rows, which is the one thing R6 forbade; a blocker that names the missing - # CALLER sends it to the three symbols in `blocked_by`. - # ⚠ ASCII ONLY IN THESE VALUES. They travel into `plan()`'s `limits` report and can reach a - # client, where standing rule 2 forbids an em dash or an en dash. - "cause": "the only relation Odoo declares between a partner and a product runs through " - "sale.order.line (order_partner_id then product_id), which is two hops, and the " - "relation engine resolves ONE. The bridge rows are NOT missing: measured 2026-08-29 " - "on the DuckDB mirror, ut_odoo_order_lines holds 256,593 in-scope rows of which " - "256,502 carry a product_id, and they are ALREADY SERVED to the grid by " - "routes_odoo_tables.GRID_SOURCES through routes_tables._read_through_rows, whose " - "reader is registered as core.perm_scope._MIRROR_READER. What is absent is a caller: " - "automation_engine._resolve_table returns the stored user_tables document first and " - "carries no read-through branch, and the link vocabulary in " - "core.user_tables._clean_link has no `through` key (its `via` is a semantic-topic " - "measure route, not a table hop, and the engine never reads it). So a link declared " - "today fails SILENTLY rather than loudly, and the engine's own docstring says why " - "that is the worst outcome available: relation_cells promises that an unreachable " - "target is skipped WITH a limit_report, because a link with no matching records and " - "a link that could not be resolved both leave the key out of cells and only that " - "report tells them apart. Its refusal is keyed on `linked is None`, and a stored " - "definition is never None, so this link lands in the one case that docstring says " - "cannot happen: no cell, no limit report, and a permanently blank column that looks " - "configured. The guard is not missing, it is keyed on the wrong sentinel. That is " - "why the link is refused here instead of declared", - "recommendation": "four parts, one ticket, and neither file is this one. (a) " - "platform/core/user_tables.py::_clean_link accepts a `through` key in the " - "link vocabulary and validates its target the way it validates `table`. " - "(b) aios-web/api/automation_engine.py opens the door and walks the second " - "hop: give _resolve_table a read-through branch that calls the registered " - "mirror reader instead of returning an empty stored document, then teach " - "relation_cells to resolve table -> through -> target. (c) both halves need " - "a fan-out display cap that is REPORTED through limit_report rather than " - "applied silently, per standing rule 1. (d) cheapest and independently " - "worth doing FIRST: re-key relation_cells' refusal from `linked is None` " - "onto `the resolved document stores no rows here`, which the readThrough " - "stamp already written by odoo_relational._ensure_table_inplace answers, so " - "that a premature link at a read-through grain reports instead of " - "blanking. Until these land, keep refusing the link: a blank configured " - "column is worse than an absent one", - # ⭐ THE BLOCKING SYMBOLS AS DATA, NOT PROSE, so `verify_odoo_relational.py` can read the - # source of each one and go RED the day the blocking property stops holding. A refusal whose - # reason has silently expired is precisely the defect that tripwire exists to prevent. - # ⚠ `_resolve_table` is in the list although the prose leads with the other two, and that is - # deliberate: it is the door BOTH of them go through, so a mirror branch landing there would - # unblock this ticket while `relation_cells` and `_clean_link` still read exactly as today. - "blocked_by": ("platform/core/user_tables.py::_clean_link", - "aios-web/api/automation_engine.py::_resolve_table", - "aios-web/api/automation_engine.py::relation_cells"), - "rows_exist_at": "aios-web/api/routes_odoo_tables.py::GRID_SOURCES['ut_odoo_order_lines'] -> " + "cause": "there are two candidate routes from an invoice to a product and BOTH are refused, " + "for reasons that have nothing to do with the relation engine (which resolves two " + "hops since W53-T40). ROUTE 1, through ut_odoo_order_lines, is THREE hops rather " + "than two: sale.order.line hangs off sale.order and not off account.move, so the " + "first hop is invoice_fields' order_link, which matches origin_order (Odoo's " + "invoice_origin, a NAME) to order_no. MEASURED 2026-08-29: invoice_origin is filled " + "on 10,706 of 31,418 posted customer invoices (34.1%), of which 10,683 match a " + "sale_order.name, and _sync_state shows the account_move.cols.invoice_origin " + "backfill still in phase running at 35,000 of 236,811 rows since 2026-08-15. A " + "depth-3 path whose first hop is free text populated on a third of the population " + "returns a confidently wrong number instead of an error. ROUTE 2 is the honest two " + "hops and is blocked on a COLUMN: account_move_line carries move_id, product_id AND " + "sale_line_id, so invoice -> ut_odoo_gl_lines -> product_data is a real two-hop " + "traversal (measured 2026-08-29: 477,852 of 511,167 posted customer invoice lines " + "carry a product_id, across 30,164 of 31,418 invoices), but gl_line_fields does not " + "expose product_id, so the bridge grain has no column for the second hop to read", + "recommendation": "take ROUTE 2 and never ROUTE 1. Expose product_id on gl_line_fields and " + "on the ut_odoo_gl_lines mirror projection in routes_odoo_tables, then " + "declare a link on ut_odoo_invoices with from=odoo_id, " + "through=ut_odoo_gl_lines, throughOn=move_id (that grain already carries " + "the account.move id under exactly that key), throughTo=product_id, " + "table=product_data and on=product_id. The engine needs no further change: " + "core.user_tables._clean_link accepts the route and " + "automation_engine.relation_cells walks it. Until that column exists, keep " + "refusing: a link declared over invoice_origin would populate and would be " + "wrong on two thirds of the rows", + # ⭐ THE BLOCKING SYMBOL AS DATA rather than prose, so `verify_odoo_relational.py` can read its + # source and go RED the day the blocking property stops holding. ⛔ IT IS ONE SYMBOL NOW, NOT + # THREE: the three the old blocker named were engine symbols and all three have changed. What + # is left is a missing COLUMN on one field builder, which is a claim about this very file. + "blocked_by": ("aios-web/api/odoo_relational.py::gl_line_fields",), + "rows_exist_at": "aios-web/api/routes_odoo_tables.py::GRID_SOURCES['ut_odoo_gl_lines'] -> " "routes_tables._read_through_rows (the same reader is registered as " "core.perm_scope._MIRROR_READER by routes_tables._register_mirror)", } @@ -1157,7 +1153,7 @@ _PRODUCT_LINK_BLOCKER = { #: the label "DBA" would put a SECOND definition of one fact under one name, which is the #: defect `module_link_fields`' own "NO ROLLUPS" note refuses for `ar_outstanding` and which #: this module refuses everywhere else. -#: ⚠ ASCII ONLY IN THESE VALUES, like `_PRODUCT_LINK_BLOCKER` above: they travel into `plan()`'s +#: ⚠ ASCII ONLY IN THESE VALUES, like `_INVOICE_PRODUCT_LINK_BLOCKER` above: they travel into `plan()`'s #: `limits` report and can reach a client, where standing rule 2 forbids an em dash or an en dash. _DBA_NAME_BLOCKER = { "subject": "ut_odoo_invoices: the DBA or trade name", @@ -1875,6 +1871,597 @@ def authoritative_schema_snapshot(): return models, errors +# The compiled TABLES tuple remains the curated mirror contract. This is the separate, per-tenant +# discovery cache for every other Odoo model the active API account may read (W55-T17 / C9). +MODEL_CATALOG_KEY = "odoo_readable_model_catalog" +MODEL_CATALOG_VERSION = 1 +MODEL_READ_PAGE = 2000 +MODEL_CATALOG_TTL = _dt.timedelta(hours=6) +_MODEL_FIELD_TYPES = { + "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency", + "date": "date", "datetime": "datetime", +} + + +def discover_readable_models(client): + """Read-only introspection over an injected Odoo client. + + ``ir.model`` supplies candidates, ``check_access_rights('read')`` narrows them to this API + account, and ``fields_get`` supplies only fields that account may inspect. No method in this + path can mutate Odoo; the production client independently blocks write-shaped method names. + """ + errors = [] + try: + candidates = client.search_read( + "ir.model", [], ["model", "name", "transient"], order="name,model") + except Exception as exc: # noqa: BLE001 + return [], [f"ir.model: {type(exc).__name__}: {exc}"] + rows = [] + for item in candidates or (): + model = str((item or {}).get("model") or "").strip().lower() + if not model: + continue + try: + readable = client.execute_kw( + model, "check_access_rights", ["read"], {"raise_exception": False}) + except Exception as exc: # noqa: BLE001 + errors.append(f"{model}: access: {type(exc).__name__}: {exc}") + continue + if not bool(readable): + continue + try: + fields = client.fields_get( + model, attributes=("string", "type", "required", "readonly", "relation")) + except Exception as exc: # noqa: BLE001 + fields = {} + errors.append(f"{model}: fields: {type(exc).__name__}: {exc}") + clean_fields = {} + for name, meta in (fields or {}).items(): + if not str(name).strip() or not isinstance(meta, dict): + continue + clean_fields[str(name)] = {k: meta.get(k) for k in + ("string", "type", "required", "readonly", "relation") + if k in meta} + rows.append({"model": model, + "name": " ".join(str((item or {}).get("name") or model).split()), + "fields": clean_fields}) + rows.sort(key=lambda row: (str(row.get("name") or "").casefold(), row["model"])) + return rows, errors + + +def _catalog_slot(connection_id): + return str(connection_id or DEFAULT_CONNECTION) + + +def readable_model_catalog(rt, connection_id=None): + """The last complete tenant-local catalogue; reading it never contacts Odoo.""" + try: + cached = rt.get(MODEL_CATALOG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cached = None + if not isinstance(cached, dict) or cached.get("version") != MODEL_CATALOG_VERSION: + return {"version": MODEL_CATALOG_VERSION, "complete": False, "capturedAt": "", + "models": [], "errors": []} + cached = (cached.get("connections") or {}).get(_catalog_slot(connection_id)) or {} + return {"version": MODEL_CATALOG_VERSION, + "complete": cached.get("complete") is True, + "capturedAt": str(cached.get("capturedAt") or ""), + "models": [dict(row) for row in (cached.get("models") or ()) + if isinstance(row, dict) and row.get("model")], + "errors": [str(value) for value in (cached.get("errors") or ())][:50]} + + +def refresh_readable_model_catalog(rt, connection_id=None, client=None, force=False): + """Populate one tenant's catalogue once, or explicitly refresh it; never widen on failure.""" + current = readable_model_catalog(rt, connection_id) + if current["capturedAt"] and not force: + try: + captured = _dt.datetime.fromisoformat(current["capturedAt"]) + if _dt.datetime.now(_dt.timezone.utc) - captured < MODEL_CATALOG_TTL: + return current + except (TypeError, ValueError): + pass + source = None + if client is None: + try: + source = (rt.odoo_source() if connection_id is None + else rt.odoo_source(connection_id)) + except Exception: # noqa: BLE001 + source = None + if source is None: + return current + try: + if client is None: + from core.odoo import get_odoo + with source._bound(): # tenant credential + scope + rows, errors = discover_readable_models(get_odoo()) + else: + rows, errors = discover_readable_models(client) + except Exception as exc: # noqa: BLE001 + rows, errors = [], [f"catalogue: {type(exc).__name__}: {exc}"] + # A partial retry is not evidence that a previously readable model disappeared. Preserve a + # complete, stamped catalogue in full whenever the new pass reports any error; otherwise one + # refused model could silently shrink the enablement roster. A first-ever partial pass may be + # retained for diagnostics, but an entirely failed pass still has no catalogue to publish. + if errors and current["complete"] and current["capturedAt"]: + return current + if not rows and errors: + return current + captured = _dt.datetime.now(_dt.timezone.utc).isoformat() + result = {"version": MODEL_CATALOG_VERSION, "complete": not errors, + "capturedAt": captured, "models": rows, "errors": errors[:50]} + slot = _catalog_slot(connection_id) + + def _cache(current_cache): + current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} + connections = dict(current_cache.get("connections") or {}) + connections[slot] = result + return {"version": MODEL_CATALOG_VERSION, "connections": connections} + + rt.update(MODEL_CATALOG_KEY, _cache, flush="sync") + return result + + +# The population fact is deliberately separate from the readable-model catalogue. Measured on +# Royal Imports on 2026-09-08: discovery found 499 readable models in 22.62 s, then exact +# search_count calls took 67.87 s for 494 models at 137 ms per model; 239 held rows. A sampled +# search_read was slower and raised on account.move, so sampling is neither the cheap nor the +# reliable fallback. The exact pass stays explicit and cached because a cold 90.5 s pass cannot +# sit on the connector detail request. +POPULATION_CENSUS_KEY = "odoo_population_census" +POPULATION_CENSUS_VERSION = 1 +FIELD_CENSUS_KEY = "odoo_field_population" +FIELD_CENSUS_VERSION = 1 +_FIELD_CENSUS_UNCOUNTABLE_TYPES = {"one2many", "many2many"} + + +def field_census(rt, connection_id=None): + """Return the cached per-model field counts; this reader never contacts Odoo.""" + try: + cached = rt.get(FIELD_CENSUS_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cached = None + if not isinstance(cached, dict) or cached.get("version") != FIELD_CENSUS_VERSION: + return {} + connections = cached.get("connections") + if not isinstance(connections, dict): + return {} + stored = connections.get(_catalog_slot(connection_id)) + if not isinstance(stored, dict): + return {} + answer = {} + for model, report in stored.items(): + if not str(model).strip() or not isinstance(report, dict): + continue + raw_counts = report.get("counts") if isinstance(report.get("counts"), dict) else {} + counts = {str(name): int(value) for name, value in raw_counts.items() + if str(name).strip() and isinstance(value, int) + and not isinstance(value, bool) and value >= 0} + raw_errors = report.get("errors") + errors = ([str(value) for value in raw_errors] + if isinstance(raw_errors, (list, tuple)) else []) + try: + duration_ms = max(0, int(report.get("durationMs") or 0)) + except (TypeError, ValueError, OverflowError): + duration_ms = 0 + answer[str(model)] = { + "capturedAt": str(report.get("capturedAt") or ""), + "durationMs": duration_ms, + "counts": counts, + "errors": errors, + "complete": report.get("complete") is True, + } + return answer + + +def refresh_field_census(rt, model, connection_id=None, client=None, force=False): + """Exactly count every countable field in one model using read-only ``search_count``.""" + model = str(model or "").strip() + current = field_census(rt, connection_id).get(model) + if current and current.get("capturedAt") and not force: + return current + + # Exact per-field domains measured 26 ms on res.partner, 41 ms on sale.order and 72 ms on + # account.move on 2026-09-08, making the full models about 5.2 s, 5.1 s and 13.7 s. A sampled + # search_read took 3.17 s and 7.68 s on the first two and failed on account.move in Odoo's + # XML-RPC marshaller, so every countable field uses exact search_count with no cap or sample. + started = _time.monotonic() + catalogue = readable_model_catalog(rt, connection_id) + row = next((item for item in (catalogue.get("models") or ()) + if isinstance(item, dict) and str(item.get("model") or "").strip() == model), None) + fields = dict(row.get("fields") or {}) if row else {} + counts = {} + errors = [] + if not model: + errors.append("model: a model name is required") + elif row is None: + errors.append(f"{model}: model is not in the readable model catalogue") + + def _count_all(bound_client): + for name, metadata in fields.items(): + field_name = str(name or "").strip() + field_meta = metadata if isinstance(metadata, dict) else {} + field_type = str(field_meta.get("type") or "") + if not field_name: + continue + if field_type in _FIELD_CENSUS_UNCOUNTABLE_TYPES: + # Relational collection fields do not answer `!= False` usefully. They remain + # unknown and are never recorded as empty or selected for automatic enablement. + errors.append(f"{model}.{field_name}: {field_type} cannot be counted usefully " + "with a non-null domain") + continue + try: + # This is the only Odoo operation in the field census. It cannot mutate data. + counts[field_name] = int(bound_client.search_count( + model, [(field_name, "!=", False)])) + except Exception as exc: # noqa: BLE001 + errors.append(f"{model}.{field_name}: {type(exc).__name__}: {exc}") + + if client is not None and fields: + _count_all(client) + elif fields: + try: + source = (rt.odoo_source() if connection_id is None + else rt.odoo_source(connection_id)) + if source is None: + errors.append("connection: Odoo source is unavailable") + else: + from core.odoo import get_odoo + with source._bound(): # tenant credential + scope + _count_all(get_odoo()) + except Exception as exc: # noqa: BLE001 + errors.append(f"connection: {type(exc).__name__}: {exc}") + + countable = sum( + 1 for name, metadata in fields.items() + if str(name or "").strip() + and str((metadata if isinstance(metadata, dict) else {}).get("type") or "") + not in _FIELD_CENSUS_UNCOUNTABLE_TYPES) + result = { + "capturedAt": _dt.datetime.now(_dt.timezone.utc).isoformat(), + "durationMs": int((_time.monotonic() - started) * 1000), + "counts": counts, + "errors": errors, + "complete": row is not None and catalogue.get("complete") is True and not errors + and len(counts) == countable, + } + slot = _catalog_slot(connection_id) + + def _cache(current_cache): + current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} + connections = dict(current_cache.get("connections") or {}) + models = dict(connections.get(slot) or {}) + models[model] = result + connections[slot] = models + return {"version": FIELD_CENSUS_VERSION, "connections": connections} + + rt.update(FIELD_CENSUS_KEY, _cache, flush="sync") + return result + + +def population_census(rt, connection_id=None): + """The last census, from the tenant store. Reading it never contacts Odoo.""" + empty = {"version": POPULATION_CENSUS_VERSION, "capturedAt": "", "complete": False, + "durationMs": 0, "counts": {}, "errors": [], "modelsCounted": 0, + "modelsWithRows": 0} + try: + cached = rt.get(POPULATION_CENSUS_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + return empty + if not isinstance(cached, dict) or cached.get("version") != POPULATION_CENSUS_VERSION: + return empty + connections = cached.get("connections") + if not isinstance(connections, dict): + return empty + slot = connections.get(_catalog_slot(connection_id)) or {} + if not isinstance(slot, dict): + return empty + raw_counts = slot.get("counts") if isinstance(slot.get("counts"), dict) else {} + counts = {str(model): int(value) for model, value in raw_counts.items() + if str(model).strip() and isinstance(value, int) and not isinstance(value, bool) + and value >= 0} + raw_errors = slot.get("errors") + errors = ([str(value) for value in raw_errors] + if isinstance(raw_errors, (list, tuple)) else []) + + def _nonnegative_int(value): + try: + return max(0, int(value)) + except (TypeError, ValueError, OverflowError): + return 0 + + return {"version": POPULATION_CENSUS_VERSION, + "capturedAt": str(slot.get("capturedAt") or ""), + "complete": slot.get("complete") is True, + "durationMs": _nonnegative_int(slot.get("durationMs")), + "counts": counts, + "errors": errors, + "modelsCounted": _nonnegative_int(slot.get("modelsCounted")), + "modelsWithRows": _nonnegative_int(slot.get("modelsWithRows"))} + + +def refresh_population_census(rt, connection_id=None, client=None, force=False): + """Count every readable model with read-only ``search_count`` and cache the result.""" + current = population_census(rt, connection_id) + if current["capturedAt"] and not force: + return current + + # Start before possible discovery so durationMs is the wall clock the person waiting sees: + # about 68 s with a warm catalogue and about 90.5 s on a fresh workspace in the measured run. + started = _time.monotonic() + catalogue = readable_model_catalog(rt, connection_id) + if not catalogue["models"]: + catalogue = refresh_readable_model_catalog( + rt, connection_id=connection_id, client=client, force=False) + + models = [] + seen = set() + for row in catalogue.get("models") or (): + model = str(row.get("model") or "").strip() if isinstance(row, dict) else "" + if model and model not in seen: + seen.add(model) + models.append(model) + + counts = {} + errors = [str(value) for value in (catalogue.get("errors") or ())] + if not models and catalogue.get("complete") is not True: + errors.append("catalogue: no complete readable model catalogue is available") + + def _count_all(bound_client): + for model in models: + try: + # search_count is the only Odoo operation in the census. It cannot mutate data. + counts[model] = int(bound_client.search_count(model, [])) + except Exception as exc: # noqa: BLE001 + errors.append(f"{model}: {type(exc).__name__}: {exc}") + + if client is not None: + _count_all(client) + elif models: + try: + source = (rt.odoo_source() if connection_id is None + else rt.odoo_source(connection_id)) + if source is None: + errors.append("connection: Odoo source is unavailable") + else: + from core.odoo import get_odoo + with source._bound(): # tenant credential + scope + _count_all(get_odoo()) + except Exception as exc: # noqa: BLE001 + errors.append(f"connection: {type(exc).__name__}: {exc}") + + result = { + "version": POPULATION_CENSUS_VERSION, + "capturedAt": _dt.datetime.now(_dt.timezone.utc).isoformat(), + "complete": catalogue.get("complete") is True and not errors + and len(counts) == len(models), + "durationMs": int((_time.monotonic() - started) * 1000), + "counts": counts, + "errors": errors, + "modelsCounted": len(counts), + "modelsWithRows": sum(1 for value in counts.values() if value > 0), + } + slot = _catalog_slot(connection_id) + + def _cache(current_cache): + current_cache = dict(current_cache) if isinstance(current_cache, dict) else {} + connections = dict(current_cache.get("connections") or {}) + connections[slot] = result + return {"version": POPULATION_CENSUS_VERSION, "connections": connections} + + rt.update(POPULATION_CENSUS_KEY, _cache, flush="sync") + return result + + +#: ⭐⭐ W53-T38 / C2 — THE CONNECTOR SCHEMA SPEC ODOO HAS NEVER HAD. +#: +#: Amazon and Meta each got a `ConnectorTableSpec` row per table in `platform/core/connector_spec.py` +#: (W53-T36). Odoo's equivalent could not be transcribed there for one reason: **`core` never +#: imports up**, so a spec that must name `JOIN_KEY`, `PRODUCT_JOIN_KEY`, `TABLES` and the field +#: builders has to be DECLARED HERE and REGISTERED into that module. `register()` exists for that. +#: +#: ⛔⛔ **THIS DECLARATION IS SEVEN KEYS ABOUT A TABLE AND IT IS NOT A LINK FIELD.** It sits BESIDE +#: `module_link_fields` and does not touch it. `module_link_fields` is reached BY STRING through +#: `getattr(_rel, "module_link_fields", None)` from `automation_engine._module_link_fields`, so +#: renaming it, moving it or folding it into anything raises NO ImportError and silently kills +#: every pre-set relation column on `customer_data`. Nothing below is emitted as a field, appears +#: in a `*_fields()` builder, or reaches `_link_bags()` in the gate. An edge here is a DECLARATION +#: the relation reader consults; a link bag is a COLUMN the automation engine resolves. They are +#: different objects with different consumers, and this file keeps them apart on purpose. +#: +#: ⛔ **R12 — DEPTH 2, EVERY HOP NAMED, AND DECLARING AN EDGE HERE CREATES NO ROLLUP.** Aggregation +#: stays at depth 1. `ORDER_LINES_KEY -> PRODUCTS_KEY` is declared below, and W53-T40 has since +#: built the LINK FIELD half of the same relation (`module_link_fields("customer_data")`'s +#: `products_ordered`, a two-hop TRAVERSAL). ⚠ THE TWO REMAIN DIFFERENT OBJECTS AND THE FACT THAT +#: BOTH NOW EXIST IS EXACTLY WHY THIS SENTENCE STAYS: an edge here is a DECLARATION the relation +#: reader consults, a link bag is a COLUMN the automation engine resolves, and neither creates a +#: rollup. Aggregation is still depth 1. This note used to read *"`_PRODUCT_LINK_BLOCKER` further +#: up this file is STILL unresolvable ... W53-T40 is the ticket that does"*; that ticket landed on +#: 2026-09-05 and the blocker is retired, so the sentence was rewritten rather than left standing. +#: ⛔ **AND THE INVOICE -> LINE -> PRODUCT ROUTE IS DELIBERATELY ABSENT.** Its first hop is +#: `invoice_origin`, free text filled on 34.1% of posted invoices (measured 2026-08-29, recorded at +#: `_INVOICE_PRODUCT_LINK_BLOCKER` above, which is the surviving half of the retired blocker). A +#: depth-3 path over a weak hop returns a confidently wrong number instead of an error, so no +#: `invoice_origin` edge is declared here at any time. +#: +#: ⛔ **R9 — `native` IS NEVER EDITABLE AND `label` IS ONLY A DEFAULT.** The labels below are our +#: SHIPPED defaults; a tenant's rename lives in `platform/core/schema_overlay.py`, per tenant, and a +#: connector re-sync must never write it. Nothing here is a place to store an override. + +#: `native`, DERIVED rather than re-typed. `ODOO_TABLE_MODELS` above already declares the Odoo model +#: behind eight of the ten keys and is the map `authoritative_schema_snapshot` is built around; a +#: second literal map beside it would be a second source for one fact. It is missing only the two +#: keys W33-T44 retired from `TABLES`, so exactly those two are supplied here. +#: ⭐ `PRODUCTS_KEY` IS `product.product`, NOT `product.template`, and the code is what settles it: +#: `read_products` reads `FROM product_product` and keys every row on `PRODUCT_JOIN_KEY`, whose own +#: field description reads "The `product.product` id. Also this row's id". A template is the family; +#: a variant is the thing an order line actually names. +_RETIRED_NATIVE = { + CUSTOMERS_KEY: "res.partner", + PRODUCTS_KEY: "product.product", +} + +#: The shipped default label for the two retired keys. The other eight come from `TABLES`' own third +#: element and are transcribed into `ODOO_SPEC` below, where the gate holds the two spellings equal. +_RETIRED_LABEL = { + CUSTOMERS_KEY: "Odoo customers", + PRODUCTS_KEY: "Odoo products", +} + +#: The identity column of a grid AS THE WIRE SPELLS IT. `routes_tables` stamps `pid` onto every row +#: it serves (`r["pid"] = int(rid)`), and `core.relations.IDENTITY_COL` is this same string with the +#: same meaning. It is taken from `CUSTOMER_JOIN_FROM` rather than re-typed, because that constant +#: already IS this value under the name the customer leg reads it by. +_ROW_ID_COL = CUSTOMER_JOIN_FROM + +#: Every edge below is `many_to_one` for one uniform reason: each is a CHILD row naming its PARENT. +#: The reverse direction is NEVER declared here. `core.relations.reverse_links` scans for tables +#: pointing at a target and its docstring says the rule outright: "Nothing declares a reverse link; +#: this scan IS the declaration". Declaring an edge from both ends is two sources for one fact. +_M2O = "many_to_one" + + +def _native_model(key): + """The Odoo model behind one `ut_odoo_*` key. `ODOO_TABLE_MODELS` first, the two retired keys + second, and a KeyError rather than a shrug for anything else.""" + return ODOO_TABLE_MODELS.get(key) or _RETIRED_NATIVE[key] + + +#: ⭐⭐ THE TEN TABLES. **`links` NAMES THE CHILD'S COLUMN, THE TARGET KEY, THE TARGET'S COLUMN AND +#: THE CARDINALITY**, in that order, matching `ConnectorTableSpec.links`. +#: +#: ⚠ THESE ARE PLAIN DICTS, NOT `ConnectorTableSpec` INSTANCES, AND THAT IS FORCED. `core` is not +#: importable at the moment this module is imported (`_ut` and `_registry` above are lazy for that +#: exact reason, and `import core.connector_spec` raises `ModuleNotFoundError` on a bare +#: `import odoo_relational` from `aios-web/api`). The dataclass is constructed inside +#: `register_connector_spec()` below, where the import has already succeeded. +#: +#: ⚠ `to_col` IS `pid` WHEREVER THE TARGET IS MATCHED ON ITS ROW IDENTITY. The one exception is +#: `ACCOUNT_JOIN_KEY`, which is an ordinary column on both sides: a GL line carries the account CODE +#: and `ut_odoo_accounts` is keyed by that code, never by a partner id. +#: ⚠ `AGENTS_KEY` and `VENDORS_KEY` are ALSO reachable on their own `agent_id` / `vendor_id` +#: columns, which is how the existing link FIELDS in `customer_fields` and `bill_fields` spell the +#: same join (`{"on": AGENT_JOIN_KEY}` / `{"on": VENDOR_JOIN_KEY}`). Both resolve to the identical +#: `res.partner` id (`read_agents` and `read_vendors` write the row id into both), so the spec uses +#: the one column every grid carries rather than a per-table spelling. +ODOO_SPEC = ( + {"key": INVOICES_KEY, "source": "account_move", "label": "Odoo invoices", + "pinned": "invoice_no", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)}, + + # ⚠ RETIRED FROM `TABLES` BY W33-T44, DECLARED HERE ANYWAY, AND THAT IS THE POINT OF A SPEC. + # `RETIRED_KEYS` pops this key and `PRODUCTS_KEY` from a tenant's document on every sync, but + # the key constants, the field builders and the readers in `_READERS` all still exist, the + # compiled twins `customer_data` / `product_data` present the same subjects, and the done-when + # asks for ten. A spec DESCRIBES a table's shape; it does not assert the table is materialised. + {"key": CUSTOMERS_KEY, "source": "res_partner", "label": _RETIRED_LABEL[CUSTOMERS_KEY], + "pinned": "customer", "identity": (_ROW_ID_COL, JOIN_KEY), + # ⚠ `AGENT_JOIN_KEY`, NEVER `JOIN_KEY`. An agent IS a `res.partner`, so both ids live in one + # namespace and joining agents through `JOIN_KEY` would link every customer to itself and look + # entirely plausible doing it. The same warning stands on `AGENT_JOIN_KEY`'s own declaration. + "links": ((AGENT_JOIN_KEY, AGENTS_KEY, _ROW_ID_COL, _M2O),)}, + + {"key": ORDERS_KEY, "source": "sale_order", "label": "Odoo orders", + "pinned": "order_no", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O),)}, + + # ⚠ RETIRED FROM `TABLES` like `CUSTOMERS_KEY` above, and declared for the same reason. + {"key": PRODUCTS_KEY, "source": "product_product", "label": _RETIRED_LABEL[PRODUCTS_KEY], + "pinned": "product", "identity": (_ROW_ID_COL, PRODUCT_JOIN_KEY), + # the catalogue's HEAD: order lines point at it, it points at nothing. + "links": ()}, + + {"key": AGENTS_KEY, "source": "res_partner", "label": "Odoo agents", + "pinned": "agent", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ()}, + + {"key": ACCOUNTS_KEY, "source": "account_account", "label": "Odoo GL accounts", + "pinned": "account_code", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ()}, + + {"key": BILLS_KEY, "source": "account_move", "label": "Odoo vendor bills", + "pinned": "bill_no", "identity": (_ROW_ID_COL, "odoo_id"), + # ⚠ `VENDORS_KEY`, NOT `CUSTOMERS_KEY`, even though both tables are `res.partner`. MEASURED and + # recorded on `vendor_fields`: 393 vendors, of which 9 also appear in the customer population, + # so pointing bills at the customers table would dangle 384 of 393 links as a mostly empty + # column rather than as an error. + "links": ((VENDOR_JOIN_KEY, VENDORS_KEY, _ROW_ID_COL, _M2O),)}, + + {"key": VENDORS_KEY, "source": "res_partner", "label": "Odoo vendors", + "pinned": "vendor", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ()}, + + # ⭐⭐ THE BRIDGE GRAIN, AND ITS THREE EDGES ARE WHAT MAKE THE DONE-WHEN'S CHAIN EXPRESSIBLE: + # customers <- orders <- order_lines -> products. Two of the three legs are REVERSE readings of + # a child naming its parent, which is why the chain needs no fourth declaration and no path + # search. ⚠ This grain stores no rows here (`READ_THROUGH_KEYS`); it is served from the mirror + # by `routes_odoo_tables.GRID_SOURCES`, whose `cols` map binds exactly `order_id`, `partner_id` + # and `product_id` to `sol.order_id`, `sol.order_partner_id` and `sol.product_id`. + {"key": ORDER_LINES_KEY, "source": "sale_order_line", "label": "Odoo order lines", + "pinned": "order_no", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O), + ("order_id", ORDERS_KEY, _ROW_ID_COL, _M2O), + (PRODUCT_JOIN_KEY, PRODUCTS_KEY, _ROW_ID_COL, _M2O))}, + + # ⚠ `move_id` REACHES MORE DOCUMENTS THAN `ut_odoo_invoices` HOLDS, and saying so is cheaper + # than discovering it: the GL line grain is deliberately UNSCOPED (a ledger whose draft and + # cancelled entries are invisible cannot be reconciled) while the invoices table is posted + # CUSTOMER documents only. So this edge resolves for the customer-document lines and dangles for + # the rest, which is the ordinary shape of a `many_to_one` into a scoped parent and is why the + # cardinality is stated rather than assumed. + {"key": GL_LINES_KEY, "source": "account_move_line", "label": "Odoo GL lines", + "pinned": "entry", "identity": (_ROW_ID_COL, "odoo_id"), + "links": ((JOIN_KEY, CUSTOMERS_KEY, _ROW_ID_COL, _M2O), + ("move_id", INVOICES_KEY, _ROW_ID_COL, _M2O), + (ACCOUNT_JOIN_KEY, ACCOUNTS_KEY, ACCOUNT_JOIN_KEY, _M2O))}, +) + + +#: ⛔⛔ **THE REGISTRATION IS AN IMPORT-ORDER DEPENDENCY AND THIS IS THE SIGN THAT SAYS SO.** +#: `platform/core/connector_spec.py` CANNOT import this file (`core` never imports up), so Odoo's +#: spec reaches that registry by exactly one route: this module being imported. Therefore +#: **`connector_spec.edges_for("odoo")` RETURNS `[]` IN ANY PROCESS THAT HAS NOT IMPORTED +#: `odoo_relational`**, and an empty answer there is an import that never happened, not a connector +#: with no edges. A reader who hits that emptiness should find this paragraph beside the call rather +#: than have to derive it. Amazon and Meta have no such dependency: their rows are literals inside +#: `connector_spec` itself. +#: ⛔ AND IT IS RECORDED RATHER THAN ASSUMED. `core` is genuinely absent from `sys.path` on a bare +#: `import odoo_relational` from `aios-web/api`, and raising there would take the whole route layer +#: down over a declaration nothing on the request path reads. So the failure is CAPTURED in +#: `SPEC_REGISTRATION` and `verify_odoo_relational`'s connector-spec section asserts it is +#: `registered`. A caller that puts `platform/` on the path later may call +#: `register_connector_spec()` again; it is idempotent, because `register()` REPLACES a connector. +SPEC_REGISTRATION = {"connector": "odoo", "registered": False, "error": ""} + + +def register_connector_spec(): + """Register `ODOO_SPEC` into `core.connector_spec` under the connector key `odoo`. + + Returns True when it landed. Records the reason in `SPEC_REGISTRATION` when it did not, and + never raises: see the banner above for why a hard failure here is the wrong trade. + """ + try: + import core.connector_spec as connector_spec + except Exception as exc: # noqa: BLE001 + # ⚠ ASCII ONLY, like `_INVOICE_PRODUCT_LINK_BLOCKER`'s values: this string is short enough to be + # quoted into a report, and standing rule 2 forbids an em dash or an en dash in copy that + # can reach a screen. + SPEC_REGISTRATION.update( + registered=False, + error="core.connector_spec is not importable from here: %s: %s" + % (type(exc).__name__, str(exc)[:120])) + return False + connector_spec.register( + SPEC_REGISTRATION["connector"], + [connector_spec.ConnectorTableSpec(**dict(row, native=_native_model(row["key"]))) + for row in ODOO_SPEC]) + SPEC_REGISTRATION.update(registered=True, error="") + return True + + +register_connector_spec() + + #: ⭐⭐ W49-T25 / C6 — THE COMPILED REGISTRY GRIDS, AND THIS MAP IS WHERE THE WHOLE TICKET LIVES. #: #: `TABLES` above declares the eight mirror grids owned by THIS file. The products and customers @@ -1899,6 +2486,103 @@ def _aios_grid(): return aios_grid +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W52-T11 — THE TWO COMPILED REGISTRY GRIDS ARE CONNECTED DATABASES, AND A SHEET CELL NEEDS +# SOMETHING TO QUERY, NOT JUST A NAME IT RECOGNISES. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# `customer_data` / `product_data` are NOT read-through (`routes_odoo_tables._sources()` names +# neither, and never will by that route — MEASURED against this box's mirror, which carries the +# raw Odoo entities and nothing named `customer_data`/`product_data` at all). Their rows are the +# STORE'S OWN, already computed with every rollup by `modules/customer_data.pool()` / +# `modules/product_data.pool()`, so there is no mirror SQL to derive them from a second time. +# `registry_grid_spec` below is what `routes_sheet.py::_for` calls instead of +# `routes_odoo_tables._source_for` for these two keys: it takes the caller's OWN scoped read +# (`core.perm_scope.scoped_table`, never re-derived and never widened here) and makes it +# queryable on the SAME mirror cursor, so a Sheet cell's aggregate SQL and `View rows`' id query +# run over it exactly as they would over a read-through grid. + +#: DuckDB column type for each field KIND the two grids' MERGED field list may carry (declared +#: JSON contract, tenant-added shared columns, and the "measure" kind CLAUDE.md keeps: R9). +#: Anything not named here stays VARCHAR — safe for text/select/status/multiselect/url/link/ +#: rollup, and for any kind this map has not met yet, the SAME "unknown degrades rather than +#: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids. +_REGISTRY_COL_TYPE = { + "int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE", + "measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP", +} + + +def _registry_col_value(value, kind): + """One field's row value, coerced to what DuckDB's parameter binder accepts for `kind`. + + ⚠ NEVER RAISES. A cell whose stored value does not fit its column's type (a stray string in a + number column, an unparsable date, a link/rollup cell carrying a list rather than a scalar) + degrades to NULL rather than aborting the whole materialisation — one bad row must not blank + `View rows` for every other one. `measure`-kind fields are NOT stored per row at all (R9: a + Metric computes fresh, it does not cache), so they degrade to NULL here too — a SQL aggregate + over an all-NULL DOUBLE column answers NULL rather than raising a DuckDB Binder error, and + `View rows` never reads this value at all (it always probes `COUNT(*)`, never the bound + field), so the id list it returns is unaffected either way. + """ + if value in (None, "") or isinstance(value, (list, tuple, dict)): + return None + try: + if kind == "int": + return int(value) + if kind in ("currency", "number", "pct", "measure"): + return float(value) + if kind == "checkbox": + return bool(value) + if kind == "date": + return str(value)[:10] + if kind == "datetime": + return str(value) + except (TypeError, ValueError): + return None + return str(value) + + +def registry_grid_spec(cur, table_key, fields, rows): + """A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway + DuckDB TEMP TABLE on THIS cursor (W52-T11). + + ⚠ `rows` MUST ALREADY BE THE CALLER'S OWN SCOPED READ — this function only makes them + queryable, it never re-derives them and never widens who may see them. + + ⚠ TEMP AND PER-CURSOR, NOT PER-TENANT. `harness.datastore.ro_con()` caches one cursor per + thread and reuses it across requests; a DuckDB TEMP TABLE lives in the CURSOR's own namespace + (a sibling cursor on the same connection cannot see it — proven empirically against this + file's own mirror), so two requests on two threads never collide, and the same thread's next + request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped + read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`. + + ⭐ THE JOIN KEY IS ALWAYS `pid`, NEVER A DECLARED FIELD. Both grains carry it as a raw row + datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id; + `product_data.pid` is `crc32(default_code)` — see `_related_source_value`'s own note), so it + is forced into the materialised columns here even when the caller's field list never + mentions it, exactly as `build_ids_query` needs `spec["id"]` to be a real column. + """ + keys = list(dict.fromkeys( + str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key"))) + if "pid" not in keys: + keys.insert(0, "pid") + by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)} + by_key.setdefault("pid", {"type": "int"}) + + table = f"_registry_grid_{table_key}" + cols_sql = ", ".join( + f'"{k}" {_REGISTRY_COL_TYPE.get((by_key.get(k) or {}).get("type"), "VARCHAR")}' + for k in keys) + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{table}" ({cols_sql})') + values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys) + for r in (rows or []) if isinstance(r, dict)] + if values: + placeholders = ", ".join(["?"] * len(keys)) + cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values) + return {"table": table, "id": '"pid"', "cols": {k: (f'"{k}"', str) for k in keys}} + + def _declared_from_spec(fields): """One grid's declared field list, reduced to the three keys the snapshot stores. @@ -2070,6 +2754,317 @@ _READERS = { READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W54-T46 / R4 / R7 / CONTRACT C4 — A TABLE BELONGS TO EXACTLY ONE CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"If a tenant is an agency that helps with clients that each have Odoo, then the +# connector for Odoo can be many depending on the API. This is why its critical."* R7 makes the +# isolation between those clients STRICT — there is no blended view, and client A must not be one +# dropped filter away from client B. +# +# ⛔⛔ THE DEFAULT CONNECTION KEEPS THE LEGACY KEYS, AND THAT IS THE WHOLE SHAPE OF THE RULE. +# Renaming tenant #0's ten live tables would move every saved view, grant, cohort, link, rollup +# and nav entry that names them. So the tenant's DESIGNATED DEFAULT connection stays exactly where +# it is and only the connections added after it are namespaced. +# +# ⛔ THE TOKEN IS THE `connection_id`, NEVER THE `slot`. MEASURED (PRD amendment A4): a pre-wave +# row's read-default slot MOVES when a sibling of the same type is deleted (2 -> 1), while its +# connection id does not. A key built from a slot would one day address a different client's rows. + +#: The separator between a base table key and the connection that owns it. TWO underscores, +#: because every base key spells its own words with ONE, so `rpartition` splits it back cleanly. +CONNECTION_SEP = "__" + +#: What `connection_for_table_key` answers for a LEGACY key when it was handed no runtime to +#: resolve the tenant's designated default against. It means "whichever connection is designated +#: default" — a real answer, not an absence, which is what keeps the inverse TOTAL. +DEFAULT_CONNECTION = "default" + +#: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a +#: document that still holds one resolves to its connection rather than reading as a stranger's +#: table — `apply_plan` has to know whose copy it is popping. +BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES] + list(RETIRED_KEYS)) + +#: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated +#: key is a key that can collide with another connection's. The longest base key is +#: `ut_odoo_order_lines` (19) and a connection id is `k_` + 12 hex (14), so a namespaced key is 35 +#: characters; the guard is here for the day one of those two numbers moves. +MAX_TABLE_KEY_LEN = 60 + +#: The characters `core.user_tables._field_key` leaves alone. A `k_` connection id is already +#: inside this set, which is why it survives the normaliser unchanged. +_KEY_CHARS = frozenset("abcdefghijklmnopqrstuvwxyz0123456789_") + + +def default_connection_id(rt): + """This tenant's DESIGNATED DEFAULT Odoo connection id, or None. Decrypts nothing. + + ⛔ NOT RE-DERIVED HERE. `core.keychain.default_connection_id` is the one resolver (W54-T44): + lowest immutable slot, ties by insertion order. A second walk in this file is exactly the + drift that resolver was written to end. + """ + if rt is None: + return None + try: + import core.keychain as keychain + return keychain.default_connection_id(rt, "odoo") + except Exception: # noqa: BLE001 + return None + + +def table_key_for(base_key, connection_id=None): + """The store key `base_key` takes under `connection_id`. + + `connection_id=None` means the tenant's DESIGNATED DEFAULT connection, and it answers the bare + legacy key unchanged: `ut_odoo_invoices` stays `ut_odoo_invoices`. + + ⚠ The result still starts with `ut_`, which is load-bearing three times over: the `ut_*` wall, + `user_tables.is_linkable_target`, and `grid_events._field_share_keys`' shared-stratum branch + all key on that prefix. + """ + base = str(base_key or "").strip() + cid = str(connection_id or "").strip().lower() + if not cid: + return base + if (CONNECTION_SEP in cid or set(cid) - _KEY_CHARS or cid != cid.strip("_")): + raise Refused(f"{cid!r} cannot be used as a table namespace: a connection id has to be " + f"lowercase letters, digits and single underscores, or the table key it " + f"builds would be stored under a different name than the one it is read by") + key = f"{base}{CONNECTION_SEP}{cid}" + if len(key) > MAX_TABLE_KEY_LEN: + raise Refused(f"{key!r} is {len(key)} characters and the store truncates a table key at " + f"{MAX_TABLE_KEY_LEN}; refusing rather than creating a key that two " + f"connections could share. Shorten the base key or the connection id.") + return key + + +def split_table_key(table_key): + """`(base_key, connection_id)` for a key this connector owns, `(None, None)` for anything + else — so the question "is this one of ours" and "whose is it" have one answer. + + A LEGACY (bare) key answers `(base, None)`: no namespace token, because the designated default + keeps its legacy key. + """ + raw = str(table_key or "").strip() + try: + from core import connector_spec + dynamic_prefix = connector_spec.ODOO_MODEL_TABLE_PREFIX + except Exception: # noqa: BLE001 + dynamic_prefix = "ut_odoo_model_" + base, sep, cid = raw.rpartition(CONNECTION_SEP) + if sep and cid and (base in BASE_TABLE_KEYS or base.startswith(dynamic_prefix)): + return base, cid + if raw in BASE_TABLE_KEYS or raw.startswith(dynamic_prefix): + return raw, None + return None, None + + +def base_table_key(table_key): + """The un-namespaced key behind `table_key`, or None when it is not one of this connector's.""" + return split_table_key(table_key)[0] + + +def connection_for_table_key(table_key, rt=None): + """WHICH connection owns this table key — exactly one, or None when the key is not one of + this connector's tables at all. + + ⭐ THE INVERSE IS THE CONTRACT. C4's clause is that a table resolves to exactly one + connection; a namespacing you cannot read backwards is a convention, not an isolation. + A legacy key belongs to the DESIGNATED DEFAULT, which is what `rt` answers; with no `rt` it + answers the `DEFAULT_CONNECTION` sentinel rather than pretending it does not know. + """ + base, cid = split_table_key(table_key) + if base is None: + return None + if cid: + return cid + return (default_connection_id(rt) or DEFAULT_CONNECTION) if rt is not None \ + else DEFAULT_CONNECTION + + +#: ⭐⭐ W54-T49 / R7 / C4 — THE SECOND IMPORT-TIME REGISTRATION IN THIS FILE, AND IT IS THE ONE +#: THAT ARMS A WALL RATHER THAN DECLARING A SPEC. +#: +#: `core.perm_scope.ConnectionScope` refuses a read of a table belonging to another connection, and +#: it answers "whose table is this" through a registration slot because `core` never imports up +#: (`platform/ARCHITECTURE.md`). The inverse lives HERE, so this module fills that slot at import, +#: exactly as `SPEC_REGISTRATION` above fills the connector-spec registry and for the identical +#: reason: the module that owns the fact is the one that declares it. +#: +#: ⛔⛔ WITHOUT THIS, THE WALL IS WHOLE, GATED AND ENFORCING NOTHING. An unfilled slot makes +#: `covers` answer False and `require` report `Unresolvable`, which is fail-closed and therefore +#: silent, so the feature would have shipped and behaved as a no-op +#: [[artifact-with-no-importer]]. `verify_scopes` asserts the slot is filled BY IMPORT, before it +#: registers anything of its own, so deleting the boot call below reds a gate. +#: +#: ⛔ IT NEVER RAISES, FOR THE REASON THE BANNER ABOVE GIVES IN FULL: `core` is genuinely absent +#: from `sys.path` on a bare import of this module from `aios-web/api`, and taking the route layer +#: down over a declaration nothing on the request path reads is the wrong trade. The failure is +#: CAPTURED here instead, in the same three keys. +OWNER_REGISTRATION = {"slot": "perm_scope.connection_owner", "registered": False, "error": ""} + + +def register_connection_owner_slot(): + """Declare this module's `connection_for_table_key` as the answer to "whose table is this". + + Returns True when it landed. Records the reason in `OWNER_REGISTRATION` when it did not, and + never raises: see the banner above. Idempotent, because the slot holds one reader and a second + call replaces it with the same function. + """ + try: + import core.perm_scope as perm_scope + except Exception as exc: # noqa: BLE001 + # ⚠ ASCII ONLY, like `SPEC_REGISTRATION`'s error string: this can be quoted into a report, + # and standing rule 2 forbids an em dash or an en dash in copy that can reach a screen. + OWNER_REGISTRATION.update( + registered=False, + error="core.perm_scope is not importable from here: %s: %s" + % (type(exc).__name__, str(exc)[:120])) + return False + perm_scope.register_connection_owner(connection_for_table_key) + OWNER_REGISTRATION.update(registered=True, error="") + return True + + +register_connection_owner_slot() + + +def table_subject_for(table_key): + """The real-world population this key presents, namespaced by its connection. + + ⛔⛔ THE SUBJECT HAS TO MOVE WITH THE KEY OR A SECOND CONNECTION CANNOT BE CREATED AT ALL. + `_ensure_table_inplace` refuses to create a table whose subject another table in the same + document already claims (one subject, one database, item 12a). Client B's invoices are a + DIFFERENT population from client A's, so the claim carries the connection with it. + """ + base, cid = split_table_key(table_key) + raw_base = base if base is not None else str(table_key or "") + subject = TABLE_SUBJECTS.get(raw_base) + if subject is None and raw_base.startswith("ut_odoo_model_"): + subject = f"odoo.model.{raw_base}" + if not subject or not cid: + return subject + return f"{subject}@{cid}" + + +def is_read_through_key(table_key): + """Does this key name one of the two grains whose rows are served from the mirror? Answered + on the BASE key, so a namespaced copy inherits the statement rather than losing it.""" + return base_table_key(table_key) in READ_THROUGH_KEYS + + +def connection_namespace(rt, connection_id=None): + """`(connection_id, namespace_token)` for a refresh of `connection_id`. + + The token is None for the DESIGNATED DEFAULT (bare, legacy keys) and the connection id for + every other connection. + + ⛔ A NAMED CONNECTION THAT DOES NOT RESOLVE IS A REFUSAL, NEVER A FALLBACK. Serving the + default to a caller that asked for a specific client is precisely the leak R7 forbids, and it + is the same asymmetry `runtime.odoo_source` already holds. + """ + default = default_connection_id(rt) + if connection_id is None: + return default, None + import core.keychain as keychain + cid = keychain.resolve_connection_id(rt, "odoo", connection_id) + if not cid: + raise Refused(f"{str(connection_id)!r} does not name an Odoo connection in this " + f"workspace, so there is nothing to spawn for it. Pick a connection in the " + f"Connectors panel and run the sync again.") + return cid, (None if cid == default else cid) + + +def connection_label(rt, connection_id=None): + """The user's own name for this connection ('Odoo 2' until they rename it), or ''.""" + if rt is None or not connection_id: + return "" + try: + import core.keychain as keychain + for row in keychain.list_entries(rt): + if row.get("type") == "odoo" \ + and str(row.get("connection_id")) == str(connection_id): + return str(row.get("label") or "") + except Exception: # noqa: BLE001 + return "" + return "" + + +def table_label_for(label, conn_label=None): + """The nav label a namespaced table shows. + + ⚠ R9 keeps `TABLES`' shipped label as the default; this only says WHICH connection the grid + belongs to, because two grids sharing a display name is how a person reads the wrong number + off the right looking tab. + """ + conn_label = " ".join(str(conn_label or "").split()) + return f"{label} ({conn_label})" if conn_label else str(label) + + +def namespaced_fields(fields, connection_id=None): + """A field contract re-pointed at ONE connection's tables. + + ⛔⛔ THIS IS WHERE R7 BECOMES STRUCTURAL RATHER THAN CONVENTIONAL. Every `link` bag in the + contracts above names a sibling by its LEGACY key, so a second connection's vendors table + would fold the FIRST connection's bills unless the target moves with the key. A namespaced + table can only ever link to a namespaced sibling of its own connection. + + ⛔ AND A TARGET THAT CANNOT BE NAMESPACED IS DROPPED, never left pointing at shared data. + `customer_link` targets the compiled registry module `customer_data`, which holds the DEFAULT + connection's customers and has no per-connection copy to point at; rendering it on another + client's invoices would show that client one of ours. An absent column is visible; a column + quietly resolving across the isolation boundary is not. + + ⛔⛔ A **RETIRED** TARGET IS DROPPED FOR EVERY CONNECTION, FOR THE IDENTICAL REASON. A retired + table has no per-connection copy any more than a registry module does: `RETIRED_KEYS` are + absent from `TABLES`, so nothing plans, creates or re-creates one under ANY connection. + Re-pointing such a link would mint `ut_odoo_customers__k_...` — a key nothing creates AND + nothing registers as absent, because `routes_grid._register_retired_tables` fills the + absent-table registry from the BARE `RETIRED_KEYS` tuple. An edge naming it would therefore + survive W54-T37's relation enumeration and refuse as a permission wall about a database that + does not exist, which is instruction 22's exact defect re-created for connection 2. + ⚠ MEASURED 2026-09-06: ZERO live contract fields link to a retired key today, so this is + LATENT, not live. It goes live the moment anybody adds one, which is precisely how those two + keys came to need retiring in the first place: they were link targets before they were dropped. + + ⚠ THE TEST IS HERE AND NOT IN `BASE_TABLE_KEYS`. That frozenset's other consumers are + `split_table_key` and `connection_for_table_key`, which NEED the retired keys present so a + retired residue key can still be ATTRIBUTED to its connection during cleanup — `apply_plan`'s + pop loop mints `table_key_for(retired, connection_id)` for exactly that purpose. Narrowing the + set would break the cleanup path; narrowing the DROP TEST is the change. + """ + if not connection_id: + return list(fields) + out = [] + for field in fields: + bag = field.get("link") if isinstance(field, dict) else None + if not isinstance(bag, dict): + out.append(field) + continue + target = str(bag.get("table") or "") + if target not in BASE_TABLE_KEYS or target in RETIRED_KEYS: + continue + out.append({**field, + "link": {**bag, "table": table_key_for(target, connection_id)}}) + return out + + +#: R6 / standing rule 1's SECOND HALF, for the case this ticket creates: a connection that is not +#: the tenant's default has no mirror of its own yet, so there is no honest row source for it. +#: Reading the default connection's mirror would be the exact cross-client leak R7 forbids, so the +#: tables are created EMPTY and the reason is reported rather than inferred from a blank grid. +def _no_mirror_report(connection_id): + return {"subject": "rows", "limit": 0, "effect": "refused", + "cause": "this connection has no mirror of its own yet, and reading the default " + "connection's mirror would put one client's rows in another client's " + "database, which strict isolation forbids", + "recommendation": "give each connection its own mirror sync, then run this refresh " + "again. The tables, their fields and their links are created now, " + "so nothing downstream has to wait for that.", + "connection": str(connection_id or "")} + + class _LentDoc: """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. @@ -2134,23 +3129,72 @@ def read_config(rt): "frozenAt": str(cur.get("frozenAt") or "")} -def grid_choices(rt): - """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from - `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). +def grid_choices(rt, connection_id=None): + """Every API-readable model, with the curated TABLES roster retained as its stable front. ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is what they have today; only an explicit untick turns one off. The alternative — an empty config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. + + W55-T17 keeps two facts separate: ``TABLES`` is what the mirror materialiser knows, while the + tenant-local catalogue is what this API account may read. A readable model already represented + by a curated table is covered there; every remaining readable model receives a stable raw-model + key and defaults off. Stored choices survive catalogue refreshes because config is keyed by the + stable database key, never by its position. """ chosen = read_config(rt)["grids"] - return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} - for _bucket, key, label, _fields in TABLES] - - -def enabled_buckets(rt): + namespace = connection_namespace(rt, connection_id)[1] if connection_id else None + catalogue = readable_model_catalog(rt, namespace) + fixed = [{"key": key, "label": label, "model": ODOO_TABLE_MODELS.get(key, ""), + "relevant": True, "defaultEnabled": True, + "enabled": bool(chosen.get(key, True))} + for _bucket, key, label, _fields in TABLES] + # Before discovery, keep the established curated defaults. A complete stamped catalogue is + # authoritative for this API account, so a curated database whose model is absent must not be + # offered: relevance never widens access. + if catalogue["complete"] and catalogue["capturedAt"]: + readable = {str(row.get("model") or "") for row in catalogue["models"]} + fixed = [row for row in fixed if not row["model"] or row["model"] in readable] + covered = {row["model"] for row in fixed if row["model"]} + dynamic = [] + try: + from core import connector_spec + for item in catalogue["models"]: + model = str(item.get("model") or "") + if not model or model in covered: + continue + key = connector_spec.odoo_model_table_key(model) + dynamic.append({"key": key, + "label": connector_spec.odoo_model_label(model, item.get("name")), + "model": model, "relevant": False, "defaultEnabled": False, + "enabled": chosen.get(key) is True}) + except Exception: # noqa: BLE001 + dynamic = [] + # Choices and config use the actual database key, including its owning connection. + for row in fixed + dynamic: + base = row["key"] + row["baseKey"] = base + row["key"] = table_key_for(base, namespace) + row["enabled"] = bool(chosen.get(row["key"], row["defaultEnabled"])) + return sorted(fixed + dynamic, + key=lambda row: (not row["enabled"], not row["relevant"], + str(row["label"]).casefold(), row["key"])) + + +def enabled_buckets(rt, connection_id=None): """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" chosen = read_config(rt)["grids"] - return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} + namespace = connection_namespace(rt, connection_id)[1] if connection_id else None + return {bucket for bucket, key, _l, _f in TABLES + if chosen.get(table_key_for(key, namespace), True)} + + +def enabled_discovered_models(rt, connection_id=None): + """The explicitly enabled long-tail model rows; curated TABLES never enter this set.""" + fixed = {str(model) for model in ODOO_TABLE_MODELS.values() if model} + return [row for row in grid_choices(rt, connection_id) + if row.get("enabled") is True and row.get("model") not in fixed + and str(row.get("key") or "").startswith("ut_odoo_model_")] def sync_seconds(rt): @@ -2173,7 +3217,7 @@ def frozen(rt): return bool(read_config(rt)["frozen"]) -def plan(cur, rt=None): +def plan(cur, rt=None, connection_id=None, buckets=None): """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and @@ -2184,6 +3228,16 @@ def plan(cur, rt=None): `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's second sentence carried as data rather than left for a reader to infer from an empty list. + ⭐⭐ W54-T46 / R4 / R7 — `connection_id` IS THE NAMESPACE TOKEN, and `None` means the + tenant's DESIGNATED DEFAULT, whose keys stay exactly the legacy ones. Every key this function + reads a cap for, reports a limit for and counts against `MAX_TABLES` is the NAMESPACED key, so + two connections are two disjoint sets of tables and neither can be planned over the other. + + ⛔ `cur` MAY BE None, and for a non-default connection it always is. There is one DuckDB mirror + per tenant and it holds the DEFAULT connection's book; handing it to another connection's plan + would build one client's rows into another client's database. So a plan with no cursor builds + NO rows and says why, per table, in `limits` (standing rule 1's second half). + ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would create some and refuse the rest — leaving a locked invoices database with no rollup host, @@ -2200,8 +3254,14 @@ def plan(cur, rt=None): # not apply on exactly the run that creates the databases. This module DECLARES these keys, so # it is the honest place to say what they are. Idempotent (a set add), and it fills the # evaluator's input rather than becoming a second evaluator. - ut.register_connected(*[key for _b, key, _l, _f in TABLES]) - excluded = excluded_ids(cur) + # Selection precedes readers, limits, shrink checks and table capacity accounting. + # A disabled or unrelated database must do no work and cannot veto this operation. + selected = set(buckets) if buckets is not None else None + tables = [row for row in TABLES if selected is None or row[0] in selected] + keys = {bucket: table_key_for(key, connection_id) + for bucket, key, _l, _f in tables} + ut.register_connected(*keys.values()) + excluded = excluded_ids(cur) if cur is not None and tables else set() # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this @@ -2245,15 +3305,24 @@ def plan(cur, rt=None): # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so # what is lent is the document as it stands the moment this plan applies. lent = _LentDoc({**existing, - **{k: {**(existing.get(k) or {}), "readThrough": True} + **{table_key_for(k, connection_id): { + **(existing.get(table_key_for(k, connection_id)) or {}), + "readThrough": True} for k in READ_THROUGH_KEYS}}, ut.STORE_KEY) built, limits = {}, {} caps = {} - for bucket, key, _label, _fields in TABLES: + for bucket, _base, _label, _fields in tables: + key = keys[bucket] reader = _READERS.get(bucket) caps[key] = cap = ut.row_limit(key, st=lent) + if cur is None and reader is not None and cap != 0: + # ⛔ A CONNECTION WITH NO MIRROR OF ITS OWN. Not silence, and not another connection's + # rows: an empty table with its reason attached (standing rule 1's second half). + built[bucket] = [] + limits[key] = _no_mirror_report(connection_id) + continue if reader is None or cap == 0: built[bucket] = [] report = ut.limit_report(key, st=lent) @@ -2264,14 +3333,16 @@ def plan(cur, rt=None): problems = [] if rt is not None: - needed = [k for _b, k, _l, _f in TABLES if k not in existing] + needed = [k for k in keys.values() if k not in existing] if needed and len(existing) + len(needed) > ut.MAX_TABLES: problems.append( f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " - f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " + f"needs {len(needed)} more ({', '.join(needed)}) for connection " + f"{connection_id or 'default'}; refusing rather than creating " f"part of a linked set") - for bucket, key, _label, _fields in TABLES: + for bucket, _base, _label, _fields in tables: + key = keys[bucket] rows = built[bucket] cap = caps[key] # asked ONCE per table, above — never re-read per loop # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED @@ -2314,7 +3385,8 @@ def plan(cur, rt=None): return built -def apply_plan(rt, built, username="automation", today=None, report=None): +def apply_plan(rt, built, username="automation", today=None, report=None, + connection_id=None, retire=True): """Create-or-merge every table in `built` and its rows. Idempotent by construction. Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the @@ -2322,6 +3394,11 @@ def apply_plan(rt, built, username="automation", today=None, report=None): statement rather than a hopeful one. ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. + + ⭐⭐ W54-T46 — `connection_id` NAMES WHOSE TABLES THESE ARE. `None` is the tenant's designated + default and writes the legacy keys, byte for byte as before. Any other connection writes its + own namespaced keys, its own namespaced LINK targets (`namespaced_fields`) and its own + namespaced subject claims, so the set it writes and the set the default writes are disjoint. """ if built.get("problems"): raise Refused("; ".join(built["problems"])) @@ -2330,7 +3407,11 @@ def apply_plan(rt, built, username="automation", today=None, report=None): # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. retired = [] - plans = [(key, label, fields(), built[bucket]) + conn_label = connection_label(rt, connection_id) + plans = [(table_key_for(key, connection_id), + table_label_for(label, conn_label), + namespaced_fields(fields(), connection_id), + built[bucket]) for bucket, key, label, fields in TABLES if bucket in built] # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. @@ -2360,11 +3441,20 @@ def apply_plan(rt, built, username="automation", today=None, report=None): # with `TypeError: list indices must be integers`. One dict, two value shapes, is a # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through # its OWN channel and the return type stays exactly what it was. - for key in RETIRED_KEYS: + # ⚠ THIS CONNECTION'S copies only. A retirement is per table, and popping the bare + # keys while refreshing connection 2 would delete the DEFAULT connection's rows. + for key in RETIRED_KEYS if retire else (): + key = table_key_for(key, connection_id) if cur.pop(key, None) is not None: retired.append(key) for key, label, fields, rows in plans: written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + from core import connector_sync + cur[key]["connector"] = { + **connector_sync.database_identity( + "odoo", connection_for_table_key(key, rt), key, + ODOO_TABLE_MODELS.get(base_table_key(key), "")), + "key": "odoo", "readOnly": True} return cur rt.update(_ut().STORE_KEY, _apply_all, flush="sync") @@ -2390,6 +3480,8 @@ def _ensure_table(rt, key, label, fields, rows, username, stamp): def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. + ``rows=None`` publishes the schema without reading or changing any existing record. + ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` was built FROM open invoices, removal meant a customer who paid their bill vanished from the @@ -2397,11 +3489,13 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. """ ut = _ut() - wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} + wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in (rows or ())} for row in wanted.values(): row["refreshed"] = stamp counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} - subject = TABLE_SUBJECTS.get(key) + # W54-T46: the subject carries the connection, or client B's invoices collide with + # client A's claim and the second connection can never be created at all. + subject = table_subject_for(key) table = cur.get(key) if table is None: if len(cur) >= ut.MAX_TABLES: @@ -2439,13 +3533,13 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): # delete records, while fields stay addable. Odoo owns this population. "recordMode": ut.AUTOMATION_RECORD_MODE, } - table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) + table["recordMode"] = ut.AUTOMATION_RECORD_MODE # W30-T35 — the durable "my rows are not in this document" statement, on the tables no # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not # `setdefault`: it is derived from the code's own structure, so the code is what it must agree # with, and a definition that somehow lost the flag should regain it rather than keep serving # an empty grid. - if key in READ_THROUGH_KEYS: + if is_read_through_key(key): table["readThrough"] = True # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same # reason `readThrough` is: it is derived from this module's own structure, so the code is what @@ -2498,6 +3592,9 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): stored[prop] = field[prop] else: stored.pop(prop, None) + if rows is None: + counts["rows"] = len(table.get("rows") or {}) + return counts stored_rows = table.setdefault("rows", {}) for rid, values in wanted.items(): current = stored_rows.get(rid) @@ -2516,27 +3613,316 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): return counts +def _model_field_specs(catalogue_row): + """Every permission-visible schema field; binary payloads stay outside the row reader.""" + fields = catalogue_row.get("fields") if isinstance(catalogue_row, dict) else {} + fields = fields if isinstance(fields, dict) else {} + ordered = sorted(fields.items(), key=lambda item: (item[0] != "id", item[0])) + out = [] + for raw_name, meta in ordered: + meta = meta if isinstance(meta, dict) else {} + raw_type = str(meta.get("type") or "char") + out.append({"key": str(raw_name), + "label": str(meta.get("string") or raw_name), + "type": _MODEL_FIELD_TYPES.get(raw_type, "text"), + "source": "odoo", + **({"description": "Binary content is not loaded into this database."} + if raw_type == "binary" else {}), + "automation": {"preset": True}}) + if not any(field["key"] == "id" for field in out): + out.insert(0, {"key": "id", "label": "ID", "type": "int", "source": "odoo", + "automation": {"preset": True}}) + for preferred in ("display_name", "name", "id"): + match = next((field for field in out if field["key"] == preferred), None) + if match is not None: + match["pinned"] = True + break + return out + + +def publish_database_schema(rt, table_key, connection_id=None, username="automation", today=None): + """Publish one cached, permission-visible database definition without source calls or row work. + + Enable owns the choice and a durable sync job owns materialization. This seam only merges the + schema, so retrying it preserves existing rows, authored fields and the navigation rename. + """ + base, key_namespace = split_table_key(table_key) + requested = connection_id if connection_id is not None else key_namespace + _cid, namespace = connection_namespace(rt, requested) + if key_namespace is not None and key_namespace != namespace: + raise Refused("the database belongs to a different connection") + key = table_key_for(base or table_key, namespace) + choice = next((row for row in grid_choices(rt, requested) if row["key"] == key), None) + if choice is None: + raise Refused("the database is not in the readable Odoo catalogue") + model = str(choice.get("model") or "") + catalogue = readable_model_catalog(rt, namespace) + row = next((row for row in catalogue["models"] if row.get("model") == model), None) + # Older default connections have a fields_get snapshot predating model discovery. A stamped + # discovery catalogue supersedes it; a different connection never borrows that snapshot. + if row is None and namespace is None and not catalogue.get("capturedAt"): + raw = (_ut().odoo_schema_snapshot(rt).get("models") or {}).get(model) + if isinstance(raw, dict): + row = {"model": model, "fields": raw} + if not row or not row.get("fields"): + raise Refused("the field catalogue is unavailable for this database. Refresh the catalogue first.") + raw_fields = _model_field_specs(row) + declared = next((fields() for _bucket, known_key, _label, fields in TABLES + if known_key == (base or table_key)), []) + fields = namespaced_fields(declared, namespace) + have = {field["key"] for field in fields} + fields.extend(field for field in raw_fields if field["key"] not in have) + from core import connector_sync + identity = connector_sync.database_identity( + "odoo", connection_for_table_key(key, rt), key, model) + stamp = today or _iso_today() + + def _publish(current): + current = current if isinstance(current, dict) else {} + _ensure_table_inplace(current, key, + table_label_for(choice["label"], connection_label(rt, namespace)), + fields, None, username, stamp) + current[key]["connector"] = {**identity, "key": "odoo", "readOnly": True} + return current + + rt.update(_ut().STORE_KEY, _publish, flush="sync") + _ut().register_connected(key) + return {"key": key, "identity": identity, "state": "catalogue-ready", + "fieldCount": len(fields), "sourceFields": [field["key"] for field in raw_fields]} + + +def _model_cell(value): + """Make XML-RPC relation/list values durable without inventing a relation to another table.""" + if isinstance(value, (list, tuple, dict)): + return _json.dumps(value, ensure_ascii=False, sort_keys=isinstance(value, dict)) + return value + + +def _read_model_rows(client, catalogue_row): + """Read one explicitly enabled model in stable pages; every call is a read method.""" + model = str(catalogue_row.get("model") or "") + metadata = catalogue_row.get("fields") or {} + fields = [field["key"] for field in _model_field_specs(catalogue_row) + if (metadata.get(field["key"]) or {}).get("type") != "binary"] + rows, offset = [], 0 + while True: + page = client.search_read(model, [], fields, limit=MODEL_READ_PAGE, + offset=offset, order="id") + for item in page or (): + rid = (item or {}).get("id") + if rid in (None, ""): + continue + rows.append({"_id": str(rid), + **{name: _model_cell((item or {}).get(name)) for name in fields}}) + if len(page or ()) < MODEL_READ_PAGE: + break + offset += len(page) + return rows + + +def sync_discovered_models(rt, source_connection_id=None, namespace=None, + username="automation", today=None, table_keys=None): + """Materialise only explicitly enabled long-tail models, atomically and tenant-bound. + + Disabled models are absent from this pass rather than represented by an empty row set. That + distinction is the preservation guarantee: no updater touches their definition, rows, fields, + grants, ownership, or tenant rename, and re-enabling resumes against the same stable key. + """ + requested = set(table_keys) if table_keys is not None else None + if requested is not None and not requested: + return {} + # An enable already selected a catalogue entry. Do not turn that one-table operation into + # discovery of every source model, even when the general discovery TTL has expired. + catalogue = (readable_model_catalog(rt, namespace) if requested is not None + else refresh_readable_model_catalog(rt, connection_id=namespace)) + by_model = {str(row.get("model") or ""): row for row in catalogue.get("models") or ()} + selected = [choice for choice in enabled_discovered_models(rt, source_connection_id) + if str(choice.get("model") or "") in by_model + and (requested is None or choice["key"] in requested)] + if not selected: + return {} + ut = _ut() + existing = dict(rt.get(ut.STORE_KEY) or {}) + keys = [str(choice["key"]) for choice in selected] + missing = [key for key in keys if key not in existing] + if missing and len(existing) + len(missing) > ut.MAX_TABLES: + raise Refused(f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} databases and " + f"needs {len(missing)} more enabled Odoo models; nothing was created") + source = (rt.odoo_source() if source_connection_id is None + else rt.odoo_source(source_connection_id)) + if source is None: + raise Refused("the selected Odoo connection is unavailable, so no discovered model was read") + prepared, errors = [], [] + from core.odoo import get_odoo + with source._bound(): # tenant credential + scope + client = get_odoo() + for choice, key in zip(selected, keys): + model = str(choice["model"]) + row = by_model[model] + try: + values = _read_model_rows(client, row) + except Exception as exc: # noqa: BLE001 + errors.append(f"{model}: {type(exc).__name__}: {exc}") + continue + stored_n = len(((existing.get(key) or {}).get("rows")) or {}) + if stored_n and len(values) < stored_n * MAX_SHRINK: + errors.append(f"{model}: read returned {len(values)} rows against {stored_n} " + "stored; preserved the prior database instead") + continue + prepared.append((key, str(choice["label"]), _model_field_specs(row), values, model)) + written = {} + stamp = today or _iso_today() + ut.register_connected(*(key for key, _label, _fields, _rows, _model in prepared)) + + def _apply(cur): + cur = cur if isinstance(cur, dict) else {} + for key, label, fields, rows, model in prepared: + written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + from core import connector_sync + cur[key]["connector"] = { + **connector_sync.database_identity( + "odoo", connection_for_table_key(key, rt), key, model), + "key": "odoo", "readOnly": True} + return cur + + if prepared: + rt.update(ut.STORE_KEY, _apply, flush="sync") + return {"tables": written, "errors": errors} + + def is_royal(tenant): + """ENTITLEMENT by slug, kept verbatim for the five callers outside this module. + + ⚠ IT IS NO LONGER THE SPAWN'S ADMISSION TEST — `available()` below is (W53-T39 / R3). + It survives because `main.py`, `routes_odoo_tables`, `routes_grid` and `routes_tables` still + gate their ROUTES on it, and because `is_royal("")` is deliberately True with + `verify_automation`'s W35-T35 leg pinning that as a fixture contract. Deleting it here would + be a silent regression in a gate this module does not run. + """ return str(tenant or "").strip().lower() in RI_SLUGS -def refresh(rt, tenant, username="automation", cur=None, today=None): +def available(rt=None, tenant_key=None, connection_id=None): + """⭐⭐ R3 — DOES THIS TENANT HAVE ODOO CONFIGURED WITH LIVE CREDENTIALS? Never a slug list. + + R3, verbatim: *"The connector table spawn gates on does this tenant have this connector + configured with live credentials, never on a tenant-slug allow-list. Applies to every + connector."* So this is the Odoo sibling of `amazon_relational.available` / + `meta_relational.available`, and the three connectors now answer the same question the same + way. + + ⛔ THE RESOLUTION IS NOT RE-DERIVED HERE, AND THAT IS THE POINT. It is + `TenantRuntime.odoo_flag_key()`, whose docstring already carries R3's three rules: first + unlocked keychain `odoo` entry (ANY tenant), else tenant #0's compiled env source, else + nothing. A second copy of those rules in this file is exactly the drift `odoo_flag_key` was + written to end — a spawn admitted against one resolution and a pause flag written against + another disagree while both report success. + + ⚠ `odoo_flag_key()` rather than `odoo_source()`: the identity is cheap (a truthy key means a + source WOULD serve) where the connector is a whole client build. + + ⚠ READ THROUGH `rt`, NEVER THROUGH A MODULE-LEVEL `os.environ`, so a fixture runtime can + answer. A runtime that cannot answer the question answers NO (fail closed): the harm this + guards is a spawn, and a spawn makes wrong numbers durable. + + ⛔ ADDING A SECOND ODOO TENANT NEEDS NO EDIT HERE. Anyone with a keychain `odoo` entry is + admitted, whatever their slug. + + ⭐ W54-T46 / R4 — `connection_id` ASKS THE SAME QUESTION OF ONE NAMED CONNECTION. + `odoo_flag_key` already answers per connection and already refuses to fall back to the + environment for a named one, so admission for connection 2 tests connection 2's credentials + and nothing else. No id is the pre-wave call, unchanged. + """ + # ⛔ THE UNNAMED CALL IS SPELLED WITH NO ARGUMENT, VERBATIM, AND THAT IS NOT STYLE. + # `verify_odoo_relational`'s `_CredRT` fixture defines `odoo_flag_key(self)` with no + # parameter, so passing even `None` positionally raises TypeError, which the handler below + # swallows as "fail closed" — eight admission legs went red for a reason that had nothing to + # do with the rule they guard. The pre-wave call stays the pre-wave call. + try: + if rt is not None: + return bool(rt.odoo_flag_key(connection_id) if connection_id is not None + else rt.odoo_flag_key()) + except Exception: # noqa: BLE001 + return False # unreadable = fail closed + if not tenant_key: + return False + # A caller holding only a slug (a script, a probe) resolves its own runtime. Never reached + # from `refresh`, which is always handed one. + try: + from harness import runtime as _runtime + _r = _runtime.get_runtime(str(tenant_key)) + return bool(_r.odoo_flag_key(connection_id) if connection_id is not None + else _r.odoo_flag_key()) + except Exception: # noqa: BLE001 + return False + + +def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None, + all_connections=False, table_keys=None): """THE entry point — the store-resync path and the route both call this. ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than silently authoritative. + + ⭐⭐ W54-T46 / R4 / R7 / C4 — ONE CONNECTION PER CALL. `connection_id=None` resolves the + tenant's DESIGNATED DEFAULT and is byte-for-byte the pre-wave behaviour, down to the return + shape (`verify_odoo_relational` asserts on it). A named connection spawns THAT connection's + namespaced tables, admitted against THAT connection's credentials, and can neither read nor + write the default connection's keys. + + ⚠ `all_connections=True` delegates to `refresh_all` below, which is the loop over every Odoo + connection this tenant holds. It is a flag on this entry point rather than a second call in + `main.py` for one measured reason: `verify_odoo_relational` pins the LITERAL `_rel.refresh(` + in `main.py`'s source and reads the 2000 characters after it to prove the relation cells are + computed after the spawn. Spelling the boot call `_rel.refresh_all(` turns two legs of a gate + this lane may not edit RED for a reason that has nothing to do with the rule they guard. """ - if not is_royal(tenant): - raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " - f"Imports only); refusing to spawn empty locked databases") - # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, - # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is - # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process - # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a - # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked - # databases from another customer's rows — a spawn, not a read, so the wrong numbers become - # durable. Entitlement is not residency. + if all_connections: + if table_keys is not None: + raise Refused("a database selection must name one connection") + return refresh_all(rt, tenant, username=username, cur=cur, today=today) + # ⛔ THE NAMESPACE IS SETTLED BEFORE ANY GUARD RUNS, because a named connection that does not + # resolve must be REFUSED rather than quietly served the default (R7). + if rt is None: + cid, ns = connection_id, (connection_id or None) + else: + cid, ns = connection_namespace(rt, connection_id) + # ⭐⭐ W53-T39 / R3 — THE ADMISSION IS A CREDENTIALS TEST, NOT A TENANT NAME. + # + # ⛔ THE OLD GATE'S REASON IS KEPT AND ONLY ITS MECHANISM IS REPLACED. It read + # `is_royal(tenant)` against a hardcoded `RI_SLUGS`, and the reason it gave was REAL: a tenant + # with no Odoo mirror behind these tables "would get empty locked databases" in their product. + # That harm is unchanged, so the refusal is unchanged; what changes is WHAT MAKES IT TRUE. A + # slug list is an allow-list that a second Odoo tenant can only join by a code edit, which is + # precisely what R3 forbids. Credentials answer the same question and answer it for everybody. + # + # ⚠ THE FROZEN CLAUSE IS NOT A LOOPHOLE, IT IS R10 SURVIVING. `disconnect` REMOVES the + # credential, so a disconnected workspace has none by construction — and the guard below + # answers a freeze with a REPORT rather than an exception, deliberately, because the resync + # loop calls this every cycle and an exception is not a status. Refusing here would hand that + # loop a raise for the one state it is meant to report. So a frozen workspace is admitted this + # far and stopped by its own guard, three lines down, with its own words. + # ⛔ THE UNNAMED CALL IS THE PRE-WAVE CALL, ARGUMENT FOR ARGUMENT. `verify_odoo_relational`'s + # R3 negative control swaps `available` for a two-parameter lambda that restores the slug + # allow-list; passing `connection_id=` unconditionally makes that swap raise TypeError instead + # of refusing, and the control stops being a control. A named connection is the only case that + # needs the third argument, so it is the only case that passes one. + _admitted = (available(rt) if connection_id is None + else available(rt, connection_id=connection_id)) + if not _admitted and not (rt is not None and frozen(rt)): + raise Refused("no Odoo credential is configured for this workspace, so there is nothing " + "behind these tables and the spawn would create empty locked databases. " + "Add an Odoo credential in the Connectors panel, then run the sync again.") + # ⭐⭐ W31-T45 / D-169 — THE CREDENTIAL GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT + # QUESTIONS, and this is the one place in the codebase where that is easy to miss. The gate + # above asks "does this tenant have Odoo configured"; it says nothing about WHICH DuckDB file + # this process has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a + # `use_path` in a provisioning script) carries tenant #0's own Odoo credential, passes the + # gate above, and then WRITES tenant #0's locked databases from another customer's rows — a + # spawn, not a read, so the wrong numbers become durable. Entitlement is not residency, and + # ⚠ W53-T39 did not weaken this by one line: the gate above changed its QUESTION, not its + # blindness. Credentials say no more about residency than a slug did. # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. if rt is not None: @@ -2552,40 +3938,114 @@ def refresh(rt, tenant, username="automation", cur=None, today=None): return {"tables": {}, "frozen": True, "note": "this workspace has disconnected Odoo; its databases are frozen as " "static data and are not being refreshed"} - if cur is None: + # ⛔⛔ THE MIRROR BELONGS TO THE DEFAULT CONNECTION, AND IT IS NOT LENT OUT. There is one + # DuckDB file per tenant (`harness/datastore` is a one-file-at-a-time process global, D-29), + # and it holds the book of whichever Odoo the tenant syncs — the default connection's. Reading + # it for connection 2 would write client A's rows into client B's database with nothing red + # anywhere, which is the exact failure R7 exists to prevent. So a non-default connection plans + # with NO cursor: its tables, fields and links are created, its rows are empty, and `limits` + # says why per table (standing rule 1's second half). + if ns is not None: + cur = None + elif cur is None: from harness import datastore cur = datastore.ro_con() - built = plan(cur, rt=rt) - # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the - # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a - # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so - # every cap, refusal and limit report is still computed over the full set — a config must not - # be able to hide a problem by hiding the table that has it. - # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from - # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not - # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. - if rt is not None: - keep = enabled_buckets(rt) - skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) - for bucket, _key, _l, _f in TABLES: - if bucket not in keep: - built.pop(bucket, None) - else: - skipped = [] + requested = set(table_keys) if table_keys is not None else None + if requested is not None: + for key in requested: + base, owner = split_table_key(key) + if base is None or owner != ns: + raise Refused("the selected database does not belong to this Odoo connection") + keep = (enabled_buckets(rt, connection_id) if rt is not None + else {bucket for bucket, _k, _l, _f in TABLES}) + if requested is not None: + keep &= {bucket for bucket, key, _l, _f in TABLES + if table_key_for(key, ns) in requested} + skipped = sorted(table_key_for(key, ns) + for bucket, key, _l, _f in TABLES if bucket not in keep) + built = plan(cur, rt=rt, connection_id=ns, buckets=keep) # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no # longer built) SAYS SO, with the keys. plan_report = {} - written = apply_plan(rt, built, username=username, today=today, report=plan_report) + written = apply_plan(rt, built, username=username, today=today, report=plan_report, + connection_id=ns, retire=requested is None) + # The fixed mirror roster above and the discovered long tail deliberately use separate + # materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled + # databases are not passed an empty set and therefore cannot be erased by reconciliation. + discovered_report = (sync_discovered_models( + rt, source_connection_id=connection_id, namespace=ns, username=username, today=today, + table_keys=({key for key in requested if base_table_key(key) not in ODOO_TABLE_MODELS} + if requested is not None else None)) + if rt is not None else {}) # ⭐⭐ W49-T25 / C6 — THE SCHEMA CATALOGUE FOLLOWS THE CODE WITH NO MANUAL STEP. Runs AFTER the # tables are applied, so the snapshot describes a pass that actually landed. The two early # returns above are respected by construction: a tenant with no Odoo entitlement raised, and a # FROZEN workspace returned before reaching here, so a disconnected connector is not # re-recorded. See `_refresh_declared_schema` for why it writes the declared half only. - declared_report = _refresh_declared_schema(rt) - return {"tables": written, - **({"schemaDeclared": declared_report} if declared_report else {}), - **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), - # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. - **({"skipped": skipped} if skipped else {}), - **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} + declared_report = _refresh_declared_schema(rt) if requested is None else None + out = {"tables": written, + **({"schemaDeclared": declared_report} if declared_report else {}), + **({"discovered": discovered_report} + if discovered_report and (discovered_report.get("tables") + or discovered_report.get("errors")) else {}), + **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), + # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. + **({"skipped": skipped} if skipped else {}), + **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} + # ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below + # are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this + # dict as it stands for the default. + if ns is not None: + out["connection"] = cid + if built.get("limits"): + out["limits"] = built["limits"] + return out + + +def refresh_all(rt, tenant, username="automation", cur=None, today=None): + """Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4). + + ⛔⛔ ONE CONNECTION'S FAILURE MUST NOT ABORT THE OTHERS. An agency with four clients cannot + have client D's databases go unwritten because client B rotated a key this morning. A refusal + or an exception is RECORDED AGAINST THE CONNECTION IT BELONGS TO and the loop continues — + exactly as `refresh` reports a frozen workspace rather than raising, and for the same reason: + this runs inside the boot rebuild and the resync loop, where an exception is not a status. + + ⭐ THE DEFAULT CONNECTION'S NUMBERS STAY AT THE TOP LEVEL, so `main.py`'s log line and every + other caller that reads `counts["invoices"]` keeps reading the same thing it read before. The + per-connection map rides beside them under `connections`. + + ⚠ ORDERED BY SLOT, which is the ordinal the tenant fixed when it added each connection, so the + log line reads in the order the Connectors panel does. The ORDER is a slot question; the KEY + is never one (see `table_key_for`). + """ + reports = {} + default = default_connection_id(rt) + rows = [] + try: + import core.keychain as keychain + rows = [r for r in keychain.list_entries(rt) if r.get("type") == "odoo"] + except Exception: # noqa: BLE001 + rows = [] + order = [str(r.get("connection_id") or r.get("id")) + for r in sorted(rows, key=lambda r: (int(r.get("slot") or 0), str(r.get("id"))))] + # ⚠ NO KEYCHAIN CONNECTION AT ALL is the pre-wave posture, not an empty loop: tenant #0 runs + # on its compiled env source, and `refresh` with no id is exactly the call it always made. + if not order: + order = [None] + elif default and default not in order: + order.insert(0, default) + primary = None + for entry_cid in order: + named = None if (entry_cid is None or entry_cid == default) else entry_cid + try: + one = refresh(rt, tenant, username=username, + cur=(cur if named is None else None), + today=today, connection_id=named) + except Exception as exc: # noqa: BLE001 + one = {"tables": {}, "failed": f"{type(exc).__name__}: {exc}"} + reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one + if named is None and primary is None: + primary = one + return {**(primary or {"tables": {}}), "connections": reports} diff --git a/api/pages.py b/api/pages.py index 5548f684b5028e59c15b04ca0b9a328c34fe66e5..2172cb03d87848251f8be139a375b0f51c004d2a 100644 --- a/api/pages.py +++ b/api/pages.py @@ -159,7 +159,7 @@ def chart(key, spec, series, fields, rows, x_order=None, drill=None, total=None, for f in fields: if f.get("type") not in VIZ_TYPES: raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a " - f"viz FieldType {VIZ_TYPES} — `money` is the kpi/table vocabulary") + f"viz FieldType {VIZ_TYPES}; `money` is the kpi/table vocabulary") rows = list(rows) out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series), "fields": list(fields), "rows": rows, @@ -317,7 +317,7 @@ _BUILDERS = {} #: goes red — which is the only way a "deliberately empty" claim can stay true. BUILDERS_EMPTY_BECAUSE = ( "the Sales page was deleted by owner ruling in wave 16; no dashboard page is published " - "today. The route, the envelope and the grant wall stay — a new page ships by registering " + "today. The route, the envelope and the grant wall stay; a new page ships by registering " "a builder here (ARCHITECTURE.md §7), not by rebuilding this file." ) diff --git a/api/rollup_sql.py b/api/rollup_sql.py index 21955a30993b5565b1d05741a7b5781c69515d63..0d0a50239ace92b4bb44bc4ce13f89169483c29a 100644 --- a/api/rollup_sql.py +++ b/api/rollup_sql.py @@ -81,7 +81,7 @@ def group_values(bag, today=None): rng = W.resolve({"kind": window}, today or _today()) if rng is None: raise RollupSourceError( - f"window {window!r} could not be resolved — refusing rather than widening this " + f"window {window!r} could not be resolved; refusing rather than widening this " f"rollup to all time") date_from, date_to = rng @@ -96,7 +96,7 @@ def group_values(bag, today=None): # ⛔ THE REFUSAL THAT MATTERS. See the module header: a truncated GROUP set is a wrong # answer per parent, not a shortened one, and it would look completely normal on screen. raise RollupSourceError( - f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups — refusing to " + f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups; refusing to " f"write cells from a truncated result") # ⛔ THE KEY COLUMN IS NOT ALWAYS `_id`, and reading it unconditionally was a SILENT diff --git a/api/routes_admin.py b/api/routes_admin.py index 7667db2b6a61c36e1c771203831e06d0ba8d67a4..e13219b90fbc0aac5e00916ac74377ce138fc7cf 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -75,10 +75,7 @@ def system_data_inspector(session: Session = Depends(require_session)): _UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") #: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. _UNAME_MAX = 254 -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 +#: Password policy lives in core.users so this route and self-service rotation cannot disagree. _ROLES = ("user", "admin") @@ -98,7 +95,7 @@ def _require_store(): ok = False if not ok: raise err(503, "store_unavailable", - "the tenant store is unavailable — accounts cannot be read or changed") + "the tenant store is unavailable; accounts cannot be read or changed") def _registry(): @@ -131,6 +128,12 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None): # The session-revocation handle. An admin needs it: it is the only visible evidence # that "sign them out everywhere" actually happened. "epoch": int(rec.get("epoch") or 0), + # Administrators need the rotation state, never the credential history behind it. + "passwordChangedAt": users.password_changed_at(rec), + "passwordExpired": users.password_expired(rec), + # The reset controls need enrolment state, never the secret, hashes or code count. + "mfaEnrolled": users.mfa_enrolled(rec), + "mfaEnrolledAt": rec.get("mfa_enrolled_at") or None, # S3 ask 4 — one sentence for the roster's Access column, so the list does not need # a /perms round trip per row to fill one cell. Computed from the same record the # editor will open, so the two cannot disagree. @@ -213,10 +216,16 @@ def _clean_username(v): return uname -def _clean_password(v): +def _clean_password(v, username=None, name=None, email=None, record=None, + allow_early_rotation=False): pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + failures = users.password_failures(pw, username=username, name=name, email=email, + record=record, + allow_early_rotation=allow_early_rotation) + if failures: + # Name every broken rule so the caller can correct the value without guessing the policy. + raise err(400, "weak_password", + "That password does not meet the policy: " + str(users.PasswordPolicyError(failures))) return pw @@ -249,7 +258,7 @@ def _clean_bus(v): raise err(400, "bad_bus", "a business-unit id must be a number") if b not in users.BU_LABELS: raise err(400, "bad_bus", - f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + f"{b} is not a known business unit {sorted(users.BU_LABELS)}; an " "unrecognised id would widen access to every BU") out.append(b) return sorted(set(out)) @@ -268,7 +277,7 @@ def _clean_modules(v): raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") if not isinstance(v, (list, tuple)) or not v: raise err(400, "bad_modules", - "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "modules must be 'all' or a NON-EMPTY list of module keys; an empty list " "reads as UNRESTRICTED, which is the opposite of locking an account down") known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) out, bad = [], [] @@ -326,17 +335,17 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi body = body or {} _require_store() uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) + name = str(body.get("name") or "").strip() or uname + email = str(body.get("email") or "").strip() or None + pw = _clean_password(body.get("password"), username=uname, name=name, email=email) role = _clean_role(body.get("role") or "user") bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None _reg_before = _registry() if uname in _reg_before: - raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + raise err(409, "user_exists", f"{uname} already exists; PATCH it to change it") # ⭐⭐ WAVE 37 · T03 (R8) — THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME. # `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only # scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was @@ -365,13 +374,16 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi f"address for this account.") # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) + try: + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) reg = _registry() if uname not in reg: # The store accepted the write and does not have it. Reporting 201 here would be the # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. - raise err(503, "store_unavailable", "the account was not saved — try again") + raise err(503, "store_unavailable", "the account was not saved; try again") return {"user": _view(uname, reg[uname])} @@ -404,12 +416,11 @@ def update_user(username: str, body: dict = Body(default=None), is_self = uname == session.uname role = _clean_role(body["role"]) if "role" in body else None active = bool(body["active"]) if "active" in body else None - # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is - # not the thing that keeps the product reachable — it just stops an administrator removing - # their own access with a toggle and having to go find the master password. + # THE ONE-CLICK LOCKOUT GUARD. There is deliberately no APP_PASSWORD bypass, so preventing a + # stored administrator from removing their own access is part of keeping recovery possible. if is_self and role is not None and role != "admin": raise err(400, "self_demote", - "you cannot remove your own administrator role — ask another admin") + "you cannot remove your own administrator role; ask another admin") if is_self and active is False: raise err(400, "self_deactivate", "you cannot deactivate your own account") kw = {} @@ -456,15 +467,65 @@ def set_password(username: str, body: dict = Body(default=None), # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — # 404, never 403, because "that account exists in another company" is itself a leak. raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) + pw = _clean_password((body or {}).get("password"), username=uname, + name=reg[uname].get("name"), email=reg[uname].get("email"), + record=reg[uname], allow_early_rotation=True) before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) + try: + # This route just checked strength AND reuse against its record. Spend the history's + # PBKDF2 passes once; the core still merges the rotation into the fresh update document. + users.set_password(uname, pw, enforce=False) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) after = int((_registry().get(uname) or {}).get("epoch") or 0) if after <= before: # The whole value of this route is the revocation. If the epoch did not move, the sessions # were not revoked, and answering 204 would say they were. raise err(503, "store_unavailable", - "the password change did not persist — outstanding sessions were NOT revoked") + "the password change did not persist; outstanding sessions were NOT revoked") + return Response(status_code=204) + + +@router.post("/admin/users/{username}/mfa/reset", status_code=204) +def reset_mfa(username: str, session: Session = Depends(admin_gate)): + """Clear a lost factor and revoke every session; the person enrols their own phone again. + + The administrator never sees or sets a user's secret. Removing it opens the enrolment door + at next sign-in; the atomic epoch bump closes every session that predates this reset. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # A cross-tenant username answers exactly like a missing one: naming another company's + # account, even only to refuse the reset, would itself disclose that the account exists. + raise err(404, "no_such_user", f"no account named {uname!r}") + before = int(reg[uname].get("epoch") or 0) + users.clear_mfa(uname) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # Revocation is the whole value of this door. A 204 without a moved epoch would claim + # the lost phone's sessions died when the store did not persist that change. + raise err(503, "store_unavailable", "the reset did not persist - outstanding sessions were NOT " + "revoked") + return Response(status_code=204) + + +@router.post("/admin/users/{username}/password/expire", status_code=204) +def expire_password(username: str, session: Session = Depends(admin_gate)): + """Force the account through password rotation without choosing its replacement secret.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Same missing-account answer across tenants: the reset door cannot enumerate others. + raise err(404, "no_such_user", f"no account named {uname!r}") + before = int(reg[uname].get("epoch") or 0) + users.set_must_change_password(uname, True) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # A flag without revocation would leave the old sessions usable. Prove the write before + # telling an administrator the reset is in effect, just like the password setter above. + raise err(503, "store_unavailable", + "the forced password rotation did not persist. Outstanding sessions were NOT revoked") return Response(status_code=204) @@ -818,7 +879,7 @@ def _module_fields(key, session=None): if session is None: raise err(400, "ungoverned_module", f"{key!r} is a tenant database, so its columns can only be read for a " - f"known tenant — this caller passed no session") + f"known tenant; this caller passed no session") import core.user_tables as user_tables defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) if not defn: @@ -1702,7 +1763,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non unknown = sorted(set(v) - governed) if unknown: raise err(400, "bad_perms", - f"unknown module keys: {unknown} — this tenant's permission editor governs " + f"unknown module keys: {unknown}; this tenant's permission editor governs " f"{sorted(governed)}") out = {} for key, raw in v.items(): @@ -1817,7 +1878,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non if tree not in (None, {}, []): if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): raise err(400, "bad_filter", - f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " + f"{key}: filter must be {{conj?, nodes:[…]}}; a bare list would lose " f"the root conjunction, and an 'or' wall read as 'and' restricts " f"nothing it was meant to") conj = tree.get("conj", "and") @@ -1885,7 +1946,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non if _leaf_count(cleaned) != _leaf_count(nodes): raise err(400, "bad_filter", f"{key}: the filter contains conditions this module cannot evaluate " - f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " + f"(an unknown field, an unknown operator, or a cohort leaf; cohorts " f"are per-user and cannot be a permanent rule). Refused rather than " f"saved with the bad conditions silently removed, which would store " f"a weaker wall than the one on screen.") @@ -1938,8 +1999,8 @@ def _refuse_unshapeable_bu(perms_out): f"{key}: this filter restricts the business unit in a way the data layer " f"cannot push into the query, so the rows would be correct while the " f"revenue figures on them stayed consolidated across both units. Express " - f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " - f"or an 'any of' over the brands — and keep any other rules alongside it.") + f"the business-unit condition as a TOP-LEVEL condition; 'DBA is Fisch', " + f"or an 'any of' over the brands; and keep any other rules alongside it.") def _guard_bu_widening(rec, cleaned, confirmed): @@ -1983,7 +2044,7 @@ def _guard_bu_widening(rec, cleaned, confirmed): raise err(400, "bu_widening", f"{key}: this would REMOVE the business-unit restriction and give the account " f"both units. If that is intended, resend with confirm_widen: true. If it is " - f"not, reload the account's permissions first — saving a filter loaded before " + f"not, reload the account's permissions first; saving a filter loaded before " f"the last change drops the conditions it did not know about.") # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees # the connection to derive_pool_scope, which is what the removed condition was feeding. @@ -2346,7 +2407,7 @@ def put_perms(username: str, body: dict = Body(default=None), # writing a wall for themselves that would bite the moment their role changed. if uname == session.uname: raise err(400, "self_perms", - "you cannot set permissions on your own account — ask another admin") + "you cannot set permissions on your own account; ask another admin") _mods = _perm_modules(session, surfaces=True) # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the @@ -2371,7 +2432,7 @@ def put_perms(username: str, body: dict = Body(default=None), # The store took the write and did not record it. A 200 here would tell an administrator # the wall is up when it is not — the one direction this must never fail in. raise err(503, "store_unavailable", - "the permissions were not saved — the account is UNCHANGED") + "the permissions were not saved; the account is UNCHANGED") return {"username": uname, "perms": rec.get("perms") or {}, "perms_v": int(rec.get("perms_v") or 0), # W46-T35 — `{module: [key, ...]}`, empty on an ordinary save. What the save cleared diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 1d653f2bc2209bc6f4955fcc017fd5c83873af85..df9c51285c0fd182a4c547de78deb1306cd1308f 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -511,7 +511,7 @@ def _require_filtered_view(session: Session, topic: str, view_id: str): # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, # the one path that reaches this branch is also the one path with nothing to debug from. raise err(503, "unavailable", - f"the table could not be read ({type(e).__name__}) — try again in a moment") + f"the table could not be read ({type(e).__name__}); try again in a moment") view = _view_by_id(g, view_id) if not isinstance(view, dict): raise err(404, "no_view", "that view does not exist on this table") @@ -572,7 +572,7 @@ def _require_filtered_view(session: Session, topic: str, view_id: str): if not _any_active(nodes): raise err(400, "no_filter", - "this view has no active filter, so no record can ever ENTER it — add a " + "this view has no active filter, so no record can ever ENTER it; add a " "condition to the view first, then create the alert") diff --git a/api/routes_assets.py b/api/routes_assets.py index 64a5ea96af0e5e42a455e33a393f5a6323c2375d..33eb66e7e8127bec6ad0a9b64e977013201dd396 100644 --- a/api/routes_assets.py +++ b/api/routes_assets.py @@ -152,14 +152,31 @@ def duplicate_application(application_id: str, @router.delete("/applications/{application_id}") -def delete_application(application_id: str, +def delete_application(application_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): try: current = _application_databases(session.runtime, application_id) allowed, _ = application_access_context(session, current) if not allowed: raise applications.ApplicationError("application does not exist") - return applications.delete(session.runtime, application_id, session.uname, session.admin, + confirm_name = (body or {}).get("confirmName") + + class NameConfirmedDelete: + """Keep confirmation inside the registry's existing atomic update and permission gate.""" + + def update(self, key, delete): + def confirmed(raw): + # The registry validates access and copies the document before removing the App. + # A refusal below never reaches persistence, even if its name changed mid-request. + deleted = delete(raw) + app = raw["applications"][application_id] + if not isinstance(confirm_name, str) or confirm_name != app.get("name"): + raise applications.ApplicationError("enter the exact App name to confirm deletion") + return deleted + + return session.runtime.update(key, confirmed) + + return applications.delete(NameConfirmedDelete(), application_id, session.uname, session.admin, allowed_databases=allowed) except (applications.ApplicationError, PermissionError) as exc: _application_error(exc) diff --git a/api/routes_auth.py b/api/routes_auth.py index 5e919c397a0e3e52a8c10c2f50d2ba5e4d2e728a..aa43d6eeb3704bc39acfb142eaf975a7c09d99cc 100644 --- a/api/routes_auth.py +++ b/api/routes_auth.py @@ -7,11 +7,7 @@ replaces, and it is deliberately the same account store so the migration is a sw CREDENTIAL check, not of the user model. """ import base64 as _b64 -import hmac -import os import re as _re -import time -from collections import OrderedDict from fastapi import APIRouter, Body, Depends, Request, Response @@ -20,49 +16,46 @@ from deps import Session, err, require_session, users, perms router = APIRouter(prefix="/api/v1/auth") -#: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so -#: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak. -#: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker, -#: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not -#: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally. -#: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a -#: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of -#: tenant A — a cross-tenant coupling in the one wave whose point is that tenants cannot reach -#: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would -#: tell you somebody else did. -_FAILS = OrderedDict() -_MAX_TRACKED = 2048 -_LOCKOUT_AFTER = 8 -_LOCKOUT_SECONDS = 60 +#: THE SERVER OWNS BACKOFF: the login page never paces itself. The budget lives in the shared +#: tenant-0 auth store, not this process, so worker rotation or a deploy cannot buy ten more tries. +#: Username and email aliases resolve to one canonical account key; only a SHA-256 digest of that +#: normalised key is persisted. Any store failure fails the authentication door closed. +def _throttle_key(identifier): + try: + return users.canonical_login_identifier(identifier) + except Exception: + raise err(503, "auth_state_unavailable", + "sign in protection is temporarily unavailable. Try again later") def _throttled(key, now=None): - now = now or time.time() - hit = _FAILS.get(key) - if not hit: - return 0 - count, last = hit - if count < _LOCKOUT_AFTER: + if key is None: # staging-only synthetic QA identity return 0 - remaining = int(_LOCKOUT_SECONDS - (now - last)) - return remaining if remaining > 0 else 0 + try: + return users.login_lockout_remaining(key, now=now) + except Exception: + raise err(503, "auth_state_unavailable", + "sign in protection is temporarily unavailable. Try again later") def _note_failure(key, now=None): - now = now or time.time() - count, last = _FAILS.get(key, (0, 0.0)) - # A window that has fully elapsed starts the count again — a user who mistypes twice today - # and twice next week is not an attacker. - if now - last > _LOCKOUT_SECONDS: - count = 0 - _FAILS[key] = (count + 1, now) - _FAILS.move_to_end(key) - while len(_FAILS) > _MAX_TRACKED: - _FAILS.popitem(last=False) + if key is None: + return 0 + try: + return users.note_login_failure(key, now=now) + except Exception: + raise err(503, "auth_state_unavailable", + "sign in protection is temporarily unavailable. Try again later") def _clear_failures(key): - _FAILS.pop(key, None) + if key is None: + return + try: + users.clear_login_failures(key) + except Exception: + raise err(503, "auth_state_unavailable", + "sign in protection is temporarily unavailable. Try again later") def _public_user(user): @@ -143,20 +136,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) body = body or {} uname = str(body.get("username") or "").strip().lower() pw = str(body.get("password") or "") - - # Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves, - # and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs. - throttle_key = ("*", uname) - wait = _throttled(throttle_key) - if wait: - raise err(429, "too_many_attempts", - f"too many failed attempts — try again in {wait} seconds") + code = str(body.get("mfaCode") or "").strip() # W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is # tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make # a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and # APP_PASSWORD are present; every ordinary username falls straight through to `verify()`. user = users.qa_identity(uname, pw) if (uname and pw) else None + # A correctly proved QA runner is a staging fixture and remains zero-touch against tenant #0. + # Every other attempt resolves username/email aliases to one durable account budget. + throttle_key = None if user else _throttle_key(uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts. Try again in {wait} seconds") if not user: user = users.verify(uname, pw) if (uname and pw) else None if not user: @@ -179,7 +172,46 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) _note_failure(throttle_key) raise err(401, "invalid_credentials", "that username and password do not match") - _clear_failures(throttle_key) + # W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or + # minting so an expired password gets the change door and no session, without counting a + # successful authentication against the guessing budget. The staging QA fixture has no + # stored password clock; every production identity does. + if not user.get("qa_runner"): + try: + rec = (users.registry() or {}).get(user["username"]) + except Exception: + rec = None + if users.password_expired(rec): + raise err(403, "password_expired", "Your password has expired. Set a new one to continue.") + if rec and not users.password_changed_at(rec): + # Legacy records start their clock at first sign-in after this ships. The background + # merge is fail-silent like touch_login, so backfilling can never deny a good login. + try: + users.stamp_password_age(user["username"]) + except Exception: + pass + + # W57-T36 / C12: every STORED account needs its second factor before any cookie is minted. + # qa_runner is the sole exception and is double-gated out of live deployments. + if not user.get("qa_runner"): + try: + rec = (users.registry() or {}).get(user["username"]) + except Exception: + rec = None + if not users.mfa_enrolled(rec): + # Correct credentials needing setup are not a guess. Counting them would lock the + # person out of the unauthenticated enrolment door that repairs their account. + raise err(403, "mfa_enrolment_required", "This account needs two step sign in. Set up your " + "authenticator app to continue.") + if not code: + # A missing factor is a prompt, not a failed attempt at guessing one. + raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.") + # TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's + # ordinary code already proves the factor, and never reset the guess budget on password. + if not (users.totp_valid(rec.get("mfa_secret"), code) + or users.consume_recovery_code(user["username"], code)): + _note_failure(throttle_key) + raise err(401, "mfa_invalid", _MFA_INVALID) # ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this # account already holds, so a forgotten sign-in on someone else's computer cannot lock the @@ -191,21 +223,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) # the bump; minting from it would issue every cookie in the product at the OLD epoch, which # `_user_for` refuses on the very next request — "200, then 401 forever". # - # TWO IDENTITIES ARE EXEMPT, both by design: - # * `qa_runner` — the synthetic staging identity has no registry record to bump (its epoch is - # derived from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes - # NOTHING on tenant #0's `users` key, which a bump's read-modify-write would violate. - # * `emergency_master` — `deps._user_for` admits that identity whatever the record's epoch - # says, so a bump revokes nothing there and only risks a store write on the break-glass - # path that exists precisely for a store outage. + # qa_runner is the sole exception: it has no registry record to bump (its epoch is derived + # from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes NOTHING on + # tenant #0's `users` key. Live deployments cannot enable this double-gated fixture. # # A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the # deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the # store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into # a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is # lost, and the log says so rather than leaving it silent. + # A complete password+factor proof clears the durable budget BEFORE a cookie is minted. If + # that security write is unavailable, authentication fails closed. + _clear_failures(throttle_key) session_epoch = int(user.get("epoch") or 0) - if not (user.get("qa_runner") or user.get("emergency_master")): + if not user.get("qa_runner"): try: bumped = users.bump_epoch(user["username"]) if bumped is not None: @@ -218,9 +249,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) aios_session.set_cookie(response, request, value) # Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier — the # person may have typed an email address and `users.verify` resolved it to the registry key; - # stamping what was typed would write onto a key that does not exist. Fail-silent inside - # `touch_login` and a no-op for the emergency-master identity (which has no record to stamp), - # so the one thing this cannot do is turn a good credential into a failed sign-in. + # stamping what was typed would write onto a key that does not exist. `touch_login` is + # fail-silent, so it cannot turn a good credential into a failed sign-in. # The synthetic QA identity has no normal account record to stamp. More importantly, a # registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema. if not user.get("qa_runner"): @@ -234,6 +264,216 @@ def login(request: Request, response: Response, body: dict = Body(default=None)) return {"user": _public_user(user)} +# ------------------------------------------------------------------ second factor (W57-T36 / C12) +_MFA_INVALID = ("That code is not right. Check your authenticator app and try the next one.") +_MFA_ALREADY_ENROLLED = ("Two step sign in is already on for this account. Ask an administrator " + "to reset it if you lost your phone.") + + +def _mfa_credentials(uname, password): + """The MFA doors share login's credential check and guessing budget, including aliases.""" + # The synthetic runner has no stored factor and is refused without touching tenant #0. + user = users.qa_identity(uname, password) if (uname and password) else None + if user: + raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.") + throttle_key = _throttle_key(uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts. Try again in {wait} seconds") + user = users.verify(uname, password) if (uname and password) else None + if not user: + try: + users._hash(password or "x", "00" * 16) + except Exception: + pass + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + return user, throttle_key + + +def _mfa_record(uname): + """A stored identity was already proved. Failure to re-read it cannot assert absence.""" + try: + rec = (users.registry() or {}).get(uname) + except Exception: + rec = None + if not rec: + raise err(503, "store_unavailable", "the account could not be read. Try again") + return rec + + +def _mfa_session_tenant(user, rec, throttle_key): + """Both MFA cookie writers retain login's tenant and T35 password-expiry walls.""" + tenant = str(user.get("tenant") or "royal-imports").strip().lower() + from harness import runtime + try: + runtime.get_runtime(tenant) + except KeyError: + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + # Confirming a phone must not become a second login door that accepts an expired password. + # Check before writing the factor, so the person can rotate and resume the same setup. + if users.password_expired(rec): + raise err(403, "password_expired", "Your password has expired. Set a new one to continue.") + return tenant + + +def _mfa_set(user, secret, codes, *, require_unenrolled=False): + """Translate a raced credential change into a retry, never a claimed successful setup.""" + try: + # Use the RESOLVED username: the posted identifier can be an email alias with no record. + return users.set_mfa(user["username"], secret, codes, + require_unenrolled=require_unenrolled, + expected_epoch=int(user.get("epoch") or 0)) + except users.MFAStateError as exc: + if str(exc) == "already_enrolled": + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + raise err(409, "mfa_changed", "This account changed during setup. Sign in again to continue.") + + +def _mfa_session(request, response, user, rec, tenant, landed, throttle_key): + """Mint only from the MFA write's landed epoch, never the stale login projection.""" + if landed is None: + # A missing record/no-op write cannot support a cookie or a promise that codes were saved. + raise err(503, "store_unavailable", "the second factor was not saved. Try again") + # Clear only after the whole factor proof and its write succeed, but before minting a cookie. + _clear_failures(throttle_key) + value, _claims = aios_session.mint(tenant, user["username"], int(landed)) + aios_session.set_cookie(response, request, value) + users.touch_login(user["username"]) + import deps as _deps + _deps.note_active(user["username"]) + if not users.password_changed_at(rec): + # This is a successful first sign-in too: preserve T35's legacy clock backfill. + users.stamp_password_age(user["username"]) + # Responses carrying a new secret or recovery codes must not survive in an HTTP cache. + response.headers["Cache-Control"] = "no-store" + + +@router.post("/mfa/enroll") +def mfa_enroll(response: Response, body: dict = Body(default=None)): + """Offer a candidate secret without changing the account or issuing recovery codes. + + Stateless by design: the client posts this candidate back with its password and phone code. + A pending-secret dict would belong to one worker and fail when confirmation reaches another. + """ + body = body or {} + uname = str(body.get("username") or "").strip().lower() + user, _ = _mfa_credentials(uname, str(body.get("password") or "")) + rec = _mfa_record(user["username"]) + if users.mfa_enrolled(rec): + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + secret = users.new_mfa_secret() + response.headers["Cache-Control"] = "no-store" + return {"secret": secret, "otpauthUri": users.otpauth_uri(secret, user["username"]), + "issuer": users.MFA_ISSUER, "digits": users.MFA_DIGITS, "period": users.MFA_PERIOD} + + +@router.post("/mfa/enroll/confirm") +def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(default=None)): + """Prove installation, enrol once and sign in under the new epoch. + + This response is the only time those codes exist in clear anywhere. The record holds SHA-256 + hashes. No other route returns them, no log line prints them, and there is no way to read them + back. Explicit regeneration issues a DIFFERENT set, never a second reading of this one. + """ + body = body or {} + uname = str(body.get("username") or "").strip().lower() + user, throttle_key = _mfa_credentials(uname, str(body.get("password") or "")) + rec = _mfa_record(user["username"]) + if users.mfa_enrolled(rec): + raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED) + secret = str(body.get("secret") or "").strip().upper() + if not users.totp_valid(secret, str(body.get("code") or "")): + _note_failure(throttle_key) + raise err(401, "mfa_invalid", _MFA_INVALID) + tenant = _mfa_session_tenant(user, rec, throttle_key) + codes = users.new_recovery_codes() + # The store guard repeats the check atomically: two workers cannot issue two first sets. + landed = _mfa_set(user, secret, codes, require_unenrolled=True) + _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + return {"user": _public_user(user), "recoveryCodes": codes} + + +@router.get("/mfa") +def mfa_status(session: Session = Depends(require_session)): + """The calling account's state only: never a secret, code or hash.""" + # This identity has no stored factor; do not read tenant #0 for the QA runner. + if session.user.get("qa_runner"): + rec = {} + else: + rec = _mfa_record(session.uname) + return {"enrolled": users.mfa_enrolled(rec), "enrolledAt": rec.get("mfa_enrolled_at") or None, + "recoveryCodesRemaining": users.recovery_codes_remaining(rec)} + + +@router.post("/mfa/recovery-codes") +def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Issue a new set once after password re-authentication, keeping the installed phone. + + Regeneration is a credential change: set_mfa bumps the epoch and signs OTHER sessions out. + Re-mint the CALLING session from that landed epoch or the user's own action signs them out. + The clear codes appear only in this response; the store and all later reads hold only hashes. + """ + uname = session.uname + user, throttle_key = _mfa_credentials(uname, str((body or {}).get("password") or "")) + # The session supplies the identity. Body username/tenant fields can never choose a target. + rec = _mfa_record(uname) + if not users.mfa_enrolled(rec): + raise err(409, "mfa_not_enrolled", "Set up your authenticator app before making recovery codes.") + tenant = _mfa_session_tenant(user, rec, throttle_key) + codes = users.new_recovery_codes() + landed = _mfa_set(user, rec["mfa_secret"], codes) + _mfa_session(request, response, user, rec, tenant, landed, throttle_key) + return {"recoveryCodes": codes} + + +@router.get("/password-policy") +def password_policy(): + """An account-independent checklist the sign-in page can show before authentication.""" + return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH, + "minAgeDays": users.PASSWORD_MIN_AGE_DAYS, + "maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY} + + +@router.post("/password", status_code=204) +def change_password(body: dict = Body(default=None)): + """Rotate with the current credential, including when expiry refuses a login session.""" + body = body or {} + uname = str(body.get("username") or "").strip().lower() + pw = str(body.get("currentPassword") or "") + new_pw = str(body.get("newPassword") or "") + # The staging runner has no stored password and is refused without touching tenant #0. + user = users.qa_identity(uname, pw) if (uname and pw) else None + if user: + raise err(400, "no_stored_password", "This account has no stored password to change here.") + # Share login's canonical durable bucket: a second unauthenticated door and an email alias + # must not buy another password-guessing budget. + throttle_key = _throttle_key(uname) + wait = _throttled(throttle_key) + if wait: + raise err(429, "too_many_attempts", + f"too many failed attempts. Try again in {wait} seconds") + user = users.verify(uname, pw) if (uname and pw) else None + if not user: + try: + users._hash(pw or "x", "00" * 16) + except Exception: + pass + _note_failure(throttle_key) + raise err(401, "invalid_credentials", "that username and password do not match") + try: + # verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed + # alias, or this door could return 204 without changing any password or revoking sessions. + users.set_password(user["username"], new_pw) + except users.PasswordPolicyError as exc: + raise err(400, "weak_password", "That password does not meet the policy: " + str(exc)) + _clear_failures(throttle_key) + return Response(status_code=204) + + @router.post("/logout", status_code=204) def logout(request: Request): """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from diff --git a/api/routes_connected_tables.py b/api/routes_connected_tables.py index f82c5ff088b4cc9f0ea024a32d796f718306eefd..1cd102b148141f08c4634be179319b9e070a5be6 100644 --- a/api/routes_connected_tables.py +++ b/api/routes_connected_tables.py @@ -32,11 +32,27 @@ behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be """ from fastapi import APIRouter, Depends, Query -from deps import Session, require_session +from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") +@router.get("/connected-tables/{table_key}/identity") +def connected_table_identity(table_key: str, session: Session = Depends(require_session)): + """Provider/database identity is separate from the row identity in the paged rows contract.""" + from core import user_tables + from routes_tables import _defn_or_refuse + + # Definitions only: this endpoint neither reads rows nor bypasses the table permission wall. + definition = _defn_or_refuse(session, table_key, defs_only=True) + identity = user_tables.connector_database_identity( + table_key, rt=session.runtime, definition=definition) + if identity is None: + raise err(404, "unknown_connector_database", "that database has no connector identity") + return {**identity, "readOnly": bool( + definition.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE)} + + @router.get("/connected-tables/{table_key}/rows") def connected_table_rows(table_key: str, offset: int = Query(default=0, ge=0), diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 6c148291b29990b68227762fc2aa55c49d29eb1d..c53e701a1dad037105daab1ba959e4468e18b11c 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -21,6 +21,8 @@ that reported another user's Gmail as "connected" would be both wrong and a smal from __future__ import annotations import os +import time +import threading from fastapi import APIRouter, Body, Depends @@ -33,11 +35,18 @@ from deps import Session, err, require_session #: of it (a gate, a future module page) must not have to import FastAPI to ask what category a #: connector is in. from core import connector_directory +from core import connector_sync +#: ⭐ W53-T27 / R9 / C2 — the label RESOLVER, imported rather than re-written. "Shipped default, +#: per-tenant override, absent means inherit" is one idea and `core/schema_overlay.py` already +#: owns it for fields; `merge_database_labels` is that idea for a database's own name. This module +#: holds the tenant's store handle and does the reading; nothing about what a blank override MEANS +#: is decided here. +from core import schema_overlay #: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a #: second correct one here would leave the same two-copies shape that produced the defect. -from routes_keychain import TENANT_WIDE_TYPES, env_odoo_available +from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available #: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a #: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any #: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router` @@ -108,7 +117,7 @@ PLANNED = [ # (Instagram arriving another way) is now visible ON the Scraper card, so keeping it would put # Instagram on this page twice, once as working and once as blocked. The Meta business # verification blocker itself is not lost — it is D-86's subject and lives in - # `.claude/wiki/research/ads-connectors-owner-actions.md`, where an owner action belongs. + # `AutoWiki/research/ads-connectors-owner-actions.md`, where an owner action belongs. ] #: ⭐ WAVE 30 · R4 — THE SCRAPER'S PLATFORMS, and the whole shape of the ruling in one list. @@ -148,6 +157,22 @@ TOKEN_CONNECTORS = [ "against your ad accounts before storing it.", "note": "The token connects the account and is verified live. The spawned databases " "(285 measured columns) land with the Meta grids."}, + # ⭐⭐ AMAZON SP-API. FIVE values rather than one token, because Login With Amazon + # exchanges a refresh token for a short-lived bearer on every call, and the seller and + # marketplace ids scope the reads. ⚠ `entry` must match `keychain.ENTRY_TYPES` or storing + # 400s, and `refresh_token` had to join `_PREVIEW_FIELDS` in the same change or the row + # renders a blank preview with nothing raising (the wave-31 defect, second edition). + # ⛔ Read-only by construction: `connectors_amazon._request` refuses any method but GET + # before it builds a request, so no route, payload or UI can reach a write. + {"key": "amazon_sp", "label": "Amazon", "entry": "amazon_sp", + "desc": "Orders, settlements and fees, FBA and AWD inventory, listings and Buy Box " + "pricing from your Amazon selling account.", + "hint": "Create a Production app client in the Solution Provider Portal, self authorize " + "it, then paste client_id, client_secret, refresh_token, seller_id and " + "marketplace_id.", + "note": "The credential is verified live against your marketplace participations before " + "it is stored. Bulk reads use the Reports API, so a backfill is one document " + "rather than a page loop."}, ] #: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many @@ -205,9 +230,18 @@ NANGO_PROVIDERS = [ #: hole the stamp closes. The spawn stamps; this only says which family may qualify. #: ⚠ `ut_odoo_` is SEEDED in `user_tables` rather than registered here, also deliberately — the #: convention leg has to answer in a worker process where nothing registered anything. +#: ⛔⛔ `ut_amzn_` JOINED IN WAVE 52, AND OMITTING IT WOULD HAVE BROKEN STANDING RULE 1 SILENTLY. +#: The rule is that a connector's records carry NO cap: `records_mutable(key)` is False for them, +#: so `MAX_ROWS` does not apply. But that answer is reached through `is_connected`, whose third leg +#: is this prefix set — so an Amazon table missing from here does not raise, does not warn, and +#: simply falls through to the 60,000-row ceiling. A seller with more than 60,000 order lines would +#: get a truncated grid and no error, which is precisely the silent cap the rule forbids. **A new +#: connector joins this dict in the same change that spawns its tables, or its no-cap guarantee is +#: decorative.** CONNECTED_PREFIXES = { "odoo": "ut_odoo_", "meta_ads": "ut_meta_", + "amazon_sp": "ut_amzn_", } @@ -241,11 +275,13 @@ def _register_probers(): Import-time and fail-quiet: an unfillable slot costs a liveness CHECK, never the ability to store a credential and never the API's ability to boot. """ - try: - import connectors_meta - return connectors_meta.register() - except Exception: # noqa: BLE001 - return False + ok = False + for mod in ("connectors_meta", "connectors_amazon"): + try: + ok = __import__(mod).register() or ok + except Exception: # noqa: BLE001 + continue + return ok _register_probers() @@ -275,6 +311,124 @@ def _table_connection_key(table_key, definition): return "" +def _module_connection_key(subject): + """Which connected card a REGISTRY MODULE's `subject` belongs to, or `""`. + + ⭐⭐ W53-T30 — DERIVED FROM `CONNECTED_PREFIXES`, NEVER A SECOND HAND-TYPED PAIR LIST. A + module's subject is namespaced by the system the records come from (`odoo:res.partner`, + `odoo:product.product` in `core/registry.py`), and the connector families this file already + knows carry that same namespace inside their table prefix (`odoo` inside `ut_odoo_`). So the + two are matched rather than mapped: a connector added to `CONNECTED_PREFIXES` in the change + that spawns its tables gets this for free, and a module whose namespace names no connected + family here simply does not appear on anybody's graph. + + ⛔ THE SUBJECT, NOT THE `source` STRING. `source` is display copy ("Odoo") and a rename of it + is a copy edit; `subject` is the identity `core.registry` already refuses to let two rows + claim at once, which is what makes it safe to key a graph off. + """ + namespace = str(subject or "").split(":", 1)[0].strip() + if not namespace: + return "" + for connection_key, prefix in CONNECTED_PREFIXES.items(): + parts = str(prefix or "").strip("_").split("_") + if len(parts) > 1 and namespace == "_".join(parts[1:]): + return connection_key + return "" + + +def _module_graph_entries(rt, user, lent, connected): + """The REGISTRY MODULES this reader may see on a connected card's graph, per connection. + + Returns `{connection key: {module key: {"dto", "visible"}}}` in exactly the shape the `ut_` + loop below builds, so the edge pass treats both kinds of node identically. + + ⭐⭐ W53-T30 — THE WHOLE TICKET. `relationship_graphs` walks `user_tables.all_defs`, and + `customer_data` / `product_data` are COMPILED REGISTRY ROWS rather than stored definitions, so + that loop could never reach them however permissive it became. They were absent from the Odoo + diagram for that reason, not for a permission one. ⛔ The retired `ut_odoo_customers` / + `ut_odoo_products` keys are NOT the subject and must not be revived: W33-T44 / A2 retired + them, `odoo_relational.RETIRED_KEYS` pops them out of every tenant's document on rebuild, and + their subjects are claimed by these two registry rows, which is now the only claim on each. + + ⛔⛔ THREE PREDICATES, THE SAME THREE `routes_tables._module_link_targets` USES ON THESE EXACT + TWO KEYS, and none of them is `is_admin`. A blanket administrator check here would silently + drop two nodes from a member's diagram for no stated reason, while every other node on the + same picture is walled per table: + 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every + provisioned tenant but #0 carries a restricted `modules:` list, so without it another + company's admin is shown a database their workspace does not have. + 2. `perm_scope.nav_may_open` — ADMISSION, the predicate `deps.py`'s module gate itself asks. + `may_read` alone would draw a node whose door then 403s. + 3. `perm_scope.may_read` — the READ wall, the same one the `ut_` loop applies per table. + + ⛔ `user_tables.may_open` IS NOT IN THAT LIST AND ITS ABSENCE IS DELIBERATE: it answers about + a STORED definition's creator and share grants, and a registry module has no such row, so it + refuses every reader including the tenant's owner. + + ⭐ THE FIELDS ARE `automation_engine._module_table`'s, WHICH IS THE ONE COMPOSER OF A MODULE'S + FIELD CONTRACT (`aios_grid` plus `odoo_relational.module_link_fields`). Restating either half + here is how this diagram would start disagreeing with the engine that resolves the very links + it draws. `want_rows=False`, so no pool is built and no store is written. + + ⚠ AND THE FIELD LIST IS THE VIEWER'S: `visible_fields` strips the hidden closure, so a column + an administrator hid from this account is neither drawn on the card nor usable as an edge + endpoint. Fail-closed everywhere: a wall that cannot be answered drops the node. + """ + try: + import automation_engine # noqa: PLC0415 - lazy, as elsewhere + import core.perm_scope as perm_scope + import core.perms as perms + import core.registry as registry + import core.user_tables as user_tables + except Exception: # noqa: BLE001 + return {} + out = {} + memo = {} + for module_key in getattr(user_tables, "LINKABLE_MODULE_KEYS", ()): + key = str(module_key or "") + row = registry.BY_KEY.get(key) or {} + connection_key = _module_connection_key(row.get("subject")) + if not key or connection_key not in connected: + continue + try: + if not perms.tenant_has_module(rt, key): + continue + if not perm_scope.nav_may_open(user, key, st=lent): + continue + if not perm_scope.may_read(user, key, st=lent): + continue + # ⛔ `_module_table`, THE ENGINE'S OWN COMPOSER (`# noqa: SLF001` for the same reason + # `_database_name_overrides` calls `routes_nav._read_nav_meta`: one reader for one + # fact beats a correct-looking second copy). + table, _limit = automation_engine._module_table(rt, key, memo) # noqa: SLF001 + declared = [dict(field) for field in ((table or {}).get("fields") or ()) + if isinstance(field, dict)] + if not declared: + continue + visible = perm_scope.visible_fields(declared, user, key, st=lent) + except Exception: # noqa: BLE001 + continue + fields = [] + for field in visible: + field_key = str(field.get("key") or "").strip() + if field_key: + fields.append({"key": field_key, + "label": str(field.get("label") or field_key).strip() or field_key}) + if not fields: + continue + # ⚠ THE REGISTRY'S OWN LABEL, READ RATHER THAN RETYPED. "Odoo customers" and "Odoo + # products" are `core/registry.py`'s `label` values (renamed there once already, W33-T42), + # and a literal copy here is a second authority that goes stale the day the row is + # renamed. There is no tenant override to merge in: `routes_nav` refuses a name for any + # key that does not start with `ut_`, so a registry module cannot be renamed at all. + label = str(row.get("label") or key).strip() or key + out.setdefault(connection_key, {})[key] = { + "dto": {"key": key, "label": label, "fields": fields}, + "visible": visible, + } + return out + + def relationship_graphs(rows, user, rt, uname, is_admin=False): """Actual, permitted tables and same-connection declared links for directory rows. @@ -350,6 +504,13 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False): "visible": visible, } + # ⭐⭐ W53-T30 — AND THE REGISTRY MODULES THIS CONNECTION STANDS BEHIND. They are not stored + # definitions, so the loop above cannot reach them; they are permitted by their own three + # predicates and then join the SAME edge pass, which is what makes an already-declared link + # between a `ut_` table and a module resolvable at both ends for the first time. + for connection_key, entries in _module_graph_entries(rt, user, lent, connected).items(): + tables_by_connection.setdefault(connection_key, {}).update(entries) + for connection_key, entries in tables_by_connection.items(): if not entries: continue @@ -383,6 +544,30 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False): "to": {"table": target_table, "field": target_field}, "label": str(field.get("label") or source_field).strip() or source_field, }) + # ⭐ W53-T30 / C2 — THE SPEC'S OWN DECLARED RELATIONS, held to the same two proofs as the + # field bags above: both endpoints must be nodes on THIS graph and both named columns must + # be visible to THIS reader. `{}` until lane D's `core.connector_spec` lands, and the + # `seen` signature is shared so a relation declared in both places is drawn once. + label_of = {(table_key, str(field.get("key") or "")): str(field.get("label") or "") + for table_key, entry in entries.items() + for field in entry["dto"]["fields"]} + for source_table, declared in _connector_spec_links(connection_key).items(): + if source_table not in table_keys: + continue + for source_field, target_table, target_field in declared: + if (source_field not in visible_keys.get(source_table, set()) + or target_table not in table_keys + or target_field not in visible_keys.get(target_table, set())): + continue + signature = (source_table, source_field, target_table, target_field) + if signature in seen: + continue + seen.add(signature) + relationships.append({ + "from": {"table": source_table, "field": source_field}, + "to": {"table": target_table, "field": target_field}, + "label": label_of.get((source_table, source_field)) or source_field, + }) graphs[connection_key] = { "connectionKey": connection_key, "status": "ready", @@ -686,6 +871,28 @@ def directory(rt, uname, is_admin=False, incomplete=None): "the connector directory's category taxonomy could not be composed", "every connector above is still listed and still correct; the category filter and " "the wider catalogue are missing from this list only") + + # ⭐⭐ W53-T29 / OWNER RULING R6 — EVERY ROW DECLARES WHICH SCOPES IT MAY EVER CARRY. + # + # R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail + # both)."* The Company / Mine toggle needs that per SOURCE, not per stored connection: a card + # may only ever appear under Company if `business` is in this list, and under Mine only if + # `personal` is — so Odoo cannot land under Mine however its `scope` was written, and a + # personal-only source cannot land under Company. + # + # ⛔ ONE PASS OVER THE COMPOSED SET, NOT A LINE IN EACH CONTRIBUTOR. Six functions above append + # rows and a seventh extends the list with a vendor catalogue; a declaration repeated in each + # is a declaration the eighth composer forgets, and the client would then treat the missing + # key as "unrestricted" with nothing going red. Stamped here, a new contributor gets the right + # answer for free. + # ⛔ AND THE ANSWER IS `routes_keychain.allowed_scopes`, IMPORTED. This module already imports + # `TENANT_WIDE_TYPES` rather than keeping a copy (see the import note at the top of the file); + # the eligibility rule built ON that tuple is fetched the same way for the same reason. + # ⚠ A ROW WITH NO KEYCHAIN TYPE (an OAuth slot, the Scraper, a catalogue listing) answers with + # both words, which is the honest default: nothing about it forbids either scope, and the one + # thing the toggle must never do is hide a connection the reader actually holds. + for row in rows: + row["allowedScopes"] = list(allowed_scopes(_entry_type_for(row.get("key")))) return rows @@ -704,6 +911,17 @@ def _oauth_row(slug, meta, row_st): else "available" if row_st.get("configured") else "unconfigured"), "connectedAs": row_st.get("email") or "", + # ⭐⭐ W53-T29 / R6 — A PER-USER SLOT IS PERSONAL, AND THE CARD NOW SAYS SO. + # `_oauth_slot_entries` below already stamps `"scope": "personal"` on these very + # credentials ("a per-user slot is personal by construction, in the vocabulary + # `routes_keychain`"), and the ROW carried no scope at all — so a connected Google card + # read as business-wide on the page while its own Keychain tab called it personal, and the + # Company / Mine split would have filed this reader's own sign-in under Company. One fact, + # said the same way in both places. + # ⚠ NO CARD MOVES TODAY: `oauth_connect.offered()` filters every provider out of the + # directory (D-45, the CASA verification nobody has bought), so this is the answer waiting + # for the day one of them is offered again rather than a change to what renders now. + "scope": "personal", "startUrl": f"/api/v1/oauth/{slug}/start", "manage": "oauth", "needs": "" if row_st.get("configured") else "the owner registers the client", @@ -851,6 +1069,19 @@ def connectors_directory(session: Session = Depends(require_session)): served = connector_directory.to_client_rows(rows) return {"connectors": served, "incomplete": incomplete, + # ⭐⭐ W53-T29 / OWNER RULING R6 — MAY THIS READER SEE THE COMPANY SIDE AT ALL. + # + # R6: *"A non-admin does not see the Company side at all; the toggle itself is hidden + # for them."* That is a PERMISSION, so it is the server's answer and not something the + # page infers from whether some row happened to carry `manage: "keychain"`. + # ⛔ THE WORD IS `routes_keychain`'s OWN. `GET /admin/keychain` has answered + # `"canBusiness": bool(session.admin)` since W32-T11, for exactly this purpose ("the + # vocabulary and the permission, so the client renders a picker it can honour"), and + # the same fact under a second name would be two vocabularies for one boolean. + # ⚠ CARRIED HERE RATHER THAN FETCHED FROM THAT ROUTE. The keychain door lists every + # visible ENTRY, so a page that wanted one boolean would pull the workspace's whole + # credential list on every visit; this route already holds the session that decides it. + "canBusiness": bool(session.admin), "categories": connector_directory.chip_counts(served), # ⛔⛔ `planned` COUNTS THE ROADMAP AGAIN, AND FOR ONE WAVE IT COUNTED 120. W49-T18 # gave every Monid listing the `planned` state because that state renders faded, so @@ -874,12 +1105,319 @@ def _odoo_admin(session): raise err(403, "not_admin", "only an administrator can inspect the Odoo schema") +def _sync_admin(session): + if not session.admin: + raise err(403, "not_admin", "only an administrator can retry a connector sync") + + +def _connection_jobs(rt, provider, connection_id): + """A status endpoint selects one connection; the jobs inventory may enumerate them all.""" + cid = str(connection_id or "").strip() or "default" + if provider == "odoo" and cid == "default": + import odoo_relational + cid = odoo_relational.default_connection_id(rt) or "default" + return connector_sync.jobs(rt, provider, cid) + + +def _sync_payload(connector_key, connection_id, row=None): + """One wire shape for every registered connector, including one not yet run.""" + meta = connector_sync.providers()[connector_key] + slot = str(connection_id or "").strip() or "default" + if row is None: + return {"provider": connector_key, "label": meta["label"], + "connectionId": slot, "recorded": False, "state": None, + "at": None, "stateAt": None, "from": None, + "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0, + "databases": [], "retryable": False, "retryHint": meta["retryHint"]} + answer = dict(row) + answer.update({"provider": connector_key, "connectionId": slot, + "label": meta["label"], "recorded": True, + "stateAt": row.get("at"), "retryHint": meta["retryHint"]}) + return answer + + +def _odoo_sync_runner(session, connection_id): + """Run Odoo through C6 without moving provider imports into the data layer.""" + connector_sync.record(session.runtime, "odoo", connection_id, "syncing") + # The current Odoo entry point plans and applies in one call, so the closest truthful seam is + # immediately before it starts applying the tenant mirror into governed database definitions. + connector_sync.record(session.runtime, "odoo", connection_id, "materializing") + import odoo_relational + result = odoo_relational.refresh(session.runtime, session.tenant, + username=session.uname or "automation", + connection_id=connection_id) + tables = result.get("tables") if isinstance(result, dict) else {} + databases = sorted(str(key) for key in (tables or {}).keys()) + sync_errors = [] + if isinstance(result, dict): + if result.get("failed"): + sync_errors.append(str(result["failed"])) + discovered = result.get("discovered") + if isinstance(discovered, dict): + sync_errors.extend(str(value) for value in (discovered.get("errors") or ())) + discovered_tables = discovered.get("tables") + if isinstance(discovered_tables, dict): + databases.extend(str(key) for key in discovered_tables) + if result.get("frozen"): + sync_errors.append(str(result.get("note") or "the connector is frozen")) + state = ("partial" if databases else "error") if sync_errors else "ready" + return connector_sync.record( + session.runtime, "odoo", connection_id, state, + detail=str(result.get("note") or "") if isinstance(result, dict) else "", + errors=sync_errors, databases=sorted(set(databases)), + ) + + +# Provider runners are an API-layer capability. The data-layer registration proves Amazon uses +# the same contract while this table honestly leaves it without an executable sync. +_SYNC_RUNNERS = {"odoo": _odoo_sync_runner} + + +class _SupersededMaterialization(Exception): + pass + + +def _invalidate_materialization(rt, key): + """Fence a publisher in the same atomic document that holds the records.""" + import core.user_tables as ut + def apply(current): + doc = dict(current or {}) + if key in doc: + doc[key] = {**doc[key], "_connectorJob": None} + return doc + rt.update(ut.STORE_KEY, apply, flush="sync") + + +def _queue_odoo_database(rt, key, *, model="", retry=False, changed=False): + import odoo_relational as rel + cid = rel.connection_for_table_key(key, rt) + revision = connector_sync.configuration_revision( + "odoo", cid, key, model, rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}) + previous = (rt.get(connector_sync.JOBS_KEY, fresh=True) or {}).get( + connector_sync.job_key("odoo", cid, key)) or {} + if previous.get("state") == "syncing" and (changed or previous.get("revision") != revision): + _invalidate_materialization(rt, key) + job = connector_sync.enqueue( + rt, "odoo", cid, key, model=model, retry=retry, changed=changed, revision=revision) + # The runtime service is started by ASGI lifespan, never by a handler or a fixture import. + _materialization_wake.set() + return job + + +class _MaterializationRuntime: + """Delegate tenant reads; fence each row publication inside the store's atomic updater.""" + def __init__(self, rt, job): + self.rt, self.job = rt, job + + def __getattr__(self, name): + return getattr(self.rt, name) + + def valid(self): + import odoo_relational as rel + key = self.job["databaseKey"] + if not connector_sync.owns(self.rt, self.job): + raise _SupersededMaterialization() + cfg = self.rt.get(rel.CONFIG_KEY, fresh=True) or {} + if cfg.get("frozen") or (cfg.get("grids") or {}).get(key) is False: + raise _SupersededMaterialization() + # Save commits field choices before the queue update. The lease alone cannot + # detect that newer configuration if enqueue fails or has not run yet. + revision = self.job.get("revision") + if revision is not None and revision != connector_sync.configuration_revision( + self.job["provider"], self.job["connectionId"], key, self.job["model"], + self.rt.get(_ODOO_SCHEMA_ENABLEMENTS, fresh=True) or {}): + raise _SupersededMaterialization() + + def update(self, name, fn, flush="sync", **kwargs): + import core.user_tables as ut + key = self.job["databaseKey"] + marker = [self.job["id"], self.job["leaseToken"]] + def apply(current): + self.valid() + if name == ut.STORE_KEY and (current.get(key) or {}).get("_connectorJob") != marker: + raise _SupersededMaterialization() + answer = fn(current) + # Lease expiry during a long merge cannot acknowledge a stale write either. + self.valid() + return answer + return self.rt.update(name, apply, flush=flush, **kwargs) + + +def _run_database_job(rt, candidate): + """Run one persisted receipt. Repeating a completed publication reconciles the same rows.""" + import core.user_tables as ut + import odoo_relational as rel + # A disconnected source is paused work, not a failed attempt. Preserve the receipt so + # reconnect can resume it; the publication fence still rechecks a mid-run disconnect. + if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"): + return + job = connector_sync.claim(rt, candidate) + if job is None: + return + fenced = _MaterializationRuntime(rt, job) + key = job["databaseKey"] + def finish(success=False): + # The pre-claim check cannot cover a disconnect during source I/O or publication. + # Only the current lease may requeue; disable and a newer Save still win. + interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen")) + if success and not interrupted: + # A newer field Save can persist after rows commit but before this acknowledgement. + # The old rows remain valid history; they cannot establish the new settings ready. + fenced.valid() + return connector_sync.finish(rt, job, success=success, interrupted=interrupted) + + stopped = threading.Event() + def heartbeat(): + while not stopped.wait(connector_sync.LEASE_SECONDS / 3): + try: + connector_sync.finish(rt, job, renew=True) + except Exception: + return # An unrenewed lease cannot publish. + heartbeat_thread = None + try: + def install(current): + fenced.valid() + if key not in (current or {}): + raise _SupersededMaterialization() + current[key]["_connectorJob"] = [job["id"], job["leaseToken"]] + return current + rt.update(ut.STORE_KEY, install, flush="sync") + heartbeat_thread = threading.Thread(target=heartbeat, daemon=True, + name="connector-lease") + heartbeat_thread.start() + cid = job["connectionId"] + source_id = None if cid == "default" else cid + if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS: + result = rel.refresh( + fenced, getattr(rt, "tenant", None), username="automation", + connection_id=source_id, table_keys={key}) + else: + # A source-only model must not depend on an unrelated DuckDB mirror being warm. + _cid, namespace = rel.connection_namespace(rt, source_id) + result = {"discovered": rel.sync_discovered_models( + fenced, source_connection_id=source_id, namespace=namespace, + username="automation", table_keys={key})} + discovered = result.get("discovered") or {} + published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})} + success = (key in published and not result.get("failed") and not result.get("frozen") + and not result.get("limits") and not discovered.get("errors")) + finish(success=success) + except _SupersededMaterialization: + # Disable/new Save already invalidated the table marker; never restore it here. + finish() + except Exception: + # Raw provider exceptions may carry credentials or source values. Store only safe copy. + finish() + finally: + stopped.set() + if heartbeat_thread: + heartbeat_thread.join(timeout=1) + + +def _resume_database_jobs(rt): + """One serial pass, also usable by isolated synthetic fixtures.""" + for candidate in connector_sync.jobs(rt): + if candidate.get("provider") == "odoo" and ( + candidate.get("state") == "queued" or + candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()): + try: + _run_database_job(rt, candidate) + except Exception: + # A failed claim/status write leaves durable work for a later scan. It must + # not starve sibling databases; never log raw store/provider exceptions. + continue + + +_materialization_wake = threading.Event() +_materialization_stop = threading.Event() +_materialization_thread = None +_materialization_lifecycle = threading.Lock() + + +@router.on_event("startup") +def _start_database_jobs(): + """Discover durable work at boot, including tenants that have no active browser session.""" + global _materialization_thread + def run(): + global _materialization_thread + from harness import runtime + while True: + with _materialization_lifecycle: + if _materialization_stop.is_set(): + # Retire under the startup lock, before this thread exits. A new lifespan + # either revives this worker or starts its replacement, never a dying one. + _materialization_thread = None + return + _materialization_wake.clear() + try: + tenants = runtime.known_tenants() + for tenant in tenants: + if _materialization_stop.is_set(): + break + try: + _resume_database_jobs(runtime.get_runtime(tenant)) + except Exception: + continue # Durable queued/leased work is retried by the next scan. + except Exception: + pass + _materialization_wake.wait(15) + with _materialization_lifecycle: + _materialization_stop.clear() + _materialization_wake.set() + if _materialization_thread and _materialization_thread.is_alive(): + return + _materialization_thread = threading.Thread(target=run, daemon=True, + name="connector-materialization") + _materialization_thread.start() + + +@router.on_event("shutdown") +def _stop_database_jobs(): + with _materialization_lifecycle: + _materialization_stop.set() + _materialization_wake.set() + + +def _sync_refusals(): + """The provider refusal types whose OWN sentence is the answer, as a tuple for `except`. + + ⚠ Resolved lazily and defensively: `odoo_relational` is imported inside functions throughout + this module to avoid an import cycle, and a refusal handler that could itself raise on import + would turn every failed sync into a different failure. + """ + try: + import odoo_relational # noqa: PLC0415 + return (odoo_relational.Refused,) + except Exception: # noqa: BLE001 + return () + + def _schema_catalog(session): """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty.""" return _schema_catalog_rt(session.runtime) -def _schema_catalog_rt(rt): +def _odoo_raw_models(rt, snapshot=None, connection_id=None): + """Raw fields from stamped discovery, or the legacy snapshot before discovery exists.""" + if snapshot is None: + import core.user_tables as user_tables + snapshot = user_tables.odoo_schema_snapshot(rt) + legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {} + try: + import odoo_relational + discovered = odoo_relational.readable_model_catalog(rt, connection_id) + except Exception: # noqa: BLE001 + discovered = {} + if not discovered.get("capturedAt"): + return legacy + # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer + # than adding snapshot-only models that the current account never proved it may read. + return {str(item.get("model") or ""): dict(item.get("fields") or {}) + for item in (discovered.get("models") or ()) + if str(item.get("model") or "")} + + +def _schema_catalog_rt(rt, database_key=None, connection_id=None): """The same catalogue, over a RUNTIME rather than a session. ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector @@ -891,35 +1429,133 @@ def _schema_catalog_rt(rt): import core.user_tables as user_tables enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} snapshot = user_tables.odoo_schema_snapshot(rt) - raw_models = snapshot.get("models") or {} + raw_models = _odoo_raw_models(rt, snapshot, connection_id) if raw_models: rows = [] for model, raw_fields in raw_models.items(): for raw_name, meta in (raw_fields or {}).items(): - key = f"{model}.{raw_name}" + legacy_key = f"{model}.{raw_name}" + key = (_odoo_field_key(rt, database_key, model, raw_name) + if database_key else legacy_key) rows.append({"key": key, "model": model, "rawName": raw_name, "label": str(meta.get("string") or raw_name), "type": str(meta.get("type") or "text"), "source": "Odoo", "emptyDiscoverable": True, "defaultEnabled": False, - "enabled": enabled.get(key) is True}) + # Legacy choices seed default-connection databases until each has + # its own explicit override. Scoped writes never change this seed. + "enabled": enabled.get(key, enabled.get(legacy_key) + if connection_id is None else False) is True}) return sorted(rows, key=lambda row: row["label"].casefold()) # Never present compiled mirror declarations as an authoritative fields_get catalogue. # Until a sync captures metadata, the schema is unavailable and enablement stays refused. return [] +def _odoo_field_key(rt, database_key, model, raw_name): + import odoo_relational + return connector_sync.field_key( + "odoo", odoo_relational.connection_for_table_key(database_key, rt), + database_key, model, raw_name) + + @router.get("/connectors/odoo/status") -def odoo_status(session: Session = Depends(require_session)): +def odoo_status(connection_id: str = None, session: Session = Depends(require_session)): _odoo_admin(session) import odoo_relational cfg = odoo_relational.read_config(session.runtime) stored = session.runtime.get(_ODOO_SYNC_STATUS) or {} errors = stored.get("errors") if isinstance(stored.get("errors"), list) else [] phase = str(stored.get("phase") or "unknown") + lifecycle = connector_sync.read(session.runtime, "odoo", connection_id) + durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id)) + if durable: + return {"schedule": cfg.get("syncEvery"), + "health": "paused" if cfg.get("frozen") else durable["state"], + "lastRun": durable["at"], "frozen": cfg.get("frozen") is True, + **durable, "stateAt": durable["at"], "connectionId": connection_id or "default"} + # The old keys stay the old default-connection answer. A caller that names a connection gets + # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect. + named = connection_id is not None + shown_errors = (list(lifecycle.get("errors") or ()) if named and lifecycle else + [] if named else errors) + shown_phase = ((lifecycle or {}).get("state") or "unknown") if named else phase + shown_last_run = ((lifecycle or {}).get("at") if named else stored.get("lastRun")) + state = ((lifecycle or {}).get("state") + or {"syncing": "syncing", "ok": "ready", "error": "error"}.get(shown_phase) + or "queued") + errors_total = ((lifecycle or {}).get("errorsTotal", 0) if named or lifecycle + else len(errors)) return {"schedule": cfg.get("syncEvery"), - "health": "paused" if cfg.get("frozen") else "error" if errors else phase, - "lastRun": stored.get("lastRun"), "errors": errors[:20], - "frozen": cfg.get("frozen") is True} + "health": "paused" if cfg.get("frozen") else + shown_phase if named else "error" if shown_errors else shown_phase, + "lastRun": shown_last_run, "errors": shown_errors[:20], + "frozen": cfg.get("frozen") is True, + "state": state, + "stateAt": (lifecycle or {}).get("at") or shown_last_run, + "databases": list((lifecycle or {}).get("databases") or ()), + "retryable": state in ("error", "partial"), + "errorsTotal": int(errors_total or 0), + "connectionId": (lifecycle or {}).get("connectionId") + or str(connection_id or "").strip() or "default"} + + +@router.get("/connectors/{connector_key}/sync") +def connector_sync_status(connector_key: str, connection_id: str = None, + session: Session = Depends(require_session)): + registered = connector_sync.providers() + if connector_key not in registered: + raise err(404, "sync_provider_unknown", + f"{connector_key} is not registered for connector sync") + row = (connector_sync.job_summary(_connection_jobs(session.runtime, connector_key, connection_id)) + or connector_sync.read(session.runtime, connector_key, connection_id)) + return _sync_payload(connector_key, connection_id, row) + + +@router.post("/connectors/{connector_key}/sync/retry") +def connector_sync_retry(connector_key: str, connection_id: str = None, + session: Session = Depends(require_session), database_key: str = None): + _sync_admin(session) + if connection_id == "default": + connection_id = None + registered = connector_sync.providers() + if connector_key not in registered: + raise err(404, "sync_provider_unknown", + f"{connector_key} is not registered for connector sync") + if connector_key != "odoo": + raise err(501, "sync_runner_unavailable", registered[connector_key]["retryHint"]) + import odoo_relational as rel + if database_key: + key, _ns, choice = _odoo_database_target(session.runtime, database_key, connection_id) + choices = [choice] + else: + choices = rel.grid_choices(session.runtime, connection_id) + receipts = [] + for choice in choices: + if not choice.get("enabled"): + continue + key = choice["key"] + existing = connector_sync.jobs(session.runtime, connector_key, + rel.connection_for_table_key(key, session.runtime)) + previous = next((j for j in existing if j["databaseKey"] == key), {}) + if previous.get("state") == "failed" and not previous.get("retryable"): + if database_key: + raise err(409, "sync_attempts_exhausted", previous["error"]) + continue + receipts.append(_queue_odoo_database(session.runtime, key, model=choice.get("model", ""), + retry=True)) + if not receipts: + raise err(409, "sync_not_retryable", "No enabled database can be retried. Check its settings.") + return {"provider": connector_key, "jobs": receipts, **connector_sync.job_summary(receipts)} + + +@router.get("/connectors/{connector_key}/jobs") +def connector_database_jobs(connector_key: str, session: Session = Depends(require_session), + connection_id: str = None): + _sync_admin(session) + if connector_key not in connector_sync.providers(): + raise err(404, "sync_provider_unknown", "This connector has no database sync runner.") + return {"provider": connector_key, "jobs": connector_sync.jobs( + session.runtime, connector_key, connection_id)} @router.get("/connectors/odoo/schema") @@ -928,10 +1564,286 @@ def odoo_schema(session: Session = Depends(require_session)): fields = _schema_catalog(session) import core.user_tables as user_tables snapshot = user_tables.odoo_schema_snapshot(session.runtime) + try: + import odoo_relational + discovered = odoo_relational.readable_model_catalog(session.runtime) + except Exception: # noqa: BLE001 + discovered = {} + has_discovery = bool(discovered.get("capturedAt")) + complete = (discovered.get("complete") is True if has_discovery + else snapshot.get("complete") is True) return {"fields": fields, "default": "off", "adminOnly": True, - "complete": snapshot.get("complete") is True, - "capturedAt": snapshot.get("capturedAt"), - "errors": list(snapshot.get("errors") or ()), "source": "Odoo"} + "complete": complete, + "capturedAt": (discovered.get("capturedAt") if has_discovery + else snapshot.get("capturedAt")), + "errors": list((discovered.get("errors") if has_discovery + else snapshot.get("errors")) or ()), + "source": "Odoo"} + + +@router.post("/connectors/odoo/census") +def odoo_population_census(session: Session = Depends(require_session)): + """Explicitly count every readable Odoo model; connector detail only reads this cache.""" + _odoo_admin(session) + import odoo_relational + census = odoo_relational.refresh_population_census(session.runtime, force=True) + return {"capturedAt": str(census.get("capturedAt") or ""), + "complete": census.get("complete") is True, + "durationMs": int(census.get("durationMs") or 0), + "modelsCounted": int(census.get("modelsCounted") or 0), + "modelsWithRows": int(census.get("modelsWithRows") or 0), + "errors": [str(value) for value in (census.get("errors") or ())][:50]} + + +def _materialisation_failure_reason(result): + """The materialiser's own reason, made safe for a sentence shown in the connector modal.""" + result = result if isinstance(result, dict) else {} + candidates = [] + if result.get("note"): + candidates.append(result.get("note")) + if result.get("failed"): + candidates.append(result.get("failed")) + errors = result.get("errors") + candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) + discovered = result.get("discovered") + if isinstance(discovered, dict): + errors = discovered.get("errors") + candidates.extend(errors if isinstance(errors, (list, tuple)) else (errors,)) + reason = str(next((value for value in candidates if str(value or "").strip()), "")).strip() + # Provider errors can carry typographic punctuation. Screen strings use plain punctuation. + reason = reason.replace("\u2014", ",").replace("\u2013", "-") + return reason[:1].upper() + reason[1:] if reason else "" + + +def _evict_odoo_product_pools(rt): + """Drop every raw-field-sensitive row pool after field enablement changes.""" + for cache_key in list(getattr(rt, "pool_cache", {}) or {}): + if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",): + rt.pool_cache.pop(cache_key, None) + + +# W57-T31 / R10 / C9: switching off starts a seven-day lock to prevent repeated toggling. +# The config document is shared with read_config, which ignores unknown keys: disabledAt +# belongs beside grids, not in a second document that could drift from enablement. +_RELOCK_DAYS = 7 +_RELOCK_SECONDS = _RELOCK_DAYS * 86400 + + +def _relock_stamps(rt): + """Read disable epochs without letting an absent or malformed map break a request.""" + try: + import odoo_relational + stamps = (rt.get(odoo_relational.CONFIG_KEY) or {}).get("disabledAt", {}) + if not isinstance(stamps, dict): + return {} + return {str(key): int(value) for key, value in stamps.items() + if isinstance(value, (int, float)) and not isinstance(value, bool)} + except Exception: # noqa: BLE001 + return {} + + +def _relock_days_left(stamp, now=None): + """Round any remaining part of a lock day up to a whole day.""" + if isinstance(stamp, bool) or not isinstance(stamp, (int, float)): + return 0 + try: + from math import ceil + remaining = stamp + _RELOCK_SECONDS - (time.time() if now is None else now) + return max(0, ceil(remaining / 86400)) + except Exception: # noqa: BLE001 + return 0 + + +def _odoo_database_target(rt, table_key, connection_id=None): + """Resolve a database once; explicit connection/key disagreement fails closed.""" + import odoo_relational + raw = str(table_key or "").strip() + base, namespace = odoo_relational.split_table_key(raw) + requested_connection = connection_id if connection_id is not None else namespace + try: + _cid, ns = odoo_relational.connection_namespace(rt, requested_connection) + choices = odoo_relational.grid_choices(rt, requested_connection) + except odoo_relational.Refused as exc: + raise err(404, "unknown_odoo_connection", str(exc)) + key = odoo_relational.table_key_for(base or raw, ns) + if namespace is not None and namespace != ns: + raise err(409, "database_connection_mismatch", + "the database belongs to a different connection") + choice = next((row for row in choices if row["key"] == key), None) + if choice is None: + raise err(404, "unknown_odoo_database", + f"Odoo does not declare a database with key {raw!r}") + return key, ns, choice + + +def _enable_odoo_database(table_key, session, connection_id=None): + """Turn on one database and publish its schema; row materialization is separate work.""" + if not session.admin: + raise err(403, "not_admin", "only an administrator can enable an Odoo database") + + import odoo_relational # noqa: PLC0415 - provider owns both paths + key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) + + # R10 / C9: only a transition back on is locked; an already-enabled database may retry + # materialisation. A successful enable clears its stamp so the next switch-off starts fresh. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + if choice.get("enabled") is not True: + days = _relock_days_left(_relock_stamps(session.runtime).get(key)) + if days > 0: + label = str(choice.get("label") or key) + unit = "day" if days == 1 else "days" + raise err(409, "database_relocked", + f"{label} was switched off less than {_RELOCK_DAYS} days ago. " + f"It cannot be switched back on for another {days} {unit}. " + f"The {_RELOCK_DAYS} day lock is what stops a database being switched " + "on and off repeatedly.") + + # Publishing the definition makes Data and rename usable before any rows have loaded. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur.setdefault("grids", {}).update({key: True}) + if isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"].pop(key, None) + return cur + + failure = None + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") + + try: + result = odoo_relational.publish_database_schema( + session.runtime, key, connection_id=namespace, username=session.uname) + except odoo_relational.Refused as exc: + # The provider owns this sentence. In particular it names missing credentials and the + # MAX_TABLES refusal without this route guessing which one happened. + failure = (409, "odoo_database_refused", str(exc)) + except Exception: # noqa: BLE001 + failure = (500, "database_schema_failed", + "The setting was enabled, but the database was not created because the " + "field catalogue could not be published.") + + facts = (_rename_rights(session.runtime, session.uname, bool(session.admin), [key]).get(key) + or {"exists": False, "canRename": False}) + exists = facts.get("exists") is True + can_rename = facts.get("canRename") is True + # ⛔⛔ THE ERROR ENVELOPE STAYS THE ONE SHAPE. `deps.err` is documented as X2's single + # non-2xx body, `{"error": {"code", "message"}}`, and an earlier draft of this door attached + # `key`, `enabled` and the fresh facts BESIDE that key so the client could read them off a + # failure. Two shapes for one thing is what this module's own comments call the defect it has + # paid for twice, and a caller that learns to read a sibling off an error will keep doing it. + # The client does not need them: the FLAG is written before materialisation is attempted, so + # every outcome of this door can change what `/detail` answers, and `flipTable` therefore + # re-reads on both paths. + if failure is not None: + raise err(*failure) + if not exists: + reason = _materialisation_failure_reason(result) + message = "The setting was enabled, but the database was not created." + if reason: + message = f"{message} {reason}" + else: + message = f"{message} The field catalogue could not be published." + raise err(409, "database_not_created", message) + + # Every permission-visible field is available from schema alone. Existing per-database + # choices survive a retry; population counts are optional background observations. + model = str(choice.get("model") or "").strip() + try: + schema_keys = {_odoo_field_key(session.runtime, key, model, name): True + for name in result["sourceFields"]} + session.runtime.update( + _ODOO_SCHEMA_ENABLEMENTS, + lambda current: { + **schema_keys, + **(current if isinstance(current, dict) else {}), + }, + flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "field_auto_enable_failed", + "The database was created, but its field choices could not be saved. " + "Try again.") + _evict_odoo_product_pools(session.runtime) + + # W56-T36: there is deliberately no navigation projection eviction here. Data's database + # list comes from + # `GET /nav` -> `core.user_tables.all_defs()` -> `TenantRuntime.get_projection(user_tables)`. + # Schema publication synchronously updates that same `user_tables` document; the HF + # backend replaces its whole-document cache as part of that write, while Postgres performs a + # fresh SELECT for every projection. `pool_cache` contains row pools for built-in databases, + # not the `ut_*` definition list, and compiled registry rows never contain tenant databases. + # The table now exists, so clients must discard navigation-derived database lists. This is a + # wire instruction rather than a guessed client row: `/nav` remains the permission-filtered + # authority for which databases the current reader may see. + try: + job = _queue_odoo_database(session.runtime, key, model=model) + except Exception: + raise err(503, "sync_queue_unavailable", + "The database settings were saved, but row sync could not be queued. Save again.") + return {"key": key, "enabled": True, "exists": exists, "canRename": can_rename, + "identity": connector_sync.database_identity( + "odoo", odoo_relational.connection_for_table_key(key, session.runtime), key, model), + "workspaceChanged": True, "state": job["state"], "job": job, + "schemaState": result["state"], + "schemaFields": result["fieldCount"], "populatedFieldsEnabled": 0, + "fieldCensusComplete": False, "fieldCensusErrors": [], + "autoEnableAlsoAffects": []} + + +@router.post("/connectors/odoo/databases/{table_key}/enable") +def odoo_database_enable(table_key: str, session: Session = Depends(require_session), + connection_id: str = None): + """Enable and create one declared Odoo database in this tenant's governed store.""" + return _enable_odoo_database(table_key, session, connection_id) + + +def _disable_odoo_database(table_key, session, connection_id=None): + """Turn off one declared Odoo database and persist its re-enable lock.""" + if not session.admin: + raise err(403, "not_admin", "only an administrator can disable an Odoo database") + + import odoo_relational + key, namespace, choice = _odoo_database_target(session.runtime, table_key, connection_id) + + # W57-T31 / R10 / C9: one config-only write starts the lock, with no sync or row deletion. + # The provider's own enabled is authoritative; re-deriving defaults is wrong for the discovered long tail. + # Repeating a disable must preserve the original deadline. + was_enabled = choice.get("enabled") is True + _invalidate_materialization(session.runtime, key) + connector_sync.cancel(session.runtime, "odoo", + odoo_relational.connection_for_table_key(key, session.runtime), key) + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") + grids = grids if isinstance(grids, dict) else {} + grids[key] = False + cur["grids"] = grids + if was_enabled: + if not isinstance(cur.get("disabledAt"), dict): + cur["disabledAt"] = {} + cur["disabledAt"][key] = int(time.time()) + return cur + + try: + session.runtime.update(odoo_relational.CONFIG_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", "the database setting was not saved. Try again.") + + stamp = _relock_stamps(session.runtime).get(key) + return {"key": key, "enabled": False, "relockDays": _RELOCK_DAYS, + "identity": connector_sync.database_identity( + "odoo", odoo_relational.connection_for_table_key(key, session.runtime), + key, choice.get("model")), + "disabledAt": stamp, + "canEnableAt": stamp + _RELOCK_SECONDS if stamp is not None else None} + + +@router.post("/connectors/odoo/databases/{table_key}/disable") +def odoo_database_disable(table_key: str, session: Session = Depends(require_session), + connection_id: str = None): + """Disable one declared Odoo database in this tenant's governed store.""" + return _disable_odoo_database(table_key, session, connection_id) @router.patch("/connectors/odoo/schema/{field_key}") @@ -939,26 +1851,70 @@ def odoo_schema_enable(field_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): _odoo_admin(session) import core.user_tables as user_tables + import odoo_relational + request = body if isinstance(body, dict) else {} + key = str(field_key or "").strip() + scoped = connector_sync.parse_field_key(key) + database_key = request.get("databaseKey") + connection_id = request.get("connectionId") + model, raw_name = key.rsplit(".", 1) if "." in key else ("", "") + if scoped: + provider, connection_id, database_key, model, raw_name = scoped + if provider != "odoo": + raise err(404, "unknown_odoo_field", "that field belongs to another connector") + if connection_id == "default": + connection_id = None + if database_key is None: + candidates = [row for row in odoo_relational.grid_choices(session.runtime, connection_id) + if row.get("model") == model] + if len(candidates) > 1: + raise err(400, "database_required", "choose the database for this field") + if candidates: + database_key = candidates[0]["key"] + namespace = None + if database_key: + database_key, namespace, choice = _odoo_database_target( + session.runtime, database_key, connection_id) + if model != choice.get("model"): + raise err(404, "unknown_odoo_field", "that model does not belong to this database") + canonical = _odoo_field_key(session.runtime, database_key, model, raw_name) + if scoped and key != canonical: + raise err(409, "database_connection_mismatch", + "the field belongs to a different connection") + key = canonical _snapshot = user_tables.odoo_schema_snapshot(session.runtime) - if _snapshot.get("complete") is not True: + try: + import odoo_relational + _discovered_complete = odoo_relational.readable_model_catalog( + session.runtime, namespace).get("complete") is True + except Exception: # noqa: BLE001 + _discovered_complete = False + if (namespace is not None or _snapshot.get("complete") is not True) and not _discovered_complete: raise err(409, "odoo_schema_unavailable", "run a successful schema sync before changing raw-field enablement") - known = {field["key"] for field in _schema_catalog(session)} - key = str(field_key or "").strip() + known = {field["key"] for field in _schema_catalog_rt( + session.runtime, database_key, namespace)} if key not in known: raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue") - enabled = (body or {}).get("enabled") + enabled = request.get("enabled") if not isinstance(enabled, bool): raise err(400, "bad_enabled", "enabled must be true or false") + previous_enabled = (session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {}).get(key) is True session.runtime.update( _ODOO_SCHEMA_ENABLEMENTS, lambda current: {**(current if isinstance(current, dict) else {}), key: enabled}, flush="sync") - for cache_key in list(getattr(session.runtime, "pool_cache", {}) or {}): - if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",): - session.runtime.pool_cache.pop(cache_key, None) + _evict_odoo_product_pools(session.runtime) + job = None + if database_key and choice.get("enabled"): + try: + job = _queue_odoo_database(session.runtime, database_key, model=model, + changed=previous_enabled != enabled) + except Exception: + raise err(503, "sync_queue_unavailable", + "The field settings were saved, but row sync could not be queued. Save again.") return {"key": key, "enabled": enabled, "source": "Odoo", - "permissionWall": "normal field permissions"} + "permissionWall": "normal field permissions", "job": job} # ═════════════════════════════════════════════════════════════════════════════════════════════ @@ -1000,9 +1956,23 @@ def _entry_type_for(connector_key): key = str(connector_key or "") if key == "odoo": return "odoo" - for token in TOKEN_CONNECTORS: - if str(token.get("key") or "") == key: - return str(token.get("entry") or "") + # ⛔⛔ FOUND ONLY ON THE MERGED TREE, AT THE WAVE-53 COMBINE, AND IT IS A 500 ON A LIVE ROUTE. + # `directory` answers 200 with a NAMED gap when any one contributor registry raises (W32-T14 / + # R6: "it used to 500 and show nothing"), and every contributor is called inside that guard. + # The `allowedScopes` pass added this wave runs AFTER the composition, OUTSIDE the guard, and + # this loop is a second unguarded read of `TOKEN_CONNECTORS` - so a broken token registry took + # the whole page back to a 500. Neither lane could see it: one owns this route, the other owns + # the gate that drives the broken registry, and only the merged tree holds both. + # ⚠ `""` is not a swallow, it is this function's own documented answer for a row with no + # keychain type, and `allowed_scopes("")` offers BOTH scopes - the permissive default the + # composer already states, chosen because the toggle must never hide a connection the reader + # actually holds. A token connector's rows are already absent from the set in this state. + try: + for token in TOKEN_CONNECTORS: + if str(token.get("key") or "") == key: + return str(token.get("entry") or "") + except Exception: # noqa: BLE001 - a registry, not a bug + return "" return "" @@ -1052,6 +2022,18 @@ def _detail_connect(entry_type, is_admin): "tenantWide": tenant_wide} +def _replacement_field_specs(entry_type): + """The write-only fields that may change on an existing credential. + + This is deliberately a projection of the Connect descriptor rather than another provider + vocabulary. URL, database, account and every future non-secret field stay server-held during + replacement; only fields the existing descriptor marks secret cross this door. + """ + spec = _detail_connect(entry_type, True) or {} + return [dict(field) for field in (spec.get("fields") or []) + if field.get("secret") is True and str(field.get("key") or "").strip()] + + def _oauth_slot_entries(rt, uname, row): """A per-user OAuth connection, rendered as C4's `keychain` entry shape. @@ -1090,7 +2072,8 @@ def _oauth_slot_entries(rt, uname, row): # A per-user slot is personal by construction, in the vocabulary `routes_keychain` # already declares. Never a third word for the same idea. "scope": "personal", - "testable": False}] + "testable": False, + "replaceable": False}] def _detail_keychain(rt, uname, is_admin, row): @@ -1111,6 +2094,7 @@ def _detail_keychain(rt, uname, is_admin, row): # already decided admin vs member vs nothing, and deciding again here is how two answers to # one question get born ([[one-question-two-normalizers]]). can_connect = manage in ("keychain", "keychain_personal") + can_replace = bool(is_admin and can_connect and _replacement_field_specs(entry_type)) entries = [] if entry_type: try: @@ -1128,6 +2112,11 @@ def _detail_keychain(rt, uname, is_admin, row): # xmlrpc check otherwise. The flag says whether a Test control would have # something to call, and it always does. "testable": True, + # W55-T18 / C10: credential replacement is an administrator action. The + # client reads this wall instead of inferring it from scope or from whether + # a masked preview happened to be visible. The replacement endpoint changes + # this row's encrypted payload in place; no secret is added to this DTO. + "replaceable": can_replace, }) except Exception: # noqa: BLE001 entries = [] @@ -1149,18 +2138,16 @@ def _detail_keychain(rt, uname, is_admin, row): "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None} -def _odoo_catalog_by_model(rt): +def _odoo_catalog_by_model(rt, connection_id=None): """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves. `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment the catalogue starts filtering instead of needing somebody to remember. """ - import core.user_tables as user_tables - snapshot = user_tables.odoo_schema_snapshot(rt) - raw_models = snapshot.get("models") or {} + raw_models = _odoo_raw_models(rt, connection_id=connection_id) offered = {} - for row in _schema_catalog_rt(rt): + for row in _schema_catalog_rt(rt, connection_id=connection_id): offered.setdefault(str(row.get("model") or ""), []).append(row) out = {} for model, raw_fields in raw_models.items(): @@ -1181,7 +2168,213 @@ def _odoo_catalog_by_model(rt): return out -def _detail_databases(rt, is_admin, connector_key): +def _spec_field(item, name): + """One attribute off a C2 table spec, whether lane D ships dataclasses or dicts.""" + if isinstance(item, dict): + return item.get(name) + return getattr(item, name, None) + + +def _connector_spec_items(connector_key): + """C2's raw table specs for one connector, as a sequence of items, or `()`. + + ⭐⭐ W53-T30 — EXTRACTED FROM `_connector_spec_tables` (W53-T27) SO THERE IS STILL EXACTLY ONE + IMPORT PATH TO LANE D'S MODULE. Two consumers now read the spec: the Databases tab wants + `native` and `label`, the relationship graph wants `links`. A second `from core import + connector_spec` with its own lookup order is how the two would start disagreeing about which + attribute of D's module is the spec, and only one of them would be exercised by any given + tenant. The lookup order, the failure posture and the reasons for both are unchanged and are + documented on `_connector_spec_tables` below. + """ + try: + from core import connector_spec # noqa: PLC0415 - lane D's module (C2) + except Exception: # noqa: BLE001 + return () + key = str(connector_key or "") + found = None + for name in ("tables_for", "for_connector", "tables", "spec", "spec_for"): + fn = getattr(connector_spec, name, None) + if not callable(fn): + continue + try: + got = fn(key) + except Exception: # noqa: BLE001 + continue + if got: + found = got + break + if not found: + for name in ("SPECS", "TABLES", "CONNECTOR_TABLES"): + blob = getattr(connector_spec, name, None) + if isinstance(blob, dict): + found = blob.get(key) or list(blob.values()) + elif isinstance(blob, (list, tuple)): + found = list(blob) + if found: + break + return list(found.values()) if isinstance(found, dict) else list(found or ()) + + +def _connector_spec_links(connector_key): + """C2's DECLARED relations for one connector: `{table key: [(from, target, to, label)]}`. + + ⭐⭐ W53-T30 / C2 — THE GRAPH'S SECOND SOURCE OF EDGES, and the contract's own words for it: + each spec row carries `links`, a list of `(from_col, target_key, to_col, cardinality)`. Today + every edge on this graph is read off a stored FIELD bag, which can only ever describe a + relation somebody wrote a column for; a spec-declared relation between two tables that carry + no link column has nowhere else to come from. + + ⛔ AN EDGE FROM HERE IS HELD TO THE SAME TWO PROOFS AS ONE FROM A FIELD BAG, in the caller: + both endpoints must be permitted NODES on this graph and both named columns must be VISIBLE + to this reader. A spec is a declaration about the shipped schema, never a permission decision + about a person, and an edge drawn on the strength of the declaration alone would name a column + an administrator hid from the account looking at it. + + ⛔ AND IT IS NOT AN OUTAGE WHEN LANE D'S MODULE IS ABSENT: `{}` means "no spec-declared + relations", the field bags still answer, and the graph is exactly what it is today. + + ⚠ `cardinality` IS NOT A LABEL and is not read as one. The wire's `label` is what the reader + sees on the line, and "one to many" is a fact about the join rather than a name for it, so the + caller labels a spec edge with the SOURCE COLUMN's own label instead. + """ + out = {} + for item in _connector_spec_items(connector_key): + table_key = str(_spec_field(item, "key") or "") + links = _spec_field(item, "links") + if not table_key or not isinstance(links, (list, tuple)): + continue + edges = [] + for link in links: + if not isinstance(link, (list, tuple)) or len(link) < 3: + continue + from_col = str(link[0] or "").strip() + target_key = str(link[1] or "").strip() + to_col = str(link[2] or "").strip() + if from_col and target_key and to_col: + edges.append((from_col, target_key, to_col)) + if edges: + out[table_key] = edges + return out + + +def _connector_spec_tables(connector_key): + """C2's spec for one connector as `{table key: {"native", "label"}}`, or `{}`. + + ⛔⛔ `core.connector_spec` IS LANE D'S MODULE AND IT IS NOT IN THIS TREE YET. C2 splits the + work: lane D declares the spec (`key`, `native`, `source`, `label`, `pinned`, `identity`, + `links`) and this lane CONSUMES it through `GET /connectors/{key}/detail`. So the import is + defensive and the fallback is the in-tree source each value already has — Odoo's native model + from `odoo_relational.ODOO_TABLE_MODELS`, its default label from `grid_choices`, Amazon's + report type from `connectors_amazon.REPORT_TYPES` and its label from `LABELS_AMAZON`. When D's + module lands, the spec WINS with no edit here. + + ⚠ D'S EXACT API IS NOT FROZEN, so the lookups tried are written down rather than guessed at by + the next reader: the callables `tables_for(key)`, `for_connector(key)`, `tables(key)`, + `spec(key)`, `spec_for(key)`, then the module attributes `SPECS`, `TABLES`, + `CONNECTOR_TABLES` (a dict keyed by connector, a dict keyed by table, or a flat sequence). + Whatever answers first is normalised by TABLE KEY, and only the keys this connector already + declares are ever consulted — so a spec carrying other connectors' tables is harmless. + ⚠ W53-T30 — THAT LOOKUP NOW LIVES IN `_connector_spec_items` ABOVE, unchanged, because the + relationship graph reads the same spec for its `links`. One import path, two readings. + + ⛔ A SPEC THAT CANNOT BE READ IS NOT AN OUTAGE. Every failure returns `{}` and the row keeps + the in-tree value it has today: C2 is a better source for these two strings, never the only + one, and a Databases tab that 500s because a sibling lane's module changed shape would be a + disclosure screen taken down by a naming question. + """ + out = {} + for item in _connector_spec_items(connector_key): + table_key = str(_spec_field(item, "key") or "") + if not table_key: + continue + out[table_key] = {"native": str(_spec_field(item, "native") or ""), + "label": str(_spec_field(item, "label") or "")} + return out + + +def _database_name_overrides(rt): + """This tenant's per-database name overrides: `{table key: {"name": ...}}`, or `{}`. + + ⛔⛔ `nav_meta` IS THE STORE, AND IT IS READ THROUGH THE NAVIGATION RAIL'S OWN READER. R9, + verbatim: *"the rename database under the Navigation rail also works, it should then sync to + the navigation in the connector too."* That rename has written `nav_meta` since wave 19 + (`routes_nav.save_nav_meta`), and `routes_tables.ut_label` already resolves a served name as + nav_meta's name, then the definition's label, then the key. Calling `routes_nav._read_nav_meta` + rather than re-deriving the document shape here is what makes the rail and this tab provably + ONE fact instead of two answers to one question. + + ⛔ AND IT IS WHY NOTHING HERE WRITES A TABLE DEFINITION'S LABEL. `user_tables.set_label` is + rewritten by a connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an + override stored there would be silently clobbered on the next sync, which is precisely the + R9 violation this ticket exists to prevent. + + ⚠ ONE READ FOR THE WHOLE LIST, never one per row: `store.get` deep copies the tenant document + on every call, and this list is eight rows on Odoo and seven on Amazon. + ⚠ A store blip is "no overrides", never a failed page. The default name is still true. + """ + try: + import routes_nav # noqa: PLC0415 - lazy, avoids a cycle + meta = routes_nav._read_nav_meta(rt) or {} # noqa: SLF001 - the ONE reader (R9) + except Exception: # noqa: BLE001 + return {} + return {str(k): v for k, v in meta.items() if isinstance(v, dict)} + + +def _rename_rights(rt, uname, is_admin, keys): + """`{table key: {exists, canRename}}` for R9's write wall and its prerequisite. + + ⛔ THE SAME PREDICATE `routes_nav.save_nav_meta` APPLIES TO A `ut_` KEY, not a second one: + `user_tables.may_open`, i.e. the table's creator, a tenant admin, or a share grant. Two ideas + of who may rename a database would show one person a control that the other door refuses. + + ⛔ A DATABASE THIS CONNECTION HAS NOT CREATED YET IS NOT RENAMEABLE, and that is not a + technicality: `nav_meta` is keyed by TABLE key, so a name stored for a table nobody can open + would sit in the tenant's document naming nothing, and the rail would never show it. + + ⭐ W56-T31: `enabled` USED TO MEAN "will be included in the next sync", while the screen + read as though the database already existed. `canRename` was telling the truth about a table + that did not exist. Returning `exists` from this same projection lets the client distinguish + that state from a reader who may not open an existing table. + + ⚠ IT IS SERVED PER ROW because the answers ARE per row. One `lend_defs` read answers the + whole list; that projection carries definitions without rows (~0.1% of the document's bytes). + """ + try: + import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere + definitions = user_tables.lend_defs(rt) + facts = {} + for key in keys: + exists = bool(user_tables.get(key, st=definitions)) + facts[key] = { + "exists": exists, + "canRename": bool(str(key).startswith("ut_") and exists + and user_tables.may_open( + key, uname, bool(is_admin), st=definitions)), + } + return facts + except Exception: # noqa: BLE001 + # Fail closed. A wall that could not be read is not a wall that admitted you. + return {key: {"exists": False, "canRename": False} for key in keys} + + +def _named_databases(rt, is_admin, uname, rows): + """The LAST step of every databases list, Odoo's and Amazon's alike: the three names. + + One place, deliberately. Each half of `_detail_databases` builds its own `native` and its own + shipped default from its own source, and then BOTH come through here — so "what is this + database called for this tenant" is answered once, by `schema_overlay.merge_database_labels`, + and a third connector cannot arrive with a fourth opinion about a blank override. + """ + named = schema_overlay.merge_database_labels(rows, _database_name_overrides(rt)) + rights = _rename_rights(rt, uname, is_admin, [str(row.get("key") or "") for row in named]) + for row in named: + facts = rights.get(str(row.get("key") or "")) or {} + row["exists"] = facts.get("exists") is True + row["canRename"] = facts.get("canRename") is True + return named + + +def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None): """C4's `databases` half: what this connector materialises, and the fields behind each one. ⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is @@ -1194,54 +2387,184 @@ def _detail_databases(rt, is_admin, connector_key): it is switched on are facts about their own workspace; the raw Odoo field catalogue is administrator-only (`_odoo_admin` gates the route that serves it) and stays that way. - ⛔⛔ W49-T14 — `sharesFieldsWith` IS NOT DECORATION, IT IS THE ONE THING THIS SCREEN WOULD - OTHERWISE LIE ABOUT. MEASURED on `odoo_relational.ODOO_TABLE_MODELS`: `ut_odoo_invoices` and - `ut_odoo_bills` are BOTH `account.move`, and `ut_odoo_agents` and `ut_odoo_vendors` are BOTH - `res.partner`. The enablement flag is keyed `"."` (`_schema_catalog_rt`), so - those pairs do not merely LOOK alike — they are one list of fields behind two names, and a - single `PATCH /connectors/odoo/schema/{field_key}` flips the field under both. A disclosure - that printed the same forty rows under two headings with no word about it would be showing a - per-database control that is not per-database, and the reader would discover it by watching - the other database change. So the row carries the sibling it shares with, by KEY, and the tab - says so above the toggles. + Fields with the same source model have independent enablement keys in each database. + `sharesFieldsWith` remains an empty list for existing clients of the older shared switches. + + ⭐⭐ W53-T27 / R9 / C2 — THREE NAMES PER ROW, AND THEY ARE THREE DIFFERENT FACTS. `native` is + the VENDOR's own identifier (`res.partner`, or Amazon's report type) and is never editable by + anybody; `defaultLabel` is OUR shipped default; `override` is this tenant's own rename, held + in the `nav_meta` bucket the navigation rail already writes. `label` stays the EFFECTIVE name + because every existing reader of it — this tab's heading, `_detail_overview`'s `creates` list + — already means "what this database is called today", and serving the un-overridden default + there would be the drift R9 exists to close. + + ⛔ ALL THREE SURVIVE FOR A NON-ADMIN, like `key` and `enabled` above them and for the same + reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE + is walled, per row, by `canRename`. """ if not connector_directory.is_introspectable(connector_key): return [] + spec = _connector_spec_tables(connector_key) + if str(connector_key) == "amazon_sp": + return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec)) try: import odoo_relational - choices = odoo_relational.grid_choices(rt) + namespace = (odoo_relational.connection_namespace(rt, connection_id)[1] + if connection_id else None) + choices = odoo_relational.grid_choices(rt, connection_id) models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {}) + # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the + # catalogue was warm, so modal detail must never turn a missing cache into inline work. + census = odoo_relational.population_census(rt, namespace) except Exception: # noqa: BLE001 return [] + census_at = str(census.get("capturedAt") or "") + counts = census.get("counts") if isinstance(census.get("counts"), dict) else {} catalog = {} if is_admin: try: - catalog = _odoo_catalog_by_model(rt) + catalog = _odoo_catalog_by_model(rt, namespace) except Exception: # noqa: BLE001 catalog = {} - # Which OTHER declared databases read the same Odoo model. Derived from the map, never a - # hand-typed pair list: a ninth table added to `TABLES` gets this disclosure for free. - shared = {} - for choice in choices: - key = str(choice.get("key") or "") - model = str(models.get(key) or "") - if model: - shared.setdefault(model, []).append(key) + enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {} out = [] for choice in choices: key = str(choice.get("key") or "") - model = str(models.get(key) or "") + model = str(choice.get("model") or models.get(key) or "") fields, excluded = catalog.get(model, ([], [])) + scoped_fields = [] + for field in fields: + field_key = _odoo_field_key(rt, key, model, field["source_field"]) + scoped_fields.append({**field, "key": field_key, + "enabled": enabled_fields.get(field_key, + field.get("enabled") if namespace is None else False) is True}) + declared = spec.get(choice.get("baseKey") or key) or {} + # Counts are model-level facts. Databases backed by the same model deliberately show the + # same sourceRows number; inventing a per-database domain here would make the number lie. + source_rows = counts.get(model) if census_at and model in counts else None out.append({"key": key, - "label": str(choice.get("label") or key), + "identity": connector_sync.database_identity( + "odoo", odoo_relational.connection_for_table_key(key, rt), key, model), + "label": str(declared.get("label") or choice.get("label") or key), + "defaultLabel": str(declared.get("label") or choice.get("label") or key), + "native": str(declared.get("native") or model), "enabled": bool(choice.get("enabled")), - "fields": [dict(field) for field in fields], + "sourceRows": source_rows, + "populated": None if source_rows is None else source_rows > 0, + "fields": scoped_fields, "excluded": [dict(item) for item in excluded], - "sharesFieldsWith": [other for other in shared.get(model, ()) - if other != key]}) + "sharesFieldsWith": []}) + return _named_databases(rt, is_admin, uname, out) + + +def _amazon_databases(rt, is_admin, spec=None): + """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM. + + Fills the SAME `ConnectorDatabase` shape Odoo does, so the Databases tab needs no client + change: `key`, `label`, `enabled`, `fields`, `excluded`, `sharesFieldsWith`. + + ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five + of the seven tables have NO published schema, because Amazon publishes artifacts only for its + JSON analytics reports and every flat-file report we depend on has none. Their columns are + measured from the report document's own header at first sync, so before that they genuinely + have zero fields. A screen that showed an empty field list with no explanation would say + *"this database has no fields"* when the truth is *"nobody has run it yet"* — the exact + confusion `_detail_introspection`'s header was written about, one layer in. + + ⛔ `sharesFieldsWith` IS REAL HERE, NOT COPIED. `orders` and `order_items` are BOTH filled from + ONE report document (`GET_FLAT_FILE_ALL_ORDERS_DATA_*`, which is one row per order ITEM), so + they share a header and therefore a field list. That is the same class of fact the Odoo pairs + carry, measured the same way: derived from the report map, never hand-typed. + + ⚠ NO FIELD TOGGLES. `canToggleFields` is False for Amazon in `_detail_introspection`; there is + no `PATCH /connectors/amazon_sp/schema/{field}` route and this wave does not build one. Fields + are disclosed, not switched. Database enablement is a separate provider capability and route, + never the generic Odoo-table flag consumed by `DatabasesTab`. + """ + try: + from harness import amazon_schema + import connectors_amazon as cx + except Exception: # noqa: BLE001 + return [] + enablement = cx.read_database_enablement(rt) + enabled = enablement.get("databases") or {} + cat = amazon_schema.catalogue() + tables = cat.get("tables") or {} + + # Which OTHER declared tables are filled from the SAME report document. Derived from the + # report map so a table added to `REPORT_TYPES` gets this disclosure for free. + by_report = {} + for name in amazon_schema.PROVENANCE: + rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + if rep: + by_report.setdefault(rep, []).append("ut_amzn_" + name) + + out = [] + for name, provenance in amazon_schema.PROVENANCE.items(): + key = "ut_amzn_" + name + spec = tables.get(name) or {} + cols = spec.get("columns") or [] + rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "") + fields = [{"key": str(c.get("name") or ""), + "label": str(c.get("name") or "").replace("_", " ").capitalize(), + "type": "text" if str(c.get("type")) == "VARCHAR" else "number", + "source_field": str(c.get("name") or ""), + # ⚠ EVERY DISCLOSED FIELD IS ON. There is no per-field switch for Amazon, and + # reporting `enabled: False` would describe a control that does not exist. + "enabled": True} + for c in cols] if is_admin else [] + excluded = [] + if provenance == "measured_at_sync" and not cols: + excluded.append({ + "source_field": "", + "reason": (amazon_schema.PROVENANCE_COPY["measured_at_sync"] + + f" The report is {rep}." if rep + else amazon_schema.PROVENANCE_COPY["measured_at_sync"])}) + # ⭐⭐ W53-T27 / C2 — THE SAME THREE FACTS AS THE ODOO ROWS, one source swapped. Amazon + # has no per-table model name; the identifier AMAZON itself uses for what fills a table is + # the REPORT TYPE (`rep`, already computed above for `sharesFieldsWith`). That is exactly + # what `native` carries: the vendor's own name for the thing, never one we invented. It is + # empty for a table whose report type is unknown, and the tab says so rather than guessing. + declared = (spec or {}).get(key) or {} + out.append({"key": key, + "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key), + "defaultLabel": str(declared.get("label") + or LABELS_AMAZON.get(key) or key), + "native": str(declared.get("native") or rep), + # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not + # read from Odoo's grid config, and disabling it leaves every stored row and + # definition untouched: only this provider config document changes. + "enabled": enabled.get(key) is True, + "fields": fields, + "excluded": excluded, + "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]}) return out +#: What an empty `excluded` list means for AMAZON, which is not what it means for Odoo. +#: +#: ⛔ A SEPARATE SENTENCE ON PURPOSE. Odoo's basis describes a DIFF of a live snapshot against a +#: catalogue. Amazon has no snapshot and no diff: a table either has a published schema or it +#: measures its columns at first sync. Reusing Odoo's wording would describe a computation that +#: does not happen here, which is the same class of error as reusing its toggles. +_AMAZON_EXCLUDED_BASIS = ( + "Amazon publishes a schema for its JSON analytics reports only. A table with no published " + "schema reads its columns from the report's own header the first time it syncs, so it lists " + "no fields until then. Nothing is being held back from you.") + + +#: Table labels for the Amazon disclosure. ⛔ NO EM DASH OR EN DASH: these reach a screen. +LABELS_AMAZON = { + "ut_amzn_orders": "Amazon orders", + "ut_amzn_order_items": "Amazon order items", + "ut_amzn_listings": "Amazon listings", + "ut_amzn_settlements": "Amazon settlements", + "ut_amzn_inventory": "Amazon inventory", + "ut_amzn_pricing": "Amazon pricing and Buy Box", + "ut_amzn_brand_analytics": "Amazon brand analytics", +} + + #: What an EMPTY `excluded` list actually means, said once, on the server that computes it. #: #: ⛔⛔ THE TRAP THIS CLOSES, MEASURED: `_odoo_catalog_by_model` builds `excluded` as a DIFF of the @@ -1286,14 +2609,55 @@ def _detail_introspection(rt, is_admin, row): "creates cannot be listed field by field.") return {"introspectable": False, "reason": reason, "fieldsVisible": False, "canToggleFields": False, "canToggleTables": False, - "catalogueComplete": False, "capturedAt": "", "excludedBasis": ""} + "catalogueComplete": False, "capturedAt": "", "excludedBasis": "", + "censusAt": "", "censusComplete": False, "censusDurationMs": 0, + "censusModelsCounted": 0, "censusModelsWithRows": 0} + if key == "amazon_sp": + # ⭐⭐ W52 · R6. Introspectable under the AMENDED test (a real schema artifact, not a + # `fields_get`), and deliberately with NO generic toggles: there is no Amazon field-toggle + # route, and `canToggleTables` belongs to Odoo's config surface. W55-T19 serves Amazon's + # database switches through the separate `enablement` capability instead, so this flag + # stays False and can never send Amazon through an Odoo endpoint. + # + # ⛔ `catalogueComplete` IS FALSE AND SAYING SO IS THE HONEST ANSWER. Only two of the seven + # tables have a published artifact; the other five measure their columns from the report's + # own header at first sync. Reporting `True` would claim a completeness that does not + # exist, and reporting `introspectable: False` would hide the two that ARE published. + try: + from harness import amazon_schema + cat = amazon_schema.catalogue() + tables = cat.get("tables") or {} + complete = bool(tables) and all((t.get("columns") or []) for t in tables.values()) + captured = str(cat.get("fetched_utc") or "") + except Exception: # noqa: BLE001 + complete, captured = False, "" + return {"introspectable": True, + "reason": "", + "fieldsVisible": bool(is_admin), + "canToggleFields": False, + "canToggleTables": False, + "catalogueComplete": complete, + # ⚠ THE CATALOGUE'S OWN FETCH DATE, NEVER TODAY'S. R6 is explicit that an + # unstamped pinned catalogue is a stale seed waiting to happen, and this screen is + # the only place the staleness becomes visible. + "capturedAt": captured, + "excludedBasis": _AMAZON_EXCLUDED_BASIS, + "censusAt": "", "censusComplete": False, "censusDurationMs": 0, + "censusModelsCounted": 0, "censusModelsWithRows": 0} complete = False captured = "" + census = {} try: import core.user_tables as user_tables snapshot = user_tables.odoo_schema_snapshot(rt) - complete = snapshot.get("complete") is True - captured = str(snapshot.get("capturedAt") or "") + import odoo_relational + discovered = odoo_relational.readable_model_catalog(rt) + census = odoo_relational.population_census(rt) + has_discovery = bool(discovered.get("capturedAt")) + complete = (discovered.get("complete") is True if has_discovery + else snapshot.get("complete") is True) + captured = str((discovered.get("capturedAt") if has_discovery + else snapshot.get("capturedAt")) or "") except Exception: # noqa: BLE001 complete, captured = False, "" return {"introspectable": True, @@ -1307,19 +2671,24 @@ def _detail_introspection(rt, is_admin, row): "canToggleTables": bool(is_admin), "catalogueComplete": complete, "capturedAt": captured, - "excludedBasis": _EXCLUDED_BASIS} + "excludedBasis": _EXCLUDED_BASIS, + "censusAt": str(census.get("capturedAt") or ""), + "censusComplete": census.get("complete") is True, + "censusDurationMs": int(census.get("durationMs") or 0), + "censusModelsCounted": int(census.get("modelsCounted") or 0), + "censusModelsWithRows": int(census.get("modelsWithRows") or 0)} def _detail_overview(row, databases, graph): """C4's `overview` half: what the modal's first tab reads, top to bottom. - `creates` is the table LABELS this connector stands behind: the declared grids when the + `creates` is the ENABLED table LABELS this connector stands behind: the declared grids when the connector declares any, and otherwise the tables its own permitted graph actually found. Both are real readings of THIS workspace; neither is a provider template. """ creates = [str(entry.get("label") or "") for entry in (databases or []) - if str(entry.get("label") or "")] - if not creates and isinstance(graph, dict) and graph.get("status") == "ready": + if entry.get("enabled") is True and str(entry.get("label") or "")] + if not databases and isinstance(graph, dict) and graph.get("status") == "ready": creates = [str(table.get("label") or "") for table in (graph.get("tables") or ()) if str(table.get("label") or "")] return {"key": str(row.get("key") or ""), @@ -1335,6 +2704,24 @@ def _detail_overview(row, databases, graph): "creates": creates} +def _detail_enablement(rt, is_admin, row): + """A provider-owned enablement capability, or ``None`` when this connector has no door. + + Amazon is intentionally the first entry and is named here rather than inferred from whether a + database list happened to be non-empty. Odoo keeps its model-catalogue and config routes; + every other connector keeps no enablement affordance until its provider declares one. + """ + if str(row.get("key") or "") != "amazon_sp": + return None + try: + import connectors_amazon as cx + return cx.enablement_capability(rt, bool(is_admin)) + except Exception: # noqa: BLE001 + # Fail closed. An unreadable provider capability is no write door. + return {"provider": "amazon_sp", "mode": "database", "canManage": False, + "updatePath": "", "preservesData": True, "available": False} + + def connector_detail(rt, user, uname, is_admin, connector_key): """The whole C4 detail DTO for one connector, or `None` when this reader has no such card. @@ -1350,10 +2737,18 @@ def connector_detail(rt, user, uname, is_admin, connector_key): graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal( key, "No actual relationship graph was returned for this connection. A provider template " "is not shown.") - databases = _detail_databases(rt, bool(is_admin), key) + # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a + # question about a PERSON and a table, not about an administrator flag alone. + databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or "")) return {"overview": _detail_overview(row, databases, graph), "keychain": _detail_keychain(rt, uname, bool(is_admin), row), "databases": databases, + "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs", + "retryPath": f"/connectors/{key}/sync/retry"} + if is_admin and key in connector_sync.providers() else None), + # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it + # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints. + "enablement": _detail_enablement(rt, bool(is_admin), row), # ⭐ W49-T14 / R24 — WHETHER THE DATABASES TAB MAY DISCLOSE ANYTHING AT ALL, and what # THIS reader may do there. An addition to C4, declared on `_detail_introspection`: # without it an empty `databases` list is "this connector has no schema" and "this @@ -1365,6 +2760,52 @@ def connector_detail(rt, user, uname, is_admin, connector_key): "relationships": graph} +def _set_amazon_database_enablement(database_key, body, session): + """The testable core of Amazon's tenant-scoped administrator enablement door.""" + if not session.admin: + raise err(403, "not_admin", + "only an administrator can change Amazon database enablement") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + request = body if isinstance(body, dict) else {} + if set(request) != {"enabled"} or type(request.get("enabled")) is not bool: + raise err(400, "bad_enabled", "enabled must be true or false") + + import connectors_amazon as cx # noqa: PLC0415 - provider owns state + key = str(database_key or "").strip() + try: + state = cx.set_database_enabled(session.runtime, key, request["enabled"]) + except KeyError: + # One refusal for blank, malformed and cross-provider keys. The response does not expose + # the rest of the tenant's Amazon roster. + raise err(404, "unknown_amazon_database", + "Amazon does not declare a database with that key") + except cx.EnablementConfigError: + raise err(409, "amazon_enablement_unavailable", + "the stored Amazon enablement settings are invalid. Nothing was changed.") + except ValueError: + raise err(400, "bad_enabled", "enabled must be true or false") + except Exception: # noqa: BLE001 + raise err(503, "enablement_unconfirmed", + "the server could not confirm the Amazon database setting. Reload before " + "trying again.") + return {"provider": cx.ENTRY_TYPE, + "key": key, + "enabled": state["databases"].get(key) is True, + # This route changes only the provider config document. It never deletes a mirror or + # a materialised table, and the wire states that invariant explicitly. + "preservesData": True} + + +# Literal and provider-owned. It appears before the generic connector detail route so future +# dynamic route additions cannot turn Amazon enablement into an Odoo-model dispatch by accident. +@router.patch("/connectors/amazon_sp/databases/{database_key}/enablement") +def amazon_database_enablement(database_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Enable or disable one declared Amazon database for this tenant, without deleting data.""" + return _set_amazon_database_enablement(database_key, body, session) + + # ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order, # and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is # the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot @@ -1385,6 +2826,265 @@ def connectors_detail(connector_key: str, session: Session = Depends(require_ses return detail +class _ReplacementConflict(Exception): + """The addressed keychain row changed between replacement preparation and its store update.""" + + +def _encrypted_replacement_payload(kc, label, entry_type, fields, username): + """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row. + + ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny + memory-only runtime lets this route reuse it while discarding the temporary identity it mints; + the tenant store sees only the later, single-row payload swap. + """ + class _Stage: + def __init__(self): + self.bucket = {} + + def get(self, name): + return self.bucket if name == kc.KEY else {} + + def update(self, name, fn, flush="sync"): + if name != kc.KEY: + raise RuntimeError("the keychain encryptor wrote an unexpected bucket") + updated = fn(self.bucket) + self.bucket = updated if updated is not None else self.bucket + return self.bucket + + stage = _Stage() + temporary = kc.add_entry(stage, label, entry_type, fields, username) + row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {}) + if not row.get("enc"): + raise RuntimeError("the keychain encryptor returned no encrypted payload") + return {"enc": str(row["enc"]), "preview": str(row.get("preview") or "")} + + +def _replace_connector_key(connector_key, entry_id, body, session): + """Replace only one connector entry's declared secrets while preserving its identity. + + The existing row remains untouched through validation, decryption, merge and encryption. The + final store update compares the complete old row before swapping only ``enc`` and ``preview``; + a concurrent rename or credential change therefore conflicts instead of being overwritten. + Scope and owner live in a separate bucket and are never written by this operation. + """ + if not session.admin: + raise err(403, "not_admin", + "only an administrator can replace a workspace connection key") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + + key = str(connector_key or "").strip() + connector = next((row for row in directory(session.runtime, session.uname, True) + if str(row.get("key") or "") == key), None) + entry_type = _entry_type_for(key) + if connector is None or not entry_type: + raise err(404, "unknown_connector", "this workspace has no connector with that key") + if str(connector.get("manage") or "") not in ("keychain", "keychain_personal"): + raise err(403, "replacement_unavailable", + "this connector has no replaceable keychain credential") + + import routes_keychain as rk # noqa: PLC0415 - circular at import + from core import keychain as kc # noqa: PLC0415 - keep boot lean + visible = rk.visible_entries(session.runtime, session.uname, True) + entry = next((row for row in visible + if str(row.get("id") or "") == str(entry_id) + and str(row.get("type") or "") == entry_type), None) + if entry is None: + # One 404 covers missing, cross-tenant, another user's personal entry and a real id bound + # to another connector. The mismatch itself is information this route does not disclose. + raise err(404, "no_entry", "no such key for this connector") + + secret_specs = _replacement_field_specs(entry_type) + secret_keys = [str(field["key"]) for field in secret_specs] + request = body if isinstance(body, dict) else {} + supplied = request.get("fields") + if set(request) != {"fields"} or not isinstance(supplied, dict): + raise err(400, "bad_replacement", + "send only the connector's new secret fields") + if set(supplied) != set(secret_keys): + raise err(400, "bad_replacement", + "send every new secret field and no account or server settings") + replacements = {} + for field in secret_keys: + value = supplied.get(field) + if not isinstance(value, str) or not value.strip(): + raise err(400, "bad_replacement", "a replacement secret cannot be blank") + replacements[field] = value.strip() + + try: + current_fields = kc.read_fields(session.runtime, entry_id) + except kc.KeychainLocked as exc: + raise err(503, "keychain_locked", + f"the keychain is locked: {exc}. Nothing was changed.") + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", + "the current key could not be read. Nothing was changed.") + if not isinstance(current_fields, dict) or not current_fields: + raise err(409, "credential_unreadable", + "the stored credential is incomplete. Nothing was changed.") + + merged_fields = {**current_fields, **replacements} + raw_bucket = session.runtime.get(kc.KEY) or {} + original = ((raw_bucket.get("entries") or {}).get(str(entry_id))) + if not isinstance(original, dict): + raise err(409, "credential_changed", + "the stored credential changed while replacement was prepared. Reload it.") + original = dict(original) + try: + encrypted = _encrypted_replacement_payload( + kc, entry.get("label"), entry_type, merged_fields, session.uname) + except kc.KeychainLocked as exc: + raise err(503, "keychain_locked", + f"the keychain is locked: {exc}. Nothing was changed.") + except ValueError as exc: + raise err(400, "bad_replacement", str(exc)) + except Exception: # noqa: BLE001 + raise err(503, "replacement_not_stored", + "the replacement could not be encrypted. The current key is unchanged.") + + def _swap(cur): + entries = cur.get("entries") if isinstance(cur, dict) else None + stored = entries.get(str(entry_id)) if isinstance(entries, dict) else None + if not isinstance(stored, dict) or stored != original: + raise _ReplacementConflict() + replacement = dict(stored) + replacement["enc"] = encrypted["enc"] + replacement["preview"] = encrypted["preview"] + entries[str(entry_id)] = replacement + return cur + + try: + session.runtime.update(kc.KEY, _swap, flush="sync") + except _ReplacementConflict: + raise err(409, "credential_changed", + "the stored credential changed while replacement was prepared. Reload it.") + except Exception: # noqa: BLE001 + # A remote store can fail after accepting a write. Claiming either outcome would be a lie; + # the safe next act is a fresh detail read, never an automatic retry or delete. + raise err(503, "replacement_unconfirmed", + "the server could not confirm whether the replacement landed. Reload this " + "connector before trying again.") + return {"ok": True} + + +@router.post("/connectors/{connector_key}/keychain/{entry_id}/replace") +def replace_connector_key(connector_key: str, entry_id: str, + body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Admin-only, write-only replacement for one visible connector credential.""" + return _replace_connector_key(connector_key, entry_id, body, session) + + +# ⚠ FIVE SEGMENTS, so it can collide with nothing already declared here: the literal Odoo routes +# are four (`/connectors/odoo/schema/{field_key}`) and `/detail` is three. +@router.patch("/connectors/{connector_key}/databases/{database_key}/name") +def connector_database_name(connector_key: str, database_key: str, + body: dict = Body(default=None), + session: Session = Depends(require_session), + connection_id: str = None): + """W53-T27 / R9 — set or CLEAR this tenant's own name for one of a connector's databases. + + ⛔⛔ IT WRITES THE BUCKET THE NAVIGATION RAIL WRITES, `nav_meta`, IN THE SAME SHAPE AND WITH + THE SAME PER-KEY MERGE. That is the whole ruling: *"the rename database under the Navigation + rail also works, it should then sync to the navigation in the connector too."* A rename made + here is the one the rail shows, and a rename made on the rail is the one this connector's + Databases tab shows, because there is one document and one reader of it + ([[one-question-two-normalizers]]). The key names, the length cap and the merge semantics are + IMPORTED from `routes_nav` rather than copied, so the two doors cannot drift apart. + + ⛔ SO WHY A SECOND DOOR AT ALL? `POST /nav/meta` cannot CLEAR a name: an empty or null `name` + 400s there ("a database needs a name"), so "clearing the override falls back to our shipped + default" has nowhere to happen today. This route is that door. `name: null` and `name: ""` + both clear; any other string sets. The day `/nav/meta` grows the same clearing capability this + becomes a thin alias of it and should be folded in. + + ⛔ AND IT NEVER TOUCHES A TABLE DEFINITION. `user_tables.set_label` would be rewritten by the + next connector re-sync (`odoo_relational.apply_plan` rebuilds definitions), so an override + stored there would vanish on a sync nobody connected to a rename. R9 forbids exactly that, and + the store choice is what makes it impossible rather than merely unlikely. + + THE WALLS, in order, and each one names what it refused: + · the database must be one THIS connector declares (404) - the door is scoped to the card + the reader opened it from, so it cannot be used to rename an unrelated table; + · the key must be a `ut_` key (400 `name_not_allowed`, the same refusal `/nav/meta` gives) - + a built-in module's label is a compiled registry literal; + · the table must EXIST in this workspace (409) - a name for a table nobody can open would + sit in the document naming nothing; + · `may_open` - the creator, an admin, or a share grant, the same predicate `/nav/meta` uses + and the same one `canRename` reports, so the control and the door always agree. + """ + import routes_nav # noqa: PLC0415 - the ONE nav owner + rt = session.runtime + key = str(database_key or "").strip() + if connector_key == "odoo": + key, connection_id, _choice = _odoo_database_target(rt, key, connection_id) + rows = _detail_databases(rt, bool(session.admin), str(connector_key or ""), + uname=session.uname, connection_id=connection_id) + row = next((candidate for candidate in rows + if str(candidate.get("key") or "") == key), None) + if row is None: + raise err(404, "unknown_database", + "this connection does not create a database with that name") + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database this connection created can be renamed. This one's name comes " + "from the module registry") + import core.user_tables as user_tables # noqa: PLC0415 - lazy, as elsewhere + definitions = user_tables.lend_defs(rt) + if not user_tables.get(key, st=definitions): + raise err(409, "database_not_created", + "this connection has not created that database in this workspace yet, so there " + "is nothing to rename") + if not user_tables.may_open(key, session.uname, session.admin, st=definitions): + raise err(403, "forbidden", "that database belongs to another user") + + body = body if isinstance(body, dict) else {} + if "name" not in body: + raise err(400, "bad_request", + "no name was sent. Send a name to set one, or null to go back to the default") + # The same normalisation `/nav/meta` applies: whitespace collapsed, capped at its own limit. + # A blank result CLEARS, which is the half that route refuses and this one exists for. + name = " ".join(str(body.get("name") or "").split())[:routes_nav._MAX_NAV_NAME] + + if not rt.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + # `routes_nav.save_nav_meta._up`, clause for clause: an entry left empty is REMOVED rather + # than stored as `{}`, and an icon set from the rail survives a rename made here. + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + if name: + entry["name"] = name + else: + entry.pop("name", None) + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + rt.update(routes_nav._NAV_META_KEY, _up) # noqa: SLF001 - the nav's own key + except Exception: # noqa: BLE001 + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + + # Read BACK through the same resolver the tab reads, so the answer cannot be the echo of a + # write that landed somewhere else. + default_label = str(row.get("defaultLabel") or "") + override = schema_overlay.override_name(_database_name_overrides(rt).get(key)) + return {"key": key, + "identity": row.get("identity"), + # ⛔ SERVED, NEVER ACCEPTED. `native` is the vendor's identifier; this door reads + # `name` and nothing else off the body (C2: `native` is never editable). + "native": str(row.get("native") or ""), + "defaultLabel": default_label, + "override": override, + "label": schema_overlay.database_label(default_label, override)} + + # ⛔⛔ W49-T16 — THE MOUNT. `dependents_router` DECLARES NO PREFIX OF ITS OWN, so including it in # this router (which carries `prefix="/api/v1"`) yields exactly `/api/v1/dependents`. Declaring # the prefix on both would serve `/api/v1/api/v1/dependents` — the mirror image of the wave-23 diff --git a/api/routes_external.py b/api/routes_external.py new file mode 100644 index 0000000000000000000000000000000000000000..0c98858f01e4ec3fff3d938f1cc8db2434f8e158 --- /dev/null +++ b/api/routes_external.py @@ -0,0 +1,399 @@ +"""Per-user credential management and the bearer-authenticated external read door.""" + +import json +from urllib.parse import quote + +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response + +import deps +from deps import Session, err, require_session +from core import external_access + + +router = APIRouter(prefix="/api/v1/external", tags=["external"]) + +_CREDENTIAL_PREFIX = "lpb1." + + +def _auth_refusal(): + raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.") + + +def require_external_session( + authorization: str | None = Header(default=None, alias="Authorization"), +) -> Session: + """Resolve a bearer credential without consulting or extending the browser session cookie.""" + if not isinstance(authorization, str): + _auth_refusal() + scheme, separator, token = authorization.strip().partition(" ") + if not separator or scheme.lower() != "bearer" or not token.strip(): + _auth_refusal() + try: + verified = external_access.verify(token.strip()) + except external_access.ExternalAuthError: + _auth_refusal() + return Session( + tenant=verified.tenant, + user=verified.user, + claims={ + "t": verified.tenant, + "u": verified.user["username"], + "external": True, + "external_key_id": verified.key_id, + }, + runtime=verified.runtime, + ) + + +def require_external_analyst( + session: Session = Depends(require_external_session), +) -> Session: + """Apply the same Analyst grant wall used by the in-app Assistant and Query.""" + session.require("analyst") + return session + + +def _can_use_analyst(session: Session) -> bool: + try: + session.require("analyst") + except HTTPException as exc: + if exc.status_code == 403: + return False + raise + return True + + +def _management_refusal(exc: external_access.ExternalAccessError): + if exc.code == "external_label_invalid": + raise err(400, exc.code, "Enter a label between 1 and 80 characters.") + if exc.code == "external_key_not_found": + raise err(404, exc.code, "That credential was not found.") + if exc.code in {"external_identity_invalid", "external_account_unavailable"}: + raise err(409, exc.code, "This account cannot create an external credential.") + raise err(503, "external_access_unavailable", "External access is unavailable right now.") + + +def _resolve_external_scope( + session: Session, + databases=None, + *, + include_schema: bool = False, + strict_schema: bool = False, +): + """Resolve the status and field contract used by every external scope description. + + `strict_schema` preserves the schema route's named refusals for unknown, forbidden and + temporarily unavailable sources. The portable document uses the non-strict form so one + unavailable source can still be reported with the reason supplied by source status. + """ + status = deps.assistant_source_status(session, databases=databases) + definitions = None + + def fallback_label(database: str) -> str: + nonlocal definitions + built_in = deps.module_cache_source(database) + if built_in: + return str(built_in.get("label") or database) + if definitions is None: + try: + candidate = session.runtime.get_projection("user_tables", drop=("rows",)) + except Exception: + candidate = {} + definitions = candidate if isinstance(candidate, dict) else {} + definition = definitions.get(database) + if isinstance(definition, dict): + return str(definition.get("label") or database) + return database + + resolved = [] + for database, raw in sorted(status.items()): + row = { + "database": database, + "permitted": bool(raw.get("permitted")), + "answerable": bool(raw.get("answerable")), + "visible": bool(raw.get("visible")), + "reason": raw.get("reason") or "", + } + if include_schema: + if row["answerable"] or strict_schema: + scoped = deps.assistant_read_scope( + session, database, fields=None, filters=None, + ) + row["schema"] = { + key: value for key, value in scoped.items() + if key not in {"records", "filters"} + } + row["label"] = str(scoped.get("label") or database) + row["fields"] = list(scoped.get("fields") or []) + else: + row["label"] = fallback_label(database) + row["fields"] = [] + resolved.append(row) + return resolved + + +def _markdown_inline(value) -> str: + text = str(value or "").replace(chr(0x2014), " - ").replace(chr(0x2013), "-") + return " ".join(text.replace("`", "'").split()) + + +def _portable_skill_document(session: Session, base_url: str) -> str: + scoped = [ + row for row in _resolve_external_scope(session, include_schema=True) + if row["permitted"] + ] + example = next((row for row in scoped if row["answerable"]), None) + if example is None and scoped: + example = scoped[0] + + surface_url = f"{base_url.rstrip('/')}/api/v1/external" + lines = [ + "---", + "name: loopable-external-data", + "description: Read this user's governed Loopable data through the external API.", + "---", + "", + "# Loopable external data", + "", + ("This surface is a read-only view of the Loopable data this user can already see. " + "It applies the same database, field, and row permissions as the Loopable app."), + "", + "## Authentication", + "", + "```http", + "Authorization: Bearer ", + "```", + "", + ("Mint the credential in Loopable Settings. It is shown once, and revoking it stops " + "the next request."), + "", + f"Base URL: `{surface_url}`", + "", + "## First call: list databases", + "", + "Endpoint: `GET /api/v1/external/databases`", + "", + "```sh", + "curl --request GET \\", + " --header 'Authorization: Bearer ' \\", + f" '{surface_url}/databases'", + "```", + "", + "Call this first. It returns only databases this user may read or already sees in the app.", + "", + "## Read a database schema", + "", + "Endpoint: `GET /api/v1/external/databases/{database}/schema`", + "", + ] + + if example is not None: + database_path = quote(str(example["database"]), safe="") + fields = example.get("fields") or [] + field_key = next( + (str(field.get("key")) for field in fields + if isinstance(field, dict) and field.get("key")), + None, + ) + query_body = { + "fields": [field_key] if field_key else [], + "filters": None, + } + lines.extend([ + "```sh", + "curl --request GET \\", + " --header 'Authorization: Bearer ' \\", + f" '{surface_url}/databases/{database_path}/schema'", + "```", + "", + "## Query a database", + "", + "Endpoint: `POST /api/v1/external/databases/{database}/query`", + "", + "```sh", + "curl --request POST \\", + " --header 'Authorization: Bearer ' \\", + " --header 'Content-Type: application/json' \\", + f" --data '{json.dumps(query_body, separators=(',', ':'))}' \\", + f" '{surface_url}/databases/{database_path}/query'", + "```", + "", + ]) + if not example["answerable"]: + lines.extend([ + ("This exact schema and query path names a permitted source, but it is not " + "answerable right now. The reason appears in the scope below."), + "", + ]) + else: + lines.extend([ + "No schema call is available because this user's current scope has no databases.", + "", + "## Query a database", + "", + "Endpoint: `POST /api/v1/external/databases/{database}/query`", + "", + "No query call is available because this user's current scope has no databases.", + "", + ]) + + lines.extend(["## This user's current read scope", ""]) + if not scoped: + lines.extend(["No databases are currently permitted.", ""]) + for row in scoped: + database = _markdown_inline(row["database"]) + label = _markdown_inline(row.get("label") or row["database"]) + lines.append(f"### `{database}`: {label}") + lines.append("") + if row["answerable"]: + fields = [ + field for field in (row.get("fields") or []) + if isinstance(field, dict) and field.get("key") + ] + lines.append("Fields this user may read:") + lines.append("") + if fields: + for field in fields: + key = _markdown_inline(field.get("key")) + field_label = _markdown_inline(field.get("label") or field.get("key")) + lines.append(f"- `{key}`: {field_label}") + else: + lines.append("- No readable fields are currently exposed.") + else: + reason = _markdown_inline(row.get("reason") or "This source is not answerable right now.") + lines.extend([ + f"Not answerable right now: {reason}", + "", + "The readable field contract is unavailable until this source is answerable.", + ]) + lines.append("") + + lines.extend([ + "## Limits", + "", + ("This surface performs no writes. It returns no data the user cannot already see in " + "Loopable, and it provides no export beyond what the user could already export from " + "the app."), + "", + ]) + document = "\n".join(lines) + if _CREDENTIAL_PREFIX in document: + raise RuntimeError("portable skill document contained a credential value") + return document + + +@router.post("/keys", status_code=201) +def create_key(body=Body(default=None), session: Session = Depends(require_session)): + """Mint a credential for the signed-in human. Bearer auth is not accepted here.""" + session.require("analyst") + label = body.get("label") if isinstance(body, dict) else None + try: + key, token = external_access.mint(session.runtime, session.uname, session.tenant, label) + except external_access.ExternalAccessError as exc: + _management_refusal(exc) + return {"key": key, "secret": token} + + +@router.get("/keys") +def list_keys(session: Session = Depends(require_session)): + """List only the signed-in human's metadata. Stored hashes and secrets never leave core.""" + try: + keys = external_access.list_for_user(session.runtime, session.uname, session.tenant) + except external_access.ExternalAccessError as exc: + _management_refusal(exc) + return {"keys": keys, "can_mint": _can_use_analyst(session)} + + +@router.post("/keys/{key_id}/revoke") +def revoke_key(key_id: str, session: Session = Depends(require_session)): + """Tombstone one key owned by the signed-in human. Bearer auth is not accepted here.""" + try: + key = external_access.revoke(session.runtime, session.uname, session.tenant, key_id) + except external_access.ExternalAccessError as exc: + _management_refusal(exc) + return {"key": key} + + +@router.get("/skill.md") +def portable_skill(request: Request, session: Session = Depends(require_session)): + """Download the signed-in human's current governed external read instructions.""" + session.require("analyst") + document = _portable_skill_document(session, str(request.base_url)) + return Response( + content=document, + media_type="text/markdown", + headers={"Content-Disposition": 'attachment; filename="skill.md"'}, + ) + + +@router.get("/whoami") +def whoami(session: Session = Depends(require_external_analyst)): + """The one observable bearer route in this ticket; later tickets add governed data reads.""" + return { + "username": session.uname, + "name": session.user.get("name") or session.uname, + "tenant": session.tenant, + "key_id": session.claims["external_key_id"], + "read_only": True, + } + + +@router.get("/databases") +def list_databases(session: Session = Depends(require_external_analyst)): + """List only sources this user may read or already sees elsewhere in the product.""" + databases = [] + for row in _resolve_external_scope(session): + if not (row.get("permitted") or row.get("visible")): + continue + public = { + "database": row["database"], + "permitted": bool(row.get("permitted")), + "answerable": bool(row.get("answerable")), + } + if not public["answerable"]: + public["reason"] = row.get("reason") or "" + databases.append(public) + return {"databases": databases, "read_only": True} + + +@router.get("/databases/{database}/schema") +def database_schema(database: str, session: Session = Depends(require_external_analyst)): + """Return the field contract after resolving the ordinary strict Assistant read wall.""" + resolved = _resolve_external_scope( + session, databases=[database], include_schema=True, strict_schema=True, + ) + return resolved[0]["schema"] + + +@router.post("/databases/{database}/query") +def query_database( + database: str, + body: dict | None = Body(default=None), + session: Session = Depends(require_external_analyst), +): + """Read one detached Assistant snapshot; FastAPI rejects non-object request bodies.""" + payload = body or {} + return deps.assistant_read_scope( + session, + database, + fields=payload.get("fields"), + filters=payload.get("filters"), + ) + + +# C7 / R3: one import-time chokepoint makes the external bearer surface structurally read only. +# POST remains available for bounded query bodies and for the cookie-only credential lifecycle +# above. No bearer dependency appears on those lifecycle routes. +_READ_ONLY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS"}) + + +def _assert_read_only_router() -> None: + for route in router.routes: + methods = set(getattr(route, "methods", set()) or set()) + refused = methods - _READ_ONLY_METHODS + if refused: + joined = ", ".join(sorted(refused)) + raise RuntimeError(f"external route {route.path} advertises refused method {joined}") + + +_assert_read_only_router() diff --git a/api/routes_forms.py b/api/routes_forms.py index 3d199fa6c174eee324e688ae56d9a17aaa52f205..8971004b3c471f25deb8ce239faade9649ab46d9 100644 --- a/api/routes_forms.py +++ b/api/routes_forms.py @@ -331,7 +331,7 @@ def _clean_values(fields: list, raw: dict): text = "" if v is None else str(v) text = text.strip() if isinstance(v, str) else text if len(text) > MAX_VALUE_LEN: - return None, f"“{label}” is too long — {MAX_VALUE_LEN} characters at most." + return None, f"“{label}” is too long; {MAX_VALUE_LEN} characters at most." if text == "": if f.get("required"): return None, f"“{label}” is required." @@ -508,7 +508,7 @@ def get_form(token: str, request: Request): """Render-time payload for a public form. Rate-limited like the POST: an unauthenticated GET that walks every tenant's workspace is the cheapest way to make this process do work.""" if not _rate_ok(_client_ip(request), time.time()): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") found = _resolve(token) if not found: raise _refuse() @@ -556,7 +556,7 @@ async def submit_form(token: str, request: Request): """ now = time.time() if not _rate_ok(_client_ip(request), now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") body = await _bounded_body(request) found = _resolve(token) if not found: @@ -600,7 +600,7 @@ async def submit_form(token: str, request: Request): # window's job, and a refusal costs the attacker the same either way. if not _daily_ok(token, time.time()): raise err(429, "form_daily_cap", - "this form has reached today's submission limit — try again tomorrow") + "this form has reached today's submission limit; try again tomorrow") import core.user_tables as user_tables @@ -616,7 +616,7 @@ async def submit_form(token: str, request: Request): actor = f"form:{submitter or token[:8]}"[:120] row_id = user_tables.add_row(table_key, clean, username=actor, st=rt) if not row_id: - raise err(409, "not_accepted", "that submission could not be saved — try again later") + raise err(409, "not_accepted", "that submission could not be saved; try again later") # W23-W7 — A's FROZEN signature, called verbatim (CP1-a). Failure here must not turn a # SAVED row into an error for the submitter: the answer is theirs and it is already stored. diff --git a/api/routes_grid.py b/api/routes_grid.py index 759f60c64366b42b96bfa831ea970035197a19d9..85e47ea32129d69c6a470da4714b2349b553ec14 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -2644,7 +2644,76 @@ def grid_hop_topics(session: Session = Depends(require_session)): _PRESET_LINK_MODULES = ("customer_data", "product_data") -def _may_open(session: Session, key: str): +def _register_retired_tables(): + """Push `odoo_relational.RETIRED_KEYS` down into `core.relations`, at import and per request. + + ⛔⛔ W54-T37 / C5 / INSTRUCTION 22 - THIS IS WHY THE OWNER SAW A PERMISSION WALL ABOUT A + DATABASE THAT DOES NOT EXIST. `ut_odoo_customers` and `ut_odoo_products` are RETIRED: popped + from every tenant document on every sync, replaced by the `customer_data` / `product_data` + registry modules, and still DESCRIBED by the connector spec, which is legitimate (a spec + describes a shape). The relation enumeration therefore minted edges at them, `_may_open` below + failed closed on a key it could not resolve, and `relations.forbidden_table_refusal` told the + reader to ask an administrator for access to a table nobody can grant. + + ⛔ ONE DECLARATION, PUSHED, NEVER COPIED. `odoo_relational.RETIRED_KEYS` stays the only place + the two keys are named; `core` cannot read it (`core` never imports up), so the app layer hands + it down exactly as `core.user_tables.register_connected_prefix` is handed its prefixes. Copying + the keys into `core` would make a second declaration of one fact. + + ⚠ GUARDED, AND CALLED TWICE FOR THE SAME REASON `_ensure_odoo_spec` exists: a process that + cannot import `odoo_relational` or `core` must still START, and one that imports them later + must still get the right answer. Registration is idempotent (it is a set union) and returns the + whole registered set, so a caller can assert what it just joined. + """ + try: + import odoo_relational + import core.relations as relations + return relations.register_retired_tables(*odoo_relational.RETIRED_KEYS) + except Exception: # noqa: BLE001 + return frozenset() + + +#: The import-time push. ⚠ Its VALUE is not read anywhere on the request path: it is here so the +#: registration has happened before the first listing, and so an operator can see what landed. +RETIRED_TABLES_AT_IMPORT = _register_retired_tables() + + +def _walled_fields(session: Session, key: str, st=None, memo: dict = None): + """`perm_scope.scoped_fields(key)`, or **None** when the field wall REFUSED — memoised. + + ⭐⭐ W54-T39 — THIS FUNCTION EXISTS BECAUSE THE SAME WALL WAS BEING ASKED TWICE PER TABLE AND + THE ANSWER WAS THROWN AWAY BOTH TIMES. `_may_open` below wanted only the verdict and + `_source_fields` below wanted only the list; each called `scoped_fields` itself, so a listing + that touches six tables paid for TWELVE wall resolutions. MEASURED against tenant #0 on + 2026-09-06: one `scoped_fields` is ~135 ms on a `ut_*` grid and ~550 ms on `product_data`, so + the duplicate pass alone was more than half of the 3.5 s the record rail took to open. + + ⛔ THE VERDICT AND THE LIST ARE ONE ANSWER, AND THEY MUST STAY ONE CALL. Resolving them + separately is not merely slow, it is two readings of one wall that can disagree if a grant + changes between them — the [[one-question-two-normalizers]] shape, on a permission. + ⚠ `None` RATHER THAN `[]` IS THE WHOLE POINT OF THE RETURN TYPE. A table the wall refuses and + a table with no columns are different facts, and `_may_open` is the caller that has to tell + them apart; collapsing both to an empty list would hide every refused target as an empty one. + + ⛔ FAIL CLOSED, exactly as before: any exception at all is a refusal, never a widened answer. + ⚠ `memo` IS ONE REQUEST'S, handed down by `_relation_context`. Passing None simply means no + memo, which is what every caller outside a listing pass wants. + """ + if memo is not None and key in memo: + return memo[key] + import core.perm_scope as perm_scope + + try: + answer = list(perm_scope.scoped_fields( + session.user, key, st=session.runtime if st is None else st) or ()) + except Exception: # noqa: BLE001 + answer = None + if memo is not None: + memo[key] = answer + return answer + + +def _may_open(session: Session, key: str, st=None, memo: dict = None): """May this session open `key` at all? The SAME two gates `_pivot_side` applies, minus the rows. ⛔ THE ROWS ARE THE POINT OF THE DIFFERENCE. `_pivot_side` calls `perm_scope.scoped_table`, @@ -2655,23 +2724,30 @@ def _may_open(session: Session, key: str): ⚠ FAIL CLOSED. Any refusal, any unknown table, any error is False. A target this account may not open must not appear in the strip, because the tab would 403 on the first click and read as a broken product rather than as a permission. - """ - import core.perm_scope as perm_scope + ⛔⛔ AND IT STILL CANNOT TELL "ABSENT" FROM "REFUSED", WHICH IS CORRECT AND IS **NOT** WHAT + INSTRUCTION 22 ASKED TO CHANGE. Conflating "not granted" with "could not be read" is deliberate + and the refusal sentence says both out loud. What was missing is a third case that never + belonged here at all: a table this deployment does not HAVE. That is answered upstream, by + `_register_retired_tables` above keeping such a key from ever being enumerated as an edge + target, and by `relations.diagnose_path` naming it when a request asks for one anyway. + + ⭐ W54-T39 — THE SECOND GATE IS NOW ASKED THROUGH `_walled_fields`, WHICH IS THE SAME QUESTION + AND THE SAME FAIL-CLOSED ANSWER, shared with `_source_fields` so one table costs one wall + resolution instead of two. ⛔ THE GRANT GATE STILL RIDES `session.require`, which resolves + against the session's OWN runtime rather than any handle passed in: a lent snapshot may make a + read cheaper and must never be able to make a grant broader. + """ try: session.require(key) except HTTPException: return False except Exception: # noqa: BLE001 return False - try: - perm_scope.scoped_fields(session.user, key, st=session.runtime) - except Exception: # noqa: BLE001 - return False - return True + return _walled_fields(session, key, st=st, memo=memo) is not None -def _source_fields(session: Session, table: str): +def _source_fields(session: Session, table: str, st=None, memo: dict = None): """The link-bearing field list for `table`, under this session's field wall. ⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS @@ -2682,13 +2758,12 @@ def _source_fields(session: Session, table: str): and look entirely correct doing it. ⚠ APPENDED, NOT MERGED BY KEY: a tenant cannot author a field on a registry module, so the two lists cannot collide, and a union keyed on `key` would hide that if they ever did. - """ - import core.perm_scope as perm_scope - try: - fields = list(perm_scope.scoped_fields(session.user, table, st=session.runtime) or ()) - except Exception: # noqa: BLE001 - fields = [] + ⚠ W54-T39 — A REFUSED WALL STILL READS AS `[]` HERE, unchanged. `_walled_fields` answers None + for that case so `_may_open` can tell it from an empty table; this door has never needed the + distinction and must not start reporting one, because a refusal is `_may_open`'s word. + """ + fields = list(_walled_fields(session, table, st=st, memo=memo) or ()) if table in _PRESET_LINK_MODULES: try: import odoo_relational @@ -2721,21 +2796,284 @@ def related_entries(session: Session, table: str): return [e for e in entries if _may_open(session, e["target"])] -@router.get("/grid/{table}/related") -def grid_related(table: str, session: Session = Depends(require_session)): - """C6 — every database related to `table`, one entry per edge, under the caller's scope. +def _ensure_odoo_spec(): + """Make sure Odoo's connector spec is registered in `core.connector_spec` before it is READ. - `{ok, table, related: [{key, label, via, target, served, where?}]}`. + ⛔⛔ AN EMPTY `edges_for("odoo")` IS AN IMPORT THAT NEVER HAPPENED, NOT A CONNECTOR WITH NO + EDGES, AND THIS FUNCTION EXISTS BECAUSE THAT FAILURE IS SILENT. `platform/core/connector_spec.py` + cannot import `odoo_relational` (`core` never imports up), so Odoo's rows reach that registry by + exactly one route: `odoo_relational` being imported IN A PROCESS WHERE `core` IS ON THE PATH. + MEASURED: a bare `import odoo_relational` from `aios-web/api` leaves `SPEC_REGISTRATION` at + `{'registered': False, 'error': "core.connector_spec is not importable from here: ..."}`, and + `edges_for("odoo")` then answers `[]` with nothing red anywhere. The relation answer would come + back short by every spec edge and look entirely correct doing it. + + ⛔ IT IS DONE HERE, NOT IN `core/relations.py`, for the layering reason above: `core` never + imports up, so the module that READS the edges cannot be the module that registers them. This + file already imports `odoo_relational` inside `_source_fields`, so the dependency is not new. + ⚠ IDEMPOTENT: `connector_spec.register` REPLACES a connector, and `register_connector_spec` + never raises. So calling it per request is cheap and the failure is recorded rather than thrown. + """ + try: + import odoo_relational + # ⚠ BESIDE THE SPEC REGISTRATION AND FOR ITS REASON. Both are pushes from this layer into + # `core`, both are idempotent, and both are wrong to skip in a process that imported this + # module before `platform/` was on the path. Registering the retired keys HERE as well as + # at import is what makes the enumeration below correct in either process shape. + _register_retired_tables() + return bool(odoo_relational.register_connector_spec()) + except Exception: # noqa: BLE001 + return False + + +def _unwalled_fields(defs: dict, key: str): + """`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only. + + ⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of + `relations.diagnose_path` / `wall_refusals`, which chooses a refusal CODE and names a table or + a hop. Nothing built from it reaches `related`, and `verify_odoo_relational` proves that by + comparing the served list with and without the whole diff rather than by trusting this note. + + ⚠ `all_defs` IS ALREADY UNSCOPED (it is the tenant's definitions, projected free of `rows`), + so the raw `fields` on a definition IS the pre-wall list; `_source_fields`' narrowing happens + later, in `perm_scope.scoped_fields`. A registry module has no `all_defs` row at all, which is + why the pre-set link fields are appended here exactly as `_source_fields` appends them. + """ + # ⚠ `isinstance` RATHER THAN `or {}`, the guard `relations.reverse_links` already carries on the + # same input: a definition that is not a dict makes `.get` an AttributeError, and this function + # runs inside the LISTING, so the crash would land on a door that is otherwise answering fine. + defn = defs.get(key) + fields = list((defn.get("fields") or ()) if isinstance(defn, dict) else ()) + if key in _PRESET_LINK_MODULES: + try: + import odoo_relational + fields = fields + list(odoo_relational.module_link_fields(key) or ()) + except Exception: # noqa: BLE001 + pass + return fields + + +def _relation_context(session: Session, table: str, memo: dict = None): + """`defs` plus the three callbacks `relations` needs, memoised for the life of ONE request. + + ⚠ BOTH WALLS ARE MEMOISED. `_may_open` runs `session.require` plus `perm_scope.scoped_fields` + and `_source_fields` runs the second of those again; a depth-2 enumeration asks about the same + dozen tables repeatedly, so without the memo one listing pays for the same permission answer + many times over. W53-T18 added a SECOND enumeration over the same tables, which is exactly why + the memo now lives out here instead of inside the builder: the diff pass costs no extra + permission call at all. + + ⛔⛔ `open_fields_for` IS A **UNION**, NOT A REPLACEMENT, AND THAT IS LOAD-BEARING. The diff in + `relations.diagnose_path` calls a path that is present unfiltered and absent filtered a WALL + removal; if the unfiltered field list could ever be NARROWER than the walled one, a path the + wall really did remove would be absent from both and get reported as an edge that never + existed. Taking the union makes the unfiltered enumeration a superset by construction, so the + diff is monotone and the only thing it can be wrong about is saying "unknown" too rarely. + + ⭐⭐ W54-T39 — **THE DEFINITIONS ARE READ ONCE AND THEN LENT, WHICH IS WHERE THE RECORD RAIL'S + THREE AND A HALF SECONDS WERE.** `perm_scope.scoped_fields` reaches `user_tables.all_defs` + for every table it is asked about, and `all_defs` is a store projection: one Neon round trip, + MEASURED at ~135 ms each on 2026-09-06. A depth-2 listing off `customer_data` asked about six + tables and paid that round trip TWELVE times for the identical document this function had + already read on its first line. `user_tables.lend` exists for exactly this — `nav_entries` has + paired `all_defs` with it since W32-T02 — so the handle below serves that one snapshot to + every wall resolution in the pass and each `ut_*` table's `scoped_fields` drops from ~135 ms + to 0. + ⛔ IT IS A READ HANDLE AND IT NEVER REACHES A WRITE. `lend_defs`' own note is the rule: every + value it serves is a `_Projected`, so a caller that wanted rows gets a loud `KeyError` rather + than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction. + Nothing here is handed to a mutation; `grid_related` is a GET. + ⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves + only `user_tables` and (lazily, once) `object_shares`, every other store key falls through to + the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all + ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list + EQUAL, key for key and value for value, to the one it returned under the bare runtime. + + ⚠ AND THE SNAPSHOT IS THE POINT, not a side effect. One listing answers about one set of + definitions; re-reading them mid-pass could enumerate an edge from a table that vanished two + hops ago and report the inconsistency as a refusal. + """ + import core.user_tables as user_tables + + try: + defs = dict(user_tables.all_defs(st=session.runtime) or {}) + except Exception: # noqa: BLE001 + defs = {} + try: + lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) + except Exception: # noqa: BLE001 + # A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback + # is: the pass simply goes back to paying a round trip per wall resolution. + lent = session.runtime + + # ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo` + # below still memoise their own composition (the grant gate, the pre-set link append); this is + # the `scoped_fields` answer underneath both of them, which is the expensive part. + # ⚠ THE CALLER MAY HAND ITS OWN IN, and `grid_related` does. That route has to answer 403 for + # the SOURCE table before it lists anything, so it resolves that one wall a moment before this + # function is built; without the shared bag the source table would be resolved twice, which on + # `product_data` is a second ~550 ms for an answer already in hand. + wall_memo = memo if memo is not None else {} + open_memo, field_memo, wide_memo = {}, {}, {} + + def may_open(key): + if key not in open_memo: + open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo) + return open_memo[key] + + def fields_for(key): + if key not in field_memo: + field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo) + return field_memo[key] + + def open_fields_for(key): + if key not in wide_memo: + walled = list(fields_for(key) or ()) + seen = {str(f.get("key")) for f in walled if isinstance(f, dict)} + extra = [f for f in _unwalled_fields(defs, key) + if isinstance(f, dict) and str(f.get("key")) not in seen] + wide_memo[key] = walled + extra + return wide_memo[key] + + return {"defs": defs, "may_open": may_open, "fields_for": fields_for, + "open_fields_for": open_fields_for} + + +def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None): + """C3's `/related` list for `table` at `depth`, narrowed to what this session may open. + + ⭐ THE WALL IS APPLIED TO EVERY TABLE ON A PATH, INTERMEDIATE INCLUDED, and it is applied by + `relations.related_paths` calling back into `_may_open` rather than by filtering the finished + list: a two-hop path through a table this reader cannot open must never be enumerated, because + the enumeration itself tells them what that table is related to. + + ⛔ THIS IS THE **SERVED** ANSWER AND W53-T18 DID NOT TOUCH IT. The diff pass that names the + refusals is `related_refusals` below; it reads this list and never feeds it. + """ + import core.relations as relations + + _ensure_odoo_spec() + table = str(table or "") + ctx = ctx or _relation_context(session, table) + return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"], + depth=depth, may_open=ctx["may_open"], + fields_for=ctx["fields_for"]) + + +def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None): + """W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`. + + `[{code, message, path, table?, hop?}]`, where `code` is one of `relations.REFUSAL_CODES`. + + ⛔⛔ *"AN EMPTY LIST IS NEVER AN ACCEPTABLE ANSWER TO A PERMISSION-FILTERED TRAVERSAL."* An + HTTP error covers the request that PINNED one path; the LISTING answers 200 with a shorter + array, and a shorter array is exactly the silent empty PostgREST hands out under RLS and Odoo + hands out for a forbidden One2many. So a path the wall removed is REPORTED here rather than + omitted, and a table with nothing on the other side says `no_related_rows` out loud. + ⛔ AND A 403 FOR THE WHOLE STRIP WOULD BE EQUALLY WRONG: the other tabs are legitimately + visible, and refusing them all in order to explain one is a wall with a bug in it. + + ⛔ IT CANNOT WIDEN `related`. The unfiltered enumeration lives entirely inside + `relations.wall_refusals`, which is handed the SERVED list and only ever reports paths that are + NOT in it. Nothing here is added to, or removed from, what the caller is served. + """ + import core.relations as relations + + table = str(table or "") + ctx = ctx or _relation_context(session, table) + return relations.wall_refusals( + table, fields=ctx["fields_for"](table), defs=ctx["defs"], depth=depth, + may_open=ctx["may_open"], fields_for=ctx["fields_for"], entries=entries, + open_fields=ctx["open_fields_for"](table), open_fields_for=ctx["open_fields_for"]) + + +#: ⭐ THE HTTP STATUS EACH REFUSAL CODE EARNS ON A **PIN**. The two permission answers are 403, the +#: same status the top-of-route `_may_open` gate on `table` itself already uses; a path that never +#: existed stays the 400 this route already answered with, unchanged; and "every hop was open and +#: there is nothing there" is a 404, because it is neither a bad request nor a refusal. +#: ⚠ `no_related_rows` cannot be reached from THIS route's pin branch (a path that resolves is +#: served, so the diff never reports it) and is mapped anyway, because `diagnose_path` is the one +#: diagnosis and the row door will raise the same bag from another ticket's fence. +#: ⭐⭐ W54-T37 / C5 - `retired_target` IS A **404**, AND IT IS DELIBERATELY NOT A 403. The whole +#: complaint in instruction 22 is a permission status on a table that does not exist, so any 4xx +#: in the permission family would repeat the lie in a header. ⚠ 410 Gone is the more precise word +#: for a database that was retired rather than never present, and it is NOT used: the client half +#: of this contract is another lane's, every existing handler here branches on the CODE rather than +#: the status, and introducing the one status nothing in this app has ever returned would risk a +#: fetch wrapper treating it as a transport failure. The code carries the meaning; the status only +#: has to be honest. +_REFUSAL_STATUS = {"forbidden_table": 403, "hop_filtered": 403, "retired_target": 404, + "no_related_rows": 404, "unknown_via": 400} + + +def _refusal_error(diag: dict): + """`err(...)` for a `diagnose_path` bag, carrying its named table, hop and candidates. + + ⛔ THE X2 SHAPE IS UNCHANGED: `{"error": {"code", "message"}}`, with the extra keys added as + SIBLINGS inside `error`. C3 asks for *"the ENUMERATED candidates with their join columns and + the exact retry strings, not prose"*, and a client cannot draw a retry button out of an English + clause, so the structure rides beside the sentence rather than inside it. + + ⭐⭐ W54-T40 — `detail` IS COPIED TOO, AND IT IS THE ROW W54-T38 BOOKED. C5 gives every refusal + a SHORT `message` and a long `detail`, and the client renders the pair as a `
` with the + sentence in the summary; this function copied four keys and dropped the diagnosis, so a refusal + arriving through THIS door rendered as the sentence alone while the identical refusal arriving + through the listing kept its disclosure. One key, and the two doors stop disagreeing about how + much of one contract they carry. + ⚠ A BAG WITH NO DIAGNOSIS STILL RENDERS AS THE SENTENCE ALONE: the loop copies a key only when + the bag carries one, so nothing gains a caret with nothing behind it. + """ + exc = err(_REFUSAL_STATUS.get(str(diag.get("code")), 400), str(diag.get("code")), + str(diag.get("message") or "")) + for key in ("table", "hop", "path", "candidates", "detail"): + if diag.get(key) not in (None, ""): + exc.detail["error"][key] = diag[key] + return exc + + +@router.get("/grid/{table}/related") +def grid_related(table: str, + depth: int = Query(default=2), + via: list[str] = Query(default=None), + session: Session = Depends(require_session)): + """C6 + C3 — every database reachable from `table` in at most `depth` NAMED hops. + + `{ok, table, depth, related: [{key, label, via, target, served, where?, cardinality, path, + hops}], refusals: [{code, message, path, table?, hop?}]}`. + + ⭐⭐ R12: *"the server enumerates what is reachable within two hops and offers it; the client + pins one by naming each edge. Never a path search."* So `depth` is 1 or 2 and nothing else, the + answer is an ENUMERATION rather than a route this server chose, and `path` is what the client + sends back to pin one. ⛔ `depth`'s default of 2 mirrors `relations.MAX_DEPTH`; + `verify_odoo_relational` asserts the two agree rather than letting two literals drift. + + ⛔ A BAD `depth` IS A NAMED 400, NEVER A 500 AND NEVER A SILENT CLAMP. Clamping 3 to 2 would + answer a different question than the one asked and call it success, which is the whole failure + mode R12's refusal exists to prevent. + + ⭐ `via` IS AN ARRAY OF EDGE KEYS, one per hop, resolved by `relations.pick_path` against THIS + enumeration, so an edge this account cannot be offered is an edge it cannot pin. A SINGLE + `?via=` arrives as a one-element list and is therefore a one-hop path, which is C3's + back-compat leg with no second matching rule behind it. + + ⭐⭐ W53-T18 / R13 — `refusals` IS A SIBLING OF `related`, AND `related` DID NOT CHANGE. R13: + *"a permission-filtered traversal must be distinguishable on the wire from an empty one"*. A + shorter `related` array is exactly the silent empty PostgREST hands out under RLS, so every + path the wall removed is NAMED here instead: `forbidden_table` naming the closed table (which + may be the INTERMEDIATE of a two-hop path), `hop_filtered` naming the hop the field wall took, + and `no_related_rows` said out loud when the answer is genuinely nothing. ⛔ THE LIST IS NOT + THE ANSWER TO A PIN: a `via` that cannot resolve fails the REQUEST with the same codes through + `_refusal_error`, and a pin that resolves reports `refusals: []`. ⭐ ONE ENTRY PER EDGE, NOT PER TARGET. R2: *"two links between the same pair give two tabs subtitled by the field"* — so `key` is the EDGE's identity and `via.key` repeats it, because - `via` is the whole of what the pinned read is handed back and it has to identify an edge on its - own. Two bridges off `customer_data` really do share `{direction: "bridge", on: "partner_id"}`. + `via` is the whole of what a resolver is handed back and it has to identify an edge on its own. + Two bridges off `customer_data` really do share `{direction: "bridge", on: "partner_id"}`. - ⭐ `served` IS PART OF THE ANSWER, AND R6'S SECOND SENTENCE IS WHY. A declared edge whose - target is served from its stored copy cannot be pinned yet (that door takes no predicate), and - the caller is told which ones those are rather than discovering it as an empty tab. Open - invoices is the one such edge today. + ⭐ `served` IS PART OF THE ANSWER, AND R6'S SECOND SENTENCE IS WHY. A declared edge whose target + is served from its stored copy cannot be pinned yet (that door takes no predicate), and the + caller is told which ones those are rather than discovering it as an empty tab. Open invoices is + the one such edge today, and a two-hop entry is `served` only when BOTH its hops are. ⛔ THE CALLER'S OWN ACCESS IS CHECKED FIRST. Listing what `table` is related to tells you what `table` is, so the listing is behind the same gate the grid is. @@ -2743,8 +3081,53 @@ def grid_related(table: str, session: Session = Depends(require_session)): table = str(table or "").strip() if not table: raise err(400, "bad_table", "name the database to list relations for") - if not _may_open(session, table): + # ⚠ W54-T39 — THE GATE IS UNMOVED AND UNCHANGED; only its ANSWER is kept. `wall_memo` rides + # into `_relation_context` below so this table's field wall, which the 403 has just resolved + # at full price, is not resolved a second time three lines later. The order is untouched: the + # caller's own access is still the first thing this route decides. + wall_memo = {} + if not _may_open(session, table, memo=wall_memo): raise err(403, "not_permitted", f"your account may not open '{table}', so it cannot be told what that database " f"is related to. Ask an administrator for access to it") - return {"ok": True, "table": table, "related": related_entries(session, table)} + import core.relations as relations + + try: + depth = int(depth) + except (TypeError, ValueError): + depth = -1 + if depth not in (1, relations.MAX_DEPTH): + raise err(400, "bad_depth", + f"a relation listing is offered at depth 1 (edges of '{table}' itself) or depth " + f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and " + f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}") + + ctx = _relation_context(session, table, memo=wall_memo) + entries = related_path_entries(session, table, depth, ctx) + if via: + wanted = [str(v) for v in via if str(v or "").strip()] + if len(wanted) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(table, wanted)) + hit = relations.pick_path(entries, wanted) + if hit is None: + # ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED + # ANSWERS, CHOSEN BY A DIFF. `relations.diagnose_path` enumerates the reachable set + # TWICE, once with this session's wall and once without it, and compares: a path the + # wall removed answers `forbidden_table` NAMING the closed table or `hop_filtered` + # NAMING the hop, while a path absent from both really is an edge that does not exist + # and keeps the `unknown_via` code this route already used, now carrying the + # enumerated candidates with their join columns and the exact retry strings. + # ⛔ THE UNFILTERED PASS CHOOSES THE CODE AND NAMES THE TABLE. IT SERVES NOTHING: + # `entries` above is the walled enumeration and is what a resolvable `via` returns. + raise _refusal_error(relations.diagnose_path( + table, wanted, fields=ctx["fields_for"](table), defs=ctx["defs"], + may_open=ctx["may_open"], fields_for=ctx["fields_for"], depth=depth, + entries=entries, open_fields=ctx["open_fields_for"](table), + open_fields_for=ctx["open_fields_for"])) + # ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other + # tabs the wall removed are the LISTING's answer, not this one, and repeating them here + # would attach an unasked-for permission report to every successful read. + return {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []} + return {"ok": True, "table": table, "depth": depth, "related": entries, + "refusals": related_refusals(session, table, depth, entries, ctx)} + diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 7c3e86ab5c45ebf435174762bad213bfecfe1ec5..1e186678eee9165932bc0c465255ff1d02aefb15 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -10,6 +10,7 @@ in the payload (`pausedNote`): pausing marks intent and warns; the source cutove the keychain cutover wave R3 staged. """ import os +import time from fastapi import Body, Depends from fastapi import APIRouter @@ -99,8 +100,10 @@ SCOPE_KEY = "keychain_scopes" #: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. #: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *a connection of that type* for +#: the TENANT — the one a caller NAMES, or the tenant's designated default when none is named +#: (W54-T45 / R4; it was "the first entry of that type" before) — that is what spawns +#: `ut_odoo_*` / `ut_meta_*` and what every measure column is #: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they #: would silently become the credential the entire workspace's databases are built from, which is #: a credential elevation wearing a scope picker. @@ -108,7 +111,40 @@ SCOPE_KEY = "keychain_scopes" #: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is #: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door #: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") +#: ⛔⛔ W52-T02 — `amazon_sp` JOINS, CLOSING A PARITY HOLE THIS FILE HAD WITH `core/keychain.py` +#: (whose own `TENANT_WIDE_TYPES` already carried three entries, `amazon_sp` included, "JOINED IN +#: WAVE 52" by its own header comment). Until this line, a non-admin could POST a PERSONAL +#: `amazon_sp` entry through THIS door — `add_key`'s refusal only fires for a type in this tuple — +#: while `core/keychain`'s shared resolver (`_first_creds` then, `_creds_for` since W54-T45) +#: already skipped a personal `amazon_sp` entry when resolving +#: the tenant's Amazon credential. So the write succeeded, the UI told the member they had +#: connected their own Amazon, and the resolver silently served nothing for it: not the elevation +#: (the resolver-side guard held), but a door that stores a credential the product then ignores, +#: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than +#: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change. +TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp") + + +#: ⭐⭐ W53-T29 / OWNER RULING R6 — WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA. +#: +#: R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail both)."* +#: ⛔ IT IS DERIVED FROM `TENANT_WIDE_TYPES`, NEVER A SECOND LIST BESIDE IT. That tuple is already +#: the authored answer to "may this source be personal" — `add_key` and `update_key` REFUSE on it +#: three lines apart — so a hand-written `{"odoo": ["business"], ...}` map here would be the +#: [[one-question-two-normalizers]] shape the day one of the two changes and the other does not. +#: The function exists so the CLIENT can be told the fact rather than re-deriving it from a +#: `tenantWideTypes` list it would have to reason about itself. +def allowed_scopes(entry_type): + """The scopes `entry_type` may legally be stored under, as a subset of `SCOPES`. + + ⚠ ELIGIBILITY, NOT PERMISSION. This is the SOURCE's own rule and it does not know who is + asking: whether THIS reader may create a business-wide connection is `session.admin`, decided + at the door in `add_key`. A card needs both answers and they are different questions, which is + why `routes_connectors._detail_connect` narrows this one by the admin flag rather than + replacing it. + """ + word = str(entry_type or "").strip().lower() + return ("business",) if word in TENANT_WIDE_TYPES else SCOPES def clean_scope(raw, default=DEFAULT_SCOPE): @@ -169,13 +205,30 @@ def visible_entries(rt, uname, is_admin=False): owner = str(e.get("createdBy") or "") if not may_see(scope, owner, uname): continue - row = {**e, "scope": scope, "owner": owner} + row = {**_identified(e), "scope": scope, "owner": owner} if not (is_admin or str(owner) == str(uname)): row["preview"] = "" out.append(row) return out +#: ⭐⭐ W54-T44 / R4 / CONTRACT C4 — THE IDENTITY, IN THE WIRE'S OWN SPELLING. +#: +#: `core/keychain.py` stamps `connection_id` and `slot` on every entry so that two connections of +#: one kind are separately addressable (R4's agency tenant: many Odoos, one workspace). This layer +#: renames the first of them to `connectionId` because that is what every other camelCase field on +#: these payloads does (`createdBy`, `canBusiness`, `tenantWideTypes`), and a payload carrying both +#: spellings would be [[a-constant-two-features-share]] on the wire, where a client picking the +#: wrong one still looks right until the day only one is filled. +#: ⚠ `slot` is one word, so it crosses unchanged. +def _identified(row): + """One keychain row, with its identity in wire spelling. Gains keys, never loses one.""" + out = dict(row) + out["connectionId"] = str(out.pop("connection_id", "") or out.get("id") or "") + out["slot"] = int(out.get("slot") or 1) + return out + + def _write_scope(rt, entry_id, scope, owner): """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the default, so the store holds one spelling of the common case.""" @@ -203,11 +256,140 @@ def _kc(): return keychain -def _resolved_odoo_key(rt): +def _ps(): + """`core.perm_scope`, lazily. Same idiom as `_kc()` and `_rel()`, and for the same reason: the + permission layer pulls the grid stack in behind it, and this router is imported at boot.""" + import core.perm_scope as perm_scope + return perm_scope + + +#: ⛔ THE ONE KIND THAT STANDS BEHIND NO CONNECTOR CARD. `_connector_by_entry_type` names every +#: other member of `keychain.ENTRY_TYPES`; `generic` is a credential a person stored by hand, so it +#: has no card and no label to borrow. Spelled here rather than dropped from the vocabulary, +#: because a grant list narrower than the door it governs is a gap nobody can see: `add_key` accepts +#: every `ENTRY_TYPES` member, so every one of them must be grantable. +CONNECTOR_KIND_LABELS = {"generic": "Other credential"} + + +def connector_grant_kinds(): + """The grant vocabulary: `[{key, label, tenantWide}]`, one row per connectable kind. + + ⭐⭐ W54-T47 / R4 — THE THIRD AXIS'S OWN WORDS. Owner: *"access becomes Database / Module / + Connector."* The first two are registry keys the permission editor already renders; this one is + a CREDENTIAL KIND, so its vocabulary is `keychain.ENTRY_TYPES` and not the registry at all. + + ⛔ DERIVED FROM `ENTRY_TYPES`, NEVER TYPED OUT. That set is what `add_entry` validates against, + so a hand-written list here would be [[one-question-two-normalizers]] the day a connector is + added: a kind the door accepts and the editor cannot grant is a permission nobody can give. + ⚠ `tenantWide` RIDES ALONG so the editor can say what a grant on that kind really means. A + tenant-wide source is never personal, so granting it hands the member the workspace's own + credential slot; that is exactly what R4 asks for and exactly what an admin should read before + ticking it. + """ + owners = _connector_by_entry_type() + out = [] + for etype in sorted(_kc().ENTRY_TYPES): + label = str((owners.get(etype) or {}).get("label") or "").strip() + out.append({"key": etype, + "label": label or CONNECTOR_KIND_LABELS.get(etype, etype), + "tenantWide": etype in TENANT_WIDE_TYPES}) + return out + + +def _grant_subject(session, username): + """The registry key for `username`, or a 404 that cannot distinguish another tenant's account + from a missing one. + + ⛔ THE 404 RULE AND THE TENANT DEFAULT ARE BORROWED, NOT RE-SPELLED. `routes_admin.get_perms` + already answers *"a cross-tenant username answers exactly like a missing one, 404, never 403, + because 'that account exists in another company' is itself a leak"*, and `_tenant_of` is + already the ONE spelling of the pre-wave default. A second copy here is how a tenant wall grows + a gap. Imported lazily, exactly as `_connector_by_entry_type` imports `routes_connectors`. + """ + import routes_admin as ra + uname = str(username or "").strip().lower() + reg = ra._registry() + if uname not in reg or ra._tenant_of(reg[uname]) != ra._tenant_of(session.user): + raise err(404, "no_such_user", f"no account named {uname!r}") + return uname + + +def _grant_admin(session): + """Deciding who may connect a source is an administrator's act, refused at the server. + + ⛔ THE CLIENT HIDES; THIS FORBIDS. `PermsEditor.tsx` renders the section only for an admin as a + courtesy, and its own header says every control there is one. + """ + if not session.admin: + raise err(403, "not_admin", + "only an administrator can decide who may add connections to this workspace") + + +@router.get("/admin/users/{username}/connector-grants") +def get_connector_grants(username: str, session: Session = Depends(require_session)): + """One account's Connector grants, plus the vocabulary the editor renders them from.""" + _grant_admin(session) + uname = _grant_subject(session, username) + return {"username": uname, + "kinds": connector_grant_kinds(), + "granted": sorted(_ps().connector_grants(uname, st=session.runtime))} + + +@router.put("/admin/users/{username}/connector-grants") +def put_connector_grants(username: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Replace one account's Connector grants wholesale (W54-T47 / R4 / contract C4). + + WHOLESALE, for the reason `put_perms` is: *"'remove this restriction' has to be expressible, + and a merge cannot express a deletion without a second vocabulary for it."* An empty list is + therefore a legal body and it REVOKES. + + ⛔ AN EMPTIED GRANT CLEARS ITS ROW rather than storing `[]`. An absent row and "no grant" mean + the same thing (`perm_scope.CONNECTOR_GRANT_KEY`'s own note), so there is one spelling of the + common case and revoking returns the store to the shape it had before anybody granted anything. + + ⛔ AN UNKNOWN KIND IS NAMED, NEVER DROPPED (W30/R6's second sentence). Silently discarding it + would tell an administrator they granted something the door will never honour. + """ + _grant_admin(session) + uname = _grant_subject(session, username) + body = body if isinstance(body, dict) else {} + raw = body.get("granted") + if not isinstance(raw, list): + raise err(400, "bad_grants", + "say which connectors this account may add, as a list of connector kinds") + known = {k["key"] for k in connector_grant_kinds()} + want = sorted({str(k).strip().lower() for k in raw if str(k or "").strip()}) + unknown = [k for k in want if k not in known] + if unknown: + raise err(400, "unknown_connector", + f"this workspace has no connector called {', '.join(unknown)}") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if want: + cur[uname] = want + else: + cur.pop(uname, None) + return cur + + try: + session.runtime.update(_ps().CONNECTOR_GRANT_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved, please try again") + return {"username": uname, "kinds": connector_grant_kinds(), "granted": want} + + +def _resolved_odoo_key(rt, connection_id=None): """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the shapes this module needs: the display string (`env` / `keychain:`) and the key the pause flag is stored under. + ⭐ W54-T45 / R4 — `connection_id` NAMES which connection is being asked about; no id keeps + meaning the tenant's designated default, so no route's behaviour moves. The mirroring is the + point: this still delegates to `TenantRuntime.odoo_flag_key`, which now refuses to fall back + to the environment for a named-but-unresolvable connection, and so does this by construction. + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, beside the `odoo_source()` it must agree with. This function had its own copy of the same three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and @@ -215,7 +397,7 @@ def _resolved_odoo_key(rt): against another, freezing nothing while the UI reports success. The two SHAPES stay here because they are this module's presentation concern; the DECISION does not. """ - flag_key = rt.odoo_flag_key() + flag_key = rt.odoo_flag_key(connection_id) if not flag_key: return None, None return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key @@ -514,6 +696,134 @@ def _with_connector(rows): return out +_REGISTERED = {"done": False} +# A stale Remove must not inherit consent forever; ten minutes leaves time to read the warning +# and retry the existing client while binding that consent to the same user and database set. +REVOKE_CONFIRM_SECONDS = 600 +_REVOCATIONS_KEY = "keychain_revocations" + + +def _odoo_table_owner(): + """Odoo's per-connection table resolver, or None when it cannot be imported. + + ⛔ ITS ABSENCE MUST NOT TAKE THE REGISTRY DOWN WITH IT, and that is why this is a function + rather than one more name in the block below's `import`. `odoo_relational` is the heavy end of + this codebase (it pulls the mirror and the connector spec); a failure to import it inside one + `try` around all three families would leave AMAZON unregistered, and an unregistered Amazon + family is a revocation that purges nothing while reporting a clean `registry: True`. That is + the silent-no-op shape instruction 30 exists to remove. + ⚠ Falling back to None is a REAL degradation and a safe one: without the resolver every + `ut_odoo_*` table resolves to the tenant's designated default connection instead of its own, + which can only ever widen the KEEP list a disclosure prints. Odoo purges nothing (R10), so a + coarse answer here cannot delete a row. + """ + try: + import odoo_relational + return odoo_relational.connection_for_table_key + except Exception: # noqa: BLE001 + return None + + +def _register_credential_tables(): + """Declare revocation policy lazily; a failed import remains retryable and reportable. + + routes_connectors imports this module at boot, and odoo_relational is heavy. An import-time + registration would put a cycle on every route before anyone even opens Keychain. + """ + if _REGISTERED["done"]: + return True + try: + from core import user_tables as ut + import routes_connectors as rc + # Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector. + ut.register_credential_tables( + "amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE, + why="The Amazon Data Protection Policy requires deleting the seller data " + "authorised by a revoked credential.") + # W32 R10's second sentence and D-259 require these grids to survive a reconnect. + ut.register_credential_tables( + "odoo", rc.CONNECTED_PREFIXES["odoo"], on_revoke=ut.REVOKE_KEEP, + owner=_odoo_table_owner(), + why="Wave 32 R10 says reconnecting resumes into the same tables. Use " + "POST /admin/connectors/odoo/disconnect to freeze the data " + "instead of deleting it.") + # No Meta ruling exists this wave. KEEP records the reversible direction explicitly. + ut.register_credential_tables( + "meta_ads", rc.CONNECTED_PREFIXES["meta_ads"], on_revoke=ut.REVOKE_KEEP, + why="No ruling has decided Meta Ads data deletion on revocation, so the reversible " + "direction is taken and its databases are kept.") + except Exception as exc: + _REGISTERED["error"] = type(exc).__name__ + return False + _REGISTERED["done"] = True + _REGISTERED.pop("error", None) + return True + + +def _credential_revocation(row, rt): + """A failed census is UNKNOWN, never evidence of an empty seller footprint.""" + registered = _register_credential_tables() + etype, cid = str(row.get("type") or ""), str(row["id"]) + if registered: + try: + from core import user_tables as ut + return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt) + except Exception as exc: + why = f"The credential footprint could not be read ({type(exc).__name__})." + else: + failure = _REGISTERED.get("error", "unavailable") + why = f"The credential table registry could not be built ({failure})." + return registered, { + "type": etype, "connectionId": cid, "action": "none", "purge": [], "keep": [], + "rows": 0, "rowsKnown": False, + "why": why + " Data cleanup is not known. " + "Restore the store or registry and retry cleanup.", + } + + +def _revocation_residue(entry_type): + """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear. + + Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror + deletion stays operator-only because it is permanent, exact-tenant work outside the product + store transaction. + """ + if entry_type != "amazon_sp": + return [] + return [{ + "where": "The tenant DuckDB mirror", + "why": "Connection removal clears product databases but not the raw Amazon mirror.", + "fix": "After every Amazon credential is revoked, an operator must run " + "platform/harness/amazon_store.py --purge --tenant " + "--confirm-tenant and retain the residue-free receipt.", + }] + + +def _write_revocation_warning(rt, entry_id, warning=None): + """Prune expired consent on every write; failed persistence requires explicit confirmation.""" + now = int(time.time()) + + def _up(cur): + kept = {} + for key, value in (cur or {}).items(): + try: + age = now - int(value.get("at", 0)) + except (AttributeError, TypeError, ValueError): + continue + if 0 <= age < REVOKE_CONFIRM_SECONDS: + kept[key] = value + kept.pop(str(entry_id), None) + if warning is not None: + kept[str(entry_id)] = warning + return kept + + try: + rt.update(_REVOCATIONS_KEY, _up, flush="sync") + return True + except Exception: + return False + + @router.get("/admin/keychain") def list_keychain(session: Session = Depends(require_session)): """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. @@ -527,7 +837,16 @@ def list_keychain(session: Session = Depends(require_session)): #: the vocabulary and the permission, so the client renders a picker it can honour #: rather than offering an option the server will refuse (R4: business is admin-only). "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} + "tenantWideTypes": list(TENANT_WIDE_TYPES), + # ⭐ W54-T47 / R4 — WHICH KINDS THIS READER MAY CREATE BUSINESS-WIDE WITHOUT BEING AN + # ADMIN. ⛔ ADDITIVE, AND `canBusiness` IS DELIBERATELY UNMOVED: that flag has meant + # "this account is an administrator" since wave 32 and three gates read it as such, so + # widening it would make one boolean answer two questions. A grant is per KIND and + # arrives as a list for the same reason the wall is per kind. + # ⚠ COURTESY, NEVER THE WALL. `add_key` consults `perm_scope.may_add_connection` + # regardless of what any client believes about this field. + "connectorGrants": sorted(_ps().connector_grants(session.uname, + st=session.runtime))} @router.post("/admin/keychain", status_code=201) @@ -540,29 +859,55 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ if scope is None: raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") etype = str(body.get("type") or "").strip().lower() + # ⭐⭐ W54-T47 / R4 / CONTRACT C4 — THE CONNECTOR GRANT IS CONSULTED HERE AND ONLY HERE, ON THE + # PATH THAT WAS ALREADY REFUSED. + # + # ⛔⛔ WHICH IS WHAT MAKES IT ABSENCE-PRESERVING, and that property is the whole design. A + # member with NO grant meets byte-identical behaviour to the day before this shipped: 403 + # `not_admin` here, 400 `scope_not_available` below, and their own personal connection of an + # ordinary type still succeeds. Nothing was storable before this ticket, so every account in + # every tenant carries no grant — reading that absence as DENY anywhere else on this route + # would revoke personal connections for the entire product with nobody having decided + # anything (`core/perm_scope.may_metrics` records the identical failure one field over). + # + # ⭐ SO THE GRANT ONLY EVER WIDENS: an explicit per-kind grant is what lets a named non-admin + # create a business-wide connection OF THAT KIND, which is R4's agency case said as a rule + # (*"many Gmail connections, or even many Odoo connections for one tenant, depending on the + # admin's configuration per user"*). + # + # ⚠ `etype` MOVED ABOVE THIS TEST rather than staying below it, because the grant is per KIND + # and a check that cannot see the kind can only ever be a second `session.admin`. + if scope == "business" and not session.admin and not _ps().may_add_connection( + session.user, etype, st=session.runtime): + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so it takes " + "an administrator, or an account an administrator has granted this connector " + "under Settings. " + + ("Ask an administrator to add it for everyone." + if etype in TENANT_WIDE_TYPES + else "You can add it as a personal connection instead.")) if scope == "personal" and etype in TENANT_WIDE_TYPES: # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this # credential is what the WHOLE workspace's databases are built from, so "personal" would # be a label on a tenant-wide key rather than a boundary around it. + # ⛔ AND THE GRANT DOES NOT REACH THIS REFUSAL. `allowed_scopes` is the SOURCE's own rule + # and knows nothing about who is asking; a grant says a person may connect this kind for + # the workspace, never that a workspace-wide credential can become one person's. raise err(400, "scope_not_available", f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " + f"through, so it is always business-wide and cannot be a personal connection. " f"An administrator can add it for everyone.") try: row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), body.get("fields"), session.uname) except kc.KeychainLocked as e: raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") + f"the keychain is locked; {e}. A secret is never stored unencrypted.") except ValueError as e: raise err(400, "bad_entry", str(e)) except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") + raise err(503, "store_unavailable", "the entry was not saved; try again") # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode # of a side bucket is to publish a credential, not to hide one. So the second write is not # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was @@ -584,49 +929,179 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_ except Exception: # noqa: BLE001 pass raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " + "the key was not saved; its sharing setting could not be stored, so " "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} + return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}} @router.put("/admin/keychain/{entry_id}") def update_key(entry_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + """Contract C1's scope door, and W54-T44 / C4's RENAME door. Both are settings ABOUT a + connection rather than the credential inside it: a stored secret is never re-openable, so + "change the key itself" still means "replace it", which is `DELETE` + `POST`. + + ⭐ R4 asks for the rename by name — *"Odoo 1, Odoo 2, and obviously user can rename this"* — + and this is where it belongs: the walls a rename needs (`_own_row`'s 404 for an entry you may + not see, `_may_touch`'s ownership rule) are already standing here, and a second door would be + a second copy of both. + ⚠ `{"label": ...}` and `{"scope": ...}` are independent; either alone is a valid request and + both together are one change. A body carrying NEITHER is a 400 rather than a silent no-op, + because "nothing happened" and "your change was saved" must never look the same. + """ row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") + body = body or {} + wants_scope, wants_label = "scope" in body, "label" in body + if not wants_scope and not wants_label: + raise err(400, "nothing_to_change", + "say what to change about this connection: a new name, or who it is shared " + "with.") + scope = None + if wants_scope: + scope = clean_scope(body.get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only " + "an administrator can make one business-wide.") if not _may_touch(session, row): raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + if wants_scope and scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: raise err(400, "scope_not_available", f"a {row.get('type')} connection is what this whole workspace's databases are " f"read through, so it is always business-wide.") + label = row.get("label") or "" + if wants_label: + # ⛔ THE NAME MOVES AND NOTHING ELSE. `rename_entry` touches one field on one row, so the + # connection keeps its `connectionId`, its `slot` and its encrypted credential, and every + # other connection of the same kind is left byte for byte as it was. + try: + renamed = _kc().rename_entry(session.runtime, entry_id, body.get("label")) + except ValueError as e: + raise err(400, "bad_label", str(e)) + except Exception: + raise err(503, "store_unavailable", "the new name was not saved, please try again") + if renamed is None: + raise err(404, "no_entry", "no such key") + label = renamed.get("label") or label owner = row.get("owner") or session.uname + if not wants_scope: + return {"entry": {**row, "label": label}} try: _write_scope(session.runtime, entry_id, scope, owner) except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + raise err(503, "store_unavailable", "the change was not saved; try again") + return {"entry": {**row, "label": label, "scope": scope, + "owner": owner if scope == "personal" else ""}} + + +@router.get("/admin/keychain/{entry_id}/revocation") +def key_revocation(entry_id: str, session: Session = Depends(require_session)): + """Disclose what revocation removes, behind the delete's own visibility and ownership walls. + + Like table_footprint, only someone who could delete may case the joint. An unavailable + census carries rowsKnown=False, because an empty list alone would fabricate reassurance. + """ + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + registered, footprint = _credential_revocation(row, session.runtime) + residue = (_revocation_residue(row["type"]) + if footprint["purge"] or not footprint["rowsKnown"] else []) + return {"entry": {key: row.get(key, "") for key in ("id", "label", "type")}, + **footprint, "registry": registered, "residue": residue} @router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): +def delete_key(entry_id: str, session: Session = Depends(require_session), confirm: str = ""): + kc, rt = _kc(), session.runtime + entries = kc.list_entries(rt) + # Instruction 30 needs idempotent revocation. An absent id's 200 is distinguishable from + # another person's personal entry's 404, but ids have 12 random hex characters and are not + # enumerable. A 404 on a lost response's retry would break this policy path's idempotence. + if not any(entry["id"] == str(entry_id) for entry in entries): + return {"ok": True, "alreadyGone": True, + "purged": {"tables": [], "rows": 0, "rowsKnown": True, + "refused": [], "failed": []}} row = _own_row(session, entry_id) if not _may_touch(session, row): raise err(403, "not_yours", "this connection is not yours to delete") + registered, footprint = _credential_revocation(row, rt) + tables = sorted(table["key"] for table in footprint["purge"]) + purges = footprint["action"] == "purge" and bool(tables) + if purges and confirm != "purge": + now = int(time.time()) + try: + warning = (rt.get(_REVOCATIONS_KEY) or {}).get(str(entry_id)) or {} + repeated = (warning.get("user") == session.uname + and 0 <= now - int(warning.get("at", 0)) < REVOKE_CONFIRM_SECONDS + and warning.get("tables") == tables) + except Exception: + repeated = False + if not repeated: + # No lane owns KeychainTab.tsx this wave. Its dialog still says "Data already read + # stays; nothing new will be read." Purging silently behind that sentence would make + # the product lie on screen. Refusing once with the count makes disclosure true for + # the client that exists; a repeat by this user over this set is the second door. + stored = _write_revocation_warning(rt, entry_id, { + "user": session.uname, "at": now, "tables": tables, + }) + # ⚠ THE KIND COMES FROM THE CREDENTIAL, NOT FROM A LITERAL. Amazon is the only + # family that purges today, so "Amazon databases" would have read correctly and + # become a lie in the one change that flips a second family to REVOKE_PURGE. + kind = kc.kind_label(row.get("type") or "") + noun = "database" if len(tables) == 1 else "databases" + count = (f" holding {footprint['rows']:,} rows. " if footprint["rowsKnown"] + else ". The row count is not known. ") + next_step = ("Choose Remove again within 10 minutes to confirm, or pass confirm=purge." + if stored else "The warning could not be saved. " + "Pass confirm=purge to confirm explicitly.") + raise err(409, "confirm_purge", + f"Removing this connection also deletes {len(tables):,} {kind} {noun}" + + count + "That data cannot be recovered. " + next_step) + + class _Ownership: + # Revoking the default changes keychain's resolver immediately. Lend only metadata from + # BEFORE revocation so core can recompute ownership after the credential is gone. Table + # definitions and deletions still use the live tenant runtime; no caller-supplied plan + # and no encrypted credential is carried into the purge. + def get(self, name): + if name == kc.KEY: + return {"entries": {entry["id"]: entry for entry in entries}} + return rt.get(name) + + # Credential FIRST, data second. A crash leaves revoked access and residual tables that can + # be swept again; the reverse leaves live credentials over deleted data we cannot read back. + # The absent-id retry above is only a no-op receipt, not proof that a crashed sweep completed. try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + kc.delete_entry(rt, entry_id) + _write_scope(rt, entry_id, DEFAULT_SCOPE, "") except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} + raise err(503, "store_unavailable", "The delete did not land. Try again.") + if not purges: + _write_revocation_warning(rt, entry_id) + if footprint["rowsKnown"]: + return {"ok": True} + # Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes + # that incomplete cleanup explicit instead of returning an invented "nothing to purge". + return {"ok": True, "registry": registered, "footprint": footprint, + "purged": {"tables": [], "rows": 0, "rowsKnown": False, + "refused": [], "failed": []}, + "residue": _revocation_residue(row["type"])} + from core import user_tables as ut + try: + result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt) + except Exception as exc: + # Access is already revoked. Report every unresolved table without leaking a store path. + result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [], + "failed": [{"key": key, "error": type(exc).__name__} for key in tables]} + _write_revocation_warning(rt, entry_id) + return {"ok": True, + "purged": {"tables": result["purged"], "rows": result["rows"], + "rowsKnown": result["rowsKnown"], "refused": result["refused"], + "failed": result["failed"]}, + "residue": _revocation_residue(row["type"])} @router.post("/admin/keychain/{entry_id}/test") @@ -666,7 +1141,7 @@ def _unsynced_customer_records(session): "shown": min(len(orphans), 50)} except Exception as e: return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} + "note": f"pool unavailable; {type(e).__name__}"} @router.get("/admin/connectors") @@ -678,21 +1153,27 @@ def connectors(session: Session = Depends(require_session)): # every colleague's personal connection on the screen next door to the one that hides them. entries = visible_entries(session.runtime, session.uname, bool(session.admin)) # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # mirrors TenantRuntime.odoo_source() exactly: the resolved keychain odoo connection (while + # unlocked), else env for tenant #0 only, else nothing (fail closed — never another tenant's + # environment). + # ⛔⛔ W54-T45 — THE WALK IS GONE AND THE RESOLUTION IS ASKED FOR. This was the third copy of + # "the first odoo entry of this tenant", written out by hand beside a comment promising it + # mirrors `odoo_source()`. Under R4 a tenant holds many Odoo connections, so a hand-rolled + # first-entry walk here would light the ACTIVE badge on one client's connection while the + # connector layer served another's — the two answers disagreeing on the one screen that + # exists to show which is which. `keychain.resolve_connection_id` is the single resolver, it + # decrypts nothing (this route runs on a locked keychain too), and it applies the same + # personal-entry skip this comment already required. # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would # make the answer depend on who opened the pane. Personal entries are excluded for the same # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + serving_odoo = kc.resolve_connection_id(session.runtime, "odoo") # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the # connectors DIRECTORY answer "does the environment serve this tenant?" with one function # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" + if serving_odoo and kc.unlocked(): + resolved = f"keychain:{serving_odoo}" elif env_odoo_available(session.runtime): resolved = "env" else: @@ -741,13 +1222,13 @@ def connectors(session: Session = Depends(require_session)): # so in as many words). So a paused connector plus a restarted server is exactly the # blank screen this sentence promised could not happen. A warning that over-promises # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "pausedNote": ("Pausing a connector never deletes data; notes, custom fields and " "views stay, and nothing reaches the source while it is paused. " "Anything this server has already read keeps showing: the customer " "workspace serves its pause-time snapshot, the product list serves " "the last copy read since startup, and measure columns keep answering " "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " + "cannot be shown; a scope with no snapshot, or the product list after " "a restart, reports that the source is paused instead of showing " "figures. Resume to start reading live again.")} # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to @@ -779,14 +1260,31 @@ def _rel(): return rel -def _odoo_entry(session): +def _odoo_entry(session, connection_id=None): """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one. + + ⭐⭐ W54-T45 / R4 / C4 — IT ASKS THE KEYCHAIN WHICH CONNECTION SERVES RATHER THAN WALKING FOR + THE FIRST ONE. `connection_id` names it; no id means the tenant's designated default, which is + the lowest-`slot` odoo connection and, for every tenant that has exactly one, the same row this + used to return. The walk this replaced was a second copy of the resolution + `keychain.resolve_connection_id` owns, and a panel resolving one connection while the connector + layer resolves another is precisely how an agency's screen comes to describe the wrong client. + ⛔ A named connection that does not resolve is None, never a sibling and never the environment. + """ kc = _kc() + cid = kc.resolve_connection_id(session.runtime, "odoo", connection_id) + if not cid: + return None scopes = _scope_rows(session.runtime) for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e + if str(e["connection_id"]) != str(cid) and str(e["id"]) != str(cid): + continue + # The personal skip belongs to the resolver above; re-asserted here because THIS module + # owns the scope vocabulary and a divergence between the two buckets must fail closed. + if entry_scope(session.runtime, e["id"], scopes)[0] == "personal": + return None + return e return None @@ -872,7 +1370,7 @@ def odoo_config_put(body: dict = Body(default=None), f"this connector has no grid called {', '.join(unknown)}") clean_grids = {str(k): bool(v) for k, v in grids.items()} if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " + notes.append("every grid is switched off; nothing will be materialised on the next " "sync, and the databases you already have are left untouched") every = body.get("syncEvery") @@ -886,7 +1384,7 @@ def odoo_config_put(body: dict = Body(default=None), # without a word is how a limit becomes invisible. clean_every = rel.DEFAULT_SYNC notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all; the " f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") server_db = body.get("serverDb") @@ -904,7 +1402,7 @@ def odoo_config_put(body: dict = Body(default=None), try: session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") + raise err(503, "store_unavailable", "the change was not saved; try again") if server_db: notes.append(_rewrite_server_db(session, server_db)) @@ -924,11 +1422,11 @@ def odoo_config_put(body: dict = Body(default=None), if clean_every is not None: if not rel.is_royal(session.tenant): notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " + "from one workspace for the whole deployment, so it will not change how " "often YOUR data refreshes until per-workspace scheduling ships") else: notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " + "uses; it changes the refresh rate for every workspace on this " "deployment, not only this one") if clean_every == "manual": notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " @@ -946,10 +1444,14 @@ def _rewrite_server_db(session, server_db): So this is add-then-delete through the module's own doors, with the side rows (pause flag, scope) carried across because they are keyed by ENTRY ID. ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. + delete this tenant has TWO odoo entries. The alternative order can leave the workspace with no + credential at all, which is worse than one stale read. + ⭐ W54-T45 MADE THAT WINDOW DETERMINISTIC RATHER THAN AMBIGUOUS, as a side effect worth + stating. It used to read "`odoo_creds` takes the first by id sort — so a resync landing inside + that window COULD read the old database", i.e. a coin flip on random hex. The resolver now + serves the tenant's designated default, the LOWEST slot, and the replacement entry is minted + high-water: so the OLD connection serves for the whole window and the new one takes over the + instant the old row is deleted. One stale read, not a coin flip. """ kc = _kc() entry = _odoo_entry(session) @@ -959,7 +1461,7 @@ def _rewrite_server_db(session, server_db): try: fields = kc.read_fields(session.runtime, entry["id"]) or {} except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") + raise err(503, "keychain_locked", f"the keychain is locked; {e}") if not fields: raise err(400, "bad_entry", "this credential could not be read back to be changed") if str(fields.get("db") or "") == server_db: @@ -969,7 +1471,7 @@ def _rewrite_server_db(session, server_db): new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) except Exception: raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") + "the server database was not changed; the existing connection is untouched") # carry the side rows across, then retire the old entry try: flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} @@ -982,7 +1484,7 @@ def _rewrite_server_db(session, server_db): _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") except Exception: # noqa: BLE001 return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") + f"not be removed; delete it under Keychains") return f"the connection now points at the {server_db} database" @@ -1047,7 +1549,7 @@ def odoo_disconnect(session: Session = Depends(require_session)): try: session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") + raise err(503, "store_unavailable", "nothing was disconnected; try again") paused_mirror = False if flag_key: @@ -1069,7 +1571,7 @@ def odoo_disconnect(session: Session = Depends(require_session)): removed = entry["id"] except Exception: raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " + "the databases are frozen but the stored credential was not removed; " "delete it under Keychains") # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the @@ -1081,11 +1583,11 @@ def odoo_disconnect(session: Session = Depends(require_session)): "still readable, and nothing is being refreshed.") if not paused_mirror and flag_key: note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "(order lines and GL lines) may keep advancing; pause the Odoo connector under " "Keychains to stop them.") note += (" Reconnecting adds the key back and resumes into the same databases." if entry else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " + "there was no stored key to remove; the databases are frozen and Reconnect " "resumes them into the same tables.") return {"frozen": True, "removedEntry": removed, # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean @@ -1107,7 +1609,7 @@ def odoo_reconnect(session: Session = Depends(require_session)): try: _rel_reconnect(session.runtime) except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") + raise err(503, "store_unavailable", "the change was not saved; try again") # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the # two read-through grids would sit at the disconnect date while the panel said "connected @@ -1130,7 +1632,7 @@ def odoo_reconnect(session: Session = Depends(require_session)): return {"frozen": False, "connected": connected, "resumedMirror": resumed, "note": ("Odoo is connected again and the databases you already had will refresh in " "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " + "The freeze is lifted, but there is no Odoo credential yet; add one under " "Keychains and the databases resume into the same tables.")} @@ -1167,5 +1669,5 @@ def pause_connector(key: str, body: dict = Body(default=None), try: session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") + raise err(503, "store_unavailable", "the change was not saved; try again") return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_nav.py b/api/routes_nav.py index 24a971488ad932f3fc4dd49e40d52b035a6ffc20..13162190d3882c910b5a4e657df93f7f218306d6 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -15,6 +15,7 @@ import time from fastapi import APIRouter, Body, Depends from deps import Session, err, perms, require_session +import application_registry router = APIRouter(prefix="/api/v1") @@ -422,6 +423,111 @@ def _granted_view_ids(session): return set() +#: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL. +#: +#: R4, verbatim: *"under the View rail for each database that comes from the same connector, they +#: need to be put in the same Folder. Odoo 1, Odoo 2, and obviously user can rename this."* The +#: rail already folds the Data level by `sourceGroup` (`dataPath.ts::groupDataDatabases`), so the +#: grouping mechanism is not the missing piece — the NAME was. Every Odoo database used to answer +#: the literal `"Odoo"`, so an agency with two Odoo connections saw its two clients' invoices +#: nested under one folder, which is the exact confusion R4 exists to prevent. +#: +#: ⛔ THE RENAME IS THE CONNECTION'S RENAME AND THERE IS NO SECOND NAMING SCHEME. The folder shows +#: the keychain row's `label`, which `keychain.add_entry` defaults to `Odoo 1` / `Odoo 2` and +#: `keychain.rename_entry` changes (W54-T44). *"Obviously user can rename this"* is therefore +#: already shipped: renaming the connection renames its folder on the next nav read, and it +#: touches no TABLE KEY — those are namespaced on the immutable `connection_id` (W54-T46), never +#: on a label a person may change twice a day. +#: +#: ⛔⛔ AND THE OVERRIDE ONLY FIRES WHEN THERE ARE TWO OR MORE CONNECTIONS TO TELL APART. With +#: exactly one Odoo connection — every tenant that exists today — the folder keeps the generic +#: `"Odoo"` it has always shown, unchanged, and nothing on anybody's rail moves. +#: +#: THE REASON IS R4 ITSELF, not caution. R4 asks for connections to be DISTINGUISHABLE: *"Odoo 1, +#: Odoo 2"*. With one connection there is one folder either way and nothing to distinguish it +#: from, so the label buys no clarity and costs the tenant a rail that suddenly reads differently. +#: Tenant #0's one connection is labelled "Odoo (migrated from this deployment)" by an August +#: migration, so an unconditional override renamed the owner's own Data folder to a truncated +#: migration artefact — a regression this ticket would have shipped in the name of a ruling about +#: agencies. The moment a SECOND connection is added the names appear on both folders together, +#: which is the only moment they mean anything. +#: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one +#: of the reserved words below is still a second connection, and its sibling must still be named. +#: +#: These two reserved words are the rail's OWN vocabulary, not a connection's to take: +#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the +#: "Loopable" group, and the branch below puts user tables in "Custom". A connection someone named +#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside +#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead. +_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"}) + +#: The registry modules that ARE the designated default connection's data rather than a namespaced +#: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link` +#: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's +#: customers"). They carry no namespace token to read backwards, so they are named here. +_DEFAULT_CONNECTION_PAGES = frozenset({"customer_data", "product_data"}) + + +def _odoo_connection_folders(runtime): + """`({connection_id: folder label}, default_connection_id)` for this tenant's Odoo + connections — resolved ONCE per `/nav`, decrypting nothing, and `({}, None)` on any failure. + + ⛔ ONCE PER REQUEST, NOT ONCE PER PAGE, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. + `odoo_relational.connection_label` and `keychain.default_connection_id` each walk + `keychain.list_entries`, which is a store read; calling either inside the page loop would put + N of them back on the one route D-175 spent a wave reducing to a single read. The loop below + asks `connection_for_table_key` WITHOUT a runtime, which is pure string work, and maps its + `DEFAULT_CONNECTION` sentinel through the default resolved here. + ⛔ AND IT FAILS SOFT ON PURPOSE. A locked keychain, a store blip or a missing import leaves the + rail painting the generic "Odoo" folder; a rail that does not paint at all is far worse than a + rail whose folder carries a generic name. + ⛔ A TENANT WITH FEWER THAN TWO ODOO CONNECTIONS GETS `{}` — the same answer as a failure, and + deliberately so: both mean "there is nothing here to tell apart, keep the generic folder". The + count is taken over the CONNECTIONS, before the reserved-word filter, so a second connection + named `Blank` still switches its sibling on. + """ + if runtime is None: + return {}, None + try: + import core.keychain as keychain + import odoo_relational as odoo_rel + connections = 0 + labels = {} + for row in keychain.list_entries(runtime): + if str(row.get("type") or "") != "odoo": + continue + connections += 1 + cid = str(row.get("connection_id") or "") + label = " ".join(str(row.get("label") or "").split()) + if cid and label and label.lower() not in _RESERVED_GROUP_LABELS: + labels[cid] = label + if connections < 2: + return {}, odoo_rel.default_connection_id(runtime) + return labels, odoo_rel.default_connection_id(runtime) + except Exception: # noqa: BLE001 + return {}, None + + +def _odoo_folder_label(key, labels, default_cid): + """The folder name an Odoo page belongs under, or `""` to keep the generic fallback. + + ⚠ `connection_for_table_key` is the ONLY sanctioned way to ask which connection owns a table, + because it is the inverse of the namespace `table_key_for` builds (C4). Parsing the key here + would be the second implementation that inverse exists to prevent. + """ + if not labels: + return "" + try: + import odoo_relational as odoo_rel + cid = (default_cid if key in _DEFAULT_CONNECTION_PAGES + else odoo_rel.connection_for_table_key(key)) + if cid == odoo_rel.DEFAULT_CONNECTION: + cid = default_cid + except Exception: # noqa: BLE001 + return "" + return labels.get(str(cid or "")) or "" + + @router.get("/nav") def nav(session: Session = Depends(require_session)): """`{pages: [{key,label,source?,chrome}], landing}`. @@ -561,12 +667,23 @@ def nav(session: Session = Depends(require_session)): # leak with no symptom until two tenants rename the same registry key. Copy the rows before # merging on the day that invariant changes. meta = _read_nav_meta(session.runtime) + # W54-T48 (R4 / C4): the tenant's Odoo connections, read ONCE for the whole page loop. + _conn_labels, _conn_default = _odoo_connection_folders(session.runtime) for page in pages: source = str(page.get("source") or "") key = str(page.get("key") or "") if key in {"customer_data", "product_data"} or source.lower() == "odoo" \ or key.startswith("ut_odoo_"): + # ⛔ THIS LITERAL IS THE FALLBACK AND IT STAYS. It is what a tenant with ONE Odoo + # connection or none still sees — every tenant that exists today — and what a database + # that resolves to no connection at all keeps. page["sourceGroup"] = "Odoo" + # ⭐ W54-T48 (R4 / C4): once the tenant has TWO connections to tell apart, each + # database is folded under its own connection's name, so two connections give two + # folders. `_odoo_connection_folders` answers `{}` below that threshold. + _folder = _odoo_folder_label(key, _conn_labels, _conn_default) + if _folder: + page["sourceGroup"] = _folder elif page.get("user_table") or source in {"", "Blank", "Custom"}: page.setdefault("sourceGroup", "Custom") # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. @@ -1165,3 +1282,40 @@ def update_schema_map_setup(key: str, body: dict = Body(default=None), return {"map": {"canConfigure": True, "fields": map_payload.get("fields") or [], **active}} + + +@router.get("/nav/application-templates") +def application_templates(session: Session = Depends(require_session)): + """The trusted template contracts the App creation flow may offer.""" + return {"templates": application_registry.template_contracts()} + + +@router.post("/nav/applications", status_code=201) +def create_nav_application(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one tenant Application from the caller's permission-filtered database schema. + + This is a thin nav/schema adapter over the existing Application registry writer. It does not + create a View or code module, and it deliberately calls ``nav_schema`` before persistence so + a field hidden from this caller cannot be smuggled into an Application definition. + """ + payload = body if isinstance(body, dict) else {} + source = str(payload.get("sourceDatabase") or "").strip() + if not source: + raise err(400, "bad_application", "sourceDatabase is required") + schema = nav_schema(source, session) + fields = schema.get("fields") if isinstance(schema, dict) else None + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") + try: + application = application_registry.create( + session.runtime, session.tenant, session.uname, payload, + field_contracts={source: fields if isinstance(fields, list) else []}, + allowed_databases={source}) + except PermissionError as exc: + raise err(403, "forbidden", str(exc)) + except application_registry.ApplicationError as exc: + raise err(400, "bad_application", str(exc)) + except Exception: + raise err(503, "store_unavailable", "the Application was not saved.") + return {"application": application} diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 8e49c41134aafde8a9738e53d63775c2d76fea88..84111558e1184cd5094de13f4ddaa0438e53cb6d 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -43,7 +43,7 @@ def _redirect_uri(request: Request, provider: str) -> str: the user actually browsed plants the session on the wrong hostname and they return logged out. Moving the app to a new hostname means moving this value AND re-registering the resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + Runbook: `AutoWiki/research/loopable-domain-runbook.md`.""" base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") if not base: base = f"{request.url.scheme}://{request.url.netloc}" diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 4321da7e102159354ce35bcfd3153b44bd77b9aa..025d65118a21cdce083f070823f0f2dd54ac028c 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -24,10 +24,10 @@ import hmac import json import time -from fastapi import Body, Depends +from fastapi import Body, Depends, HTTPException from fastapi import APIRouter, Query -from deps import Session, err, require_session +from deps import (Session, err, module_cache_rows, module_cache_source, require_session) router = APIRouter(prefix="/api/v1") @@ -507,11 +507,19 @@ def _resync_state(cur): # reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as # though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]). # -# ⚠ ONE BUCKET IS BOUND HERE (orders). The other seven still serve from the materialised copy and -# say so on the wire (`readThrough: false`), because R6's second sentence — *"if there is lag or -# it can't be done, you need to explicitly tell me why and recommend a fix"* — makes an unconverted -# grid something to REPORT, not something to leave looking converted. Adding one is a spec row plus -# a green oracle check. +# ⭐⭐ W53-T19 / RULING R5 — EVERY ODOO BUCKET IS BOUND HERE NOW, AND THAT IS THE WHOLE TICKET. +# R5: *"Every `ut_odoo_*` table becomes read-through bound. The stored-copy refusal is deleted, +# not hidden."* Before this the registry held four of the ten declared keys +# (orders, order lines, GL lines, accounts) and the other four DECLARED grains — invoices, agents, +# vendors, bills — answered `409 not_read_through` on the rows door, which is what every Linked +# database tab onto them showed a person. They are bound below, off the SAME SQL their readers in +# `odoo_relational` already run, so the differential oracle has something to hold them to. +# +# ⛔ BINDING IS NOT UN-MATERIALISING, and conflating the two would delete rows six link rollups +# still fold. `_unmaterialisable` is the separate, gated decision about who may STOP STORING rows, +# and none of the four newly bound grains becomes eligible by being bound: each is small enough to +# live in the document and each is folded by a link declared on another table. What changes is +# that the WINDOW door can now answer them, which is what a relation pin needs. #: `ut_*` key -> how to read that grid straight out of the mirror. #: @@ -589,9 +597,12 @@ def _odoo_sources(): # ⭐ R6 / W30-T29 — TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops # being a fact about them. `core` never imports up, so the declaration goes this way round. # ⚠ ALL EIGHT, not just the read-through-bound ones — every row in these tables comes from - # Odoo, which is what R6 is about; being served from the stored copy today is our conversion - # state, not a property of the data. `_sources`' own `register_connected` covers only the keys - # a provider RETURNS, so this call is not redundant with it and must not be folded into it. + # Odoo, which is what R6 is about; where its rows are SERVED FROM is our conversion state, not + # a property of the data. `_sources`' own `register_connected` covers only the keys a provider + # RETURNS, so this call is not redundant with it and must not be folded into it. + # ⚠ W53-T19: the two sets now coincide (every `TABLES` key is bound below), and the call still + # stands rather than being deleted as redundant — a grain declared but not yet bound is the + # normal state of the next connector, and this is the line that keeps it uncapped meanwhile. _ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES]) GRID_SOURCES = {} # the LOCAL registry this builder fills and returns _s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)), @@ -734,6 +745,186 @@ def _odoo_sources(): "THEN '1' ELSE '' END", _s), }, } + + # ═════════════════════════════════════════════════════════════════════════════════════════ + # ⭐⭐ W53-T19 / RULING R5 — THE LAST FOUR DECLARED GRAINS: invoices, agents, vendors, bills. + # + # Each one is the SQL its reader in `odoo_relational` already runs, restated as a window. That + # is a second statement of one question and it is gated rather than trusted: `verify_scopes`'s + # `section_unmaterialise` holds every binding against its own `*_fields()` declaration in BOTH + # directions (a `cols` key with no declaration is a silent no-cell, because `rows_from_pool` + # projects strictly; a declaration with no `cols` key is an INACTIVE filter leaf, which + # WIDENS), and `section_read_through` is the differential oracle shape for the row values. + # + # ⛔⛔ NOT ONE `read_*` STATEMENT WAS TOUCHED TO MAKE THESE FIT, AND THAT IS A HARD RULE HERE. + # `verify_odoo_relational.FixtureCursor` DISPATCHES ON THE SELECT-CLAUSE HEAD, because + # `read_bills` reads `account_move` with a different projection and the opposite move types + # from `read_invoices`; aliasing either projection once already made the fixture recognise + # nothing and return a CONFIDENT EMPTY on every box with no mirror, six legs red. The specs + # below are new SQL beside those readers, never a rewrite of them. + # + # ⛔ THE PREDICATES ARE **IMPORTED**, NEVER RETYPED. `_POSTED_DOCS` and `_VENDOR_DOCS` are the + # only statement of which `account.move` documents belong to which grain, and a fifth copy of + # `move_type IN (...)` is how a grid and its reader start describing different populations. + # ⚠ They open with the bare column `state`, and they are left UNQUALIFIED on purpose: the one + # joined spec here (invoices) joins only a derived commission aggregate, which carries no + # `state`, `move_type` or `payment_state` column, so SQL resolution is unambiguous. Prefixing + # `am.` would qualify only the FIRST term of a two-term predicate, which is worse than not + # qualifying at all. + inv_key, ag_key = rel.INVOICES_KEY, rel.AGENTS_KEY + ven_key, bill_key = rel.VENDORS_KEY, rel.BILLS_KEY + _ut().register_connected(inv_key, ag_key, ven_key, bill_key) + + # `checkbox` cells are the overlay's `'1'` / `''` convention (`odoo_relational._in_scope`), so + # a boolean column is coerced HERE rather than being CASEd in SQL: read as a bare column it + # goes through `_degrade`, which answers NULL when the mirror has not backfilled it, and NULL + # then lands as an honestly unticked cell instead of a Binder error. + _flag = (lambda v: "1" if v else "") + # ⛔ BLANK, NOT ZERO, AND `read_invoices` SAYS WHY: an invoice carrying more than one agent has + # NO single agent id, and `int(None or 0)` would render `0` — a plausible, wrong, clickable + # foreign key. The reader writes `""` for exactly that row and so does this. + _i_blank = (lambda v: "" if v in (None, "") else int(v)) + _s_strip = (lambda v: str(v or "").strip()) + + # ── INVOICES ───────────────────────────────────────────────────────────────────────────── + # `read_invoices`' default projection: every POSTED customer document, the channel TAGGED + # rather than dropped. The commission aggregate is the reader's own subquery, verbatim, and it + # is the reason this grain needs `optional`: `account_invoice_line_agent` is an OCA module + # entity a mirror hydrated from an older seed can lack entirely, and the reader degrades to + # blank agent columns rather than refusing. So does this. + _AGENT_FOLD = ( + "(SELECT invoice_id, count(DISTINCT agent_id) AS n_agents, " + " min(agent_id) AS one_agent, " + " string_agg(DISTINCT agent_name, ', ' ORDER BY agent_name) AS agent_names " + " FROM account_invoice_line_agent WHERE invoice_id IS NOT NULL GROUP BY invoice_id) ag") + GRID_SOURCES[inv_key] = { + "from_sql": f"(account_move am LEFT JOIN {_AGENT_FOLD} ON ag.invoice_id = am.id)", + # ⚠ `ag` IS ABSENT FROM THIS MAP ON PURPOSE: it is a derived alias, not a mirror table, so + # `_degrade` must leave its projections alone rather than look for a `PRAGMA table_info` + # that does not exist. `_source_for` refuses a spec whose declared tables are missing, and + # declaring `ag` here would turn a degradable absence into no binding at all. + "tables": {"am": "account_move"}, + "where": f"{rel._POSTED_DOCS} AND partner_id IS NOT NULL", + "id": "am.id", + "needs_excluded": True, + "cols": { + "invoice_no": ("am.name", _s), + "odoo_id": ("am.id", _i), + "customer": ("am.partner_name", _s), + rel.JOIN_KEY: ("am.partner_id", _i), + "invoice_date": ("am.invoice_date", rel._as_date), + "due_date": ("am.invoice_date_due", rel._as_date), + # ⚠ THE SIGNED PAIR, exactly as the reader reads them: Odoo's `_signed` fields already + # carry a refund's direction, so nothing here re-derives a sign. + "residual": ("am.amount_residual_signed", _n), + "amount_untaxed": ("am.amount_untaxed_signed", _n), + "payment_state": ("am.payment_state", _s), + "move_type": ("am.move_type", _s), + "origin_order": ("am.invoice_origin", _s_strip), + "agent": ("ag.agent_names", _s), + rel.AGENT_JOIN_KEY: ("CASE WHEN ag.n_agents = 1 THEN ag.one_agent END", _i_blank), + "wholesale_scope": ( + "CASE WHEN am.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + "optional": ( + {"absent": (("account_invoice_line_agent",),), + "without": {"from_sql": "account_move am", + "cols": {"agent": ("''", _s), + rel.AGENT_JOIN_KEY: ("CAST(NULL AS BIGINT)", _i_blank)}}}, + ), + } + + # ── AGENTS ─────────────────────────────────────────────────────────────────────────────── + # ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES and `agent_fields`' docstring holds + # the measurement: 16 partners carry commission lines, 17 carry `res_partner.agent`, the union + # is 19. Either source alone silently drops real agents, so BOTH legs are here and each one is + # separately optional. The three `optional` entries are the three degraded mirrors, and the + # combined one is written LAST because entries apply in order and the last merge wins. + _COMMISSIONED = ("id IN (SELECT agent_id FROM account_invoice_line_agent " + " WHERE agent_id IS NOT NULL)") + GRID_SOURCES[ag_key] = { + "table": "res_partner", + "where": f"({_COMMISSIONED} OR COALESCE(agent, FALSE))", + "id": "id", + "cols": { + "agent": ("name", _s), + "odoo_id": ("id", _i), + rel.AGENT_JOIN_KEY: ("id", _i), + "flagged": ("agent", _flag), + "commissioned": (f"CASE WHEN {_COMMISSIONED} THEN '1' ELSE '' END", _s), + }, + "optional": ( + {"absent": (("account_invoice_line_agent",),), + "without": {"where": "COALESCE(agent, FALSE)", + "cols": {"commissioned": ("''", _s)}}}, + {"absent": (("res_partner", "agent"),), + "without": {"where": _COMMISSIONED, "cols": {"flagged": ("''", _s)}}}, + # ⛔ NEITHER SOURCE ⇒ NO POPULATION, SPELLED `1=0` RATHER THAN LEFT EMPTY. An empty + # `where` is EVERY partner, which is 3,649 customers wearing the Agents label; the + # reader answers `[]` on the same mirror and so must this. + {"absent": (("account_invoice_line_agent",), ("res_partner", "agent")), + "without": {"where": "1=0", + "cols": {"flagged": ("''", _s), "commissioned": ("''", _s)}}}, + ), + } + + # ── VENDORS ────────────────────────────────────────────────────────────────────────────── + # ⚠ DERIVED FROM THE BILLS, exactly as `read_vendors` is, and the asymmetry with customers is + # deliberate there: a partner with no posted bill has no payable history to show. MEASURED in + # that reader's docstring: 393 vendors, 0 bills with a null partner, nothing dangles. + # ⚠ ALIASED `p` AND EVERY COLUMN QUALIFIED, matching the reader, because `zip` unqualified is + # a bare identifier a SQL dialect is free to read as something else. + GRID_SOURCES[ven_key] = { + "from_sql": "res_partner p", + "tables": {"p": "res_partner"}, + "where": (f"p.id IN (SELECT partner_id FROM account_move WHERE {rel._VENDOR_DOCS} " + " AND partner_id IS NOT NULL)"), + "id": "p.id", + "cols": { + "vendor": ("p.name", _s), + "odoo_id": ("p.id", _i), + rel.VENDOR_JOIN_KEY: ("p.id", _i), + "country": ("p.country_name", _s), + "email": ("p.email", _s), + "phone": ("p.phone", _s), + "mobile": ("p.mobile", _s), + "website": ("p.website", _s), + "vat": ("p.vat", _s), + # ⚠ `vendor_ref` HERE, `ref` IN THE MIRROR, and `vendor_fields` explains the rename: + # the bills grid already spends `ref` on the vendor's own invoice number. + "vendor_ref": ("p.ref", _s), + "street": ("p.street", _s), + "street2": ("p.street2", _s), + "city": ("p.city", _s), + "zip": ("p.zip", _s), + }, + # `account_move` is a SECONDARY dependency here (it defines the population, it projects + # nothing), so its absence degrades to an empty grid rather than to no binding. The reader + # answers `[]` on that mirror for the same reason. + "optional": ({"absent": (("account_move",),), "without": {"where": "1=0"}},), + } + + # ── BILLS ──────────────────────────────────────────────────────────────────────────────── + # ⚠ DOCUMENT GRAIN, and `bill_fields` records why it is the only one that fits: 6,538 bills + # against 154,917 expense GL lines. This is the payable; the line ledger is `ut_odoo_gl_lines` + # above, and they are different tables answering different questions. + GRID_SOURCES[bill_key] = { + "table": "account_move", + "where": f"{rel._VENDOR_DOCS} AND partner_id IS NOT NULL", + "id": "id", + "cols": { + "bill_no": ("name", _s), + "odoo_id": ("id", _i), + "vendor": ("partner_name", _s), + rel.VENDOR_JOIN_KEY: ("partner_id", _i), + "invoice_date": ("invoice_date", rel._as_date), + "due_date": ("invoice_date_due", rel._as_date), + "amount_untaxed": ("amount_untaxed_signed", _n), + "residual": ("amount_residual_signed", _n), + "payment_state": ("payment_state", _s), + "move_type": ("move_type", _s), + }, + } return GRID_SOURCES @@ -918,11 +1109,52 @@ def _degrade(expr, have_by_alias, default_alias=""): def _source_for(cur, table_key): - """The resolved spec for one connected grid, or None when this grid is not read-through yet.""" - spec = _sources().get(str(table_key or "")) + """The resolved spec for one connected grid, or None when this grid is not read-through yet. + + ⭐⭐ W53-T19 — `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists + because R5 asks every Odoo grain to be bound while two of them read a table or a column a + mirror can honestly lack. `_degrade` already answers the missing-COLUMN case for a BARE + projection; it cannot answer a missing TABLE named in the FROM or in the WHERE, and a SELECT + naming an absent table is a DuckDB Binder error reaching a user as a bare 500 (D-107's shape). + + An entry is `{"absent": ((table[, column]), ...), "without": {}}`: when EVERY + requirement it names is absent from the mirror, its `without` is merged over the spec, with + `cols` merged key by key rather than replacing the map. Entries apply IN DECLARATION ORDER, so + a combined case (both of `read_agents`' two sources missing) is written last and wins. + + ⛔ DEGRADE, NEVER REFUSE, which is the posture `odoo_relational.columns` already sets for the + readers: without its commission table the invoices grid serves blank agent cells, not a 409, + and `read_invoices` degrades the same way over the same absence. ⚠ THE **BASE** TABLE IS A + DIFFERENT QUESTION and keeps its existing answer: `tables`/`table` must all resolve, and a + spec whose base table the mirror lacks is no binding at all (None, and the caller refuses). + """ + key = str(table_key or "") + # Registry-module pools are databases, but never mirror bindings. In particular, + # `product_data` must be reached through its scoped product pool after a relation has resolved + # target ids; letting it fall into `_sources()` makes a cache alias look like an absent Odoo + # table and turns Products ordered into the wrong 404. + if module_cache_source(key) is not None: + return None + spec = _sources().get(key) if not spec: return None from harness import datastore + for opt in (spec.get("optional") or ()): + wants = tuple(opt.get("absent") or ()) + if not wants: + continue + gone = True + for need in wants: + need = tuple(need) + have = datastore.columns_of(str(need[0]), cur=cur) + if have and (len(need) < 2 or str(need[1]).lower() in have): + gone = False + break + if not gone: + continue + without = dict(opt.get("without") or {}) + merged = {**spec["cols"], **(without.pop("cols", None) or {})} + spec = {**spec, **without, "cols": merged} # ⚠ `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table # spec declares none and its columns are bare, so it degrades against `table` as before. aliases = dict(spec.get("tables") or {}) @@ -1284,6 +1516,301 @@ def _related_source_value(session, entry, record): return None +def _cache_relation_plan(session, source, via, target): + """The offered module link and its explicit bridge, or a cause-specific refusal. + + A module/cache target is resolved from its module field declaration because the public + relation entry intentionally carries only the edge needed by ordinary one-hop pins. The + `through` metadata is not a second inferred graph: it is the exact link bag that declared + Products ordered. Resolving against `related_entries` first keeps hidden link fields outside + this door just as they are outside the drawer's tab list. + """ + import core.relations as relations + from routes_grid import _may_open, related_entries + + source = str(source or "") + target = str(target or "") + via = str(via or "") + if relations.is_retired_table(target): + raise err(404, "relation_target_retired", + f"'{target}' is retired and is no longer a relation target in this workspace") + if module_cache_source(target) is None: + raise err(404, "relation_target_missing", + f"'{target}' is not a current module/cache relation target in this workspace") + if not _may_open(session, source): + raise err(403, "not_permitted", + f"'{source}' could not be opened for your account, so this relation cannot " + "be followed") + + offered = next((entry for entry in related_entries(session, source) + if str(entry.get("key") or "") == via + and str(entry.get("target") or "") == target + and str((entry.get("via") or {}).get("direction") or "") == "forward"), + None) + if offered is None: + raise err(400, "unknown_relation", + f"'{via}' is not an available relation from '{source}' to '{target}'") + + field = next((item for item in _rel().module_link_fields(source) + if str(item.get("key") or "") == via), None) + link = (field or {}).get("link") if isinstance(field, dict) else None + if not isinstance(link, dict) or str(link.get("table") or "") != target: + raise err(404, "relation_target_missing", + f"the relation '{via}' no longer has a current target in this workspace") + through = str(link.get("through") or "") + through_on = str(link.get("throughOn") or "") + through_to = str(link.get("throughTo") or "") + on = str(link.get("on") or relations.IDENTITY_COL) + if not through or not through_on or not through_to: + raise err(409, "relation_target_unavailable", + f"the relation '{via}' has no complete stored path to '{target}' right now") + if relations.is_retired_table(through): + raise err(404, "relation_target_retired", + f"'{through}' is retired, so the relation to '{target}' is no longer available") + if not _may_open(session, through): + raise err(403, "forbidden_table", + f"the relation to '{target}' crosses '{through}', which your account cannot " + "open") + if not _may_open(session, target): + raise err(403, "forbidden_table", + f"the relation ends at '{target}', which your account cannot open") + return dict(offered, __source__=source), link, through, through_on, through_to, on + + +def _cache_relation_source_value(session, entry, record): + """Read a cache-backed source value only from a row visible to this session.""" + import core.relations as relations + + source = str(entry.get("__source__") or "") + if module_cache_source(source) is None: + return _related_source_value(session, entry, record) + try: + rows = module_cache_rows(session, source) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + raise err(503, "relation_source_unavailable", + f"'{source}' could not be read right now; try this relation again") from exc + row = next((item for item in rows if str(item.get("pid")) == str(record)), None) + if row is None: + raise err(404, "relation_source_missing", + f"record {record} is no longer available in '{source}'") + frm = str((entry.get("via") or {}).get("from") or relations.IDENTITY_COL) + value = row.get("pid") if frm == relations.IDENTITY_COL else row.get(frm) + return value if value not in (None, "") else None + + +def _cache_relation_target_ids(session, source, record, via, target): + """Resolve one explicit source -> mirror bridge -> module-cache path to target row pids.""" + entry, _link, through, through_on, through_to, on = _cache_relation_plan( + session, source, via, target) + source_value = _cache_relation_source_value(session, entry, record) + if source_value in (None, ""): + raise err(400, "unknown_relation", + f"record {record} has no readable source value for relation '{via}'") + + compiled = _compiled_read(through, session) + spec = compiled["spec"] + left = (spec.get("cols") or {}).get(through_on) + right = (spec.get("cols") or {}).get(through_to) + if not left or not right: + raise err(409, "relation_target_unavailable", + f"the stored path through '{through}' is missing a required join column") + try: + source_value = left[1](source_value) + except Exception: # noqa: BLE001 + source_value = str(source_value) + + frm = spec.get("from_sql") or spec.get("table") + where = str(compiled.get("where") or "").strip() + params = list(compiled.get("params") or ()) + predicate = f"({where}) AND " if where else "" + sql = (f"SELECT DISTINCT {right[0]} AS related_value FROM {frm} WHERE " + f"{predicate}{left[0]} = ? AND {right[0]} IS NOT NULL") + values = {str(row[0]) for row in compiled["cur"].execute( + sql, params + [source_value]).fetchall() if row and row[0] not in (None, "")} + + try: + target_rows = module_cache_rows(session, target) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + raise err(503, "relation_target_unavailable", + f"'{target}' could not be read right now; try this relation again") from exc + ids = [] + seen = set() + for row in target_rows: + if str(row.get(on)) not in values: + continue + try: + pid = int(row.get("pid")) + except (TypeError, ValueError): + continue + if pid > 0 and pid not in seen: + seen.add(pid) + ids.append(pid) + return {"ok": True, "sourceKind": "module_cache", "target": str(target), + "ids": ids, "path": [str(source), through, str(target)]} + + +@router.get("/odoo-tables/relation-targets") +def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge=1), + via: str = Query(...), target: str = Query(...), + session: Session = Depends(require_session)): + """Target pids for a declared relation whose destination is a runtime-cache database.""" + return _cache_relation_target_ids(session, source, record, via, target) + + +def _via_path(via): + """The list of EDGE KEYS a `via` names, or None when it is one edge's bag instead. + + ⭐⭐ W54-T40 / R12 — *"the client pins one by naming each edge (`via = ["orders", + "order_lines"]`)"*. THREE SPELLINGS ARRIVE AND ALL THREE ARE THE SAME THING, which is why + exactly one function decides: + + * an ARRAY of keys, which is what `RecordDetail.tsx::relatedPinKey` sends for a two-hop tab + and what `verify_wiring.py` pins that client literal to; + * a bare STRING, C3's back-compat leg, which `relations.pick_path` already reads as a + one-element path; + * the two-hop entry's own `via` BAG, `{direction: 'path', key, path}`, which is what a client + holding the whole entry naturally hands back. Reading `path` out of it costs one key and + removes an entire class of "the shape was almost right" refusal. + + ⛔ A ONE-HOP BAG STAYS A BAG AND ANSWERS None. `relations.pick` is the resolver for that shape + and it matches on `direction` plus `key`; routing it here instead would resolve a one-hop pin + against the depth-aware listing for no gain and change the cost of the six tabs that already + worked. ⚠ AN EMPTY OR PARTLY-BLANK PATH IS **NOT** A PATH: it answers None and falls into the + `bad_relation` guard, which is the honest refusal for `via = []`. + """ + if isinstance(via, str): + return [via] if via.strip() else None + if isinstance(via, (list, tuple)): + keys = [str(v).strip() for v in via] + return keys if keys and all(keys) else None + if isinstance(via, dict) and isinstance(via.get("path"), (list, tuple)): + keys = [str(v).strip() for v in via["path"]] + return keys if keys and all(keys) else None + return None + + +def _related_path_legs(session, source, entry, entries): + """`(first, mid_key, last)` — the two ONE-HOP entries a two-hop composite is made of. + + ⛔⛔ EACH LEG IS RESOLVED FROM **ITS OWN TABLE'S** SCOPED LISTING, AND THAT IS R13 AT PIN TIME. + An edge key is unique per SOURCE table and not across a connector (`relations`' own banner says + so, with the ten-edges/seven-keys measurement behind it), so hop 2 is looked up in the + intermediate's listing and nowhere else. Resolving both against the source's list would be the + ambiguous-key bug the entry key exists to prevent, and it would also skip the intermediate's + permission wall on the way past. + + ⚠ HOP 1 COSTS NOTHING EXTRA: it is in `entries`, the very listing the composite was picked + from, because a depth-2 enumeration returns the one-hop entries first. Only hop 2 needs a + second listing, and it is asked at depth 1. + """ + import core.relations as relations + from routes_grid import related_path_entries + + hops = [str(k) for k in (entry.get("path") or ())] + first = relations.pick_path(entries, hops[:1]) + if first is None: + raise err(400, "unknown_relation", relations.unknown_via_refusal(source, hops[:1])) + mid_key = str(first.get("target") or "") + last = relations.pick_path(related_path_entries(session, mid_key, 1), hops[1:2]) + if last is None: + raise err(400, "unknown_relation", relations.unknown_via_refusal(mid_key, hops[1:2])) + return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key) + + +def _related_hop_values(session, source, record, first, mid_key, last, notes): + """The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows. + + ⭐⭐ W54-T40 — THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE. + Hop 1 turns the source record into a set of intermediate rows; this returns the value each of + those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them. + + ⛔⛔ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR + USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its + administrator row wall and its hidden-field closure all run before a single id is produced, so + a traversal cannot be used to learn about rows of the middle database a reader may not see. A + second, lighter read here would be a second permission wall, and this file already carries the + rule that a second copy of that block does not drift towards refusing. + ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled + registry module with no read-through binding, so the one enumerated path that passes through it + answers a sentence carrying the cause and a way round it, rather than an empty grid. + + ⚠ NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back, + however many there are, and the reader is told through `limits` that this tab resolves a second + database on every page. + """ + import core.relations as relations + from fastapi import HTTPException # noqa: PLC0415 + + value = _related_source_value(session, first, record) + if value in (None, ""): + raise err(400, "unknown_relation", + relations.unresolvable_source_refusal(source, first)) + try: + mid = _compiled_read(mid_key, session) + except HTTPException as exc: + why = "" + if isinstance(getattr(exc, "detail", None), dict): + why = str((exc.detail.get("error") or {}).get("message") or "") + raise err(409, "relation_not_served", + f"this relation reaches '{last.get('target')}' through '{mid_key}', and that " + f"database could not be read here to follow it" + + (f" ({why})" if why else "") + + f". Open '{mid_key}' on its own, find the records this one is linked to, and " + f"follow the next relation from there") + mid_spec = mid["spec"] + on1 = str((first.get("via") or {}).get("on") or "") + step = (mid_spec.get("cols") or {}).get(on1) + if not step: + raise err(409, "relation_not_served", + f"'{mid_key}' has no live expression for '{on1}', which is the column the first " + f"hop of this relation joins on, so the traversal cannot be resolved. Open " + f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection") + # ⭐ THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is + # not a column of the mirror spec, so it resolves to the spec's own identity expression, which + # is the same one this route trusts everywhere else and is per table rather than a bare `id`. + frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL) + take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL + else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0]) + if not take: + raise err(409, "relation_not_served", + f"'{mid_key}' has no live expression for '{frm2}', which is the value the second " + f"hop of this relation is keyed on, so the traversal cannot be resolved. Open " + f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection") + try: + value = step[1](value) + except Exception: # noqa: BLE001 + value = str(value) + mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"] + parts = [p for p in (mid["base_where"], first.get("where")) if p] + parts.append(f"{step[0]} = ?") + pred = " AND ".join(f"({p})" for p in parts) + rows = mid["cur"].execute( + f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}", + list(mid["wall_params"]) + [value]).fetchall() + values = [r[0] for r in rows if r and r[0] is not None] + # ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS** + # SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one + # fixed client phrase and falls through to the SERVER's own first sentence for an effect it has + # not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking + # every row above it"*, which is a true statement about a deep page and a FALSE one about a + # seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the + # cause below, which is what actually happened. + # ⚠ THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full + # stop and truncates past 120 characters, so the mechanism is stated before the count. + notes.append({ + "subject": "related", "effect": "traversal", + "cause": f"these rows are reached through '{mid_key}', so every page resolves that " + f"database first. This record matched {len(values):,} of its records, and this " + f"database is then narrowed to the rows those reach", + "recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or " + f"filter this database beside the relation to ask a smaller question"}) + return values + + def _related_pin(table_key, session, spec, base_where, wall_params, cur, related): """`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned. @@ -1309,6 +1836,30 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`, so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY. + + ⭐⭐ W54-T40 / INSTRUCTION 22 / R12 — **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.** + The owner's second quoted error was this function's own `bad_relation` sentence, and it fired + on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming + each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the + guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and + 18 answered 400 on the first click. + + ⛔⛔ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT. + `relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction + is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So + a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is + then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values + hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered + `400 unknown_relation` and changed nothing the reader could see. + + ⭐ EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The + intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for + an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a + single id crosses into this table's predicate. A second, laxer read of the intermediate would + be exactly the permission wall with a hole in it that `related_entries` exists to prevent. + ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty: + `product_data` is a registry module with no read-through binding, so the one enumerated path + that passes through it refuses by name with a cause and a recommendation. """ import core.relations as relations @@ -1324,13 +1875,18 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related source = str(related.get("table") or "") record = related.get("record") via = related.get("via") - if not source or record in (None, "") or not isinstance(via, dict): + # ⭐⭐ W54-T40 — `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the + # one place that decides which. It answers None for the bag, which is what keeps the guard + # below spelling the SAME refusal for a genuinely incomplete pin. + path = _via_path(via) + if not source or record in (None, "") or (path is None and not isinstance(via, dict)): raise err(400, "bad_relation", "a pinned read needs the record's database, the record id and the relation to " - "follow, as {table, record, via}. Ask GET /api/v1/grid/" + (source or "{table}") + "follow, as {table, record, via}, where `via` is one relation or the array of " + "edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}") + "/related for the relations this account may follow") - from routes_grid import _may_open, related_entries + from routes_grid import _may_open, related_entries, related_path_entries if not _may_open(session, source): # ⚠ THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False @@ -1346,8 +1902,20 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related # ⛔ RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be # offered is an edge you cannot pin" structural rather than remembered. `related_entries` is # the same function `GET /grid/{table}/related` answers with; there is no second resolution. - entries = related_entries(session, source) - entry = relations.pick(entries, via) + # ⭐ W54-T40 — AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE + # LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers + # with, so a traversal this account is not offered is a traversal it cannot pin either. + # ⚠ THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs + # the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the + # difference between roughly half a second and roughly two. + if path is None: + entries = related_entries(session, source) + entry = relations.pick(entries, via) + else: + if len(path) > relations.MAX_DEPTH: + raise err(400, "over_depth", relations.over_depth_refusal(source, path)) + entries = related_path_entries(session, source, len(path)) + entry = relations.pick_path(entries, path) if entry is None: raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via)) entry = dict(entry, __source__=source) @@ -1361,12 +1929,33 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related if not entry.get("served", True): raise err(409, "relation_not_served", relations.unserved_target_refusal(entry)) - on = str((entry.get("via") or {}).get("on") or "") + # ⭐⭐ W54-T40 — THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is + # not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no + # `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates + # onto the pair, because one of them may carry a predicate the other does not (the open-invoice + # bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped + # listing, and the last one is what this block below reads. + notes = [] + first, mid_key, last = entry, "", entry + if len(entry.get("path") or ()) > 1: + first, mid_key, last = _related_path_legs(session, source, entry, entries) + + on = str((last.get("via") or {}).get("on") or "") # ⚠ `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value # this response returns: a relation must keep working when an administrator has hidden the id # column it runs over, and no cell of it is ever projected. The hidden-field closure still # governs everything that reaches the wire, one layer up. - binding = (spec.get("cols") or {}).get(on) + # ⭐⭐ W54-T40 — AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge + # with no declared `on` to `pid`, which is the wire's name for the row identity and is never a + # key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one + # carries it). Every such edge therefore refused with "has no live expression for 'pid'" while + # the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this + # route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and + # deliberately not a bare `id`, which is ambiguous on a join-shaped grain. + if on == relations.IDENTITY_COL: + binding = (spec["id"], int) + else: + binding = (spec.get("cols") or {}).get(on) if not binding: raise err(409, "relation_not_served", f"'{table_key}' has no live expression for '{on}', which is the column this " @@ -1374,14 +1963,23 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related f"this database directly, or ask for '{on}' to be added to its live connection") on_expr, on_cast = binding[0], binding[1] - value = _related_source_value(session, entry, record) - if value in (None, ""): - raise err(400, "unknown_relation", - relations.unresolvable_source_refusal(source, entry)) - try: - value = on_cast(value) - except Exception: # noqa: BLE001 - value = str(value) + if mid_key: + # ⭐ HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See + # `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through + # that database) and compiles to `1=0` two blocks down, never to "no pin". + values = _related_hop_values(session, source, record, first, mid_key, last, notes) + else: + value = _related_source_value(session, entry, record) + if value in (None, ""): + raise err(400, "unknown_relation", + relations.unresolvable_source_refusal(source, entry)) + values = [value] + cast = [] + for one in values: + try: + cast.append(on_cast(one)) + except Exception: # noqa: BLE001 + cast.append(str(one)) # ── the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ──────────────── # ⛔ NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer @@ -1391,22 +1989,27 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related # ⚠ THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can # produce is already narrowed to rows they may see, so the pin cannot be used to learn how many # rows exist beyond their boundary. + # ⭐ W54-T40 — `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept + # BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The + # many-value arm is only ever reached by the second hop of a traversal. frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] - parts = [p for p in (base_where, entry.get("where")) if p] - parts.append(f"{on_expr} = ?") - pred = " AND ".join(f"({p})" for p in parts) - ids = [int(r[0]) for r in cur.execute( - f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}', - list(wall_params) + [value]).fetchall() if r and r[0] is not None] + ids = [] + if cast: + parts = [p for p in (base_where, last.get("where")) if p] + parts.append(f"{on_expr} = ?" if len(cast) == 1 + else f"{on_expr} IN ({','.join('?' for _ in cast)})") + pred = " AND ".join(f"({p})" for p in parts) + ids = [int(r[0]) for r in cur.execute( + f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}', + list(wall_params) + list(cast)).fetchall() if r and r[0] is not None] - notes = [] # ⭐ STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited # here; the reader is told that this particular relation is answered by reading the source # database once per page, because it is keyed on a column rather than on the record id. - if str((entry.get("via") or {}).get("from") or "") != relations.IDENTITY_COL: + if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL: notes.append({ "subject": "related", "effect": "slow", - "cause": f"this relation is keyed on the '{entry['via']['from']}' column of " + "cause": f"this relation is keyed on the '{first['via']['from']}' column of " f"'{source}' rather than on its record id, so every page of these rows is " f"resolved by reading that database first", "recommendation": "read this database on its own and filter it on " @@ -1423,6 +2026,279 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related return sql, params, notes +# ── ⭐⭐ W53-T20 / D-377 — THE VIEW-MEMBERSHIP LEAF, ON A SERVER-WINDOWED GRID ───────────────── +# +# D-377: *"`Where View is [x]` is unavailable on any server-windowed (read-through Odoo) grid"* — +# W37-T26 shipped the leaf for client-windowed grids and the SQL compiler never learned it. T08 +# makes every Odoo grid server-windowed, so the latent gap became a visible one. +# +# ⛔⛔ THE FIX IS A REWRITE, NOT A NEW SQL ARM, AND THAT IS THE WHOLE POINT. +# `harness/filter_sql.py::_compile_cohort` ALREADY compiles "the row identity is in one of these +# hand-supplied pid sets", and it already gets the three hard cases right: an empty set is `1=0`, +# an ABSENT set is `1=0` **for `noneOf` too** (answering "everything" to `is none of [a set we +# cannot see]` is the widening sin wearing a plausible face), and the membership test is wrapped in +# `COALESCE(..., FALSE)` so a NULL row id does not make `NOT (id IN (...))` NULL and drop a row the +# client would keep. A `__view__` leaf asks EXACTLY that question about a different kind of set. So +# each leaf is rewritten into the `__cohort__` shape with the view's resolved pids supplied through +# `cohort_sets=`, and `noneOf` is correct for free because it is the same compiler that already got +# it right. `filter_sql.py` is not this lane's fence and does not need to be. +# +# ⭐ EXACTLY ONE PLACE LEARNS THE LEAF: `_compiled_read`, immediately before the ONE +# `compile_filter_tree` call it makes. `/rows` and `/aggregates` are the same door (C1), so both +# inherit it, and `totalUnfiltered` keeps counting the population with the reader's conditions +# dropped because the rewrite happens to `tree` and never to `base_where`. + +#: The two RESERVED pseudo-columns a filter tree may name. Neither is a column, and `_leaf_cols` +#: cannot tell them apart from a real `colId` — it reads the key, not the vocabulary. Subtracting +#: them is what keeps a membership leaf out of the `filter_ignored` sentence that explains a column +#: with no mirror expression: that sentence would be FALSE here, and it reads as "your condition was +#: dropped", which is the one thing a membership leaf must never be. +_PSEUDO_COLS = frozenset(("__cohort__", "__view__")) + + +def _active_cohort_leaves(nodes): + """Every ACTIVE `__cohort__` leaf a tree names, at any depth. + + "Active" in `filter_sql.is_rule_active`'s own sense for this leaf: driven by the SET it names, + so a leaf with a blank value is half-built and ignored on both engines. Inactive leaves are + excluded deliberately — reporting one would refuse a read the client renders happily. + """ + out = [] + for n in (nodes or []): + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + out.extend(_active_cohort_leaves(n.get("children"))) + elif str(n.get("colId") or "") == "__cohort__" and _fs().parse_cohort_ids(n.get("value")): + out.append(n) + return out + + +def _view_store(session, table_key): + """The workspace document a saved view of THIS database lives in. + + ⛔ `table_store.find_view`, NOT a second resolver. `done-when` is a row-set EQUALITY between + this path and the client-windowed one, so a second implementation is the one thing guaranteed + to fail it: the client reads the views `/workspace` served out of this same document. + """ + import core.table_store as table_store + return table_store.make(f"{table_key}_table_workspace", st=session.runtime) + + +def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits): + """Rewrite every `__view__` leaf into the `__cohort__` shape. `(tree, cohort_sets)`. + + `cohort_sets` is **None** when the tree names no view, and that is load-bearing rather than + tidy: `_compile_cohort` REFUSES when `cohort_sets` is missing, which is how a genuine cohort + leaf keeps answering `400 filter_unsupported` on this door exactly as it does today instead of + quietly compiling to "matches nothing". + + HOW A VIEW IS RESOLVED, mirroring `CustomerGrid.tsx::viewSets` step for step, because the + acceptance is that both paths answer the same rows: + + * the view is found in this database's workspace and the caller may see it, or it is + UNRESOLVABLE; + * a view LOCKED to a cohort is unresolvable here (the lock is resolved from this tenant's + records, not from the connection) and is reported; + * `config.filters` is compiled through the SAME `compile_filter_tree` the reader's own + conditions go through, with `member_ids=config.memberPids` — which reproduces the client's + `curated.has(pid) || matchFilterTree(...)` union exactly; + * no filters and a curated `memberPids` is the client's `else if (curated)` branch: the + membership IS the view; + * no filters and no members means the view holds every row in scope; + * the ids are materialised in ONE `SELECT DISTINCT` under `base_where`, uncapped, the same + way `_related_pin` does it, so the wall rides along and standing rule 1 is kept. + + THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY: + resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets` + holds every row -> `is` becomes `noneOf` over an EMPTY set (`NOT (1=0)`, true of every row) + and `is not` becomes a key that is absent (`1=0`) + unresolvable -> a key that is ABSENT, which `_compile_cohort` answers `1=0` for BOTH ops. + That is `evalNode`'s `if (!set) return false;` byte for byte, and it is + D-377's negative control: `is not [a view we cannot resolve]` must match + NOTHING, never everything. + """ + import aios_grid + + if not aios_grid.view_refs_of(tree): + return tree, None + + # ⛔ A GENUINE COHORT LEAF STILL REFUSES. Supplying `cohort_sets` for the views would otherwise + # make a real `__cohort__` leaf beside them compile to `1=0` in silence, where today it answers + # 400 with a cause. Narrowing is the safe direction, but an unanswerable condition that reports + # nothing is how a reader ends up trusting an empty table. + if _active_cohort_leaves(tree): + raise err(400, "filter_unsupported", + "a condition on cohort membership cannot be evaluated on this database's live " + "connection, because a cohort is resolved from this tenant's own records rather " + "than from the connection. Filter on a saved view instead, or open this " + "database's stored copy") + + store = _view_store(session, table_key) + sets, memo, unresolved = {}, {}, [] + counter = [0] + + def key_for(): + counter[0] += 1 + # ⚠ A SYNTHETIC KEY, NEVER THE VIEW ID. `parse_cohort_ids` splits `value` on commas and + # truncates at 120 characters; a view id is free text and may carry either. A key we mint + # cannot collide, cannot split, and cannot be mistaken for a cohort the tenant owns. + return "__view_%d__" % counter[0] + + def resolve(vid, seen): + """`('ids', [...])` / `('all', None)` / `(None, reason)` for one view id.""" + if vid in memo: + return memo[vid] + if vid in seen: + # `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left + # ABSENT, so a stored loop degrades to an explained empty table, not a hung request. + return None, "it refers to itself through another saved view" + found = store.find_view(vid, prefer_owner=session.uname) + if not found: + got = (None, "it is no longer there") + memo[vid] = got + return got + owner, view = found + if owner != session.uname and not _may_see_view(session, vid, store): + got = (None, "your account cannot open it") + memo[vid] = got + return got + cfg = (view.get("config") or {}) if isinstance(view, dict) else {} + if str(cfg.get("cohortLock") or "").strip(): + got = (None, "it is locked to a cohort, and a cohort is resolved from this tenant's " + "own records rather than from this database's live connection") + memo[vid] = got + return got + members = [] + for raw in (cfg.get("memberPids") or ()): + try: + members.append(int(raw)) + except (TypeError, ValueError): # noqa: PERF203 + continue + raw_nodes = cfg.get("filters") or [] + if not isinstance(raw_nodes, list): + raw_nodes = [] + if not raw_nodes: + # `filters.length` is the client's own test, and it is the NODE COUNT rather than + # "anything active": with no nodes at all the curated set IS the view, and with nodes + # that all happen to be half-built the view holds everything. Two different answers + # from one list, so the emptiness is checked before the compiler sees it. + got = ("ids", members) if members else ("all", None) + memo[vid] = got + return got + nodes = rewrite(raw_nodes, seen + (vid,)) + unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols)) + if unknown: + # ⛔ UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it + # cannot see as INACTIVE, which WIDENS the view's row set - and `cols` has the field + # wall's hidden columns already stripped, so for a scoped reader that widening would be + # the permission door leaking through the back. Reported below, never silently applied. + got = (None, "it filters on " + ", ".join(unknown) + + ", which this database's live connection cannot read for your account") + memo[vid] = got + return got + try: + pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or" + else "and"), + columns=cols, member_ids=(members or None), + id_sql=spec["id"], today=_today(), cohort_sets=sets) + except ValueError as exc: + got = (None, "it uses a condition this database's live connection cannot evaluate (" + + str(exc) + ")") + memo[vid] = got + return got + if pred is None: + # Every condition in it is half-built, so it matches every row - which is what + # `matchFilterTree` answers for an inactive tree, and the `member_ids` union rides + # inside that same `if (filters.length)` on both engines. + got = ("all", None) + memo[vid] = got + return got + if pred.uses_aggregate: + got = (None, "it filters on an aggregate column, which this database's live connection " + "cannot evaluate one row at a time") + memo[vid] = got + return got + frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"] + where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql + ids = [int(r[0]) for r in cur.execute( + f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}', + list(wall_params) + list(pred.params)).fetchall() if r and r[0] is not None] + got = ("ids", ids) + memo[vid] = got + return got + + def rewrite(nodes, seen): + out = [] + for n in (nodes or []): + if not isinstance(n, dict): + out.append(n) + continue + if isinstance(n.get("children"), list): + out.append(dict(n, children=rewrite(n.get("children"), seen))) + continue + if str(n.get("colId") or "") != aios_grid.VIEW_FIELD: + out.append(n) + continue + vid = str(n.get("value") or "").strip()[:aios_grid.MAX_VIEW_ID] + if not vid: + # Half-built, and half-built is IGNORED on both engines. Left as it stands: its + # `colId` is not a column, so `is_rule_active` already answers False for it. + out.append(n) + continue + # `normalizeViewOp`: anything that is not `notInView` reads as `inView`, so a leaf + # carrying a corrupted op asks the narrower question rather than the wider one. + not_in = str(n.get("op") or "") == "notInView" + key = key_for() + kind, got = resolve(vid, seen) + if kind == "ids": + sets[key] = got + op = "noneOf" if not_in else "anyOf" + if len(got) >= _PIN_REPORT_AT: + limits.append({ + "subject": "view", "effect": "slow", + "cause": f"the saved view this condition names holds {len(got):,} rows, " + f"and each page of this database names every one of them in the " + f"query that narrows it", + "recommendation": "add a condition beside the view to ask a smaller " + "question, or open the saved view itself"}) + elif kind == "all": + if not_in: + op = "anyOf" # key absent -> 1=0 + else: + sets[key] = [] + op = "noneOf" # NOT (1=0) -> true of every row + else: + op = "anyOf" # key absent -> 1=0 for `is` AND for `is not` + unresolved.append((vid, got)) + out.append({"colId": "__cohort__", "op": op, "value": key}) + return out + + tree = rewrite(tree, ()) + if unresolved: + limits.append({ + "subject": ", ".join(vid for vid, _why in unresolved), + "effect": "filter_unresolved", + "cause": "this condition asks whether a row is in a saved view that could not be " + "resolved on this database's live connection (" + + "; ".join(f"{vid}: {why}" for vid, why in unresolved) + + "), so it matches no rows at all, for `is not` as well as for `is`", + "recommendation": "pick a saved view you can open on this database, or remove the " + "condition"}) + return tree, sets + + +def _may_see_view(session, vid, store): + """May this caller open that saved view? The same predicate the view rail asks. + + ⛔ NOT SKIPPED FOR BEING INCONVENIENT. A view a caller cannot see resolves to nothing on the + client (it is not in the payload at all), so resolving it here would make this door answer a + question the other door cannot, out of a row set somebody else curated. + """ + import core.shares as shares + return shares.may_see("view", vid, session.uname, is_admin=session.admin, st=store.st) + + def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None, search=None, tree=_TREE_UNSET, related=None): """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once. @@ -1516,9 +2392,17 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= spec = _source_for(cur, table_key) if spec is None: + # ⭐⭐ W53-T19 / R5 — THE SENTENCE IS REWRITTEN AND THE REFUSAL IS NOT. R5 retires the + # phrase, not the guard: `verify_scopes` drives `_source_for` to None as a negative control + # and requires a 409 here rather than a grid quietly served from somewhere else, and every + # Odoo grain is bound today only on a mirror that carries the tables they read. The cause + # is now named as what it is, an absent binding, in the same words + # `relations.unserved_target_refusal` uses for the same absence. + # ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). raise err(409, "not_read_through", - "this connected database is still served from its stored copy; it has no " - "read-through binding on this deployment yet") + "this connected database has no read-through binding on this deployment, so " + "its rows cannot be read a window at a time here. Open it from the Data " + "section, or ask for a read-through binding to be added for it") cols = _filter_columns(spec, fields) @@ -1636,8 +2520,20 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= "recommendation": "ask an administrator to set the restriction on a whole number, " "or on a column that is not an amount"}) + # ── W53-T20 / D-377 — the view-membership leaf, rewritten into the cohort shape ─────────── + # ⭐ HERE, AND NOWHERE ELSE. After the wall and the pin (so a view resolves inside the scope + # the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and + # before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with + # no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler. + tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols, + base_where, wall_params, cur, limits) + # ── the predicate, pushed down ──────────────────────────────────────────────────────────── - named = _leaf_cols(tree) + # ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__` + # are not columns and never will be in `cols`, so leaving them in would report every membership + # condition as a column the mirror cannot read - a false sentence, and one that reads as "your + # filter was dropped" beside a filter that was in fact applied. + named = _leaf_cols(tree) - _PSEUDO_COLS missing = sorted(named - set(cols)) if missing: limits.append({ @@ -1687,9 +2583,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= where, params = base_where, list(wall_params) try: + # ⭐ `cohort_sets` IS PASSED ONLY WHEN A VIEW LEAF WAS REWRITTEN (it is None otherwise), and + # that is what keeps a genuine cohort leaf REFUSING here rather than compiling to `1=0` in + # silence. `id_sql` rides along because `_compile_cohort` raises without it. pred = _fs().compile_filter_tree( tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols, - today=_today()) + id_sql=spec["id"], today=_today(), cohort_sets=cohort_sets) except ValueError as e: # ⛔ A REFUSAL IS AN ANSWER, NOT A CRASH — and it must not become "no filter". Ranking ops # ("top 10") have no SQL form in this compiler; saying so beats returning every row. diff --git a/api/routes_platform_admin.py b/api/routes_platform_admin.py index df1ea25ec8f74523f39adbee62c674041d5addfa..731414254b55d7548864c76b0a3ab13c67396389 100644 --- a/api/routes_platform_admin.py +++ b/api/routes_platform_admin.py @@ -537,7 +537,7 @@ def _aws_report(days=7): """ if not _AWS_REPORT.is_file(): return {"available": False, "text": "", - "note": ("The AWS usage report is not part of this deployment — `ops/` ships " + "note": ("The AWS usage report is not part of this deployment; `ops/` ships " "with the repository, not with the container image. Run " "`python ops/aws_usage_report.py` locally for the live figures.")} try: @@ -547,7 +547,7 @@ def _aws_report(days=7): cwd=str(_AWS_REPORT.parent.parent)) except subprocess.TimeoutExpired: return {"available": False, "text": "", - "note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s — CloudWatch may " + "note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s; CloudWatch may " f"be unreachable from here. Nothing was assumed about usage.")} except Exception as e: # noqa: BLE001 return {"available": False, "text": "", @@ -556,7 +556,7 @@ def _aws_report(days=7): if proc.returncode != 0 or not out: detail = (proc.stderr or "").strip().splitlines() return {"available": False, "text": out, - "note": ("AWS credentials are not available here, or boto3 is not installed — " + "note": ("AWS credentials are not available here, or boto3 is not installed; " "no usage could be read, and none is guessed. " + (detail[-1][:200] if detail else ""))} return {"available": True, "text": out, "note": ""} @@ -655,7 +655,7 @@ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)): """ rows = _user_rows(tenant) return {"users": rows, "count": len(rows), - "stampsNote": ("Login and activity stamps started with this release — accounts that " + "stampsNote": ("Login and activity stamps started with this release; accounts that " "have not signed in since show no date rather than an invented one. " "Activity is recorded at most once an hour per account.")} @@ -700,7 +700,7 @@ def platform_connectors(tenant: str = "", session: Session = Depends(padmin_gate out += rows return {"connectors": out, "count": len(out), "keychainLocked": locked_any, "errors": errors, - "note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY` — stored " + "note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY`; stored " "credentials cannot be read, so a tenant's sources fail closed rather than " "falling back to anyone else's.")} diff --git a/api/routes_products.py b/api/routes_products.py index ac2f1a7bf15685429177236239bd16d9d4b9204c..9c0537a9c7905d102c332a189180243de1f38fca 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -94,7 +94,7 @@ def _pool_for(rt, team_id): if hit: return hit[1] raise err(503, "connector_paused", - "this data source is paused and nothing has been read since — " + "this data source is paused and nothing has been read since; " "an admin can resume it under Settings → Connectors") def build(): @@ -127,7 +127,7 @@ def scoped_pool(session: Session): rows_src = _pool_for(session.runtime, team_id) except Exception as e: raise err(503, "pool_unavailable", - f"the product catalogue could not be built — {str(e)[:160]}") + f"the product catalogue could not be built; {str(e)[:160]}") # Image is a tenant-wide created field. Seed/repair its definition before # the field wall runs so every reader receives the same metadata and the # Leadership account owns the permission editor from the first request. @@ -636,7 +636,7 @@ def patch_product(pid: int, body: dict = Body(default=None), "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", - "the tenant store is unavailable — your change was not saved") + "the tenant store is unavailable; your change was not saved") # What actually landed, read back from the PRODUCT bucket rather than echoed from the # request: a refused key or a truncated value must not be reported as accepted. diff --git a/api/routes_publish.py b/api/routes_publish.py index bbdb9d3507f4d4d96d1e9e030baaf9b238e8cdd2..a97ba6581c5c5cd3196609ebedd932f569213546 100644 --- a/api/routes_publish.py +++ b/api/routes_publish.py @@ -301,12 +301,12 @@ def _may_publish_source_view(session: Session, table_key: str, view_id: str): if cfg.get("cohortLock"): raise err(400, "reader_scoped", "this view is locked to a cohort, and a cohort's membership is resolved for " - "the person reading it — a public link has no reader, so the page would show " + "the person reading it; a public link has no reader, so the page would show " "no rows at all. Publish a copy without the cohort lock.") if _needs_a_reader(cfg.get("filters")): raise err(400, "reader_scoped", "this view filters on a cohort or a measure rule, and each is resolved for the " - "person reading it — a public link has no reader, so " + "person reading it; a public link has no reader, so " "the page would show no rows at all. Publish a copy filtered on columns.") return view, mode @@ -818,7 +818,7 @@ async def mint_view_link(request: Request, session: Session = Depends(require_se # If the write cannot be read back, say so; do not describe a state nobody verified. if not entry: raise err(503, "not_saved", - "the link was minted but could not be read back — reload and try again") + "the link was minted but could not be read back; reload and try again") return _state_of(token, entry) @@ -864,7 +864,7 @@ async def revoke_view_link(request: Request, session: Session = Depends(require_ def get_published(token: str, request: Request): """The published view. Read-only, unauthenticated, projected to the view's own columns.""" if not _rate_ok(_client_ip(request), time.time()): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") found = _resolve(token) if not found: # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. @@ -906,7 +906,7 @@ async def open_published(token: str, request: Request): """ now = time.time() if not _rate_ok(_client_ip(request), now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") body = await _bounded_body(request) found = _resolve(token) if not found: diff --git a/api/routes_records.py b/api/routes_records.py index ef0bb2c8153db1e40b2656a715f545d80b1fe22b..fb4c818d7c5995f8c5d8205efa3a4179a0596a5d 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -47,7 +47,7 @@ def _scope_or_400(raw): if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): return "customer" if scope in _CUSTOMER_SCOPES else scope raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") + "scope must be customer, cohort, product or a ut_ database; refusing to guess") def _pool_or_refuse(session: Session, scope: str): @@ -117,7 +117,7 @@ def _in_book(pid, session, scope): # out loud rather than silently admitting: R6's second sentence is that a limit which # cannot be removed gets REPORTED, and this is the one place the report has no envelope to # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window); " f"admitting on the table wall alone, per D-72") return if pid not in pids: @@ -129,7 +129,7 @@ def _unavailable(): return err( 503, "store_unavailable", - "record comments are temporarily unavailable — no change was saved", + "record comments are temporarily unavailable; no change was saved", ) diff --git a/api/routes_sheet.py b/api/routes_sheet.py index 1c0ccf8a6a1dc98c1fa3564a2a5597d3e8eaedfb..a763843d106bada64acd62447e2d3066fda27f65 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -35,6 +35,8 @@ while its surface is a 404. (standing rule 2), and each names both the cause and the subject. """ +import hashlib +import json import os import tempfile import threading @@ -88,17 +90,30 @@ def _mirror(session: Session): `RuntimeError` means the mirror is completing its first sync, so it is a 503 the caller may retry. `DatastoreMismatch` SUBCLASSES `RuntimeError`, so the guard must run ABOVE the try, and collapsing them would turn the loudest refusal in the system into a spinner. + + ⭐⭐ W53-T13 — `_for` TAKES A `metric_window` NOW, AND EVERY EXISTING CALL SITE IS UNCHANGED. + It is keyword-only with a None default, so `_mirror(session)(database)` still means "leave + every Metric column at its own declared window". See `_registry_mirror` for what a window does. """ import routes_odoo_tables as rot import odoo_relational as rel - def _for(table_key): + def _for(table_key, *, metric_window=None, needed=None): fields = None for _bucket, key, _label, mk_fields in rel.TABLES: if key == table_key: fields = mk_fields() break - if fields is None: + # ⭐⭐ W52-T11 — `TABLES` NAMES EIGHT ODOO GRIDS AND DELIBERATELY NOT THE TWO COMPILED + # REGISTRY GRIDS (W33-T44: `customer_data` / `product_data` present the same subjects and + # would duplicate them). `_for` used to check `TABLES` alone, so a cell bound to either of + # those two 404'd as "not a connected Odoo database" even though both plainly are one — + # it was consulting the wrong roster, not reporting a real refusal. `_REGISTRY_GRID_FIELDS` + # is the OTHER roster; a key found there is exactly as connected as one found in `TABLES`. + # ⚠ `TABLES` ITSELF IS UNTOUCHED — it has other consumers (`declared_schema_fields`, + # `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows. + is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS + if fields is None and not is_registry_grid: raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.") try: cur, mismatch = rot._mirror_cur(session) @@ -106,19 +121,389 @@ def _mirror(session: Session): raise err(503, "store_not_ready", str(exc)) if mismatch: raise err(409, "cross_tenant_store", mismatch) + if is_registry_grid: + # ⛔⛔ ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet` + # resolves a binding CELL BY CELL, and this branch now costs a full `/workspace` + # assembly rather than a `scoped_table` read: measured on the running app, 40 to 70 + # seconds each. Forty cells on one database would rebuild that forty times and time the + # door out, which is a worse failure than the one this ticket fixes because it takes the + # whole workbook with it. `_view_filter`'s own `memo` argument exists for exactly this + # shape; the cache hangs off the SESSION because that object is built fresh by + # `require_session` on every request and dies with it, so nothing can outlive its own + # cursor or leak across users. The temp table it names lives in that same cursor. + # + # ⭐⭐ W54-T36 — AND ONLY THE COLUMNS THE CELL ACTUALLY READS ARE MATERIALISED. + # MEASURED on tenant #0's `customer_data` (3,673 records, 47 columns), warm, through + # the real `drill_cell`: 8,437 ms total, of which `odoo_relational.registry_grid_spec` + # is 6,137 ms. That step turns every row into a tuple of 47 Python values and + # `executemany`s it, so its cost is per COLUMN, and a drill reads none of them: it + # counts records and lists ids. `needed` is the honest set (the aggregate's column, + # plus every column the cell's filter and the row wall name); `None` still means the + # whole grid, so a caller that has not worked out its set loses nothing. + # + # ⛔ THE MEMO WIDENS, IT NEVER NARROWS, and that is what keeps `refresh_sheet` safe. + # A slot already built for a wider set answers a narrower request unchanged; a request + # naming a column the slot does not carry rebuilds at the UNION, so a workbook whose + # cells read different columns converges instead of rebuilding per cell. A `None` + # entry means every column is present and is reusable by anything. + memo = getattr(session, "_sheet_mirrors", None) + if memo is None: + memo = {} + session._sheet_mirrors = memo + slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str)) + want = None if needed is None else {str(k) for k in needed} + entry = memo.get(slot) + if entry is not None: + built, made = entry + if built is None or (want is not None and want <= built): + return made + if want is not None: + want = set(want) | built + made = _registry_mirror(session, cur, table_key, metric_window, needed=want) + memo[slot] = (None if want is None else set(want), made) + return made spec = rot._source_for(cur, table_key) if spec is None: raise err(409, "not_read_through", - f"{table_key} is still served from its stored copy, so a cell cannot read " - "through to it on this deployment yet.") - return cur, spec, rot._filter_columns(spec, fields) + f"{table_key} has no read-through binding on this deployment, so a cell cannot " + "read through to it yet.") + # ⚠ THE FIELD LIST RIDES ON THE SPEC (W53-T13). `_resolve_binding` has to ask whether the + # bound column can be TOTALLED at all, and that question needs the field's own `type` and + # `rollup` bag, which `_filter_columns` reduces away. Copying rather than mutating is + # `_source_for`'s own idiom (it returns `{**spec, "cols": resolved}`) and matters for the + # same reason: the spec dict handed back by `_source_for` is not this call's to keep. + return cur, {**spec, "fields": fields}, rot._filter_columns(spec, fields) return _for -def _view_filter(session: Session, database: str, view_id: str, view_name: str = ""): +def _registry_assembly(session: Session, table_key: str): + """The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal. + + ⭐⭐ W53-T13 — THIS IS THE FIX, AND IT IS A SECOND CALLER OF AN ESTABLISHED PATH RATHER THAN A + NEW ONE. `_offered_view` above already reaches for these same two assemblies. Measured on the + running app as `admin@royal-imports`: `/workspace?scope=customer` serves **47** fields, while + `perm_scope.scoped_fields` resolves through `routes_grid.py::_customer_rows`, which returns + `list(aios_grid.FIELDS)` - the **static 37-key contract**. So the Sheet was building its temp + table out of 37 of the tenant's 47 columns and `aggregate_sql` then refused the other 10 as + `field_unknown`, Metric and plain `custom_` alike. `count` was the only aggregate that worked + because it returns `COUNT(*)` before it ever looks at `columns`, which is exactly the owner's + report: *"It seems only Count works."* + + ⛔⛔ IT WIDENS NOTHING, AND THAT IS CHECKED RATHER THAN HOPED. `grid_assembly` applies + `perm_scope.apply_row_scope` (`routes_customers.py:358`) and `perm_scope.hidden_keys` + (`routes_customers.py:496`), narrowing the field list, the rows AND the overlay dict together; + `product_assembly` does the same. It is the wall this person's OWN grid goes through, so a + Sheet cell sourced from it can only ever see what they are already shown. + + ⚠ `consume_corrections=False` IS NOT OPTIONAL, for `_offered_view`'s reason: those one-shot + field-name correction acks belong to the `/workspace` refresh that exists to show them to the + person who made the edit, and a bound cell resolving its scope must not eat them. + + ⭐⭐ W54-T36 — ONE ASSEMBLY PER DATABASE PER REQUEST, memoised on the SESSION exactly as + `_mirror` memoises the mirror and for the same lifetime reason: `require_session` builds that + object per request and it dies with the request, so nothing here outlives its own cursor or + reaches another account. MEASURED on tenant #0's `customer_data`, warm: 2,145 ms a call. It + already ran once per (database, window); it now also survives a mirror rebuilt at a wider + column set, and every caller BELOW reuses it. `_offered_view` reaches for `grid_assembly` and + `product_assembly` on its own and is a separate cost; see its own note. + + ⛔ AN UNEXPECTED FAILURE REFUSES LOUDLY AND DOES NOT DEGRADE TO THE STATIC CONTRACT. Falling + back would answer the cell out of 37 columns while the person is looking at 47, which is the + defect this function exists to end, and it would do it with a 200. + """ + import core.perm_scope as perm_scope + memo = getattr(session, "_sheet_assemblies", None) + if memo is None: + memo = {} + session._sheet_assemblies = memo + if table_key in memo: + return memo[table_key] + try: + if table_key == "product_data": + from routes_products import product_assembly + memo[table_key] = product_assembly(session, scope="product", + consume_corrections=False) + return memo[table_key] + from routes_customers import grid_assembly + memo[table_key] = grid_assembly(session, scope="customer", consume_corrections=False) + return memo[table_key] + except perm_scope.Denied: + raise err(403, "database_unreadable", f"You cannot read {table_key}.") + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + # ⛔ THE SENTENCE A PERSON READS SAYS WHAT HAPPENED; THE LOG SAYS WHY. Without this line + # the refusal is honest and undiagnosable at the same time: it names the database and the + # act, and nothing anywhere records which call inside a 300 line assembly failed. + # `harness.telemetry.error` is the same channel `routes_customers._measure_err` uses. + try: + import harness.telemetry as _tel + _tel.error("api:sheet-registry-assembly", exc) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_not_ready", + f"{table_key} could not be assembled for this cell, so the number is not being " + "shown rather than shown from an older set of columns. Try again in a moment.") + + +def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None): + """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period. + + ⭐⭐ R7's SECOND HALF LIVES HERE: *"must also be bounded by date time lookup"*. `g["derived"]` + resolves each Metric column at ITS OWN declared window (`{kind: "ltm"}` on both of the live + ones), NOT at the cell's period, so a cell labelled one month would otherwise print a twelve + month number. When the binding carries a period, every Metric column is re-resolved at THAT + window through the SAME evaluator the grid renders from (`core/measure_resolve.py:: + column_values` for the customer grain, `routes_products._measure_cells` for the product one), + so a Sheet cell and the grid cannot disagree about a number. + + ⛔ A METRIC THAT CANNOT BE RESOLVED AT THE REQUESTED WINDOW GOES BLANK AND IS REPORTED. It + does NOT fall back to its own window: that would answer a wider question under a narrower + label, with nothing on screen saying so. The report is a `limits` row (standing rule 1). + + ⛔ AND A METRIC THIS ACCOUNT MAY NOT SEE IS NEVER RE-RESOLVED. `grid_assembly` skips + `column_values` entirely when `may_metrics` is off, so a column with no value anywhere in + `g["derived"]` is one the capability wall has already answered for. Re-resolving it here would + be a second answer to that question, from the one door that never asked it. + + ⭐⭐ W54-T36 — `needed` IS THE COLUMN SET THIS CELL CAN POSSIBLY READ, AND `None` IS EVERY + COLUMN. It narrows what gets MATERIALISED, never what gets assembled: `g` is the same scoped + assembly, `rows` are the same scoped rows, and `spec["fields"]` below is still the FULL list + so a refusal can name a column the mirror did not carry. + + ⛔⛔ IT IS INTERSECTED WITH THE ASSEMBLY'S OWN FIELDS AND MUST STAY THAT WAY. `cols` is what + `_resolve_binding` subtracts a row wall's columns from to decide whether the wall can be + applied at all, and that check is FAIL CLOSED (409 `scope_not_applicable`). Materialising a + column just because a filter named it would invent an empty column for a name this database + does not have, and the wall would then compile against nothing and quietly stop restricting. + """ + import aios_grid + import odoo_relational as rel + import routes_odoo_tables as rot + from modules import sheet as sheet_mod + + g = _registry_assembly(session, table_key) + fields = list(g.get("fields") or []) + derived, bounded, limits = _metric_derived(session, table_key, g, metric_window) + rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"), + derived=derived) + # The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces + # `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column. + kept = (fields if needed is None + else [f for f in fields if str(f.get("key") or "") in needed]) + # ⛔ THE TYPES ARE NORMALISED FOR THE MIRROR ONLY. See `modules/sheet.py::mirror_field_type`: + # `metric`, `rating` and a numeric `rollup` are in neither `_REGISTRY_COL_TYPE` nor + # `filter_sql.NUMERIC_TYPES`, so without this they land as VARCHAR and compare as strings. + # ⚠ THE SAME LIST GOES TO BOTH CALLS, deliberately. `registry_grid_spec` picks the SQL type and + # `_filter_columns` picks the compiler's type; handing them two different lists is how a column + # ends up stored as one thing and compared as another. + spec_fields = sheet_mod.mirror_fields(kept) + spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows) + # ⛔⛔ THE MATERIALISED TABLE IS GIVEN THE WINDOW'S OWN NAME, AND WITHOUT THIS THE MEMO ABOVE + # SERVES ONE WINDOW'S ROWS UNDER ANOTHER WINDOW'S ANSWER (found by W53-T14, live). + # `odoo_relational.registry_grid_spec` names its temp table `_registry_grid_` and + # `CREATE OR REPLACE`s it, so the name carries the DATABASE and not the PERIOD, while + # `_mirror`'s memo is keyed by `(table_key, window)`. Resolve one customer_data cell at no + # period and a second at `{month, 2026-08}` inside ONE request and the second build replaces + # the first one's table underneath the spec the memo is still holding; a third cell reading + # the first slot then runs its SQL against August's rows. + # ⚠ MEASURED, and it is silent: `avg` over `measure_sales_the_last_12_months_6hlkf` answered + # **138.3942281513749** in a batch that had already resolved that month (508,322.0 / 3,673) + # and **1710.679825755513** on its own, which is the all-time number the preview door and the + # grid both give. No refusal, no limits row, a plausible number under the wrong label. + # ⛔ IT IS FIXED HERE RATHER THAN IN `registry_grid_spec` BECAUSE THAT FILE BELONGS TO ANOTHER + # LANE THIS WAVE. The copy is one `CREATE TABLE AS SELECT` over rows already in memory, next + # to an assembly that costs 20 to 130 seconds, so it is not a cost worth conditioning on. + # ⚠ ALWAYS, INCLUDING THE `None` WINDOW: the unqualified name is REPLACED by every later + # build, so leaving the no-period case pointing at it is exactly the case measured above. + # ⛔⛔ W54-T36 — THE COLUMN SET IS PART OF THAT NAME FOR THE SAME REASON THE WINDOW IS. Two + # cells on one database at one window may now be materialised over DIFFERENT columns, and a + # narrower later build would otherwise replace a wider earlier one under a spec the memo is + # still holding, so a query would address a column its own table no longer has. + slot = hashlib.sha1( + json.dumps([metric_window, None if needed is None else sorted(needed)], + sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12] + at_window = f'{spec["table"]}__at_{slot}' + cur.execute(f'CREATE OR REPLACE TEMP TABLE "{at_window}" AS ' + f'SELECT * FROM "{spec["table"]}"') + spec = {**spec, "table": at_window} + # ⚠ `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal` + # has to name WHY a column cannot be totalled, and the normalised copy has already thrown that + # away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds + # text" instead of "it folds its linked records into text rather than into a number". + spec = {**spec, "fields": fields, "unbounded": limits, "bounded": bounded} + return cur, spec, rot._filter_columns(spec, spec_fields) + + +def _metric_derived(session: Session, table_key: str, g, metric_window): + """`(derived, bounded_keys, unbounded)` for one assembly at the cell's period. + + `unbounded` is `{field key: limits row}` and is KEYED rather than listed on purpose: a mirror + is built once for a database and a cell reads one or two of its columns, so reporting every + Metric column that failed would bury the one the reader is actually looking at under a list + about columns their cell never touches. `_resolve_binding` picks the rows for the columns the + binding names. + + With no period this is `g["derived"]` exactly as the grid uses it, no metric keys claimed as + bounded, and nothing to report. With one, every Metric column this account can already see is + recomputed at that window, ONE COLUMN AT A TIME so a failure can be attributed to the column + it belongs to rather than reported as "something went wrong". + + ⚠ THE MEMO IS PER CALL. `core/measure_resolve.py`'s memo key carries a POOL STAMP this function + does not hold, and inventing one would park an answer under a stamp that never invalidates: + the cell would keep serving a number from before the last sync. A fresh dict costs one query + per Metric column and cannot go stale. + """ + import aios_grid + from modules import sheet as sheet_mod + + derived = g.get("derived") or {} + mfields = aios_grid.measure_fields_of(g.get("fields") or []) + if not metric_window or not mfields: + return derived, (), {} + + # Which Metric columns carry a value at all today: the capability wall's own answer, reused. + seen = set() + for cells in derived.values(): + seen.update(cells or ()) + live = [f for f in mfields if f.get("key") in seen] + + out = {pid: dict(cells or {}) for pid, cells in derived.items()} + for f in mfields: # every Metric column starts blank at a new window + for cells in out.values(): + cells.pop(f.get("key"), None) + + bounded, unbounded = [], {} + for f in live: + key = f.get("key") + label = str(f.get("label") or key) + rewritten = dict(f, measure=dict(f.get("measure") or {}, window=metric_window)) + try: + cells = _measure_cells_at(session, table_key, g, rewritten) + except Exception: # noqa: BLE001 + cells = {} + got = False + for pid, values in (cells or {}).items(): + if key in values: + out.setdefault(pid, {})[key] = values[key] + got = True + if got: + bounded.append(key) + else: + unbounded[key] = sheet_mod.metric_window_limit( + label, + "this column's metric could not be measured over the period this cell names, so " + "it was left blank rather than answered over its own wider window") + for f in mfields: + if f.get("key") not in seen: + # ⚠ THE CAUSE NAMES BOTH POSSIBILITIES because this door cannot tell them apart, and + # a sentence that picks one would be confidently wrong half the time. A column with no + # value anywhere in `derived` is either one the metrics capability withholds from this + # account or one whose own declared window already failed to resolve. + unbounded[f.get("key")] = sheet_mod.metric_window_limit( + str(f.get("label") or f.get("key")), + "this column has no value on this database for this account, either because the " + "metric behind it is not shared with you or because it could not be measured at " + "all, so there was nothing to bound to the period") + return out, tuple(bounded), unbounded + + +def _measure_cells_at(session: Session, table_key: str, g, field): + """`{pid: {field key: value}}` for ONE rewritten Metric column, through its grain's evaluator. + + ⛔ TWO GRAINS, TWO EVALUATORS, AND NEITHER IS RE-IMPLEMENTED HERE. `core/measure_resolve.py` + is CUSTOMER grain (it resolves through `harness/measure_filter.py`, whose entity is the + partner); the product grain goes through `semantic.entity_measure_values`, which is what + `routes_products._measure_cells` wraps and which joins by SKU CODE rather than by pid. Calling + the wrong one would answer a column of blanks on one of the two grids. + """ + if table_key == "product_data": + import routes_products as rp + return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"), + g.get("today"), "sheet-period", {}, + rp.product_measures(session)) + from core import measure_resolve + return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(), + g.get("today"), "sheet-period", {}) + + +def _offered_view(session: Session, database: str, view_id: str): + """The view THIS database's own workspace assembly offers under `view_id`, or None. + + ⛔⛔ WAVE 53 FOLLOW-UP TO W53-T12 — `find_view` ALONE CANNOT SEE A PINNED VIEW NOBODY HAS + TOUCHED, AND THE BINDING EDITOR OFFERS EXACTLY THOSE. `all-customers` is pinned PER USER + (`core/grid_events.py:1652`: `_pinned_per_user = (_kind == 'system' or view_id in + ('all-customers', _ag_overview_id()))`), so every user of a table gets their own copy of that + id, but only once they have touched it — an autosave, a column resize. An account that never + customised it has NO stored record, `find_view` answers None, and this door then refused, by + name, a view the `/workspace` door had just offered the same session. Measured on the running + app as `admin@royal-imports`: `/workspace?scope=customer` served exactly one view, + `all-customers`, and a cell bound to it answered `409 view_missing: View All customers no + longer exists.` That sentence is the owner's report of D-662's sibling, word for word. + + ⭐⭐ SO THE FALLBACK BUILDS THE SAME ASSEMBLY THE `/workspace` DOOR BUILDS AND PICKS THE VIEW + OUT OF IT. The mechanism is copied from `routes_alerts.py::_require_filtered_view`, including + its `_view_by_id` — IMPORTED rather than re-written, because two hand-rolled "find the view in + an assembly" walks is exactly the shape that let `g["views"] IS A LIST` raise on every call for + thirteen waves ([[one-question-two-normalizers]]). + + ⚠ `consume_corrections=False` IS NOT OPTIONAL (`routes_alerts.py:498`). Those one-shot + field-name correction acks belong to the `/workspace` refresh that exists to show them to the + person who made the edit; a bound cell resolving its scope must not eat them. + + ⚠ `with_rows=False` on the `ut_` leg for the reason W31-T20 built the flag: only `g["views"]` + is read here, and building the table would make a read-through grain refuse + `409 window_required` over rows nobody asked for. + + ⛔ AN UNEXPECTED FAILURE ANSWERS None RATHER THAN ESCAPING. This function only ever runs after + `find_view` has already missed, so the cell was on its way to a refusal either way; letting a + raw exception out would turn one cell's bad luck into a 500 for the whole workbook refresh. + A refusal that already NAMES ITS OWN CAUSE (`HTTPException`) is let through instead, because + "the store is not ready" is a truer answer than "that view no longer exists". + """ + key = str(database or "") + try: + if key.startswith("ut_"): + from routes_tables import ut_assembly + g = ut_assembly(session, key, + storage_key=f"{session.tenant}:{key}:{session.uname}", + consume_corrections=False, with_rows=False) + elif key == "product_data": + from routes_products import product_assembly + g = product_assembly(session, scope="product", consume_corrections=False) + elif key == "customer_data": + from routes_customers import grid_assembly + g = grid_assembly(session, scope="customer", consume_corrections=False) + else: + return None + except HTTPException: + raise + except Exception: # noqa: BLE001 + return None + from routes_alerts import _view_by_id + return _view_by_id(g, view_id) + + +def _view_filter(session: Session, database: str, view_id: str, view_name: str = "", + memo=None): """`(nodes, conj)` for a saved view's filter, or a `view_missing` refusal that NAMES it. + ⭐⭐ TWO LOOKUPS, CHEAPEST FIRST, AND THE ORDER IS THE CONTRACT (wave 53, follow-up to + W53-T12). `find_view` answers from the caller's OWN stored record and is authoritative when it + hits, trashed record included (R11). Only on a MISS does `_offered_view` rebuild the workspace + assembly, which is what resolves a pinned view this account has never customised. A miss in + BOTH still refuses by name: an id nobody can resolve must never quietly mean "every row", which + would serve a whole database under a scope nobody wrote. + + ⚠ `memo` is a PER-REQUEST dict, keyed `(database, view_id)`, and it exists because the two + per-cell loops in this file resolve a binding CELL BY CELL. `refresh_sheet`'s own docstring + claimed "each distinct `view_id` is resolved ONCE" and the code never did that: `seen_db` + deduplicates the DATABASE WALL only, and the resolve underneath it ran per cell. That was + survivable while the whole lookup was one `find_view`; with an assembly behind the fallback, + forty cells on one view would rebuild it forty times. The memo caches the REFUSAL too, so a + workbook full of cells on one deleted view costs one lookup rather than forty. + ⛔⛔ `prefer_owner=session.uname` IS LOAD-BEARING AND OMITTING IT IS A SILENT CROSS-USER READ. The system view id `all-customers` is pinned on every database, and every user who has ever touched that view's config holds their OWN record under the same id, so a bare first-hit scan @@ -130,10 +515,42 @@ def _view_filter(session: Session, database: str, view_id: str, view_name: str = ⚠ The refusal names the view, which is why `clean_binding` keeps `view_name` on the binding: once the record is deleted there is nothing left to look a name up from. """ - from core import table_store + slot = (str(database or ""), str(view_id or "")) + if memo is not None and slot in memo: + cached = memo[slot] + if isinstance(cached, Exception): + raise cached + return cached + try: + out = _view_filter_once(session, database, view_id, view_name) + except HTTPException as exc: + if memo is not None: + memo[slot] = exc + raise + if memo is not None: + memo[slot] = out + return out + + +def _view_filter_once(session: Session, database: str, view_id: str, view_name: str = ""): + """`_view_filter` without the memo. Read that docstring first; this is only its body.""" + from core import table_store, view_templates + # ⛔⛔ THE BUCKET IS `_table_workspace`, AND THIS DOOR WAS ASKING FOR THE BARE KEY (wave + # 53, follow-up to W53-T12). `view_templates.workspace_key` is the ONE normaliser between a + # database key and the store bucket its saved views live in — `customer_data` lives in + # `customer_table_workspace`, `product_data` in `product_table_workspace`, a `ut_` table in + # `_table_workspace` — and every other reader in the app already goes through it or + # spells the suffix out (`routes_starred`, `routes_tables`, `routes_shares`, `routes_grid`, + # `routes_automation`). `TableStore("customer_data")` reads a bucket nothing writes, so this + # lookup could not hit for ANY database: `prefer_owner` and `include_trashed` below were both + # unreachable, and R11's promise that a trashed view keeps a cell working for 30 days was + # dead code. Fixing the key is what makes the cheap leg able to answer at all. + # ⚠ `workspace_key` answers None for a key that is not a table; the bare key is kept in that + # case so nothing that used to resolve stops resolving. # ⚠ `TableStore(table_key, st=...)`, in that order. `st` is the STORE HANDLE, and passing the # session's runtime is what makes the lookup read THIS tenant's bucket rather than tenant #0's. - store = table_store.TableStore(database, st=session.runtime) + store = table_store.TableStore(view_templates.workspace_key(database) or str(database or ""), + st=session.runtime) # ⭐⭐ `include_trashed=True` IS R11, AND IT IS ONE KEYWORD BECAUSE THE DEFAULT MUST NOT MOVE # (wave 51, W51-T26). A deleted view is MOVED into `.trash`, so the bare scan answered # None and this door refused a cell whose view somebody deleted an hour ago and can still @@ -144,11 +561,50 @@ def _view_filter(session: Session, database: str, view_id: str, view_name: str = # permission-adjacent (shares, starred, publish, forms); widening there would open doors no # ticket asked about, with every gate green. found = store.find_view(view_id, prefer_owner=session.uname, include_trashed=True) - if not found: + view = found[1] if found else None + if view is None: + # ⭐⭐ THE SECOND LOOKUP, AND IT RUNS ONLY ON A MISS. See `_offered_view`: a pinned view + # this account has never touched has no stored record anywhere, so the workspace assembly + # is the only place it exists. A miss HERE too is a genuinely gone view and still refuses + # by name. + view = _offered_view(session, database, view_id) + if view is None: raise err(409, "view_missing", f"View {view_name or view_id} no longer exists.") - _owner, view = found import routes_odoo_tables as rot - return rot._fe().tree_parts((view or {}).get("filters")) + # ⛔⛔ D-662 — THE FILTER TREE LIVES AT `config.filters`, NOT AT THE RECORD'S TOP LEVEL. + # Reading `view["filters"]` answered None on every stored view, and `tree_parts(None)` is + # `([], "and")` — a legal EMPTY filter, so EVERY view-scoped cell resolved with no filter at + # all and reported the whole database, with nothing anywhere going red. + # + # ⭐⭐ AND THE READER IS `core.sheets.adopted_scope_filter`, NOT A FOURTH HAND-ROLLED WALK. + # That function is this repo's canonical answer to "which rows does this view mean": its own + # docstring names THIS door as the one that got it wrong, and it already carries the three + # shapes a stored record comes in — `config.filters` as a `{conj, nodes}` tree, `config.filters` + # as a BARE LIST whose conjunction lives beside it in `config.filterConj` (a bare-list read + # through `tree_parts` alone would silently answer `and` for an OR view), and a LOCKED view, + # which is a curated ROW SET and adopts its COHORT leaf rather than the incidental filters its + # config happens to hold (`DESIGN.md` section 4, the three locks). The cohort leaf does not + # compile on the Sheet path and refuses by name, which is the honest answer: a locked view's + # membership is not a predicate, and inventing one here would put a plausible wrong number on + # a screen. It deep-copies, so nothing here can mutate the stored view. + from core import sheets as sheets_mod + return rot._fe().tree_parts(sheets_mod.adopted_scope_filter(view)) + + +def _materialize_copy_binding(raw_binding, filter_tree): + """A copied saved-View binding with its own filter, without mutating either input. + + R5 is deliberately a COPY-TIME action. The source cell keeps following its saved View, while + the copy receives a deep local shadow and can step a chosen leaf without editing shared View + state. A binding already scoped to a local filter passes through the normal cleaner unchanged. + """ + from modules import sheet as sheet_mod + import copy as _copy + + binding = sheet_mod.clean_binding(raw_binding) + if not binding["scope"].get("view_id"): + return binding + return {**binding, "scope": {"filter": _copy.deepcopy(filter_tree)}} #: ⛔ THE TWO CAP CODES ANSWER 413, NOT 400 (contract C3). A 400 says the request was malformed and @@ -335,6 +791,23 @@ def _wire_index(index): return out +def _binding_scope_key(session: Session, database: str, binding: dict) -> str: + """The scope hash `_resolve_binding` would compute under, for THIS caller, right now (R3, + W52-T12). + + Mirrors the wall half of `_resolve_binding` without opening the mirror or running SQL, so a + cache hit can be decided before any database work runs. Both this function and + `_resolve_binding` derive `wall_tree` the same way and both feed `core.sheets.scope_hash`, the + ONE function that turns those pieces into a key, so a future change to the wall's shape cannot + make the two drift silently apart. + """ + from core import sheets + import core.perm_scope as perm_scope + wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter") + if perm_scope.row_scope_applies(session.user, database) else None) + return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}) + + def _reresolved_for_reader(session: Session, book): """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5). @@ -350,14 +823,29 @@ def _reresolved_for_reader(session: Session, book): ⚠ `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it. + + ⭐⭐ W52-T12 / R3: A CACHE HIT SKIPS `_resolve_binding` ENTIRELY, AND A MISS FILLS ONLY THE + CALLER'S OWN SLOT. The cache lives in a document `core.sheets` owns and keys by + `(database, wall, binding scope)` (`_binding_scope_key`), never by user id or role name, so + two readers who share a row restriction share an answer and a reader whose restriction differs + never reads the other's slot. Only `refresh_sheet` may ever clear it; a GET here only ever + fills the ONE slot the caller themselves is entitled to. """ + from core import sheets import copy as _copy fresh = _copy.deepcopy(book) stamp = _now_iso() seen_db = {} + # ⭐ The same per-request memo as `refresh_sheet`, for the same reason and with the same + # lifetime: this loop resolves cell by cell too, and a reader's GET must not rebuild one + # database's workspace assembly once per bound cell. + view_memo = {} touched = 0 + store = _store(session) + wid = book.get("id") or "" for tab in fresh.get("sheets") or (): - for _addr, cell in (tab.get("cells") or {}).items(): + tab_ref = str(tab.get("id") or tab.get("name") or "") + for addr, cell in (tab.get("cells") or {}).items(): if not isinstance(cell, dict) or cell.get("kind") != "bound": continue touched += 1 @@ -377,8 +865,29 @@ def _reresolved_for_reader(session: Session, book): "message": f"You cannot read {db}."} cell.pop("limits", None) continue + + cell_key = tab_ref + "!" + addr + scope_key = _binding_scope_key(session, db, binding) + hit = None + if wid and scope_key: + try: + hit = sheets.cached_scope_value(store, wid, cell_key, scope_key) + except Exception: # noqa: BLE001 + hit = None + if hit is not None: + cell["state"] = "fresh" + cell["value"] = hit.get("value") + cell["display"] = "" if hit.get("value") is None else str(hit.get("value")) + cell["refreshed_at"] = hit.get("refreshedAt") + cell.pop("refusal", None) + #: ⚠ NOT CLAIMED ON A CACHE HIT: `limits` describes THIS resolve's own truncation, + #: and a cached slot never recorded one, so a prior (possibly stale) value on the + #: cell is dropped rather than shown as if it described the answer just served. + cell.pop("limits", None) + continue + try: - out = _resolve_binding(session, binding) + out = _resolve_binding(session, binding, view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None @@ -398,6 +907,15 @@ def _reresolved_for_reader(session: Session, book): cell["limits"] = out["limits"] else: cell.pop("limits", None) + #: ⚠ FILLED AFTER THE CELL ABOVE IS DONE READING `out`, and best effort: a cache-write + #: failure only costs a future recompute, never a wrong number, so it never raises past + #: here. + if wid and scope_key: + try: + sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp, + out["database"]) + except Exception: # noqa: BLE001 + pass if not touched: return fresh # ⭐ EVERY FORMULA IS RE-EVALUATED TOO, over the numbers this reader is entitled to. Leaving @@ -611,8 +1129,7 @@ def _run_import(job_id, path, filename, runtime, uname): stamp = _now_iso() made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname) saved = sheets.put_workbook(runtime, made["id"], - {"name": book.get("name"), "sheets": book.get("sheets"), - "refreshed_at": stamp, "rev": made["rev"]}) + dict(book, refreshed_at=stamp, rev=made["rev"])) _job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"], sentence=_import_sentence(report)) except sheet_import.ImportRefusal as exc: @@ -777,7 +1294,8 @@ def read_sheet_tab(workbook_id: str, tab: str, found = sheets.find_tab(served, tab) except sheets.SheetRefusal as exc: raise _refuse(exc) - out = {"id": book.get("id"), "rev": book.get("rev"), "tab": found} + out = {"id": book.get("id"), "rev": book.get("rev"), "tab": found, + "schemaVersion": book["schemaVersion"]} if served.get("refreshedFor"): out["refreshedFor"] = served["refreshedFor"] return out @@ -806,6 +1324,12 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None `workbook_read_only`, not 409, because 409 names a problem they could fix by refetching and this one they cannot fix at all. An editor's write falls straight through to the rev guard, which is exactly what C5 asks for. + + ⭐⭐ W52-T10 — A FORMULA EVALUATES WHEN YOU COMMIT IT. `refresh_sheet` used to be the ONLY + caller of `_recompute_formulas`, so `=SUM(A1)` landed in the tab as the literal text and sat + there until somebody pressed Refresh. `computed` now rides out on every accepted patch: the + whole workbook is recomputed IN MEMORY (never a second write, never a second `rev`, see + `_recomputed_tab_cells`) and every FORMULA cell on THIS tab is handed back with its answer. """ from core import sheets raw = dict(body or {}) @@ -822,6 +1346,67 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None expect_rev=expect) except sheets.SheetRefusal as exc: raise _refuse(exc) + accepted = out.pop("accepted", []) + out["computed"] = _recomputed_tab_cells(session, workbook_id, tab, changes=accepted) + return out + + +def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, changes=None): + """W52-T10: every FORMULA cell on `tab_ref`, recomputed over the WHOLE workbook, IN MEMORY. + + ⭐⭐ THIS IS THE WHOLE FIX. The parser and the evaluator were already correct + (`formula_runtime.py::_normal` upper-cases the function name and `evaluate_sheet` already + resolves dependencies across every tab, W51-T23); the defect was that nothing on the commit + path ever CALLED either one. This is that call. + + ⚠ IN MEMORY, LIKE `_reresolved_for_reader`, AND FOR THE SAME REASON: this is a READ over what + the PATCH above just persisted, not a second write. A caller that persisted the recomputed + values would have to bump `rev` a second time for one commit, which is a revision the client + never asked for and would have to guess at on its NEXT patch. + + ⭐ A BOUND CELL NEEDS NOTHING EXTRA HERE. `_recompute_formulas` reads any non-formula cell's + stored `value` (a literal's or a bound cell's, whichever it is) straight off the document, so + `=SUM(A1)` sees A1's live database-bound number the moment A1 itself carries one, with no + branch here that knows or cares which kind of cell A1 is. + + ⭐ RECOMPUTED OVER THE WHOLE BOOK, RETURNED FOR ONE TAB. A cross-tab formula needs every tab's + cells to resolve its reference (W51-T23), so the pass runs over the whole workbook; C3's + client opens one tab at a time, so only the tab this commit landed on is worth sending back. + A formula sitting on a tab nobody has open still shows its last answer until that tab is + opened or the workbook is refreshed, exactly as a stale database-bound cell does. + """ + from core import sheets + from modules import sheet as sheet_mod + import formula_runtime as fr + + try: + book = sheets.get_workbook(_store(session), workbook_id) + found = sheets.find_tab(book, tab_ref) + except sheets.SheetRefusal: + return {} + edits = None + if changes is not None: + tab_at = sheets._tab_index(book["sheets"], tab_ref) + scope = _tab_scope_names(book["sheets"])[tab_at] + edits = {fr.a1_key(str(addr).strip().upper(), scope): + (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes} + _recompute_formulas(book, fr, sheet_mod, + cache_key=(session.tenant, session.uname, workbook_id), changes=edits) + found = sheets.find_tab(book, tab_ref) + out = {} + for addr, cell in (found.get("cells") or {}).items(): + if not isinstance(cell, dict) or (cell.get("kind") != "formula" + and "spill" not in cell and "spillParent" not in cell): + continue + row = {"value": cell.get("value"), + "display": "" if cell.get("value") is None else str(cell.get("display") or ""), + "state": cell.get("state") or "fresh"} + if cell.get("refusal"): + row["refusal"] = cell["refusal"] + for key in ("error", "style", "formulaMeta", "spill", "spillParent"): + if key in cell: + row[key] = cell[key] + out[addr] = row return out @@ -976,8 +1561,11 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) forty cells to one database asks once; a reader who has lost access to that database gets a refusal on every one of its cells rather than a stale number quietly repainting. - ⚠ Each distinct `view_id` is resolved ONCE. `table_store.find_view` runs a legacy migration on - every call, so resolving per cell would make a forty-cell refresh do forty migrations. + ⚠ Each distinct `view_id` is resolved ONCE, and as of the wave-53 follow-up to W53-T12 that + sentence is finally TRUE OF THE CODE. It was written of `seen_db`, which deduplicates the + DATABASE WALL and nothing else: the resolve underneath it ran per cell, so a forty-cell + workbook did forty `table_store.find_view` calls and forty legacy migrations with them. + `view_memo` below is the dedupe this sentence describes. ⛔⛔ C5 — THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed @@ -999,6 +1587,12 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) stamp = _now_iso() seen_db = {} + # ⭐ ONE MEMO FOR THE WHOLE REFRESH, keyed `(database, view_id)`. `seen_db` deduplicates the + # WALL and never the scope, so forty cells on one saved view used to resolve that view forty + # times. See `_view_filter`: the second lookup behind it builds a workspace assembly, which is + # work no workbook should pay for per cell. Per REQUEST, never on the runtime: a view's filter + # may be edited between two refreshes and a longer lived cache would serve the old rows. + view_memo = {} for tab in book.get("sheets") or (): for addr, cell in (tab.get("cells") or {}).items(): if not isinstance(cell, dict) or cell.get("kind") != "bound": @@ -1019,7 +1613,7 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) "message": f"You cannot read {db}."} continue try: - out = _resolve_binding(session, binding) + out = _resolve_binding(session, binding, view_memo=view_memo) except HTTPException as exc: detail = getattr(exc, "detail", None) or {} info = detail.get("error") if isinstance(detail, dict) else None @@ -1059,6 +1653,10 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)) "refreshed_at": stamp, "rev": book.get("rev")}) except sheets.SheetRefusal as exc: raise _refuse(exc) + # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE + # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET + # recomputes once, under their own scope, and fills its own slot again from there. + sheets.clear_scope_cache(_store(session), workbook_id) return {"workbook": saved, "recalc_ms": recalc_ms} @@ -1096,7 +1694,7 @@ def _tab_scope_names(tabs): return names -def _recompute_formulas(book, fr, sheet_mod): +def _recompute_formulas(book, fr, sheet_mod, *, cache_key=None, changes=None): """Re-evaluate every formula cell across the WORKBOOK, in one memoized dependency pass. ⭐⭐ ONE PASS OVER EVERY TAB, NOT ONE PASS PER TAB (W51-T23). Per tab, `DATA!B12` was never in @@ -1110,6 +1708,13 @@ def _recompute_formulas(book, fr, sheet_mod): A cell whose value is None because the database refused is handed to the engine as absent, so arithmetic over it stays None rather than silently counting it as nothing. """ + # The document engine owns calculation, including spills and error traces. Pass the + # whole workbook (names, epoch, formats and array metadata) and keep its in-place result. + # This additive adapter becomes active when the engine ticket supplies the callable. + recompute_document = getattr(fr, "recompute_document", None) + if callable(recompute_document): + return recompute_document(book, cache_key=cache_key, changes=changes) + tabs = [t for t in (book.get("sheets") or ()) if isinstance(t, dict)] names = _tab_scope_names(tabs) spec = {} @@ -1190,7 +1795,7 @@ def _formula_message(code, subject=""): }.get(code, "This formula could not be calculated.") -def _resolve_binding(session: Session, raw_binding, *, today=None): +def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None): """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`. The order below is the contract, not a preference: @@ -1198,6 +1803,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None): 2. the mirror cursor, whose two failures mean opposite things; 3. the ROW wall, fail-closed, refusing 409 when it names a column the mirror cannot answer; 4. the binding's own scope, whose unanswerable columns are REPORTED rather than dropped. + + ⚠ W54-T36 — THE WALL AND THE SCOPE ARE NOW READ BEFORE STEP 2 SO THE MIRROR KNOWS WHICH + COLUMNS TO MATERIALISE. Steps 3 and 4 still happen where they did: the wall is still checked + against `cols` after the mirror exists, and it still refuses fail closed. + + ⭐⭐ W53-T13 / R7 — THE PERIOD IS RESOLVED BEFORE THE MIRROR IS OPENED, because it decides what + the Metric columns of that mirror MEAN. `modules/sheet.py::period_window` turns the cell's + `{unit, start}` into the custom window `harness/windows.py` speaks, and `_registry_mirror` + re-resolves every Metric column at it. A period that bounds nothing is reported rather than + left silent: see `period_limits`. """ from modules import sheet as sheet_mod import core.perm_scope as perm_scope @@ -1207,13 +1822,53 @@ def _resolve_binding(session: Session, raw_binding, *, today=None): except sheet_mod.SheetRefusal as exc: raise err(400, exc.code, exc.message) - database = _require_database(session, binding["database"]) - cur, spec, cols = _mirror(session)(database) + # ⛔ C2's OTHER AGGREGATE ARM IS ANSWERED HERE, NOT LEFT TO FALL THROUGH TO `aggregate_sql`, + # where it died as `field_unknown` with an empty subject. See `metric_key_refusal`. + metric_refusal = sheet_mod.metric_key_refusal(binding["aggregate"]) + if metric_refusal is not None: + raise err(400, metric_refusal.code, metric_refusal.message) - # ── the ROW wall, fail-closed ──────────────────────────────────────────────────────────── + database = _require_database(session, binding["database"]) + window = sheet_mod.period_window(binding.get("period")) + + # ⭐⭐ W54-T36 — THE WALL AND THE SCOPE ARE READ BEFORE THE MIRROR IS OPENED, so the mirror can + # be built over the columns this cell can possibly read instead of over the whole database. + # Neither read touches the mirror: the wall comes from `perm_scope` and the scope from the + # stored view document, which is why the order can move at all. + # ⛔ WHAT MUST NOT MOVE IS THE WALL'S OWN CHECK. It subtracts the wall's columns from `cols` + # and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below. wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter") if perm_scope.row_scope_applies(session.user, database) else None) wall_nodes, wall_conj = _fe_parts(wall_tree) + scope = binding["scope"] + if scope.get("view_id"): + scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"], + scope.get("view_name", ""), memo=view_memo) + else: + scope_nodes, scope_conj = _fe_parts(scope.get("filter")) + needed = (sheet_mod.filter_columns_named(scope_nodes) + | sheet_mod.filter_columns_named(wall_nodes)) + if binding["aggregate"].get("field"): + needed.add(str(binding["aggregate"]["field"])) + + cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed) + + # ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a + # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder + # error and a `sum` over a browser-computed formula column reaches it as a confident 0. + bound_field = next((f for f in (spec.get("fields") or []) + if isinstance(f, dict) + and f.get("key") == (binding["aggregate"].get("field") or None)), None) + agg_refusal = sheet_mod.aggregate_refusal(binding["aggregate"], bound_field) + if agg_refusal is not None: + raise err(409, agg_refusal.code, agg_refusal.message) + if bound_field and bound_field.get("measure") and not binding.get("period"): + label = bound_field.get("label") or bound_field.get("key") + raise err(409, "metric_period_required", + f"{label} is measured over a period, so this cell needs one. " + "Open the cell's Period section and choose a period.") + + # ── the ROW wall, fail-closed ──────────────────────────────────────────────────────────── if wall_nodes: unknown = sorted(sheet_mod.filter_columns_named(wall_nodes) - set(cols)) if unknown: @@ -1228,12 +1883,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None): "you on a column this database carries") # ── the binding's own scope ────────────────────────────────────────────────────────────── - scope = binding["scope"] - if scope.get("view_id"): - scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"], - scope.get("view_name", "")) - else: - scope_nodes, scope_conj = _fe_parts(scope.get("filter")) + # ⚠ W54-T36 — READ ABOVE, WITH THE WALL, because the mirror is now built over the columns the + # two of them name. The contract's ORDER is unchanged: the wall is still applied and still + # checked against `cols` before the scope reaches the compiler. # ⚠ NOTHING IS DONE HERE ABOUT `needs_excluded`. `_source_for` has already resolved it into the # spec's column expressions; applying it a second time would double-count the exclusion and @@ -1264,6 +1916,21 @@ def _resolve_binding(session: Session, raw_binding, *, today=None): row = cur.execute(sql, params).fetchone() value = row[0] if row else None + # ⭐⭐ EVERY WAY THIS ANSWER CAME OUT WIDER THAN THE BINDING ASKED FOR TRAVELS WITH IT + # (standing rule 1's second sentence). Three sources, one list: a filter leaf the compiler + # could not push down (`build_aggregate_query`), a Metric column that could not be measured at + # the cell's period (`_metric_derived`), and a period that bounds nothing at all because the + # bound column is stored per record rather than measured over time (`period_limits`). + # ⚠ ONLY THE COLUMNS THIS BINDING DEPENDS ON. See `_metric_derived`: the mirror knows about + # every Metric column on the database and a cell reads one or two of them, so the aggregate's + # own column plus every column its filter names is the honest set. + depends = set(sheet_mod.filter_columns_named(scope_nodes)) + if binding["aggregate"].get("field"): + depends.add(str(binding["aggregate"]["field"])) + limits = (list(limits) + + [row for key, row in (spec.get("unbounded") or {}).items() if key in depends] + + sheet_mod.period_limits(binding.get("period"), binding["aggregate"], bound_field, + spec.get("bounded") or (), depends)) return {"value": value, "limits": limits, "database": database, "binding": binding} @@ -1288,6 +1955,182 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None), "limits": out["limits"]} +@router.post("/sheets/{workbook_id}/binding-shadows") +def binding_shadows(workbook_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Canonical saved-View filters needed before copied bindings step (C5, R5). + + request: `{"bindings": [, ...]}` + response: `{"shadows": [{database, viewId, filter}, ...]}` + + The source bindings and saved Views are read only. The browser materialises each returned tree + into the copied binding, commits the stale cells immediately, then schedules Save's evaluator. + The editor wall is intentional: this door exists only to prepare a workbook write, and a + viewer must never turn a reader-specific filter into the owner's stored document. + """ + from core import sheets + from modules import sheet as sheet_mod + + try: + book = sheets.get_workbook(_store(session), workbook_id) + _require_editor(session, book) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + + raw_bindings = (body or {}).get("bindings") + if not isinstance(raw_bindings, list): + raise err(400, "sheet_bindings_required", + "Choose the database cells whose saved Views should be copied.") + + shadows = [] + seen = set() + view_memo = {} + for raw in raw_bindings: + try: + binding = sheet_mod.clean_binding(raw) + except sheet_mod.SheetRefusal as exc: + raise err(400, exc.code, exc.message) + scope = binding["scope"] + view_id = str(scope.get("view_id") or "") + if not view_id: + continue + database = _require_database(session, binding["database"]) + slot = (database, view_id) + if slot in seen: + continue + seen.add(slot) + nodes, conj = _view_filter(session, database, view_id, scope.get("view_name", ""), + memo=view_memo) + local = _materialize_copy_binding( + binding, {"conj": conj, "nodes": nodes}) + shadows.append({"database": database, "viewId": view_id, + "filter": local["scope"]["filter"]}) + return {"shadows": shadows} + + +@router.post("/sheets/{workbook_id}/cells/{addr}/compute") +def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """W54-T32 (instruction 9): recompute ONE stored bound cell and persist just that cell. + + ⭐⭐ THE WHOLE OF THE TICKET IS THAT `refresh_sheet` WAS THE ONLY DOOR THAT COMPUTED. Saving a + binding wrote a `stale` cell and nothing ever resolved it, so the shimmer that means "a number + is coming" ran until somebody pressed Refresh. Refresh answers by recomputing EVERY bound cell + and every formula in the workbook, which is why it is slow and why the save path must not call + it: one cell over `ut_odoo_orders` measured 55 to 90 ms through `preview`, against a whole + workbook pass plus a full document write. + + ⛔ IT IS A WRITE DOOR, so `_require_editor` runs FIRST, exactly as it does on `refresh_sheet` + and for the same C5 reason: this ends in a store write, and a viewer's numbers must never land + in the owner's document. A viewer's own re-resolve still happens in memory on the READ path. + + ⛔⛔ THE STORED `binding` IS NEVER REWRITTEN HERE, AND THAT IS D-674's HALF OF THIS TICKET. + Only `state`, `value`, `display`, `refreshed_at` and `refusal` are touched on a COPY of the + cell; the binding object is carried through verbatim. So a binding that arrived without a + `period` cannot acquire one by being computed, whatever a later editor does with the panel. + + ⛔⛔ AND IT FILLS THE ONE SCOPE-CACHE SLOT IT JUST COMPUTED, WHICH IS THE TRAP IN THIS FILE. + `_reresolved_for_reader` serves a non-owner from `core.sheets.cached_scope_value`, keyed + `(database, wall, binding scope)`; persisting a new number and leaving the old slot behind + would make the very next GET serve the number this door was called to replace. There is no + per-cell clear, and `clear_scope_cache` belongs to `refresh_sheet` alone under R3, so the + surgical answer is `fill_scope_cache` on exactly the slot this caller is entitled to, which is + the same call and the same key `_reresolved_for_reader` makes after its own resolve. + + ⚠ NO FORMULA PASS RUNS HERE, DELIBERATELY. `_recomputed_tab_cells` recomputes the WHOLE + workbook's formulas, which the prototype measured at 2.9 s as a FLOOR for the owner's 11,300 + formulas: fifty times the cost of the one number this door exists to deliver. The client + evaluates its own formulas as the value lands (`sheetState.ts`), the ordinary cell PATCH + already runs the server-side pass on the very next commit, and Refresh still runs it for + everyone. A formula's STORED value therefore stays as last computed until one of those, which + is what a stale bound cell already did before this ticket. + + ⚠ `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or + its next PATCH would carry a revision the server has already moved past. + """ + from core import sheets + + try: + book = sheets.get_workbook(_store(session), workbook_id) + _require_editor(session, book) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + + key = str(addr or "").replace("$", "").upper() + #: ⚠ THE TAB IS NAMED BY THE CALLER WHEN IT KNOWS ONE, and looked up by address when it does + #: not, which is `drill_cell`'s behaviour. Two tabs may legally hold the same address, so a + #: client that has a tab in hand should say which, or the first one wins. + ref = str((body or {}).get("tab") or "").strip() + tab, cell = None, None + if ref: + try: + tab = sheets.find_tab(book, ref) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + cell = (tab.get("cells") or {}).get(key) + else: + for candidate in book.get("sheets") or (): + if key in (candidate.get("cells") or {}): + tab, cell = candidate, candidate["cells"][key] + break + if not isinstance(cell, dict): + raise err(404, "cell_missing", f"Cell {key} is not filled in.") + if cell.get("kind") != "bound": + raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.") + + tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "") + binding = cell.get("binding") or {} + stamp = _now_iso() + saved = dict(cell) + limits, resolved = [], None + try: + resolved = _resolve_binding(session, binding) + except HTTPException as exc: + #: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door + #: and a cell refused by that one read identically. It is PERSISTED rather than raised: + #: a cell left `stale` would shimmer forever, and the reason belongs on the cell. + detail = getattr(exc, "detail", None) or {} + info = detail.get("error") if isinstance(detail, dict) else None + saved["state"] = "refused" + saved["value"] = None + saved["display"] = "" + saved["refusal"] = { + "code": (info or {}).get("code", "sheet_unresolved"), + "message": (info or {}).get("message", "This cell could not be refreshed."), + } + else: + limits = resolved["limits"] + saved["state"] = "fresh" + saved["value"] = resolved["value"] + saved["display"] = "" if resolved["value"] is None else str(resolved["value"]) + saved["refreshed_at"] = stamp + saved.pop("refusal", None) + + try: + landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved}) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + + if resolved is not None: + #: Best effort, exactly as `_reresolved_for_reader` fills its own slot: a cache write that + #: fails costs a future recompute and never a wrong number. + try: + sheets.fill_scope_cache(_store(session), workbook_id, tab_ref + "!" + key, + _binding_scope_key(session, + str(binding.get("database") or ""), + binding), + resolved["value"], stamp, resolved["database"]) + except Exception: # noqa: BLE001 + pass + + wire = dict(saved) + #: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no + #: such key: it describes THIS resolve's truncation, not the cell. + if limits: + wire["limits"] = limits + return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key} + + @router.get("/sheets/{workbook_id}/cells/{addr}/drill") def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)): """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17). @@ -1330,8 +2173,12 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s raise err(400, exc.code, exc.message) database = _require_database(session, binding["database"]) - cur, spec, cols = _mirror(session)(database) + # ⭐⭐ W54-T36 — THE WALL AND THE SCOPE ARE READ FIRST, AND THE MIRROR IS BUILT OVER WHAT THEY + # NAME. The drill probes `COUNT(*)` and lists ids, so on a cell whose scope is a plain view or + # an empty filter the honest column set is EMPTY and only the id column is materialised. + # MEASURED on tenant #0's `customer_data` (3,673 records, 47 columns), warm, through this + # route: 8,437 ms, of which 6,137 ms was materialising columns no query here reads. wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter") if perm_scope.row_scope_applies(session.user, database) else None) wall_nodes, wall_conj = _fe_parts(wall_tree) @@ -1343,6 +2190,19 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s else: scope_nodes, scope_conj = _fe_parts(scope.get("filter")) + # ⚠ THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never + # reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a + # different window would answer that condition against different values. The popup would then + # list rows the cell's own number never counted. + # ⭐ W54-T36 MEASURED WHAT THAT WINDOW COSTS, because the cheaper mirror was the obvious guess: + # `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the + # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from + # the column set instead, where the time actually is. + cur, spec, cols = _mirror(session)( + database, metric_window=sheet_mod.period_window(binding.get("period")), + needed=(sheet_mod.filter_columns_named(scope_nodes) + | sheet_mod.filter_columns_named(wall_nodes))) + count_sql, count_params, limits = sheet_mod.build_aggregate_query( spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes, scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj) @@ -1360,3 +2220,156 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s else: out["filter"] = scope.get("filter") return out + + +def _resolve_home_tab(book, tabs, names, ref): + """The tab an UNQUALIFIED address in a `resolve-bindings` request belongs to. + + ⚠ `ref` MAY BE A TAB `id` OR ITS NAME, exactly as `GET /sheets/{id}/tabs/{tab}` accepts both, + and a ref this workbook does not carry is a NAMED 404 rather than an empty answer. The flat + `{addr: value}` body of contract C1 has no room to say "that tab is gone", so the only place + that sentence can be told is the status line, and an answer of `{}` would read to the client + as "none of those cells are bound" - a different problem with a different fix. + + ⚠ NO `tab` AT ALL MEANS THE FIRST TAB, which is what `_tab_scope_names` already makes an + unqualified reference mean everywhere else in this file. + """ + from core import sheets + ref = str(ref or "").strip() + if not ref: + return names[0] if names else "" + try: + found = sheets.find_tab(book, ref) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + for tab, name in zip(tabs, names): + if tab is found: + return name + return names[0] if names else "" + + +def _asked_key(asked, home, fr): + """One requested address, as the `TAB!ADDR` key the whole-workbook pass already uses. + + ⭐ A QUALIFIED REFERENCE KEEPS ITS OWN TAB and an unqualified one lands on `home`, which is + the same rule `_recompute_formulas` gives a formula's own references. `'Sales data'!B4` is the + quoted spelling a name with a space wears inside a formula, so the quotes are stripped here + rather than making that cell unaddressable through this door. + """ + text = str(asked or "").strip() + if "!" not in text: + return fr.a1_key(text, home) + sheet, _, addr = text.rpartition("!") + sheet = sheet.strip() + if len(sheet) >= 2 and sheet[0] == "'" and sheet[-1] == "'": + sheet = sheet[1:-1].replace("''", "'") + return fr.a1_key(addr, sheet or home) + + +@router.post("/sheets/{workbook_id}/resolve-bindings") +def resolve_bindings(workbook_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{addr: value}` for the NAMED bound cells only (contract C1, ruling R2). + + ⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL. + Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the + client still needs the server for is the handful of DATABASE-BOUND cells that formula leans + on. `POST /sheets/{id}/refresh` keeps recomputing the WHOLE workbook and is untouched: load + and import still need it, and lane A still calls it for those two paths. What moved off the + commit path is the loop, not the door. + + request: `{"cells": ["B4", "'Sales data'!C9", ...], "tab": ""}` + `tab` is the tab an unqualified address belongs to and defaults to the first tab. + + response: `{"B4": 6283327.0, "Sales data!C9": null}` - the flat map contract C1 names, keyed + by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the + reference it already holds and never has to re-derive this file's key spelling. + + ⛔⛔ A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS, AND THAT IS THE ONE + DISTINCTION THE SHAPE CAN CARRY. An address that is empty, holds a typed literal or holds a + formula is not this door's business - lane A's evaluator already has those values in hand - + so asking for one costs nothing and answers nothing. + + ⛔⛔ AND HERE IS WHAT THE SHAPE CANNOT CARRY, STATED RATHER THAN HIDDEN: a bound cell whose + binding was REFUSED just now (a database wall, a row wall, a scope condition this database + cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching + rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it. + `POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full + `{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is + the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD + amendment rather than fixed by widening the shape on one lane's authority, because lane A is + the consumer and a second shape agreed by nobody is worse than a known gap. + + ⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a + refusal is caught per cell and the siblings still answer. + + ⭐⭐ THE BATCH PAYS FOR ITS ASSEMBLY ONCE, WHICH IS MOST OF THE REASON THIS DOOR TAKES A LIST + AT ALL. A bound cell on a compiled registry grid (`customer_data`, `product_data`) costs a + full workspace assembly - measured at 20 to 130 seconds on this box after W53-T13 - so N cells + resolved one request each would cost N of them. Three memos make one request pay once: + - `session._sheet_assemblies`, keyed by `table_key` inside `_registry_assembly` (W54-T36), + so the assembly itself is built once per database however many mirrors are cut from it. + - `session._sheet_mirrors`, keyed `(table_key, metric_window)` inside `_mirror`, so every + cell on one database at one period shares one temp table. W54-T36 also gave each slot the + COLUMN SET it was materialised over: a narrower cell reuses a wider slot, and a cell + naming a column the slot lacks rebuilds at the union rather than per cell. Both hang off + the Session, which `require_session` builds fresh per request and drops with it. + - `view_memo`, keyed by `view_id`, so a saved-view scope is looked up once for the batch. + ⚠ AND IDENTICAL BINDINGS RESOLVE ONCE. Two cells carrying the same database, aggregate, scope + and period cannot disagree inside one request, so the second one reads the first one's answer + rather than running the same SQL again. Per REQUEST only, for the same reason `view_memo` is: + a saved view's filter may be edited between two calls. + + ⛔ A READ DOOR, LIKE `preview_cell` AND `drill_cell`, NOT A WRITE ONE LIKE `refresh_sheet`. + It calls neither `_require_editor` nor `put_workbook`, so C5's hazard - a viewer's narrower + numbers being SAVED into the owner's document - cannot arise here: nothing this door computes + is ever stored. The per-database wall still runs on every cell, inside `_resolve_binding`. + + ⚠ NO CAP ON HOW MANY CELLS MAY BE ASKED FOR (standing rule 1). The client asks for the cells + one commit actually touched; the memos above are what keep a large batch from multiplying + database work, and a batch's cost is dominated by the number of distinct (database, period) + pairs in it rather than by the number of cells. + """ + from core import sheets + import formula_runtime as fr + + try: + book = sheets.get_workbook(_store(session), workbook_id) + except sheets.SheetRefusal as exc: + raise _refuse(exc) + + payload = dict(body or {}) + tabs = [t for t in (book.get("sheets") or ()) if isinstance(t, dict)] + names = _tab_scope_names(tabs) + home = _resolve_home_tab(book, tabs, names, payload.get("tab")) + + bound = {} + for tab, name in zip(tabs, names): + for addr, cell in (tab.get("cells") or {}).items(): + if isinstance(cell, dict) and cell.get("kind") == "bound": + bound[fr.a1_key(addr, name)] = cell + + view_memo = {} + by_binding = {} + answer = {} + for asked in (payload.get("cells") or ()): + label = str(asked or "") + if not label: + continue + cell = bound.get(_asked_key(label, home, fr)) + if cell is None: + continue + binding = cell.get("binding") or {} + slot = json.dumps(binding, sort_keys=True, default=str) + if slot not in by_binding: + try: + by_binding[slot] = _resolve_binding(session, binding, + view_memo=view_memo)["value"] + except HTTPException: + # ⛔ SWALLOWED HERE AND NOWHERE ELSE. See the docstring: the flat body has no + # room for a cause, and failing the whole batch on one walled cell would take + # every sibling's number down with it. The same refusal still travels in full on + # the refresh and read doors, which is where the cell's own sentence comes from. + by_binding[slot] = None + answer[label] = by_binding[slot] + return answer diff --git a/api/routes_slack.py b/api/routes_slack.py index 6f49fe88dd7fc8679153bd9ffacadef5999a890d..cb549f279229b6cc6fa156ee0990d9d7c5e3df92 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -18,7 +18,7 @@ live specimen of identity that LOOKS like a permission and is not. `core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` below, which is the whole projection and is nine lines. Study: -`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). +`AutoWiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a @@ -450,7 +450,7 @@ def create_channel_agent(body: dict = Body(default=None), for a in existing.values(): if isinstance(a, dict) and str(a.get("channel") or "") == channel: raise err(409, "channel_taken", - "that channel already has an agent — edit it instead of adding a second") + "that channel already has an agent; edit it instead of adding a second") aid = secrets.token_urlsafe(9) rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) @@ -554,7 +554,7 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), if (fresh.get("perms") or {}) != cleaned: # The one direction this must never fail in: telling an administrator the wall is up when # it is not. - raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") + raise err(503, "store_unavailable", "the permissions were not saved; the agent is UNCHANGED") return {"id": str(agent_id), "perms": fresh.get("perms") or {}, "perms_v": int(fresh.get("perms_v") or 0)} @@ -615,7 +615,7 @@ def _is_empty(value): False is precisely the kind of write somebody needs to approve, and a naive `if not value` drops both rows silently — the approver then reads a card that does not describe the write they are approving. The MIT licence gate for this borrowing is recorded in - `.claude/wiki/research/opentag-adoption.md`. + `AutoWiki/research/opentag-adoption.md`. """ return value is None or (isinstance(value, str) and value.strip() == "") @@ -636,7 +636,7 @@ def approval_card(action, values, agent=None): "action": str(action or "")[:80], "rows": shown, "truncated": dropped, - "note": (f"{dropped} more field(s) not shown — open the record to review them before " + "note": (f"{dropped} more field(s) not shown; open the record to review them before " f"approving." if dropped else ""), "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", } @@ -905,7 +905,7 @@ async def slack_events(request: Request): team = str(payload.get("team_id") or slug) if not _rate_ok(f"{slug}:{team}", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") event = payload.get("event") if isinstance(payload.get("event"), dict) else {} channel = str(event.get("channel") or "") @@ -953,7 +953,7 @@ async def slack_interact(request: Request): if not isinstance(payload, dict): raise _refuse() if not _rate_ok(f"{slug}:interact", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + raise err(429, "too_many_requests", "too many requests; wait a moment and try again") actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) @@ -974,7 +974,7 @@ async def slack_interact(request: Request): "rejected": "Rejected. Nothing was changed.", # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, # so an approved card today performs nothing and says exactly that rather than "Approved." - "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", + "no_committer": "Approved, but this workspace has nothing configured to carry it out yet.", "failed": "That change could not be completed. Nothing was saved; ask the bot again.", "committed": "Approved.", }.get(verdict, "That approval is no longer available. Ask the bot again.")} diff --git a/api/routes_tables.py b/api/routes_tables.py index 64f462cb2d4e744bc680605e566c0436d4327d09..c6b1d7cd39880750563dcff17fc96d5158bae101 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -53,6 +53,119 @@ from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") +class _MetricReadRuntime: + """Lend the tenant's read doors while refusing lazy schema/workspace migrations.""" + def __init__(self, runtime): + self._runtime = runtime + + def __getattr__(self, name): + if name in {"get", "get_projection", "revision", "available", "exists", "tenant", "pool_cache", + "key", "assert_datastore_matches", "odoo_paused", "odoo_source", "odoo_flag_key", + "datastore_path", "store_namespace"}: + return getattr(self._runtime, name) + if name in {"put", "update", "upload_bytes", "delete_path"}: + raise RuntimeError("Scoped Metrics require an existing, readable database definition.") + raise AttributeError(name) + + +def _related_product_fields(session, rows): + """Resolve authored Metric definitions through the existing workspace and field wall.""" + import aios_grid + from core import grid_events, perm_scope, shared_overlay + from modules import product_data as pd + + pids = frozenset(r["pid"] for r in rows) + ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[], + hidden_keys=set(), admin=session.admin, fallback_ws=None, + seen_ids={}, st=session.runtime, scope_key="product", + table=pd.table_ops(session.runtime)) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) + _, fields, _, _ = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key="product", with_cohorts=False, + fields_base=aios_grid.product_fields(), st=session.runtime) + have = {f["key"] for f in fields} + fields = list(fields) + [dict(f, source="overlay", shared=True) + for key, f in shared_overlay.fields(pd.TABLE_KEY, st=session.runtime).items() + if key not in have] + hidden = perm_scope.hidden_keys(session.user, "product_data", fields, st=session.runtime) + return [f for f in fields if f.get("measure") and f["key"] not in hidden] + + +@router.post("/tables/{table_key}/related-metrics") +def related_metrics(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C6: a visible Products ordered edge, origin customer and real destination ids. + + The request supplies only grid identities and a named relation. Fields, periods, source + identities and values are read from this tenant; nothing is authored by this endpoint. + """ + import copy + from core import perm_scope, relations + import routes_grid + import routes_products + import automation_engine + + body = body if isinstance(body, dict) else {} + pin = body.get("related") or {} + pids = body.get("pids") + if (table_key != "product_data" or not isinstance(pin, dict) + or pin.get("table") != "customer_data" + or type(pin.get("record")) is not int or pin["record"] <= 0 + or not isinstance(pids, list) or not pids + or any(type(pid) is not int or pid <= 0 for pid in pids)): + raise err(400, "bad_metric_scope", "Name the origin customer and related product records.") + viewer = copy.copy(session) + viewer.runtime = _MetricReadRuntime(session.runtime) + try: + # Both endpoints and the actual bridge are permission-bearing databases. + for key in ("customer_data", "product_data", "ut_odoo_order_lines"): + viewer.require(key) + entries = routes_grid.related_path_entries(viewer, "customer_data", 1) + via = pin.get("via") + edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via) + if (edge is None or edge.get("target") != "product_data" + or edge.get("via", {}).get("field") != "products_ordered"): + raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") + source_fields = routes_grid._source_fields(viewer, "customer_data") + hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime) + if "products_ordered" in hidden: + raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.") + customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime) + if not any(r.get("pid") == pin["record"] for r in customers): + raise err(403, "metric_record_refused", "The origin record is not available to this account.") + products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime) + requested = set(pids) + selected = [r for r in products if r.get("pid") in requested] + if (len(selected) != len(requested) + or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)): + raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.") + # A bridge restriction that cannot be translated must never be silently dropped. + bridge_fields = perm_scope.scoped_fields(viewer.user, "ut_odoo_order_lines", st=viewer.runtime) + declared = perm_scope.entry(viewer.user, "ut_odoo_order_lines") or {} + if (perm_scope.row_scope_applies(viewer.user, "ut_odoo_order_lines") + or (not viewer.admin and declared.get("hiddenFields")) + or not {"partner_id", "product_id"}.issubset({f["key"] for f in bridge_fields})): + raise err(409, "metric_scope_unsupported", "Order-line permission restrictions cannot be applied to these Metrics yet.") + scopes = [perm_scope.derive_pool_scope(viewer.user, key) + for key in ("customer_data", "product_data", "ut_odoo_order_lines")] + teams = {team for team, _ in scopes if team is not None} + agents = {agent for _, agent in scopes if agent is not None} + if len(teams) > 1 or len(agents) > 1: + raise err(403, "metric_scope_conflict", "The related databases have incompatible permission scopes.") + fields = _related_product_fields(viewer, products) + values, reasons = automation_engine.related_product_metric_values( + viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer), + team_id=next(iter(teams), None), agent=next(iter(agents), None)) + return {"ok": True, "values": values, "fields": [f["key"] for f in fields], + "scope": {"customer": pin["record"], + "products": {r["pid"]: r["product_id"] for r in selected}}, + "reason": " ".join(reasons)} + except (perm_scope.Denied, perm_scope.UnknownTable): + raise err(403, "metric_record_refused", "The related records are not available to this account.") + except (perm_scope.Unresolvable, RuntimeError, ValueError): + raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.") + + def _ut(): import core.user_tables as user_tables return user_tables diff --git a/api/routes_uploads.py b/api/routes_uploads.py index 59612b0480088456b93e71d1407c7f72b405b049..c9de0e7ed81783a05ed937549221cd75d377975a 100644 --- a/api/routes_uploads.py +++ b/api/routes_uploads.py @@ -60,7 +60,7 @@ async def tabular_preview(file: UploadFile = File(...), data = await file.read() if len(data) > MAX_BYTES: raise err(400, "too_large", - "that file is over 5 MB — export just the column you need and retry") + "that file is over 5 MB; export just the column you need and retry") if name.endswith(".csv"): text = data.decode("utf-8-sig", errors="replace") rows = iter(csv.reader(io.StringIO(text))) @@ -70,7 +70,7 @@ async def tabular_preview(file: UploadFile = File(...), wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True) ws = wb.worksheets[0] except Exception: - raise err(400, "bad_file", "that workbook could not be read — is it a real .xlsx?") + raise err(400, "bad_file", "that workbook could not be read; is it a real .xlsx?") rows = ws.iter_rows(values_only=True) else: raise err(400, "bad_type", diff --git a/api/test_odoo_agent_snapshot.py b/api/test_odoo_agent_snapshot.py new file mode 100644 index 0000000000000000000000000000000000000000..757ab55b42123ff7373920b6dadad46b49d393da --- /dev/null +++ b/api/test_odoo_agent_snapshot.py @@ -0,0 +1,111 @@ +"""Exercise the real invoice reader and its oracle on synthetic DuckDB snapshots.""" +from contextlib import redirect_stdout +import io +import socket +import unittest +from unittest.mock import patch + +import duckdb +import verify_odoo_relational as gate + + +class AgentSnapshotTests(unittest.TestCase): + def setUp(self): + self.network = patch.object(socket.socket, "connect", side_effect=AssertionError("network forbidden")) + self.network_spy = self.network.start() + self.addCleanup(self.network.stop) + self.con = duckdb.connect(":memory:") + self.addCleanup(self.con.close) + self.con.execute(""" + CREATE TABLE res_partner (id INTEGER, name VARCHAR, agent BOOLEAN); + INSERT INTO res_partner VALUES (10, 'Synthetic customer', false), + (70, 'Alpha', true), (80, 'Beta', true), (90, 'Gamma', true); + CREATE TABLE account_move ( + id INTEGER, name VARCHAR, partner_id INTEGER, partner_name VARCHAR, + invoice_date DATE, invoice_date_due DATE, amount_untaxed_signed DOUBLE, + amount_residual_signed DOUBLE, payment_state VARCHAR, move_type VARCHAR, + state VARCHAR, invoice_origin VARCHAR); + INSERT INTO account_move VALUES + (1,'I1',10,'Synthetic customer','2026-09-01','2026-10-01',10,10,'not_paid','out_invoice','posted',''), + (2,'I2',10,'Synthetic customer','2026-09-01','2026-10-01',20,20,'not_paid','out_invoice','posted',''), + (3,'R3',10,'Synthetic customer','2026-09-01','2026-10-01',-5,-5,'not_paid','out_refund','posted',''), + (4,'Draft',10,'Synthetic customer','2026-09-01','2026-10-01',5,5,'not_paid','out_invoice','draft',''), + (5,'Bill',10,'Synthetic customer','2026-09-01','2026-10-01',5,5,'not_paid','in_invoice','posted',''), + (6,'No partner',NULL,'','2026-09-01','2026-10-01',5,5,'not_paid','out_invoice','posted',''); + CREATE TABLE account_invoice_line_agent (invoice_id INTEGER, agent_id INTEGER, agent_name VARCHAR); + INSERT INTO account_invoice_line_agent VALUES + (1,70,'Alpha'), (1,70,'Alpha'), (2,80,'Beta'), (2,70,'Alpha'), + (4,90,'Gamma'), (5,90,'Gamma'), (6,90,'Gamma'); + """) + self.saved_pass, self.saved_fail = gate.PASS[:], gate.FAIL[:] + self.addCleanup(self.restore_verdict) + + def restore_verdict(self): + gate.PASS[:] = self.saved_pass + gate.FAIL[:] = self.saved_fail + + def prove(self, mutate=None): + real = gate.rel.read_invoices + + def read(cur, *args, **kwargs): + rows = real(cur, *args, **kwargs) + return mutate(rows) if cur is self.con and mutate else rows + + gate.PASS.clear() + gate.FAIL.clear() + with patch.object(gate.rel, "read_invoices", side_effect=read), redirect_stdout(io.StringIO()): + gate._prove_invoice_agent_and_links(self.con) + self.network_spy.assert_not_called() + return gate.FAIL[:] + + def test_healthy_new_snapshot_is_not_compared_to_august_counts(self): + self.assertEqual(self.prove(), []) + + def test_growth_and_new_distinct_agent_still_reconcile(self): + self.con.execute("INSERT INTO account_move SELECT 7,'I7',partner_id,partner_name,invoice_date,invoice_date_due,amount_untaxed_signed,amount_residual_signed,payment_state,move_type,state,invoice_origin FROM account_move WHERE id=1") + self.con.execute("INSERT INTO account_invoice_line_agent VALUES (7,90,'Gamma')") + self.assertEqual(self.prove(), []) + + def assert_row_parity_refuses(self, mutate): + failures = self.prove(mutate) + self.assertTrue(any("raw commission rows" in name for name in failures), failures) + + def test_wrong_existing_agent_with_equal_counts_is_refused(self): + def mutate(rows): + return [dict(r, agent_id=80) if r['_id']=='1' else r for r in rows] + self.assert_row_parity_refuses(mutate) + + def test_wrong_name_with_equal_counts_is_refused(self): + self.assert_row_parity_refuses(lambda rows: [dict(r, agent='Wrong') if r['_id']=='1' else r for r in rows]) + + def test_multi_agent_truncation_is_refused(self): + self.assert_row_parity_refuses(lambda rows: [dict(r, agent='Alpha', agent_id=70) if r['_id']=='2' else r for r in rows]) + + def test_dropped_invoice_is_refused(self): + self.assert_row_parity_refuses(lambda rows: [r for r in rows if r['_id']!='3']) + + def test_duplicate_invoice_is_refused(self): + self.assert_row_parity_refuses(lambda rows: rows + [dict(rows[0])]) + + def test_missing_commission_source_is_a_failure(self): + self.con.execute('DROP TABLE account_invoice_line_agent') + self.assertTrue(any('raw commission oracle' in name for name in self.prove())) + + def test_always_false_comparator_fails_its_positive_control(self): + with patch.object(gate, '_invoice_agents_match', return_value=False): + self.assertTrue(any('NC baseline:' in name for name in self.prove())) + + def test_always_true_comparator_fails_all_three_negative_controls(self): + with patch.object(gate, '_invoice_agents_match', return_value=True): + failures = self.prove() + self.assertEqual(sum(name.startswith('NC: one ') for name in failures), 3) + + def test_dated_measurement_is_preserved(self): + from copy import deepcopy + historical = deepcopy(gate.rel._INVOICE_COLUMN_FILL) + self.assertEqual(self.prove(), []) + self.assertEqual(gate.rel._INVOICE_COLUMN_FILL, historical) + + +if __name__ == '__main__': + unittest.main() diff --git a/api/test_odoo_customer_population.py b/api/test_odoo_customer_population.py new file mode 100644 index 0000000000000000000000000000000000000000..7437557914297663da53030dda29b4585fd97b27 --- /dev/null +++ b/api/test_odoo_customer_population.py @@ -0,0 +1,204 @@ +"""Synthetic source/mirror tests of the real population oracle, never live proof.""" +from contextlib import redirect_stdout +import io +import socket +import unittest +from unittest.mock import patch + +import duckdb +import verify_odoo_relational as gate +import core.odoo as source + + +class Source: + def __init__(self, rows): + self.rows = rows + self.calls = [] + self.ignore_drop = False + self.count_calls = 0 + + def select(self, model, domain): + assert model == 'res.partner' + rows = self.rows + for field, op, value in domain or []: + predicates = { + '=': lambda item: item == value, + '>': lambda item: item > value, + '<=': lambda item: item <= value, + 'in': lambda item: item in value, + 'not in': lambda item: item not in value, + } + rows = [row for row in rows if predicates[op](row[field])] + return sorted(rows, key=lambda row: row['id']) + + def search_count(self, model, domain=None): + self.count_calls += 1 + self.calls.append(('count', domain)) + # A deliberately broken provider double leaves the deletion invisible. + if self.ignore_drop and self.count_calls == 4: + return 1 + return len(self.select(model, domain)) + + def search_read(self, model, domain=None, fields=None, limit=None, order=None): + self.calls.append(('read', domain, fields, limit)) + return [{key: row[key] for key in fields} + for row in self.select(model, domain)[:limit]] + + +class CustomerPopulationTests(unittest.TestCase): + def setUp(self): + self.network = patch.object(socket.socket, 'connect', side_effect=AssertionError('network forbidden')) + self.network_spy = self.network.start() + self.addCleanup(self.network.stop) + self.con = duckdb.connect(':memory:') + self.addCleanup(self.con.close) + self.con.execute(""" + CREATE TABLE res_partner (id INTEGER, name VARCHAR, customer_rank INTEGER, + active BOOLEAN, write_date VARCHAR); + INSERT INTO res_partner VALUES + (90101,'Synthetic document partner',0,true,'2026-09-01 00:00:00'), + (90102,'Synthetic customer A',1,true,'2026-09-01 00:00:00'), + (90103,'Synthetic customer B',2,true,'2026-09-01 00:00:00'); + CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR); + INSERT INTO sale_order VALUES (90101,'sale'); + CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR); + CREATE TABLE _sync_state (entity VARCHAR, cursor_wd VARCHAR); + INSERT INTO _sync_state VALUES ('res_partner','2026-09-03 01:02:25'); + """) + self.source = Source([ + dict(id=90101, customer_rank=0, active=True, write_date='2026-09-01 00:00:00'), + dict(id=90102, customer_rank=1, active=True, write_date='2026-09-01 00:00:00'), + dict(id=90103, customer_rank=2, active=True, write_date='2026-09-01 00:00:00'), + ]) + self.saved_pass, self.saved_fail = gate.PASS[:], gate.FAIL[:] + self.addCleanup(self.restore) + + def restore(self): + gate.PASS[:], gate.FAIL[:] = self.saved_pass, self.saved_fail + + def prove(self, mutate=None): + real = gate.rel.read_customers + + def read(cur, *args, **kwargs): + rows = real(cur, *args, **kwargs) + if cur is self.con: + # SQL has no ORDER BY: put a valid document-only row first. + rows.sort(key=lambda row: int(row[gate.rel.JOIN_KEY])) + if mutate: + rows = mutate(rows) + return rows + + gate.PASS.clear() + gate.FAIL.clear() + output = io.StringIO() + with patch.object(source, 'get_odoo', return_value=self.source), \ + patch.object(gate.rel, 'read_customers', side_effect=read), redirect_stdout(output): + gate._prove_customer_population(self.con) + self.network_spy.assert_not_called() + self.output = output.getvalue() + return gate.FAIL[:] + + def test_document_only_first_row_does_not_break_healthy_control(self): + self.assertEqual(self.prove(), []) + + def test_already_red_population_cannot_make_ineffective_drop_pass(self): + self.con.execute('DELETE FROM res_partner WHERE id=90103') + self.source.ignore_drop = True + failures = self.prove() + self.assertTrue(any('AS OF THE MIRROR' in name for name in failures)) + self.assertTrue(any('NC:' in name and 'ONE in-scope' in name for name in failures), failures) + + def assert_gap(self, marker, mutate=None): + failures = self.prove(mutate) + name = next(name for name in failures if 'AS OF THE MIRROR' in name) + self.assertIn('got=1, want=2', name) + self.assertIn(marker, name) + self.assertNotIn('90103', name) + self.assertNotIn('Synthetic', name) + reads = [call for call in self.source.calls if call[0] == 'read'] + self.assertTrue(reads) + self.assertTrue(all(call[2] == ['id'] for call in reads)) + + def test_missing_raw_partner_is_classified_without_identity(self): + self.con.execute('DELETE FROM res_partner WHERE id=90103') + self.assert_gap('mirror_absent=1') + + def test_stale_rank_is_distinguished_from_missing_partner(self): + self.con.execute('UPDATE res_partner SET customer_rank=0 WHERE id=90103') + self.assert_gap('mirror_rank_not_positive=1') + + def test_stale_active_is_distinguished_from_missing_partner(self): + self.con.execute('UPDATE res_partner SET active=false WHERE id=90103') + self.assert_gap('mirror_inactive=1') + + def test_eligible_raw_partner_omitted_by_reader_is_classified(self): + self.assert_gap('mirror_eligible_omitted=1', + lambda rows: [row for row in rows if row[gate.rel.JOIN_KEY] != 90103]) + + def test_newer_source_partner_stays_outside_both_counts(self): + self.source.rows.append(dict(id=90104, customer_rank=1, active=True, + write_date='2026-09-04 00:00:00')) + self.assertEqual(self.prove(), []) + self.assertEqual(sum(call[0] == 'read' for call in self.source.calls), 1) + + def test_diagnostic_does_not_cap_the_missing_population(self): + self.con.execute('DELETE FROM res_partner WHERE id=90103') + self.prove() + diagnostic = [call for call in self.source.calls if call[0] == 'read' + and any(clause[1] == 'not in' for clause in call[1])] + self.assertEqual(len(diagnostic), 1) + self.assertIsNone(diagnostic[0][3]) + + def test_diagnostic_rejects_changed_or_incomplete_source_sets(self): + self.con.execute('DELETE FROM res_partner WHERE id=90103') + original = self.source.search_read + for response in ([], [{'id': 90103}, {'id': 90103}], [{'id': 90102}], + [{'id': '90103'}], [{'id': True}], [{'id': -1}]): + with self.subTest(response=response): + def read(model, domain, **kwargs): + if any(clause[1] == 'not in' for clause in domain): + return response + return original(model, domain, **kwargs) + with patch.object(self.source, 'search_read', side_effect=read): + failures = self.prove() + self.assertTrue(any('source_changed_or_incomplete' in name for name in failures)) + + def test_diagnostic_failure_preserves_red_and_hides_exception_contents(self): + self.con.execute('DELETE FROM res_partner WHERE id=90103') + original = self.source.search_read + + def read(model, domain, **kwargs): + if any(clause[1] == 'not in' for clause in domain): + raise RuntimeError('Synthetic secret client details') + return original(model, domain, **kwargs) + + with patch.object(self.source, 'search_read', side_effect=read): + failures = self.prove() + self.assertTrue(any('gap_diagnostic=unavailable' in name for name in failures)) + self.assertNotIn('Synthetic secret client details', self.output) + + def test_duplicate_mirror_identity_is_not_counted_as_a_valid_diagnosis(self): + self.con.execute('UPDATE res_partner SET customer_rank=0 WHERE id=90103') + self.con.execute('INSERT INTO res_partner SELECT * FROM res_partner WHERE id=90103') + self.assertTrue(any('duplicate_mirror_identity' in name for name in self.prove())) + + def test_empty_reader_has_no_vacuous_negative_control(self): + failures = self.prove(lambda rows: []) + self.assertTrue(any('AS OF THE MIRROR' in name for name in failures)) + self.assertTrue(any('NC: ONE in-scope' in name for name in failures)) + + def test_forged_witness_cannot_remove_an_unrelated_row(self): + with patch.object(self.source, 'search_read', return_value=[{'id': 99999}]): + failures = self.prove() + self.assertTrue(any('NC: ONE in-scope' in name for name in failures)) + + def test_one_control_cannot_hide_two_missing_partners(self): + self.con.execute('DELETE FROM res_partner WHERE id IN (90102,90103)') + failures = self.prove() + self.assertTrue(any('got=0, want=2' in name and 'mirror_absent=2' in name + for name in failures)) + self.assertTrue(any('NC: ONE in-scope' in name for name in failures)) + + +if __name__ == '__main__': + unittest.main() diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py new file mode 100644 index 0000000000000000000000000000000000000000..9b304ffb9971690ee33dc31e39fec9165a67ceea --- /dev/null +++ b/api/test_odoo_rank_refresh.py @@ -0,0 +1,411 @@ +"""Exercise the real mirror writer with synthetic Odoo rows and temporary DuckDB only.""" +from contextlib import ExitStack, redirect_stdout +import ast +import copy +import io +import os +from pathlib import Path +import socket +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import ANY, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'platform')) +import duckdb +from harness import datastore as ds + + +class RankRefreshTests(unittest.TestCase): + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.net = self.stack.enter_context(patch.object( + socket.socket, 'connect', side_effect=AssertionError('external network forbidden'))) + self.temp = self.stack.enter_context(tempfile.TemporaryDirectory()) + self.path = str(Path(self.temp) / 'synthetic.duckdb') + self.spec = copy.deepcopy(ds.ENTITIES['res_partner']) + self.spec['backfill_fields'] = [] + self.spec['m2m'] = [] + self.rows = [dict(id=pid, name='Synthetic partner', customer_rank=rank, + active=active, write_date='2026-09-01 00:00:00') + for pid, rank, active in [(1, 0, True), (2, 1, True), (3, 2, False)]] + with duckdb.connect(self.path) as con: + ds._ensure_table(con, 'res_partner', self.spec) + ds._upsert(con, 'res_partner', self.spec, self.rows) + con.execute('''CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, + phase VARCHAR, last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)''') + ds._save_state(con, 'res_partner', 'live', 3, '2026-09-03 01:02:25') + self.calls = [] + self.stack.enter_context(patch.object(ds, 'connect', side_effect=lambda: duckdb.connect(self.path))) + self.stack.enter_context(patch.dict(ds.ENTITIES, res_partner=self.spec)) + self.stack.enter_context(patch.object(ds, 'source_paused', return_value=False)) + self.stack.enter_context(patch.object(ds.O, 'search_read', side_effect=self.read)) + + def read(self, model, domain, fields, limit=None, order=None): + self.assertEqual(model, 'res.partner') + self.calls.append((copy.deepcopy(domain), list(fields), limit)) + rows = self.rows + for field, op, value in domain: + if op == 'in': + rows = [r for r in rows if r[field] in value] + elif op == '>': + rows = [r for r in rows if r[field] > value] + elif op == '>=': + rows = [r for r in rows if r[field] >= value] + else: + self.fail('unexpected source domain') + return [{k: row.get(k) for k in ['id'] + list(fields)} + for row in sorted(rows, key=lambda r: r['id'])[:limit]] + + def sync(self): + result = ds.sync_entity('res_partner', max_batches=1, log=lambda *_: None) + self.net.assert_not_called() + return result + + def ranks(self): + with duckdb.connect(self.path) as con: + return con.execute('SELECT id, customer_rank FROM res_partner ORDER BY id').fetchall() + + def report_and_rebuild_reason(self): + # Execute the production decision without importing main's app/startup effects. + source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8')) + decision = next(node for node in source.body if isinstance(node, ast.FunctionDef) + and node.name == '_resync_rebuild_reason') + age = next(node for node in source.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) + and target.id == '_RESYNC_REBUILD_MAX_AGE_SECONDS' + for target in node.targets)) + ns = {} + exec(compile(ast.Module(body=[age, decision], type_ignores=[]), + 'main.py:resync-decision', 'exec'), ns) + report = ds.sync_all(entities=['res_partner'], max_batches=1, log=lambda *_: None) + self.net.assert_not_called() + return report, ns['_resync_rebuild_reason'](report, {}, 100, 101) + + def test_rank_only_change_triggers_resident_projection_rebuild(self): + self.rows[0]['customer_rank'] = 4 + report, reason = self.report_and_rebuild_reason() + self.assertEqual(self.ranks(), [(1, 4), (2, 1), (3, 2)]) + self.assertEqual(reason, 'source-change') + self.assertEqual(report['res_partner']['pulled'], 1) + + def observe_resident_recovery(self, failed_passes): + # Run the actual loop with a synthetic clock/source/store. No server is imported. + import harness + import odoo_relational as rel + source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8')) + functions = {'_store_resync_loop', '_resync_rebuild_reason', + '_meta_rebuild_force_reason'} + nodes = [node for node in source.body + if isinstance(node, ast.FunctionDef) and node.name in functions + or isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) + and target.id == '_RESYNC_REBUILD_MAX_AGE_SECONDS' + for target in node.targets)] + with duckdb.connect(self.path) as con: + con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)') + con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)') + cycle, current_pass = [0], [0] + attempts, projection, sleeps, reports, events = [], [], [], [], [] + total = max(failed_passes, default=1) + 2 + def sleep(seconds): + if len(reports) >= total: + raise StopIteration('bounded synthetic resident loop') + events.append(('sleep', seconds)) + cycle[0] += 1 + sleeps.append(seconds) + def sync(manual, passes): + events.append(('sync', passes)) + self.assertFalse(manual) + self.assertEqual(passes, len(reports) + 1) + current_pass[0] = passes + if passes == 2: + self.rows[0]['customer_rank'] = 4 + report = ds.sync_all(entities=['res_partner'], max_batches=1, log=lambda *_: None) + reports.append(report['res_partner']['pulled']) + return report, {}, None + def rebuild(why, previous, meta_state): + attempts.append(current_pass[0]) + if current_pass[0] in failed_passes: + return False + with duckdb.connect(self.path) as con: + projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con)) + return True + clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + current_pass[0] * 60, + strftime=lambda *_: 'synthetic-time', gmtime=lambda: ()) + ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None], + _resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False, + tenant='synthetic', tenantSource='test', tenantSeconds=60, + envSeconds=60, disagreement=None), + _resync_mirror_pass=sync, _pull_meta=lambda *_: True, + _rebuild_odoo_relational=rebuild, + _rebuild_meta_relational=lambda *_: {}, + _rebuild_amazon_relational=lambda *_: {}) + exec(compile(ast.Module(body=nodes, type_ignores=[]), + 'main.py:resident-loop', 'exec'), ns) + runtime = SimpleNamespace(get_runtime=lambda *_: object()) + with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \ + patch.object(harness, 'runtime', runtime, create=True), \ + redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(StopIteration, 'bounded synthetic'): + ns['_store_resync_loop']() + self.net.assert_not_called() + self.resident_events = events + return attempts, projection, sleeps, reports + + def test_resident_loop_runs_first_pass_before_cadence_sleep(self): + self.observe_resident_recovery(set()) + self.assertEqual(self.resident_events[0], ('sync', 1)) + + def test_trusted_gate_refreshes_only_its_marked_isolated_copy(self): + import core.odoo as odoo + import verify_odoo_relational as gate + gate.PASS.clear() + gate.FAIL.clear() + path = Path(self.path) + no_client = object() + previous_client = getattr(odoo._tlocal, 'client', no_client) + with patch.dict(os.environ, {'AIOS_TRUSTED_GATE_SOURCE_ISOLATED': '1'}), \ + patch.object(ds, 'path_for', return_value=path), \ + patch.object(ds, 'sync_entity', return_value=('live', 2)) as sync, \ + patch.object(ds, 'use_path') as use_path, \ + patch.object(odoo, 'OdooClient', return_value=object()): + self.assertTrue(gate._prepare_isolated_trusted_mirror()) + sync.assert_called_once_with('res_partner', log=ANY) + use_path.assert_called_once_with(path) + self.assertEqual(gate.FAIL, []) + if previous_client is no_client: + self.assertFalse(hasattr(odoo._tlocal, 'client')) + else: + self.assertIs(odoo._tlocal.client, previous_client) + self.net.assert_not_called() + + def test_failed_trusted_gate_refresh_still_releases_writer_and_client(self): + import core.odoo as odoo + import verify_odoo_relational as gate + gate.PASS.clear() + gate.FAIL.clear() + path = Path(self.path) + no_client = object() + previous_client = getattr(odoo._tlocal, 'client', no_client) + with patch.dict(os.environ, {'AIOS_TRUSTED_GATE_SOURCE_ISOLATED': '1'}), \ + patch.object(ds, 'path_for', return_value=path), \ + patch.object(ds, 'sync_entity', side_effect=RuntimeError('synthetic failure')), \ + patch.object(ds, 'use_path') as use_path, \ + patch.object(odoo, 'OdooClient', return_value=object()): + self.assertFalse(gate._prepare_isolated_trusted_mirror()) + use_path.assert_called_once_with(path) + self.assertEqual(gate.PASS, []) + self.assertEqual(gate.FAIL, [ + 'trusted isolated mirror completed the product customer-rank refresh before live oracles' + ]) + if previous_client is no_client: + self.assertFalse(hasattr(odoo._tlocal, 'client')) + else: + self.assertIs(odoo._tlocal.client, previous_client) + self.net.assert_not_called() + + def test_unmarked_gate_never_writes_a_mirror(self): + import verify_odoo_relational as gate + with patch.dict(os.environ, {}, clear=True), \ + patch.object(ds, 'sync_entity') as sync: + self.assertTrue(gate._prepare_isolated_trusted_mirror()) + sync.assert_not_called() + + def test_failed_rank_rebuild_retries_on_next_unchanged_pass(self): + attempts, projection, sleeps, reports = self.observe_resident_recovery({2}) + self.assertEqual(reports, [0, 1, 0, 0]) + self.assertEqual(attempts, [1, 2, 3]) + self.assertEqual(projection, [1, 2]) + self.assertEqual(sleeps, [60] * 3) + + def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self): + attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3}) + self.assertEqual(reports, [0, 1, 0, 0, 0]) + self.assertEqual(attempts, [1, 2, 3, 4]) + self.assertEqual(projection, [1, 2]) + self.assertEqual(sleeps, [60] * 4) + + def test_successful_rank_rebuild_still_skips_unchanged_pass(self): + attempts, projection, sleeps, reports = self.observe_resident_recovery(set()) + self.assertEqual(reports, [0, 1, 0]) + self.assertEqual(attempts, [1, 2]) + self.assertEqual(projection, [1, 2]) + self.assertEqual(sleeps, [60] * 2) + + def test_unchanged_refresh_remains_a_confirmed_noop(self): + for _ in range(2): + report, reason = self.report_and_rebuild_reason() + self.assertEqual(report['res_partner']['pulled'], 0) + self.assertIsNone(reason) + + def test_cleared_and_null_ranks_report_only_changed_existing_rows(self): + self.rows[0]['customer_rank'] = False # Odoo's unset numeric becomes SQL NULL. + self.rows[1]['customer_rank'] = 0 + self.rows.append(dict(id=4, customer_rank=9, active=True, + name='Absent mirror partner', write_date='2026-09-01 00:00:00')) + report, reason = self.report_and_rebuild_reason() + self.assertEqual(self.ranks(), [(1, None), (2, 0), (3, 2)]) + self.assertEqual(reason, 'source-change') + self.assertEqual(report['res_partner']['pulled'], 2) + report, reason = self.report_and_rebuild_reason() + self.assertEqual(report['res_partner']['pulled'], 0) + self.assertIsNone(reason) + + def test_refresh_and_delta_both_reach_the_existing_report(self): + self.rows[0]['customer_rank'] = 5 + self.rows[1].update(name='Delta source name', write_date='2026-09-04 00:00:00') + report, reason = self.report_and_rebuild_reason() + self.assertEqual(reason, 'source-change') + self.assertEqual(report['res_partner']['pulled'], 2) + with duckdb.connect(self.path) as con: + self.assertEqual(con.execute('SELECT name FROM res_partner WHERE id=2').fetchone(), + ('Delta source name',)) + + def test_bounded_refresh_reports_each_changed_page_then_zero(self): + original = self.read + def small_page(model, domain, fields, limit=None, order=None): + return original(model, domain, fields, limit=1, order=order) + self.rows[0]['customer_rank'] = 5 + self.rows[2]['customer_rank'] = 6 + with patch.object(ds.O, 'search_read', side_effect=small_page), \ + patch.object(ds, 'COL_BACKFILL_MAX_BATCHES', 1): + for expected in (1, 0, 1, 0): + report, reason = self.report_and_rebuild_reason() + self.assertEqual(report['res_partner']['pulled'], expected) + self.assertEqual(reason, 'source-change' if expected else None) + + def test_lost_refresh_signal_mutant_is_detected_at_resident_consumer(self): + actual = ds._backfill_columns + def suppress(*args, **kwargs): + actual(*args, **kwargs) + return 0 + with patch.object(ds, '_backfill_columns', side_effect=suppress): + with self.assertRaises(AssertionError): + self.test_rank_only_change_triggers_resident_projection_rebuild() + + def test_false_refresh_signal_mutant_is_detected_by_noop_control(self): + actual = ds._backfill_columns + def inflate(*args, **kwargs): + result = actual(*args, **kwargs) + return 1 if kwargs.get('repeat') else result + with patch.object(ds, '_backfill_columns', side_effect=inflate): + with self.assertRaises(AssertionError): + self.test_unchanged_refresh_remains_a_confirmed_noop() + + def test_rank_change_behind_cursor_reaches_customer_reader(self): + self.rows[0]['customer_rank'] = 4 + self.assertEqual(self.sync()[0], 'live') + self.assertEqual(self.ranks(), [(1, 4), (2, 1), (3, 2)]) + import odoo_relational as rel + with duckdb.connect(self.path) as con: + con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)') + con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)') + self.assertEqual({r[rel.JOIN_KEY] for r in rel.read_customers(con)}, {1, 2}) + self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25') + + def test_completed_refresh_repeats_and_clears_rank(self): + self.sync() + self.rows[1]['customer_rank'] = 0 + self.rows[2]['customer_rank'] = 7 + self.sync() + self.assertEqual(self.ranks(), [(1, 0), (2, 0), (3, 7)]) + refresh = [call for call in self.calls if 'write_date' not in call[1]] + self.assertTrue(refresh) + self.assertTrue(all(call[1] == ['customer_rank'] for call in refresh)) + self.assertTrue(all(('active', 'in', [True, False]) in call[0] for call in refresh)) + + def test_bounded_refresh_resumes_then_starts_another_cycle(self): + original = self.read + def small_page(model, domain, fields, limit=None, order=None): + return original(model, domain, fields, limit=1, order=order) + for row in self.rows: + row['customer_rank'] += 10 + with patch.object(ds.O, 'search_read', side_effect=small_page), \ + patch.object(ds, 'COL_BACKFILL_MAX_BATCHES', 1): + self.sync() + self.assertEqual(self.ranks(), [(1, 10), (2, 1), (3, 2)]) + self.sync() + self.assertEqual(self.ranks(), [(1, 10), (2, 11), (3, 2)]) + self.sync() + self.assertEqual(self.ranks(), [(1, 10), (2, 11), (3, 12)]) + self.sync() # Empty source page proves this cycle finished. + self.rows[0]['customer_rank'] = 20 + self.sync() + self.assertEqual(self.ranks()[0], (1, 20)) + + def test_undeclared_entity_does_not_add_recurring_source_work(self): + self.spec.pop('refresh_fields', None) + self.sync() + self.assertEqual(len(self.calls), 1) + self.assertIn('write_date', self.calls[0][1]) + + def test_completed_schema_backfill_does_not_suppress_rank_refresh(self): + self.spec['backfill_fields'] = ['customer_rank'] + marker = 'res_partner.cols.customer_rank' + with duckdb.connect(self.path) as con: + con.execute('INSERT INTO _sync_state VALUES (?,?,?,?,?,?)', + [marker, 'done', 3, None, 3, '2026-09-02 00:00:00']) + self.rows[0]['customer_rank'] = 8 + self.sync() + self.assertEqual(self.ranks()[0], (1, 8)) + with duckdb.connect(self.path) as con: + self.assertEqual(con.execute('SELECT * FROM _sync_state WHERE entity=?', [marker]).fetchone(), + (marker, 'done', 3, None, 3, '2026-09-02 00:00:00')) + + def test_failed_page_resumes_without_advancing_delta_cursor(self): + original = self.read + connections = [] + def connect(): + con = duckdb.connect(self.path) + connections.append(con) + return con + def interrupted(model, domain, fields, limit=None, order=None): + if ('id', '>', 1) in domain: + raise RuntimeError('synthetic source interruption') + return original(model, domain, fields, limit=1, order=order) + self.rows[0]['customer_rank'] = 5 + self.rows[1]['customer_rank'] = 6 + with patch.object(ds, 'connect', side_effect=connect), \ + patch.object(ds.O, 'search_read', side_effect=interrupted): + with self.assertRaisesRegex(RuntimeError, 'synthetic source interruption'): + self.sync() + with self.assertRaises(duckdb.ConnectionException): + connections[0].execute('SELECT 1') + with duckdb.connect(self.path) as con: + self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25') + self.assertEqual(con.execute("SELECT phase,last_id FROM _sync_state " + "WHERE entity='res_partner.refresh.customer_rank'").fetchone(), + ('running', 1)) + self.calls.clear() + self.sync() + self.assertEqual(self.ranks(), [(1, 5), (2, 6), (3, 2)]) + self.assertIn(('id', '>', 1), self.calls[0][0]) + + def test_only_rank_changes_and_missing_rows_are_not_invented(self): + self.rows[0].update(customer_rank=9, name='Changed source name', active=False) + self.rows.append(dict(id=4, customer_rank=9, active=True, + name='Absent mirror partner', write_date='2026-09-01 00:00:00')) + self.sync() + self.assertEqual(self.ranks(), [(1, 9), (2, 1), (3, 2)]) + with duckdb.connect(self.path) as con: + self.assertEqual(con.execute('SELECT name,active,write_date FROM res_partner WHERE id=1').fetchone(), + ('Synthetic partner', True, '2026-09-01 00:00:00')) + + def test_refresh_removal_mutant_is_detected_by_customer_membership(self): + self.spec.pop('refresh_fields', None) + with self.assertRaises(AssertionError): + self.test_rank_change_behind_cursor_reaches_customer_reader() + + def test_pause_never_opens_mirror_or_source(self): + with patch.object(ds, 'source_paused', return_value=True), \ + patch.object(ds, 'connect') as connect: + self.assertEqual(self.sync(), (ds.PAUSED_PHASE, 0)) + connect.assert_not_called() + self.assertEqual(self.calls, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/api/test_odoo_relational_verdict.py b/api/test_odoo_relational_verdict.py new file mode 100644 index 0000000000000000000000000000000000000000..8b0d5a294493cc1d33f2b6b677ad3cdc64f79fd1 --- /dev/null +++ b/api/test_odoo_relational_verdict.py @@ -0,0 +1,128 @@ +"""Hermetic tests of relational gate verdicts; these never prove source parity.""" +from contextlib import ExitStack, redirect_stdout +import io +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +import verify_odoo_relational as gate +from harness import datastore + + +class RelationalVerdictTests(unittest.TestCase): + def setUp(self): + self.original_pass = gate.PASS[:] + self.original_fail = gate.FAIL[:] + self.original_taken = gate._MIRROR_TAKEN[:] + gate.PASS.clear() + gate.FAIL.clear() + gate._MIRROR_TAKEN[0] = False + self.capture = io.StringIO() + self.output = redirect_stdout(self.capture) + self.output.__enter__() + + def tearDown(self): + self.output.__exit__(None, None, None) + gate.PASS[:] = self.original_pass + gate.FAIL[:] = self.original_fail + gate._MIRROR_TAKEN[:] = self.original_taken + + def main_verdict(self, cursor_result): + # Run the real gate orchestration. Source oracles are disabled only in + # this unit test; the prerequisite check and final exit stay real. + with ExitStack() as stack: + for name in vars(gate): + if name.startswith("_prove_"): + stack.enter_context(patch.object(gate, name, return_value=None)) + stack.enter_context(patch.object(gate, "run", return_value={})) + stack.enter_context(patch.object(gate, "real_cursor", return_value=cursor_result)) + stack.enter_context(patch.object(datastore, "ready", return_value=False)) + return gate._main(SimpleNamespace(negative_control=False)) + + def test_absent_mirror_cannot_pass_readability(self): + self.assertEqual(self.main_verdict((None, "missing")), 1) + self.assertTrue(any("[missing]" in name for name in gate.FAIL)) + + def test_unopenable_mirror_cannot_pass_readability(self): + self.assertEqual(self.main_verdict((None, "open-failed")), 1) + self.assertTrue(any("[open-failed]" in name for name in gate.FAIL)) + + def test_copy_failure_remains_fatal_even_if_source_can_open(self): + gate._MIRROR_TAKEN[0] = True + self.assertEqual(self.main_verdict((Mock(), "synthetic")), 1) + + def test_readable_mirror_passes_and_releases_probe_cursor(self): + cursor = Mock() + self.assertEqual(self.main_verdict((cursor, "synthetic")), 0) + cursor.close.assert_called_once_with() + + def test_probe_releases_real_duckdb_before_different_configuration(self): + import duckdb + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "synthetic.duckdb" + duckdb.connect(str(path)).close() + with patch.object(datastore, "path_for", return_value=path): + self.assertTrue(gate._check_mirror_admission()) + with duckdb.connect(str(path), read_only=False) as con: + self.assertEqual(con.execute("SELECT 42").fetchone(), (42,)) + + def test_probe_close_failure_remains_red(self): + cursor = Mock() + cursor.close.side_effect = RuntimeError("PRIVATE_SENTINEL") + self.assertEqual(self.main_verdict((cursor, "synthetic")), 1) + self.assertTrue(any("[close-failed]" in name for name in gate.FAIL)) + self.assertNotIn("PRIVATE_SENTINEL", self.capture.getvalue()) + + def test_missing_path_has_safe_machine_readable_reason(self): + with tempfile.TemporaryDirectory() as tmp: + with patch.object(datastore, "path_for", return_value=Path(tmp) / "absent.duckdb"): + self.assertEqual(gate.real_cursor(), (None, "missing")) + + def test_open_error_does_not_disclose_exception_or_path(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "synthetic.duckdb" + path.touch() + with patch.object(datastore, "path_for", return_value=path), patch( + "duckdb.connect", side_effect=RuntimeError("PRIVATE_SENTINEL /private/source") + ): + result = gate.real_cursor() + self.assertEqual(result, (None, "open-failed")) + self.assertNotIn("PRIVATE_SENTINEL", repr(result)) + + def control_verdict(self, *, background=False, detect=True): + def observed_run(corrupt=None): + gate.check("synthetic ordinary passing assertion", True) + if background: + gate.check("synthetic source parity failure", False) + if corrupt is not None and detect: + gate.check("synthetic corruption detected", False) + with patch.object(gate, "run", side_effect=observed_run): + return gate._negative_control() + + def test_negative_controls_preserve_prior_failures(self): + gate.check("synthetic prerequisite refused", False) + self.assertEqual(self.control_verdict(), 0) + self.assertIn("synthetic prerequisite refused", gate.FAIL) + self.assertNotIn("synthetic corruption detected", gate.FAIL) + + def test_uncorrupted_control_baseline_failure_survives(self): + self.assertEqual(self.control_verdict(background=True), 0) + self.assertIn("synthetic source parity failure", gate.FAIL) + self.assertNotIn("synthetic corruption detected", gate.FAIL) + + def test_control_replays_do_not_inflate_passing_population(self): + gate.check("synthetic original passing assertion", True) + self.assertEqual(self.control_verdict(), 0) + self.assertEqual(gate.PASS.count("synthetic ordinary passing assertion"), 0) + self.assertIn("synthetic original passing assertion", gate.PASS) + self.assertFalse(gate.FAIL) + + def test_background_red_cannot_kill_an_undetected_mutation(self): + self.assertEqual(self.control_verdict(background=True, detect=False), 1) + self.assertIn("synthetic source parity failure", gate.FAIL) + + +if __name__ == "__main__": + unittest.main() diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 3bcfbc449bd9d93b18e97fc6d2051613a8f62fc4..d9906fc0a4c6fa055d3e7edc1d9e331e8785877b 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -1594,7 +1594,7 @@ def cohort_field(label='Locked views'): 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', 'default': False, 'filterable': False, 'derived': True, 'multi': True, 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' - 'so grouping by this column puts a customer under EVERY view they are in — the ' + 'so grouping by this column puts a customer under EVERY view they are in; the ' 'group counts therefore add up to more than the record count, which stays the ' 'number of distinct customers. Read-only: membership changes only by adding or ' 'removing customers on the locked view itself.', @@ -3940,10 +3940,10 @@ def scope_counts(shown, matched, total): """ shown, matched, total = int(shown), int(matched), int(total) if matched > total: - raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " + raise ValueError(f"matched ({matched}) exceeds total ({total}); a filter cannot match " f"more rows than the scope holds") if shown > matched: - raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " + raise ValueError(f"shown ({shown}) exceeds matched ({matched}); the window cannot hold " f"more rows than the filter matched") return {"shown": shown, "matched": matched, "total": total, "windowed": True} diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 1e8957c1ae82bb852a9abcc376000663474ca9e0..9073470106cf5678964c36ad62db315246b1f678 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -118,6 +118,22 @@ "default": true, "description": "A Google Maps link to this customer's address." }, + { + "key": "lat", + "label": "Latitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, + { + "key": "lon", + "label": "Longitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, { "key": "customer_since", "label": "Customer since", @@ -166,17 +182,6 @@ "default": true, "description": "How many days late this customer is running against their own usual ordering rhythm." }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, { "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", "key": "ar_open", diff --git a/platform/core/auth_google.py b/platform/core/auth_google.py index 43f656d44078f18c71eaf50fb179e6a6a189f3b3..2ee8fc8c34f735793c0e0210502626949d17d8c9 100644 --- a/platform/core/auth_google.py +++ b/platform/core/auth_google.py @@ -11,8 +11,8 @@ DESIGN (recorded now so the full build has a contract to meet): core/users.py account via its `email` attribute (set_access(..., email=...)); an unknown email FAILS CLOSED with "no account for this Google identity" — Google login never creates users, so the per-BU/per-module grant model stays the only door. - - The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the - bootstrap master; removing password auth is a separate, owner-approved step. + - The password gate stays beside it. APP_PASSWORD may seed a fresh registry, but it has no + authentication bypass; removing stored-password auth is a separate, owner-approved step. ENV (placeholders — see .env.example additions in docs/GOOGLE_LOGIN_TODO.md): GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console @@ -50,7 +50,7 @@ def auth_url(state: str, code_challenge: str) -> str: session. Template guard: raises until the TODO checklist lands. """ raise NotImplementedError( - "Google login is a template — complete docs/GOOGLE_LOGIN_TODO.md, then implement " + "Google login is a template; complete docs/GOOGLE_LOGIN_TODO.md, then implement " "auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, " "code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)." ) @@ -61,7 +61,7 @@ def exchange_code(code: str, code_verifier: str) -> dict: Template guard: raises until implemented (requests.post with client_id/secret, redirect_uri, grant_type=authorization_code, code, code_verifier).""" raise NotImplementedError( - "Google login is a template — implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md." + "Google login is a template; implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md." ) @@ -70,7 +70,7 @@ def verify_id_token(id_token: str) -> dict: return its claims ({'email', 'email_verified', 'name', ...}). MUST reject email_verified=False. Template guard: raises until implemented.""" raise NotImplementedError( - "Google login is a template — implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md." + "Google login is a template; implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md." ) diff --git a/platform/core/connector_directory.py b/platform/core/connector_directory.py index 317f09ff681372da6d93dd94ddd2d6bf58f2b144..44d1091244c03c39c2074372fe99def3486d9e48 100644 --- a/platform/core/connector_directory.py +++ b/platform/core/connector_directory.py @@ -18,7 +18,7 @@ it, and there is nothing to find. Measured against the live Monid API on 2026-09 tags, categories, metrics, hints`, and none of those carries a sidebar name. So the 14 categories are reachable only from Monid's WEB UI SIDEBAR, which is what - `.claude/wiki/waves/wave49/proto/monid-catalogue.md` captured by hand. `MONID_PROVIDERS` is + `AutoWiki/waves/wave49/proto/monid-catalogue.md` captured by hand. `MONID_PROVIDERS` is regenerated by re-reading that file, not by calling an endpoint. ⚠ AND THE CATALOGUE IS A FLOOR, NOT A CENSUS. `discover` returns providers absent from the proto @@ -87,7 +87,7 @@ MONID_CATEGORY_ENDPOINTS = { #: ⭐⭐ THE AUTHORED CROSSWALK: (provider name, endpoint count, the categories it is filed under). #: #: REGENERATION, because there is no generator: re-read the `PROVIDERS` line of -#: `.claude/wiki/waves/wave49/proto/monid-catalogue.md`, which is `name count` pairs separated by +#: `AutoWiki/waves/wave49/proto/monid-catalogue.md`, which is `name count` pairs separated by #: the middle dot. The NAME and the COUNT come from there unchanged and `verify_catalog.py` proves #: it, pair by pair, so a typo here goes red rather than quiet. The CATEGORIES are the authored #: half: a judgement about which sidebar rows a provider sits under, made once, here. @@ -228,6 +228,13 @@ MONID_PROVIDERS = ( #: NO category, appears only under `All`, and `verify_catalog.py` names it and goes red. NATIVE_CATEGORIES = { "odoo": ("Data", "Finance", "E-commerce"), + # ⭐⭐ W52-T01 — FOUND UNFILED WHILE BUILDING THE `spawns_tables` TIER, AND FIXED HERE RATHER + # THAN LEFT: `amazon_sp` joined `TOKEN_CONNECTORS` (routes_connectors.py) in wave 31 and this + # dict never gained a matching row, so `verify_catalog.py`'s C4 leg + # (`every-native-connector-is-filed`) was red before this ticket touched the file. Filed the + # same way Odoo is: it reaches orders, fees and listings, which is Data plus the two shelves + # Odoo itself sits under for the commerce side of the same story. + "amazon_sp": ("Data", "E-commerce", "Finance"), "webhooks": ("Utilities", "Web Tools"), "scraper": ("Scrapers", "Social Media"), "stripe": ("Finance", "E-commerce"), @@ -300,6 +307,12 @@ MONID_ACTION_STATE = "available" #: state with the neutral tone by construction (the wave-9 law: no client union over a server #: vocabulary), so the client renders it without a change. Any gate holding a HAND-TYPED copy of #: the five-word set has to learn the sixth. +#: +#: ⚠ W53-T31 (R11) — TELEGRAM IS ONE OF THE 117, AND IT STAYS ONE. `MONID_PROVIDERS` carries +#: `("Telegram", 7, ...)`: a vendor listing, not a connector we have built or planned. Wave 53 +#: adopted only its CREDENTIAL SHAPE on paper and built no runtime path, so the row keeps this +#: word and Telegram is deliberately absent from `routes_connectors.PLANNED`. The build plan, +#: with the two keychain steps it needs first, is `AutoWiki/research/ideation/telegram-bot.md`. CATALOGUE_STATE = "catalogue" #: ⭐⭐ W49-T14 · RULINGS R21 + R24 — WHICH CONNECTORS HAVE A FIELD CATALOGUE, AUTHORED IN CODE. @@ -320,10 +333,26 @@ CATALOGUE_STATE = "catalogue" #: behind them to introspect. Membership is by KEY, and every catalogue key wears #: `MONID_KEY_PREFIX`, so the two sets cannot overlap by construction. #: -#: ⚠ ONE ENTRY TODAY, AND ADDING A SECOND IS A REAL DECISION. A connector named here promises the -#: Databases tab a field catalogue AND a route that can enable one of its fields; without both, -#: the tab renders toggles that lead nowhere, which is R8's fake affordance. -INTROSPECTABLE_CONNECTORS = ("odoo",) +#: ⚠ ADDING AN ENTRY IS A REAL DECISION. A connector named here promises the Databases tab a +#: field catalogue; whether it also gets FIELD TOGGLES is a separate flag, because a toggle needs +#: a write route behind it and one that always 409s is R8's fake affordance. +#: +#: ⭐⭐ W52 · R6 AMENDS R24, AND `amazon_sp` IS WHY. R24 restricted this screen to *introspectable* +#: connectors, meaning a live field catalogue — a `fields_get`. Owner ruling R6 (wave 52): +#: **the test is now "is there a real schema artifact", not "is there a `fields_get`"**. SP-API +#: has NO introspection endpoint at any price, and Amazon publishes versioned, vendor-authored +#: JSON Schemas on GitHub which satisfy the ruling's purpose better than a probe would: they are +#: pinned by blob sha and cannot drift under us between reads. +#: +#: ⛔ AND AMAZON GETS THE SCREEN WITHOUT THE TOGGLES, DELIBERATELY. The field-toggle door is +#: `PATCH /connectors/odoo/schema/{field_key}`; there is no Amazon equivalent and this wave does +#: not build one, so `_detail_introspection` answers `canToggleFields: False` for it. Naming it +#: here buys the DISCLOSURE and nothing more, which is exactly what R6 asked for. +#: +#: ⚠ Its catalogue is honestly INCOMPLETE and says so: only two of the seven tables have a +#: published artifact. The other five measure their columns from the report's own header at first +#: sync, so before that they have none — see `harness/amazon_schema.PROVENANCE`. +INTROSPECTABLE_CONNECTORS = ("odoo", "amazon_sp") def is_introspectable(connector_key) -> bool: @@ -334,6 +363,38 @@ def is_introspectable(connector_key) -> bool: return str(connector_key or "") in INTROSPECTABLE_CONNECTORS +#: ⭐⭐ W52-T01 · RULING R1 / CONTRACT C1 — WHICH CONNECTORS SPAWN A RELATIONAL DATABASE OF THEIR +#: OWN, THE TEST THAT TIERS THE CONNECTORS PAGE. +#: +#: R1, verbatim: *"A connector is a DATABASE connector when it spawns `ut_*` tables browsable +#: under Data. Odoo, Amazon and Meta Ads all qualify, and a future connector qualifies +#: automatically the day it spawns tables."* +#: +#: ⛔⛔ THIS IS A DIFFERENT QUESTION FROM `INTROSPECTABLE_CONNECTORS`, AND REUSING THAT TUPLE WAS +#: THE TRAP THE TICKET NAMES. `INTROSPECTABLE_CONNECTORS` answers "does this connector publish a +#: LIVE FIELD CATALOGUE" (R24) and Meta Ads is deliberately absent from it, because Meta has no +#: `fields_get` to disclose. But Meta Ads DOES spawn `ut_meta_*` tables +#: (`aios-web/api/meta_relational.py`) even though it publishes no schema, so a Databases tier +#: computed from `INTROSPECTABLE_CONNECTORS` would silently drop Meta Ads from R1's own example. +#: +#: Ground truth, read from the relational spawn modules directly rather than guessed: +#: `odoo_relational.py`, `amazon_relational.py` and `meta_relational.py` each carry a `# THE SPAWN` +#: section registering a `ut__*` prefix through `user_tables.register_connected_prefix` +#: (mirrored for the row-cap question in `routes_connectors.CONNECTED_PREFIXES`, a different +#: subsystem answering a different question — see that constant's own header). A future connector +#: joins this tuple in the same change that ships its relational spawn, exactly as C1 says. +SPAWNS_TABLES_CONNECTORS = ("odoo", "amazon_sp", "meta_ads") + + +def spawns_tables(connector_key) -> bool: + """Does this connector spawn a `ut__*` relational database, browsable under Data (R1)? + + Pure and total: an unknown key does not spawn tables, the fail-closed direction — a connector + is tiered into "Databases" only once its spawn is real, never by a guessed default. + """ + return str(connector_key or "") in SPAWNS_TABLES_CONNECTORS + + #: ⛔ THE KEYS A ROW CARRIES FOR US AND NEVER FOR THE CLIENT (C4). One tuple, read by #: `to_client_row` and by the gate's negative control, so "which keys are server-only" has one #: answer wherever it is asked. @@ -455,8 +516,17 @@ def to_client_row(row): catalogue's, we reason with it on the server, and R27's design rule is that it never reaches a screen in any form. Stripping it here rather than at each composer means there is ONE place to read to know what the wire carries, and one place for the gate to break. + + ⭐⭐ W52-T01 / C1 — AND THIS IS ALSO WHERE `spawns_tables` IS STAMPED, for the same reason. + Every row that reaches this function carries a `key`, native or catalogue alike, so computing + the tier HERE rather than at each composer means there is one place to read for "does this + card belong under Databases" and one place for `verify_catalog.py` to break if it stops being + true. A catalogue row's key is never in `SPAWNS_TABLES_CONNECTORS` (they all wear + `MONID_KEY_PREFIX`), so every one of the 100+ listings answers `False` by construction. """ - return {k: v for k, v in row.items() if k not in SERVER_ONLY_KEYS} + out = {k: v for k, v in row.items() if k not in SERVER_ONLY_KEYS} + out["spawns_tables"] = spawns_tables(row.get("key")) + return out def to_client_rows(rows): diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py new file mode 100644 index 0000000000000000000000000000000000000000..45aa4249d8573e27187f5bd0800b37185f73d06c --- /dev/null +++ b/platform/core/connector_spec.py @@ -0,0 +1,424 @@ +"""connector_spec.py — ONE shared schema spec for every connector's tables, carrying the vendor's +own name for each of them. + +⭐⭐ **THIS FILE IS THE SINGLE SOURCE OF TRUTH FOR A CONNECTOR'S TABLE SHAPE FROM NOW ON.** The +shape already existed twice before it had a name: `aios-web/api/amazon_relational.py` and +`aios-web/api/meta_relational.py` each carry loose `SOURCE` / `LABELS` / `PINNED` / `IDENTITY` / +`LINKS` module dicts saying the same six things about their tables in two different spellings, and +`odoo_relational.py` carries a third. Nobody called it a spec, so nothing could read it generically +and every new consumer (the relation engine, the connector detail door, the Databases tab) had to +learn each connector's dialect. This module generalises those dicts; it does not invent a new model. + + ConnectorTableSpec one table: key, native, source, label, pinned, identity, links + register(connector, ...) add a connector's tables to the registry + tables_for(connector) that connector's specs, in declaration order + spec_for(ut_key) one table's spec, or None + connector_of(ut_key) which connector owns a ut_* key, or None + all_specs() flat ut_key -> spec across every connector + connectors() the connector keys, in registration order + edges_for(key) the relation edges of a CONNECTOR or of ONE table + +⭐ **THE FIFTH KEY IS `native`, AND IT IS THE ONE THAT DID NOT EXIST BEFORE.** `SOURCE` is the +MIRROR table — `amzn_orders`, `meta_campaigns` — which is OUR name for the copy we hold. The +vendor's own identifier for the same thing (`GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL`, +`AdAccount`, `res.partner`) lived only in comments, so no screen and no gate could reach it. + +⛔⛔ **R9 — `label` IS A DEFAULT, `native` IS NEVER EDITABLE.** The `label` here is our SHIPPED +default display name and nothing else. A tenant's override lives in `core/schema_overlay.py`, is +stored per tenant, and **a connector re-sync must never write it**; renaming a database from the +navigation rail is that same override. Nothing in this module is a place to store an override, and +a caller that writes one here has put tenant state in shipped code. + +⛔ **LAYERING: `core` NEVER IMPORTS UP.** `aios-web/api/*` is above `platform/core/`, so this file +must not import `amazon_relational`, `meta_relational` or `odoo_relational`, and it does not import +`harness` either. Every value below is therefore written out LITERALLY, transcribed from the modules +named beside it. + +⭐ **THE TRANSCRIPTION IS HELD TRUE BY A GATE, NOT BY A COMMENT.** `aios-web/api/verify_meta.py` +(`section_connector_spec`) asserts, per table, that this module's `source` / `label` / `pinned` / +`identity` / `links` agree exactly with the live `amazon_relational` and `meta_relational` dicts. +That gate lives up in the api layer precisely because it may import both sides. It is the codebase's +own idiom: `core/relations.py::compiled_pin_matches_filter_sql` proves two SQL spellings agree by a +gate rather than by a comment, and `relations.OPEN_INVOICE_WHERE` is held against +`odoo_relational._AR_WHERE` the same way. +⚠ **WHEN W53-T37 LANDS** and `amazon_relational` / `meta_relational` READ their dicts from here, +those parity checks become tautological and this note should be revisited — but **the checks stay**, +because a tautology is exactly what catches a hand edit that re-introduces a literal on either side. + +⛔ **ODOO IS NOT HERE YET.** W53-T38 writes Odoo's spec in its own file fence. There is deliberately +no `"odoo"` entry and no stub for one; `edges_for("odoo")` returns `[]` until that ticket lands. +""" +from dataclasses import dataclass, replace +import hashlib +import re + +#: ⭐ THE CARDINALITY VOCABULARY, AS A CONSTANT SO EVERY CONSUMER CHECKS MEMBERSHIP RATHER THAN +#: SPELLING. The relation wire (W53-T17) and the parity gate both read this set; a fifth value +#: added by hand in a link tuple goes red instead of travelling silently to a screen. +CARDINALITIES = ("many_to_one", "one_to_many", "one_to_one", "many_to_many") + +# The discoverable Odoo catalogue is runtime data, because two API accounts can read different +# installed modules. Only its stable naming rule belongs in this tenant-independent spec layer. +# A digest closes the otherwise-real collision between model names such as ``x.a_b`` and +# ``x_a.b`` after punctuation is normalised for a ``ut_*`` key. +ODOO_MODEL_TABLE_PREFIX = "ut_odoo_model_" + + +def odoo_model_table_key(model): + """Stable ``ut_*`` key for one vendor model name; blank/invalid input is refused.""" + native = str(model or "").strip().lower() + if not native or not re.fullmatch(r"[a-z0-9_.]+", native): + raise ValueError("an Odoo model name may contain only lowercase letters, digits, dots and underscores") + # 18 + the 14-character prefix + separator + digest leaves room for the connector's + # ``__k_<12 hex>`` namespace under user_tables' 60-character key ceiling. + stem = re.sub(r"[^a-z0-9]+", "_", native).strip("_")[:18] or "model" + digest = hashlib.sha256(native.encode("utf-8")).hexdigest()[:10] + return f"{ODOO_MODEL_TABLE_PREFIX}{stem}_{digest}" + + +def odoo_model_label(model, vendor_label=""): + """Our default database label, while preserving Odoo's own label when it supplied one.""" + label = " ".join(str(vendor_label or "").split()) + if label: + return label[:80] + return str(model or "").replace(".", " ").replace("_", " ").strip().capitalize()[:80] + + +@dataclass(frozen=True) +class ConnectorTableSpec: + """One connector table. ⛔ EXACTLY SEVEN FIELDS, IN THIS ORDER — contract C2 says "exactly", + and `verify_meta.section_connector_spec` reads `dataclasses.fields` to make that word + mechanical. Grouping by connector is done by the two-level `SPECS` registry below, NOT by an + eighth field. + """ + key: str #: our `ut_*` key — the grid a person opens + native: str #: the vendor's OWN identifier. ⛔ NEVER editable (R9) + source: str #: the mirror table `harness/*_store.py` writes + label: str #: our shipped DEFAULT display name. A DEFAULT, never an override (R9) + pinned: str = None #: the title column, or None when the table has no natural one + identity: tuple = () #: the id columns, in FALLING preference + #: ⚠ A TUPLE OF 4-TUPLES `((from_col, target_key, to_col, cardinality), ...)`, AND THE TUPLE IS + #: DELIBERATE: a `list` default on a frozen dataclass is a mutable shared default and defeats + #: the freeze. C2's word "LIST" is about `links` being a SEQUENCE per key — several links from + #: one table, which is Amazon's shape — rather than Meta's single scalar 3-tuple; a tuple + #: satisfies that. **Do not "fix" this back to a list.** `register()` normalises a caller's + #: list into a tuple, so passing either is safe and only tuples are ever stored. + links: tuple = () + + +#: ⭐ THE REGISTRY IS TWO-LEVEL: `{connector_key: {ut_key: ConnectorTableSpec}}`. That is what makes +#: a connector groupable without an eighth dataclass field. Insertion order is declaration order, so +#: `tables_for` and `edges_for` return a connector's tables in the order its store syncs them. +SPECS = {} + + +def _normalise_links(links): + """A caller's list-of-lists becomes a tuple-of-4-tuples. Explicit, so `links` is ONE type + everywhere downstream and the gate can assert 4-tuple-ness without a shape apology.""" + return tuple(tuple(entry) for entry in (links or ())) + + +def register(connector, specs): + """Add one connector's tables. `specs` may be a mapping of `ut_key -> spec` or any iterable of + specs; `links` is normalised to tuples on the way in. Re-registering a connector REPLACES it, + which is what lets a gate's negative control swap the registry and restore it in a `finally`. + """ + rows = list(specs.values()) if isinstance(specs, dict) else list(specs) + SPECS[connector] = {s.key: replace(s, links=_normalise_links(s.links)) for s in rows} + return SPECS[connector] + + +def connectors(): + """The registered connector keys, in registration order.""" + return tuple(SPECS.keys()) + + +def tables_for(connector): + """One connector's specs, in declaration order. Unknown connector -> `()`.""" + return tuple(SPECS.get(connector, {}).values()) + + +def all_specs(): + """Flat `ut_key -> spec` across every connector.""" + out = {} + for tables in SPECS.values(): + out.update(tables) + return out + + +def spec_for(ut_key): + """One table's spec, or None. Never raises.""" + for tables in SPECS.values(): + if ut_key in tables: + return tables[ut_key] + return None + + +def connector_of(ut_key): + """Which connector owns a `ut_*` key, or None.""" + for connector, tables in SPECS.items(): + if ut_key in tables: + return connector + return None + + +def _label(col): + """`amazon_order_id` -> `Amazon order id`. ⛔ THE SAME RULE BOTH CONNECTORS ALREADY USE + (`amazon_relational._label`, `meta_relational._label`), copied because `core` cannot import + them. The parity gate holds the two spellings together.""" + return col.replace("_", " ").strip().capitalize() + + +def edges_for(key): + """The relation edges of a CONNECTOR key (`"amazon"`) or of ONE `ut_*` table key. + + ⭐ **ONE FUNCTION, TWO NAMESPACES, AND THEY CANNOT COLLIDE:** every table key is `ut_`-prefixed + and no connector key is, so `edges_for("amazon")` (every edge of the connector, in table + declaration order) and `edges_for("ut_amzn_order_items")` (just that table's) are never + ambiguous. W53-T38's `edges_for("odoo")` is the connector form. + + Each edge is `{"key", "label", "source", "target", "from_col", "to_col", "cardinality"}` where + `source` is the `ut_*` table the edge starts FROM — needed because the connector form mixes + tables — and: + · `key` is `f"{target}_link__{from_col}"`, Amazon's shape. ⛔ THE SOURCE COLUMN IS IN THE + KEY ON PURPOSE: two links from one table to the same target (orders by id and by sku) would + otherwise collide on one key and the second would silently overwrite the first. Meta's older + `f"{target}_link"` cannot express that, which is why Amazon's convention is the shared one. + · `label` is `f"{target_label} by {_label(from_col)}"`. + + ⚠ **NEVER RAISES.** An unknown key returns `[]`, and a link whose target is not registered still + emits its edge, labelled off the raw target key. A lenient reader is what keeps a partially + ported connector set from blowing up in a caller's hands; the GATE is what asserts that every + target resolves, and that is the right place for that judgement. + """ + specs = all_specs() + if key in SPECS: + rows = list(SPECS[key].values()) + elif key in specs: + rows = [specs[key]] + else: + return [] + out = [] + for spec in rows: + for from_col, target, to_col, cardinality in spec.links: + target_spec = specs.get(target) + target_label = target_spec.label if target_spec else target + out.append({ + "key": f"{target}_link__{from_col}", + "label": f"{target_label} by {_label(from_col)}", + "source": spec.key, + "target": target, + "from_col": from_col, + "to_col": to_col, + "cardinality": cardinality, + }) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# AMAZON — transcribed from `aios-web/api/amazon_relational.py` + `harness/amazon_client.py` +# ═════════════════════════════════════════════════════════════════════════════════════════════ +#: ⭐ `native` IS THE SP-API REPORT TYPE, from `harness/amazon_client.py::REPORT_TYPES[t]["report"]` +#: — the identifier Amazon itself names the data by. ⚠ `orders` and `order_items` share one report +#: on purpose: the flat-file order report is already one row per order ITEM, so orders is the +#: deduplicated head and order_items the lines (see `amazon_store.TABLES`). Two of our tables, one +#: vendor object, which is exactly why `native` and `source` had to be separate keys. +#: ⚠ `identity` is `amazon_relational.IDENTITY` verbatim: a flat-file report has no `id` column the +#: way Graph does, so each table names the columns that identify one of its rows, in falling +#: preference. +#: ⛔ R12 — AMAZON JOINS ONLY TO AMAZON. Nothing here targets `ut_odoo_*`, `product_data` or the +#: hand-maintained ASIN mastersheet. +#: ⚠ EVERY LINK BELOW IS `many_to_one`, and the reason is uniform: each one is a CHILD row naming +#: its PARENT (a line names its order, a pricing row names its listing). Amazon's `LINKS` carries +#: 3-tuples today with the cardinality left implicit; this is where it becomes explicit. +_AMZN = "ut_amzn_" + +AMAZON_TABLES = ( + ConnectorTableSpec( + key=_AMZN + "orders", + native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + source="amzn_orders", + label="Amazon orders", + pinned="amazon_order_id", + identity=("amazon_order_id",), + links=( + ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_AMZN + "order_items", + native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + source="amzn_order_items", + label="Amazon order items", + pinned="amazon_order_id", + identity=("order_item_id", "amazon_order_id"), + links=( + ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), + # ⚠ the order report carries `sku`; listings key on `seller_sku`. Same value, two + # spellings — the reason a link names BOTH columns rather than one shared name. + ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_AMZN + "listings", + native="GET_MERCHANT_LISTINGS_ALL_DATA", + source="amzn_listings", + label="Amazon listings", + pinned="seller_sku", + identity=("seller_sku", "asin1", "asin"), + links=(), # the product spine's HEAD: everything points at it, it points nowhere + ), + ConnectorTableSpec( + key=_AMZN + "settlements", + native="GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", + source="amzn_settlements", + label="Amazon settlements", + pinned="settlement_id", + identity=("settlement_id",), + # ⚠ THE MONEY-TO-ORDER BRIDGE — what turns "an order happened" into "this is what we were + # actually paid for it". The V2 settlement report carries the order id per line. + links=( + ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), + ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_AMZN + "inventory", + native="GET_LEDGER_SUMMARY_VIEW_DATA", + source="amzn_inventory", + label="Amazon inventory", + pinned="asin", + identity=("asin", "fnsku", "msku"), + # ⚠ THE LEDGER KEYS ON `msku`, NOT `sku` — FBA calls the seller's own SKU the MSKU. Same + # value, a third spelling. + links=( + ("msku", _AMZN + "listings", "seller_sku", "many_to_one"), + ("asin", _AMZN + "listings", "asin1", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_AMZN + "pricing", + native="GET_SALES_AND_TRAFFIC_REPORT", + source="amzn_pricing", + label="Amazon pricing and Buy Box", + pinned="parent_asin", + identity=("child_asin", "parent_asin"), + links=( + ("child_asin", _AMZN + "listings", "asin1", "many_to_one"), + ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_AMZN + "brand_analytics", + native="GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", + source="amzn_brand_analytics", + label="Amazon brand analytics", + pinned="search_term", + identity=("search_term",), + # ⚠ keyed by SEARCH TERM, not by product, so it joins to nothing by default. Its ASIN + # columns are the top-clicked ASINs for a term, a different relationship from "this row is + # about this product", and a link would be misleading. + links=(), + ), +) + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# META ADS — transcribed from `aios-web/api/meta_relational.py` +# ═════════════════════════════════════════════════════════════════════════════════════════════ +#: ⭐ `native` IS THE GRAPH API NODE NAME — `AdAccount`, `Campaign`, `AdSet`, `Ad`, `AdCreative`, +#: `AdsInsights`. Meta's own vocabulary for the objects `meta_store` mirrors. +#: ⛔⛔ **`identity` WAS DECLARED HERE FOR THE FIRST TIME, NOT TRANSCRIBED.** It is honest rather +#: than invented: `meta_store._read` keys every row on the object's own Graph `id`, so `("id",)` is +#: what the five object tables are already identified by in practice. `ut_meta_insights` has no +#: single `id` — the daily insights grain is one row per ad per day — so it takes +#: `("ad_id", "date_start")`. +#: ⭐ W53-T37 AMENDMENT, 2026-09-05 — TWO SENTENCES HERE WERE TRUE FOR ONE TICKET AND ARE NOW +#: FALSE, so they are corrected rather than left to be falsified in one grep. +#: * *"`meta_relational` has NO `IDENTITY` dict at all"* — it has one now, DERIVED FROM THIS +#: DECLARATION, so the value is no longer unverifiable against the live module; what is true +#: instead is that nothing in `meta_relational` READS it yet (`_read` keys rows on the Graph +#: `id` inline), so it is reachable by name for the caller that will ask. +#: * *"Meta's `LINKS` is a SCALAR 3-tuple per key today and its generated edge key is +#: `f"{target}_link"`"* — W53-T37 made it a SEQUENCE keyed by `edges_for`, which is what +#: retired the collision described below. It is no longer latent, it is fixed. +#: The sequence-plus-source-column shape is still the reason this file holds links the way it +#: does: two edges to one target collide on both key and label the moment a scalar spelling +#: declares them, and Meta was one declaration away from that. +_META = "ut_meta_" + +META_TABLES = ( + ConnectorTableSpec( + key=_META + "ad_accounts", + native="AdAccount", + source="meta_ad_accounts", + label="Meta ad accounts", + pinned="name", + identity=("id",), + links=(), # the hierarchy's HEAD: account -> campaign -> ad set -> ad + ), + ConnectorTableSpec( + key=_META + "campaigns", + native="Campaign", + source="meta_campaigns", + label="Meta campaigns", + pinned="name", + identity=("id",), + links=( + ("account_id", _META + "ad_accounts", "account_id", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_META + "adsets", + native="AdSet", + source="meta_adsets", + label="Meta ad sets", + pinned="name", + identity=("id",), + links=( + ("campaign_id", _META + "campaigns", "id", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_META + "ads", + native="Ad", + source="meta_ads", + label="Meta ads", + pinned="name", + identity=("id",), + links=( + ("adset_id", _META + "adsets", "id", "many_to_one"), + ), + ), + ConnectorTableSpec( + key=_META + "creatives", + native="AdCreative", + source="meta_creatives", + label="Meta creatives", + pinned="name", + identity=("id",), + # a creative is referenced BY an ad rather than naming its parent, so it declares no link + # of its own — the same posture as `ut_amzn_listings`. + links=(), + ), + ConnectorTableSpec( + key=_META + "insights", + native="AdsInsights", + source="meta_insights_daily", + label="Meta insights (daily)", + pinned="date_start", + identity=("ad_id", "date_start"), + links=( + ("ad_id", _META + "ads", "id", "many_to_one"), + ), + ), +) + + +register("amazon", AMAZON_TABLES) +register("meta", META_TABLES) + +__all__ = ["CARDINALITIES", "ConnectorTableSpec", "SPECS", "AMAZON_TABLES", "META_TABLES", + "ODOO_MODEL_TABLE_PREFIX", "odoo_model_table_key", "odoo_model_label", + "register", "connectors", "tables_for", "all_specs", "spec_for", "connector_of", + "edges_for"] diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..457a3ae1539d6ab509e32769934148a2fa7d4db6 --- /dev/null +++ b/platform/core/connector_sync.py @@ -0,0 +1,365 @@ +"""Tenant-and-connection scoped connector sync lifecycle state. + +This data-layer module records what a provider runner did. It deliberately does not import or +call providers: the API layer owns runners while every connector shares this durable vocabulary. +""" +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +import time +import uuid + +import core.store as store + + +STATE_KEY = "connector_sync_state" +STATES = ("queued", "syncing", "materializing", "ready", "partial", "error") +_ERROR_LIMIT = 50 +_DEFAULT_CONNECTION = "default" +_PROVIDERS = {} + +# One receipt per immutable database identity. No credentials, session, or source values. +JOBS_KEY = "connector_materialization_jobs" +MAX_ATTEMPTS = 3 +LEASE_SECONDS = 300 +LIFECYCLE_STATES = ("configured", "catalogue-ready", "queued", "syncing", "ready", "failed") + + +def job_key(provider, connection_id, database_key): + return json.dumps([_key(provider), _slot(connection_id), _key(database_key)], + separators=(",", ":")) + + +def _job_public(row): + answer = {k: v for k, v in row.items() if k != "leaseToken"} + answer["retryable"] = (row.get("state") == "failed" + and int(row.get("attempts", 0)) < MAX_ATTEMPTS) + answer["maxAttempts"] = MAX_ATTEMPTS + if row.get("state") == "failed" and not answer["retryable"]: + answer["error"] = "Sync attempts exhausted. Check the connection and update its settings." + return answer + + +def jobs(rt, provider=None, connection_id=None): + """Persisted status, including receipts left by a previous process.""" + document = rt.get(JOBS_KEY, fresh=True) or {} + return [_job_public(row) for row in document.values() + if isinstance(row, dict) + and (provider is None or row.get("provider") == provider) + and (connection_id is None or row.get("connectionId") == _slot(connection_id))] + + +def job_summary(receipts): + """Aggregate only durable facts; a retry of ready work must never claim it is queued.""" + if not receipts: + return None + states = {row.get("state") for row in receipts} + state = next((value for value in ("syncing", "queued", "failed", "ready", "catalogue-ready", "configured") + if value in states), "configured") + errors = [row["error"] for row in receipts if row.get("error")] + stamp = max(row.get("at", 0) for row in receipts) + return {"state": state, "at": _dt.datetime.fromtimestamp(stamp, _dt.timezone.utc).isoformat(), + "retryable": any(row.get("retryable") for row in receipts), + "databases": [row["databaseKey"] for row in receipts if row.get("state") == "ready"], + "detail": f"Database sync {state}", "errors": errors, "errorsTotal": len(errors)} + + +def configuration_revision(provider, connection_id, database_key, model, field_choices): + """Fingerprint this database's field choices, never rows or sibling configuration.""" + identity = database_identity(provider, connection_id, database_key, model) + selected = [] + for key, value in (field_choices or {}).items(): + scoped = parse_field_key(key) + if scoped and scoped[:4] == (identity["provider"], identity["connectionId"], + identity["databaseKey"], identity["model"]): + selected.append((scoped[4], value is True)) + return hashlib.sha256(json.dumps(sorted(selected), separators=(",", ":")).encode()).hexdigest() + + +def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False, + changed=False, revision=None, now=None): + """A Save is idempotent. Failed work requires an explicit, bounded retry.""" + identity = database_identity(provider, connection_id, database_key, model) + key = job_key(provider, connection_id, database_key) + stamp, receipt = time.time() if now is None else now, uuid.uuid4().hex + + def apply(current): + doc = dict(current or {}) + previous = doc.get(key) or {} + updated = changed or (revision is not None and previous.get("revision") != revision) + if not updated and previous.get("state") in ("queued", "syncing", "ready"): + return doc + attempts = int(previous.get("attempts", 0)) + if not updated and previous.get("state") == "failed" and (not retry or attempts >= MAX_ATTEMPTS): + return doc + doc[key] = {**identity, "id": receipt, "state": "queued", "at": stamp, + "attempts": attempts if retry and not updated else 0, "leaseUntil": 0, + "revision": revision, + "leaseToken": "", "error": "", "workspaceChanged": False} + return doc + + landed = rt.update(JOBS_KEY, apply, flush="sync") + return _job_public(landed[key]) + + +def claim(rt, candidate, *, now=None): + """Atomically claim queued or abandoned work; replay callbacks have no side effects.""" + key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) + stamp, token = time.time() if now is None else now, uuid.uuid4().hex + + def apply(current): + doc = dict(current or {}) + row = dict(doc.get(key) or {}) + if row.get("id") != candidate.get("id"): + return doc + if row.get("state") != "queued" and not ( + row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp): + return doc + if int(row.get("attempts", 0)) >= MAX_ATTEMPTS: + row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0, + error="Sync attempts exhausted. Check the connection and update its settings.") + else: + row.update(state="syncing", at=stamp, attempts=int(row.get("attempts", 0)) + 1, + leaseToken=token, leaseUntil=stamp + LEASE_SECONDS) + doc[key] = row + return doc + + landed = rt.update(JOBS_KEY, apply, flush="sync") + row = landed.get(key) or {} + return dict(row) if row.get("leaseToken") == token else None + + +def owns(rt, job, *, now=None): + row = (rt.get(JOBS_KEY, fresh=True) or {}).get(job_key( + job["provider"], job["connectionId"], job["databaseKey"])) or {} + return (row.get("id") == job.get("id") and row.get("state") == "syncing" + and row.get("leaseToken") == job.get("leaseToken") + and row.get("leaseUntil", 0) > (time.time() if now is None else now)) + + +def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None): + """A stale worker cannot acknowledge, renew, or overwrite a newer receipt.""" + key = job_key(job["provider"], job["connectionId"], job["databaseKey"]) + stamp = time.time() if now is None else now + + def apply(current): + doc = dict(current or {}) + row = dict(doc.get(key) or {}) + if (row.get("id") != job.get("id") or row.get("state") != "syncing" + or row.get("leaseToken") != job.get("leaseToken") + or row.get("leaseUntil", 0) <= stamp): + return doc + if renew: + row["leaseUntil"] = stamp + LEASE_SECONDS + elif interrupted: + # Disconnect suspends the current attempt, preserving earlier failures and identity. + row.update(state="queued", at=stamp, attempts=max(0, int(row.get("attempts", 0)) - 1), + leaseUntil=0, leaseToken="", workspaceChanged=False, error="") + else: + row.update(state="ready" if success else "failed", at=stamp, + leaseUntil=0, leaseToken="", workspaceChanged=bool(success), + error="" if success else "Rows could not be synced. Check the connection and retry.") + doc[key] = row + return doc + + landed = rt.update(JOBS_KEY, apply, flush="sync") + return _job_public(landed.get(key) or {}) + + +def cancel(rt, provider, connection_id, database_key): + key = job_key(provider, connection_id, database_key) + stamp = time.time() + + def apply(current): + doc = dict(current or {}) + if key in doc: + doc[key] = {**doc[key], "state": "configured", "at": stamp, + "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False, + "error": ""} + return doc + + rt.update(JOBS_KEY, apply, flush="sync") + + +def _st(st): + return st if st is not None else store + + +def _key(value): + return str(value or "").strip() + + +def _slot(connection_id): + return _key(connection_id) or _DEFAULT_CONNECTION + + +def _target(rt, st): + return st if st is not None else rt + + +def database_identity(provider, connection_id, database_key, model=""): + """Source-neutral identity; labels and sync state never participate in a database key.""" + if not _key(provider) or not _key(database_key): + raise ValueError("provider and database key are required") + return {"provider": _key(provider), "connectionId": _slot(connection_id), + "databaseKey": _key(database_key), "model": _key(model)} + + +def field_key(provider, connection_id, database_key, model, raw_name): + """An opaque, collision-free key for one source field in one database.""" + identity = database_identity(provider, connection_id, database_key, model) + if not identity["model"] or not _key(raw_name): + raise ValueError("model and raw field name are required") + return json.dumps([identity["provider"], identity["connectionId"], + identity["databaseKey"], identity["model"], _key(raw_name)], + separators=(",", ":"), ensure_ascii=True) + + +def parse_field_key(value): + """Read a scoped field key, or return None for a legacy model.field key.""" + try: + parts = json.loads(value) + except (TypeError, ValueError): + return None + if not isinstance(parts, list) or len(parts) != 5 or not all( + isinstance(part, str) and part for part in parts): + return None + return (*parts,) + + +def _public(provider, connection_id, row): + if not isinstance(row, dict): + return None + answer = dict(row) + answer["provider"] = provider + answer["connectionId"] = connection_id + # RULE: retryability follows the state. Storing it would let the two answers drift. + answer["retryable"] = answer.get("state") in ("error", "partial") + return answer + + +def record(rt, provider, connection_id, state, *, detail="", errors=(), databases=(), st=None): + """Record one lifecycle transition and return its defensive public representation.""" + provider_key = _key(provider) + if not provider_key: + raise ValueError("provider is required") + if state not in STATES: + # RULE: unknown states are refused. A free-form status eventually stores a typo that the + # UI cannot render as one of C6's six observable states. + raise ValueError(f"unknown connector sync state: {state}") + connection_key = _slot(connection_id) + all_errors = [str(value) for value in (errors or ())] + visible_errors = all_errors[:_ERROR_LIMIT] + now = _dt.datetime.now(_dt.timezone.utc).isoformat() + written = {} + + def _apply(current): + document = dict(current or {}) + providers_doc = dict(document.get(provider_key) or {}) + previous = providers_doc.get(connection_key) + previous_state = previous.get("state") if isinstance(previous, dict) else None + row = { + "state": state, + # RULE: stamp every transition and name its predecessor. A resting state alone cannot + # tell the UI whether work advanced, retried, or regressed. + "at": now, + "from": previous_state, + "detail": str(detail or ""), + # RULE: partial and error stay distinct. Partial exposes databases usable now; + # error means no usable set was produced. + "errors": visible_errors, + # RULE: retain up to 50 errors and report the full count. A storage limit must be + # observable rather than silently turning additional failures into zero information. + "errorsTotal": len(all_errors), + # These are governed table keys, not copied provider rows. In Odoo, row_limit == 0 + # keeps 963,783 GL lines out of the editable user_tables document. + "databases": [str(value) for value in (databases or ()) if str(value)], + } + providers_doc[connection_key] = row + # RULE: key state by provider and connection. The measured legacy defect is that + # odoo_sync_status is one tenant document, so a second Odoo connection overwrites the + # first one's status. The default slot matches Odoo's existing catalog convention. + document[provider_key] = providers_doc + written.update(row) + return document + + _st(_target(rt, st)).update(STATE_KEY, _apply, flush="sync") + return _public(provider_key, connection_key, written) + + +def read(rt, provider, connection_id=None, st=None): + """Read one connection defensively; store failures mean no recorded state, never a 500.""" + try: + provider_key = _key(provider) + connection_key = _slot(connection_id) + document = _st(_target(rt, st)).get(STATE_KEY) or {} + providers_doc = document.get(provider_key) if isinstance(document, dict) else None + row = providers_doc.get(connection_key) if isinstance(providers_doc, dict) else None + return _public(provider_key, connection_key, row) + except Exception: # noqa: BLE001 - status reads cannot take down their caller + return None + + +def read_all(rt, st=None): + """Read every provider and connection defensively.""" + try: + document = _st(_target(rt, st)).get(STATE_KEY) or {} + answer = {} + for provider_key, connections in document.items(): + if not isinstance(connections, dict): + continue + public_connections = {} + for connection_key, row in connections.items(): + public = _public(str(provider_key), str(connection_key), row) + if public is not None: + public_connections[str(connection_key)] = public + if public_connections: + answer[str(provider_key)] = public_connections + return answer + except Exception: # noqa: BLE001 - status reads cannot take down their caller + return {} + + +def clear(rt, provider, connection_id, st=None): + """Remove only the named provider/connection state.""" + provider_key = _key(provider) + connection_key = _slot(connection_id) + + def _drop(current): + document = dict(current or {}) + connections = dict(document.get(provider_key) or {}) + connections.pop(connection_key, None) + if connections: + document[provider_key] = connections + else: + document.pop(provider_key, None) + return document + + _st(_target(rt, st)).update(STATE_KEY, _drop, flush="sync") + + +def register_provider(provider, *, label, retry_hint): + """Register provider metadata; runners remain in the API layer.""" + provider_key = _key(provider) + if not provider_key: + raise ValueError("provider is required") + _PROVIDERS[provider_key] = { + "key": provider_key, + "label": str(label or provider_key), + "retryHint": str(retry_hint or ""), + } + + +def providers(): + """Return a copy of registered provider metadata.""" + return {key: dict(value) for key, value in _PROVIDERS.items()} + + +register_provider("odoo", label="Odoo", retry_hint="Retry the Odoo database sync.") +register_provider( + "amazon_sp", + label="Amazon", + retry_hint="No sync runner is wired for this connector yet.", +) diff --git a/platform/core/external_access.py b/platform/core/external_access.py new file mode 100644 index 0000000000000000000000000000000000000000..cf2cbb6dcb6fa95c14255033e30ab72fe502b0b0 --- /dev/null +++ b/platform/core/external_access.py @@ -0,0 +1,328 @@ +"""Per-user credentials for the governed external read surface. + +The credential is authentication only. Verification reconstructs the same PUBLIC user record +the normal API session carries, so every later database read still goes through ``perm_scope`` +and the existing Assistant mirror. This module never reads product rows and never caches a +credential, user, bucket, or tenant runtime. +""" +import calendar +from dataclasses import dataclass +import hashlib +import hmac +import secrets +import time + +import core.users as users +from harness import runtime + + +BUCKET = "external_access" +ALGORITHM = "sha256-salt-v1" +AUTH_ERROR = "invalid_external_credential" +_DUMMY_SALT = bytes(32) +_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest() +#: Seconds between two durable `last_used` stamps for ONE credential. +#: +#: ⛔⛔ THE STAMP IS THROTTLED, AND THIS IS NOT AN OPTIMISATION. `st.update(..., flush="sync")` is +#: a fresh strict read plus a BLOCKING UPLOAD, i.e. one durable commit per verified request. An +#: external AI reads in bursts and the store's budget is 256 commits an hour, so an unthrottled +#: stamp would spend a tenant's whole write budget on telemetry and put an upload inside the +#: latency of every read. It would also make a surface contract C7 calls READ ONLY write on every +#: single call, which is the claim this module exists to keep true. The pane renders `last_used` to +#: the minute, so a quarter of an hour of staleness costs the reader nothing. +#: ⚠ REVOCATION IS NOT THROTTLED AND MUST NEVER BE: `revoke` and `mint` stay `flush="sync"`, and +#: verification still reads the bucket FRESH on every request, which is what makes "revoking stops +#: the next request" true. +_STAMP_EVERY = 900.0 + + +class ExternalAccessError(RuntimeError): + """A stable core-layer refusal. HTTP status and copy belong to the API layer.""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +class ExternalAuthError(ExternalAccessError): + """A uniform authentication refusal that reveals no credential or tenant detail.""" + + +@dataclass(frozen=True) +class VerifiedCredential: + """The authenticated public principal plus the request-scoped tenant runtime.""" + + tenant: str + key_id: str + user: dict + runtime: object + + +def _now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _stamp_is_due(last_used, now: float) -> bool: + """Is this credential's `last_used` stale enough to be worth a durable write? + + An absent, unparseable or future stamp is DUE, so a corrupt value self-heals on the next + request rather than freezing the field forever. + """ + if not last_used: + return True + try: + seen = calendar.timegm(time.strptime(str(last_used), "%Y-%m-%dT%H:%M:%SZ")) + except (TypeError, ValueError): + return True + return not (0.0 <= now - seen < _STAMP_EVERY) + + +def _digest(salt: bytes, secret: str) -> str: + """Hash one external secret with SHA-256 and its per-record random salt. + + The secret has 256 bits of machine-generated randomness, not human-password entropy. A slow + password KDF therefore buys nothing against an offline attacker and would deliberately delay + every request on the read surface. + """ + return hashlib.sha256(salt + secret.encode("utf-8")).hexdigest() + + +def _book(st, *, fresh: bool = True) -> dict: + try: + value = st.get(BUCKET, fresh=fresh) + except Exception as exc: + raise ExternalAccessError("external_access_unavailable") from exc + if value is None: + return {} + if not isinstance(value, dict): + raise ExternalAccessError("external_access_unavailable") + return value + + +def _normalise_owner(username: str, tenant: str) -> tuple[str, str]: + owner = str(username or "").strip().lower() + tenant_key = str(tenant or "").strip().lower() + if not owner or not tenant_key: + raise ExternalAccessError("external_identity_invalid") + return owner, tenant_key + + +def _assert_bound_runtime(st, tenant: str) -> None: + if str(getattr(st, "key", "") or "").strip().lower() != tenant: + raise ExternalAccessError("external_identity_invalid") + + +def _active_public_user(username: str, tenant: str) -> dict: + try: + registry = users.registry() or {} + except Exception as exc: + raise ExternalAccessError("external_access_unavailable") from exc + if not isinstance(registry, dict): + raise ExternalAccessError("external_access_unavailable") + record = registry.get(username) + if not isinstance(record, dict) or not record.get("active", True): + raise ExternalAccessError("external_account_unavailable") + public = users._public(username, record) + if public.get("tenant") != tenant: + raise ExternalAccessError("external_account_unavailable") + return public + + +def _metadata(record: dict) -> dict: + """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam.""" + return { + "id": record.get("id"), + "label": record.get("label"), + "created": record.get("created"), + "last_used": record.get("last_used"), + "revoked_at": record.get("revoked_at"), + } + + +def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]: + """Create one credential and return ``(metadata, token)``; the token is never persisted.""" + owner, tenant_key = _normalise_owner(username, tenant) + _assert_bound_runtime(st, tenant_key) + _active_public_user(owner, tenant_key) + clean_label = str(label or "").strip() + if not clean_label or len(clean_label) > 80: + raise ExternalAccessError("external_label_invalid") + + secret = secrets.token_urlsafe(32) + salt = secrets.token_bytes(32) + created = _now() + minted: dict = {} + + def _set(value): + if value is None: + book = {} + elif isinstance(value, dict): + book = dict(value) + else: + raise ExternalAccessError("external_access_unavailable") + key_id = secrets.token_urlsafe(18) + while key_id in book: + key_id = secrets.token_urlsafe(18) + record = { + "id": key_id, + "username": owner, + "tenant": tenant_key, + "label": clean_label, + "salt": salt.hex(), + "hash": _digest(salt, secret), + "algo": ALGORITHM, + "created": created, + "last_used": None, + "revoked_at": None, + } + book[key_id] = record + minted["record"] = record + return book + + try: + st.update(BUCKET, _set, flush="sync") + except ExternalAccessError: + raise + except Exception as exc: + raise ExternalAccessError("external_access_unavailable") from exc + record = minted.get("record") + if not isinstance(record, dict): + raise ExternalAccessError("external_access_unavailable") + token = f"lpb1.{tenant_key}.{record['id']}.{secret}" + return _metadata(record), token + + +def list_for_user(st, username: str, tenant: str) -> list[dict]: + """List only this user's active credential metadata, newest first.""" + owner, tenant_key = _normalise_owner(username, tenant) + _assert_bound_runtime(st, tenant_key) + records = [ + _metadata(record) + for record in _book(st).values() + if isinstance(record, dict) + and record.get("username") == owner + and record.get("tenant") == tenant_key + and not record.get("revoked_at") + ] + return sorted(records, key=lambda row: str(row.get("created") or ""), reverse=True) + + +def revoke(st, username: str, tenant: str, key_id: str) -> dict: + """Tombstone one credential owned by this user; foreign and absent IDs are identical.""" + owner, tenant_key = _normalise_owner(username, tenant) + _assert_bound_runtime(st, tenant_key) + target = str(key_id or "").strip() + revoked: dict = {} + + def _set(value): + if not isinstance(value, dict): + raise ExternalAccessError("external_key_not_found") + book = dict(value) + current = book.get(target) + if (not isinstance(current, dict) + or current.get("username") != owner + or current.get("tenant") != tenant_key): + raise ExternalAccessError("external_key_not_found") + record = dict(current) + if not record.get("revoked_at"): + record["revoked_at"] = _now() + book[target] = record + revoked["record"] = record + return book + + try: + st.update(BUCKET, _set, flush="sync") + except ExternalAccessError: + raise + except Exception as exc: + raise ExternalAccessError("external_access_unavailable") from exc + record = revoked.get("record") + if not isinstance(record, dict): + raise ExternalAccessError("external_access_unavailable") + return _metadata(record) + + +def _parse_token(token: str) -> tuple[str, str, str]: + parts = str(token or "").strip().split(".") + if (len(parts) != 4 or parts[0] != "lpb1" or not all(parts[1:]) + or len(parts[1]) > 128 or len(parts[2]) > 128 or len(parts[3]) > 256): + raise ExternalAuthError(AUTH_ERROR) + return parts[1].lower(), parts[2], parts[3] + + +def _comparison_material(record) -> tuple[bool, bytes, str]: + usable = isinstance(record, dict) and record.get("algo") == ALGORITHM + if usable: + try: + salt = bytes.fromhex(str(record.get("salt") or "")) + expected = str(record.get("hash") or "") + if len(salt) != 32 or len(expected) != 64: + raise ValueError("bad credential digest") + int(expected, 16) + return True, salt, expected + except (TypeError, ValueError): + pass + return False, _DUMMY_SALT, _DUMMY_HASH + + +def verify(token: str) -> VerifiedCredential: + """Verify a bearer token from fresh state and resolve its ordinary public-user principal.""" + try: + tenant_key, key_id, secret = _parse_token(token) + try: + st = runtime.get_runtime(tenant_key) + except KeyError as exc: + raise ExternalAuthError(AUTH_ERROR) from exc + + book = _book(st) + record = book.get(key_id) + material_ok, salt, expected = _comparison_material(record) + matched = hmac.compare_digest(_digest(salt, secret), expected) + record_ok = ( + material_ok + and isinstance(record, dict) + and record.get("id") == key_id + and record.get("tenant") == tenant_key + and bool(record.get("username")) + and not record.get("revoked_at") + ) + if not record_ok or not matched: + raise ExternalAuthError(AUTH_ERROR) + + owner = str(record["username"]).strip().lower() + try: + public = _active_public_user(owner, tenant_key) + except ExternalAccessError as exc: + raise ExternalAuthError(AUTH_ERROR) from exc + + if not _stamp_is_due(record.get("last_used"), time.time()): + return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st) + + used_at = _now() + + def _stamp(value): + if not isinstance(value, dict): + raise ExternalAuthError(AUTH_ERROR) + current = value.get(key_id) + if (not isinstance(current, dict) or current.get("revoked_at") + or current.get("username") != owner + or current.get("tenant") != tenant_key + or current.get("hash") != record.get("hash")): + raise ExternalAuthError(AUTH_ERROR) + book_now = dict(value) + stamped = dict(current) + stamped["last_used"] = used_at + book_now[key_id] = stamped + return book_now + + try: + st.update(BUCKET, _stamp, flush="sync") + except ExternalAuthError: + raise + except Exception as exc: + raise ExternalAuthError(AUTH_ERROR) from exc + return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st) + except ExternalAuthError: + raise + except Exception as exc: + raise ExternalAuthError(AUTH_ERROR) from exc diff --git a/platform/core/keychain.py b/platform/core/keychain.py index 44df0f427f687f9f234e4159fa35f212941b8fd7..3e595010eaf85907eceee6cabe19fe9350b423c8 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -2,7 +2,7 @@ An admin stores a data source's secrets (Royal's Odoo API key today; any tenant's sources tomorrow) encrypted at rest in the TENANT's store bucket. The runtime posture is R3's -"live with env fallback": the connector layer asks `odoo_creds(rt)` FIRST and falls back to +"live with env fallback": the connector layer asks `odoo_creds(rt, connection_id)` FIRST and falls back to the process environment (tenant #0's `.env` / Space secrets), so Royal keeps working with an empty keychain and a new tenant needs no code to bring keys. The FULL cutover (env retired) is staged for a later wave — R3, verbatim. @@ -14,7 +14,7 @@ decrypts and writes are refused — fail closed, never a plaintext fallback. SHAPE (bucket `keychain`, per-tenant via the runtime handle): - {entries: {id: {id, label, type, enc, preview, created, createdBy}}} + {entries: {id: {id, connection_id, slot, label, type, enc, preview, created, createdBy}}} `enc` = Fernet token over the JSON fields dict. `preview` = a masked hint ("····1234") computed ONCE at write from the entry's most secret-looking field — the ONLY part of a @@ -45,7 +45,7 @@ KEY = 'keychain' #: ⚠ R8 keeps the two doors separate on purpose: this is the TENANT's own token (GTM Lab pastes #: theirs), while R5's "Connect Facebook" is the delegated-OAuth door Nango owns. Neither replaces #: the other, and the schema work is blocked on neither. -ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads'} +ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads', 'amazon_sp'} #: which stored field feeds the masked preview, per type (first present wins) #: ⚠ `access_token` JOINED IN WAVE 31, AND ITS ABSENCE WAS A REAL DEFECT FOUND BY BUILDING THE #: SEAM RATHER THAN BY READING IT. Meta's credential is conventionally an `access_token`, and with @@ -53,7 +53,14 @@ ENTRY_TYPES = {'odoo', 'generic', 'stripe', 'shopify', 'meta_ads'} #: — so the keychain table would show a Meta row with no hint at all, and an admin holding two #: tokens could not tell which one is stored or whether the paste worked. Nothing raises; the #: entry is fine; the only symptom is a blank cell on the screen the feature exists for. -_PREVIEW_FIELDS = ('api_key', 'access_token', 'value', 'password', 'token') +#: ⛔ `refresh_token` JOINED FOR AMAZON SP-API, AND ITS ABSENCE WOULD HAVE BEEN THE WAVE-31 +#: DEFECT A SECOND TIME. An `amazon_sp` entry's only secret is its refresh token; with the +#: tuple as it stood, `add_entry` stored the credential correctly and computed an EMPTY +#: preview, so the keychain table would show an Amazon row with no hint at all. Nothing +#: raises; the entry is fine; the only symptom is a blank cell on the screen the feature +#: exists for. Matching is EXACT-KEY (`clean.get(k)` below), so a near-miss name silently +#: yields the same blank. +_PREVIEW_FIELDS = ('api_key', 'access_token', 'value', 'password', 'token', 'refresh_token') MAX_ENTRIES = 40 MAX_FIELD_LEN = 500 @@ -85,27 +92,129 @@ def _bucket(rt): return rt.get(KEY) or {} +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W54-T44 / OWNER RULING R4 / CONTRACT C4 — A CONNECTION HAS AN IDENTITY. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"If a tenant is an agency that helps with clients that each have Odoo, then the +# connector for Odoo can be many depending on the API. This is why its critical."* So MANY +# connections of ONE kind coexist, and each one has to be addressable on its own. +# +# ⭐ THE STORAGE ALREADY ALLOWED THIS AND THE READER DID NOT. `add_entry` has always minted a +# fresh `k_` per call with no uniqueness constraint, so a second `odoo` entry was already +# storable — it was simply never READ (every resolver takes the first of its type). This block +# adds the two facts a reader needs in order to tell two connections of one kind apart: +# +# `connection_id` the entry's own id, stamped EXPLICITLY rather than left implicit. It is +# stable, unique, never reused and never re-derived by sorting. +# `slot` an IMMUTABLE per-(tenant, type) ordinal, minted HIGH-WATER, which is what +# makes the default label `Odoo 1` / `Odoo 2` stable and what a namespaced +# table can be keyed on. Deleting a connection must never let a later one +# reuse a retired ordinal, or two different connections would answer to one +# address at two different times. +# +# ⛔ THE READ DEFAULTS ARE READ DEFAULTS AND NEVER A MIGRATION, the same posture `SCOPE_KEY` +# takes below. Rows written before this wave carry neither field: they read back as +# `connection_id = id` and as their 1-based position among entries of their type in the existing +# insertion-id sort. Materialising that on the next read would rewrite a tenant's encrypted +# store to record something already derivable ([[a-migration-that-runs-on-the-next-write]]) — +# and this store's rows are Fernet tokens, so a rewrite is the one operation that can lose a +# credential. Nothing here decrypts anything. + +#: type -> the human word the default label is built from. `generic` has no product name behind +#: it, so it says what it is rather than inventing one. +_KIND_LABELS = {'odoo': 'Odoo', 'meta_ads': 'Meta Ads', 'amazon_sp': 'Amazon', + 'stripe': 'Stripe', 'shopify': 'Shopify', 'generic': 'Connection'} + + +def kind_label(etype): + """The human word for a connection type, for a default label. Never blank.""" + return _KIND_LABELS.get(str(etype or '').strip().lower()) or 'Connection' + + +def connection_id_of(entry, entry_id): + """A stored row's stable connection id — its own `connection_id`, else its entry id. + + ⚠ A NAMED FUNCTION RATHER THAN AN INLINE `or`, so that the identity has ONE derivation a gate + can break in memory: the negative control for "two connections stay two" needs somewhere to + collapse them. + """ + return str((entry or {}).get('connection_id') or entry_id) + + +def slot_of(entry, ordinal): + """A stored row's slot — its own `slot` when it has one, else the read default `ordinal`. + + ⚠ `ordinal` is the caller's 1-based position among entries of the SAME type, in the same + insertion-id sort every resolver already uses. A stored slot always wins, which is what makes + the field immutable once written. + """ + try: + n = int((entry or {}).get('slot') or 0) + except (TypeError, ValueError): + n = 0 + return n if n >= 1 else int(ordinal) + + def list_entries(rt): - """Metadata ONLY — never a decrypted field, never `enc` itself.""" + """Metadata ONLY — never a decrypted field, never `enc` itself. + + Every row carries `connection_id` and `slot` (W54-T44 / C4). Rows may GAIN keys here and must + never LOSE one: `verify_api` asserts the listing has no `enc` / `url` / `api_key` / `fields`, + and several callers read these rows by key. + """ out = [] + seen = {} for eid, e in sorted((_bucket(rt).get('entries') or {}).items()): if not isinstance(e, dict): continue - out.append({'id': eid, 'label': e.get('label') or eid, - 'type': e.get('type') or 'generic', + etype = e.get('type') or 'generic' + seen[etype] = ordinal = seen.get(etype, 0) + 1 + out.append({'id': eid, + 'connection_id': connection_id_of(e, eid), + 'slot': slot_of(e, ordinal), + 'label': e.get('label') or eid, + 'type': etype, 'preview': e.get('preview') or '', 'created': e.get('created') or '', 'createdBy': e.get('createdBy') or ''}) return out +def next_slot(rt, etype): + """The slot a NEW connection of `etype` takes: one above the highest this tenant has ever + shown for that type. Never decrypts anything. + + ⛔ HIGH-WATER, NOT A COUNT. `len(entries of this type) + 1` would hand a retired ordinal to a + new connection the moment one is deleted — two different credentials answering to `Odoo 2`, + a month apart, with every table W54-T46 namespaces on that ordinal pointing at the wrong one. + ⚠ It reads through `list_entries`, so a row written before this wave counts at its READ + DEFAULT rather than at zero: a tenant with two un-stamped Odoo entries gets 3, not 1. + """ + etype = str(etype or '').strip().lower() + top = 0 + for row in list_entries(rt): + if row.get('type') == etype: + try: + top = max(top, int(row.get('slot') or 0)) + except (TypeError, ValueError): + continue + return top + 1 + + def add_entry(rt, label, etype, fields, username): """Encrypt + store. Returns the metadata row, or raises ValueError on shape problems and - KeychainLocked when there is no key (a secret must never be stored in the clear).""" + KeychainLocked when there is no key (a secret must never be stored in the clear). + + ⭐ W54-T44 / C4 — the row is stamped with its `connection_id` and its `slot`, and a caller + that supplies NO label gets the default the slot names: the tenant's first Odoo connection is + `Odoo 1`, the second `Odoo 2`. An explicitly supplied label always wins, and the label stays + the user's to change afterwards (`rename_entry`). + ⚠ THE TYPE IS VALIDATED FIRST NOW, because the default label is built from it: a blank label + on an unknown type must still be told about the type, which is the fixable half. + """ label = ' '.join(str(label or '').split())[:80] etype = str(etype or '').strip().lower() - if not label: - raise ValueError('give the key a label') if etype not in ENTRY_TYPES: raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}") if not isinstance(fields, dict) or not fields: @@ -121,7 +230,11 @@ def add_entry(rt, label, etype, fields, username): secretish = next((clean[k] for k in _PREVIEW_FIELDS if clean.get(k)), '') preview = ('····' + secretish[-4:]) if len(secretish) >= 4 else ('····' if secretish else '') eid = f'k_{_secrets.token_hex(6)}' - row = {'id': eid, 'label': label, 'type': etype, 'enc': token.decode('ascii'), + slot = next_slot(rt, etype) + if not label: + label = f'{kind_label(etype)} {slot}' + row = {'id': eid, 'connection_id': eid, 'slot': slot, + 'label': label, 'type': etype, 'enc': token.decode('ascii'), 'preview': preview, 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), 'createdBy': str(username or '')} @@ -134,10 +247,42 @@ def add_entry(rt, label, etype, fields, username): return cur rt.update(KEY, _up, flush='sync') - return {'id': eid, 'label': label, 'type': etype, 'preview': preview, + return {'id': eid, 'connection_id': eid, 'slot': slot, + 'label': label, 'type': etype, 'preview': preview, 'created': row['created'], 'createdBy': row['createdBy']} +def rename_entry(rt, entry_id, label): + """Give ONE connection a new name. Returns its metadata row, or None when there is no such + entry. Raises ValueError on a blank name. + + ⛔ THE LABEL AND NOTHING ELSE. `enc`, `connection_id`, `slot`, `preview`, `created` and + `createdBy` are left exactly as they were, and no other row is touched — that is the whole + contract of a rename under R4, where two connections of one kind sit side by side and + renaming one must not disturb the other. + ⛔ AND IT DECRYPTS NOTHING. A stored secret is never re-openable and a rename has no business + asking for the key: this works on a LOCKED keychain, where re-encrypting a row to change a + display string would refuse (or, worse, rewrite a Fernet token to alter a label). + """ + label = ' '.join(str(label or '').split())[:80] + if not label: + raise ValueError('give this connection a name') + eid = str(entry_id) + found = [] + + def _up(cur): + row = (cur.get('entries') or {}).get(eid) + if isinstance(row, dict): + row['label'] = label + found.append(True) + return cur + + rt.update(KEY, _up, flush='sync') + if not found: + return None + return next((e for e in list_entries(rt) if e['id'] == eid), None) + + def delete_entry(rt, entry_id): def _drop(cur): (cur.get('entries') or {}).pop(str(entry_id), None) @@ -167,12 +312,19 @@ def read_fields(rt, entry_id): def first_entry_of_type(rt, etype): - """The metadata row of the FIRST entry of `etype`, or None. Never decrypts anything. - - ⭐ W35-T45 — the same "first, by insertion-id sort" rule `_first_creds` resolves on, exposed as - an IDENTITY question rather than a credential one. `ensure_entry_of_type` needs to know whether - an entry exists WITHOUT unlocking it, and `list_entries` + a hand-rolled `next()` at each caller - is how two places come to disagree about which entry is "the" one. + """DOES THIS TENANT HAVE ANY ENTRY OF `etype` AT ALL? The first such metadata row, or None. + Never decrypts anything. + + ⛔⛔ W54-T45 — THIS IS AN EXISTENCE QUESTION AND NOTHING ELSE, AND THE NAME IS THE ONLY THING + LEFT THAT SUGGESTS OTHERWISE. It used to be the identity twin of `_first_creds`: "which entry + serves" and "does one exist" were the same walk, which is exactly the coupling contract C4 + exists to break. Under R4 a tenant may hold MANY Odoo connections, so "the first one" is no + longer an answer to "which credential serves this connector" — that question is now + `default_connection_id` / `_creds_for`, which resolve on a `connection_id` and order on the + immutable `slot`. ⛔ Do not route any credential path back through here. + What survives is the migration/idempotency question `ensure_entry_of_type` asks below, and + `verify_api` asks in three places as a "nothing was written" probe: both of those genuinely + mean *is there one*, not *which one*. """ return next((e for e in list_entries(rt) if e.get('type') == str(etype or '')), None) @@ -181,9 +333,11 @@ def ensure_entry_of_type(rt, etype, label, fields, username): """Create ONE entry of `etype` only if the tenant has none. `(row, why)`; `row` is None on skip. ⭐ W35-T45 / R11 — the primitive tenant #0's env-to-keychain migration is built from, and the - reason it lives here is that only this module knows what "already has one" means: `odoo_creds` - resolves to the FIRST entry of a type, so a second one would be stored, invisible in every - resolver, and impossible to tell from the one that serves. + reason it lives here is that only this module knows what "already has one" means. + ⚠ W54-T45 SOFTENED WHAT IS AT STAKE WITHOUT CHANGING THIS FUNCTION. A second entry of a type + is no longer invisible: under R4 it is a second CONNECTION with its own `connection_id`, and + every resolver can address it. What this still prevents is a MIGRATION running twice and + quietly minting `Odoo 2` out of the same environment variables `Odoo 1` came from. ⛔ IDEMPOTENT BY TYPE, NOT BY A FLAG. A migration that keys idempotency on a marker it writes itself is one lost marker away from running twice ([[a-migration-that-runs-on-the-next-write]]); @@ -212,13 +366,24 @@ def ensure_entry_of_type(rt, etype, label, fields, username): return None, f'{type(exc).__name__}: {exc}' -def odoo_creds(rt): - """R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls - back to the environment). Deterministic order = insertion-id sort, so 'first' is stable.""" - return _first_creds(rt, 'odoo') +def odoo_creds(rt, connection_id=None): + """R3's resolver seam: the fields of the odoo connection NAMED by `connection_id`, else None. + + ⭐⭐ W54-T45 / R4 / C4 — IT TAKES A CONNECTION NOW, AND THAT IS THE WHOLE TICKET. An agency + tenant holds one Odoo connection per client, so "the tenant's Odoo credential" is not a + question with one answer any more. `connection_id` names WHICH one; a `connection_id` that + names no entry of this type resolves to None and never to a sibling. + ⚠ NO ID MEANS THE DESIGNATED DEFAULT (`default_connection_id`), which is the lowest-`slot` + connection of the type. Every existing one-argument call site therefore keeps working and now + means exactly that, rather than "whichever row happened to sort first". + ⛔ None here still means "not configured", and the CALLER decides what that entails: + `harness/runtime.py::odoo_source` falls back to tenant #0's compiled environment only when no + connection was NAMED. + """ + return _creds_for(rt, 'odoo', connection_id) -def meta_creds(rt): +def meta_creds(rt, connection_id=None): """W31 (R2): the FIRST `meta_ads` entry's fields, else None — the Meta connector's resolver. ⭐⭐ W49-T13 / D-202 — IT HAS A PRODUCTION CALLER NOW, AND THAT WAS THE WHOLE DEFECT. @@ -239,8 +404,38 @@ def meta_creds(rt): is no equivalent for Meta: `META_ADS_ACCESS_TOKEN` is the OWNER's measurement token (R8), and handing it to a tenant whose admin has not stored one is precisely the leak the R3 rule exists to prevent. A tenant with no entry gets None, and the connector says so. + + ⭐ W54-T45 / R4 — `connection_id` NAMES WHICH Meta connection, and no id means the tenant's + DESIGNATED DEFAULT (the lowest-`slot` `meta_ads` connection), not "whichever row sorts first". + An id that names no `meta_ads` entry resolves to None, never to a sibling. + """ + return _creds_for(rt, 'meta_ads', connection_id) + + +def amazon_creds(rt, connection_id=None): + """W52: the FIRST `amazon_sp` entry's fields, else None — the Amazon connector's resolver. + + ⛔ THE SAME SHAPE AND THE SAME IMPLEMENTATION AS `odoo_creds` / `meta_creds`, deliberately. + "Which stored credential serves this connector?" is ONE question, and this module has already + been one copy-paste away from having several answers to it. What differs between connectors + belongs in the connector. + + ⚠ FAILS CLOSED WITH NO ENVIRONMENT FALLBACK, like Meta and unlike Odoo. `AMAZON_SP_*` in the + environment is an OPERATOR's proving credential (`connectors_amazon.py --check`), not a + tenant's, and serving it to a workspace whose admin has stored nothing would hand one tenant + another's Amazon seller account. `amazon_store.sync` therefore refuses rather than guessing. + + ⚠ AND IT RETURNS THE WHOLE FIELD SET, not one secret: an Amazon credential is five values + (`client_id`, `client_secret`, `refresh_token`, `seller_id`, `marketplace_id`) and the client + needs three of them just to mint an access token. + + ⭐ W54-T45 / R4 — `connection_id` NAMES WHICH Amazon connection, and no id means the tenant's + DESIGNATED DEFAULT (the lowest-`slot` `amazon_sp` connection), not "whichever row sorts + first". An id that names no `amazon_sp` entry resolves to None, never to a sibling. + ⚠ `amazon_creds(None)` — a MISSING RUNTIME, not a missing id — still answers None rather than + raising; `verify_amazon` asserts exactly that, and the guard lives in `_creds_for`. """ - return _first_creds(rt, 'meta_ads') + return _creds_for(rt, 'amazon_sp', connection_id) #: ⭐⭐ WAVE 32 (R4 / contract C1) — THE SCOPE SIDE-BUCKET, READ HERE, WRITTEN NOWHERE IN THIS FILE. @@ -257,12 +452,19 @@ def meta_creds(rt): SCOPE_KEY = 'keychain_scopes' #: The credential types a WHOLE WORKSPACE reads through, which therefore cannot be personal. -#: ⛔ THE ELEVATION THIS PREVENTS, in B's words: `odoo_creds`/`meta_creds` resolve to *the first -#: entry of that type for the TENANT* — that is what spawns every `ut_odoo_*` / `ut_meta_*` grid and +#: ⛔ THE ELEVATION THIS PREVENTS, in B's words: `odoo_creds`/`meta_creds` resolve to *a connection +#: of that type for the TENANT* (W54-T45: the one a caller NAMES, or the tenant's designated +#: default when none is named) — that is what spawns every `ut_odoo_*` / `ut_meta_*` grid and #: what every measure column is answered from. So a member storing a personal Odoo key would not get #: "their own Odoo": they would silently become the credential the entire workspace's databases are #: built from. **A credential elevation wearing a scope picker.** -TENANT_WIDE_TYPES = ('odoo', 'meta_ads') +#: ⛔⛔ `amazon_sp` JOINED IN WAVE 52, AND IT IS THE SAME ELEVATION FOR THE SAME REASON. An +#: `amazon_sp` credential is what `amazon_store.sync` resolves to, and that sync spawns every +#: `ut_amzn_*` grid the workspace reads. A member storing a personal Amazon key would not get +#: "their own Amazon": they would become the seller account the entire workspace's Amazon databases +#: are built from. The list was written to prevent exactly this and a new tenant-wide connector has +#: to join it in the same change that adds the connector, or the guard silently does not cover it. +TENANT_WIDE_TYPES = ('odoo', 'meta_ads', 'amazon_sp') def _personal_ids(rt): @@ -277,9 +479,123 @@ def _personal_ids(rt): if isinstance(v, dict) and str(v.get('scope') or '') == 'personal') -def _first_creds(rt, etype): - """The first entry of `etype`, decrypted — or None. Deterministic order = insertion-id sort, - so "first" is stable across reads rather than dict-order luck. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W54-T45 / R4 / CONTRACT C4 — EVERY READER NAMES WHICH CONNECTION IT WANTS. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# What stood here was `_first_creds(rt, etype)`, and one line of it — "return the first entry of +# this type" — was the whole ceiling on R4. Storage already held many connections of one kind +# (W54-T44 gave each of them an id and a slot); every READER collapsed them back to one. An +# agency with two clients on Odoo could store both credentials, see both rows, rename both, and +# be served the first one for every question either client asked. That is not a missing feature, +# it is R7's isolation rule failing silently: one client's numbers answering another's screen. +# +# ⛔ THE HELPER IS NOT NAMED "FIRST" ANY MORE, ON PURPOSE. A shared body called `_first_creds` is +# how the defect would grow back one connector at a time: the next resolver copies the name and +# inherits the rule. `_creds_for` asks the question the contract asks — *which stored credential +# serves this connector, for this connection?* +# +# ⛔ RESOLVE ON IDS, ORDER ON SLOTS, and the distinction is MEASURED, not stylistic: a pre-wave +# row's read-default `slot` MOVES when a sibling of its type is deleted (2 -> 1) while its +# `connection_id` does not. So `connection_id` is the ADDRESS a caller passes and holds, and +# `slot` is only how the tenant's DESIGNATED DEFAULT is picked and how connections are ordered +# on a screen. A caller that stored a slot as an address would eventually address another +# client's credential. + + +def default_connection_id(rt, etype): + """THE TENANT'S DESIGNATED DEFAULT connection of `etype` — its `connection_id`, or None. + Never decrypts anything. + + ⛔⛔ A DESIGNATED DEFAULT PINNED BY AN IMMUTABLE ORDINAL, NOT "WHICHEVER ROW HAPPENS TO SORT + FIRST", AND THAT DISTINCTION IS THE POINT OF THIS WHOLE TICKET. The rule is the LOWEST `slot` + — the per-(tenant, type) ordinal `add_entry` mints high-water and never rewrites — with ties + broken by the existing insertion-id order so the answer is total and stable. "First of its + type" was an accident of dict ordering that a delete, a re-key or a store rewrite could move + under a caller's feet; the lowest slot is a decision the tenant made when it added its first + connection. + ⚠ Personal entries of a tenant-wide type are skipped exactly as `_creds_for` skips them, so + the default and the credential can never disagree about which connection serves. + ⚠ AND IT IS A DEFAULT, NEVER A FALLBACK. It answers "no id was named"; it is not consulted + when an id WAS named and failed to resolve, because serving a sibling to a caller that asked + for a specific client is the leak R7 forbids. + """ + if rt is None: + return None + etype = str(etype or '').strip().lower() + skip = _personal_ids(rt) if etype in TENANT_WIDE_TYPES else frozenset() + best = None + for pos, row in enumerate(list_entries(rt)): + if row['type'] != etype or str(row['id']) in skip: + continue + rank = (int(row['slot']), pos) + if best is None or rank < best[0]: + best = (rank, str(row['connection_id'])) + return best[1] if best is not None else None + + +def _row_for(rt, connection_id, etype=None): + """The metadata row `connection_id` addresses, or None. Never decrypts anything. + + Matches on the row's `connection_id` and, for rows written before W54-T44, on its entry id — + which `connection_id_of` makes the same value anyway. When `etype` is asserted, a row of a + DIFFERENT type is None rather than a match: an id is only an address within the connector + that was asked for. + ⛔ A personal entry of a tenant-wide type is None here too, even when named EXPLICITLY. The + wave-32 skip is a credential-elevation guard, not a tie-breaker, and letting an explicit id + walk past it would reopen the door from the one side that stayed shut. + """ + if rt is None: + return None + want = str(connection_id or '') + if not want: + return None + etype = str(etype or '').strip().lower() + for row in list_entries(rt): + if str(row['connection_id']) != want and str(row['id']) != want: + continue + rtype = str(row['type'] or '') + if etype and rtype != etype: + return None + if rtype in TENANT_WIDE_TYPES and str(row['id']) in _personal_ids(rt): + return None + return row + return None + + +def resolve_connection_id(rt, etype, connection_id=None): + """WHICH connection of `etype` would serve — an IDENTITY, decrypting nothing. None when + nothing would. + + ⛔ IT EXISTS SO THERE IS GENUINELY ONE RESOLVER. `harness/runtime.py::odoo_flag_key` needs the + identity of the serving connection without opening it (a pause flag must still resolve on a + LOCKED keychain), and its docstring has always claimed to be that one resolver while carrying + an inline first-entry walk of its own. This is the walk it now calls instead. + ⚠ The answer is a `connection_id`, which for every stored row equals that row's entry id + (`add_entry` stamps them the same and `connection_id_of` defaults the older rows to it), so + the pause flags written against the old resolution keep resolving. + """ + if connection_id is None: + return default_connection_id(rt, etype) + row = _row_for(rt, connection_id, etype) + return str(row['connection_id']) if row else None + + +def connection_creds(rt, connection_id, etype=None): + """THE fields of the ONE connection `connection_id` addresses, decrypted — or None. + + ⭐⭐ W54-T45 / C4 — the reader the contract asks for: *"`keychain.odoo_creds` and every other + first-entry reader must take a `connection_id`."* Two connections of one kind therefore return + two different credential sets for two different ids, which is the agency case R4 names. + + ⚠ A MISSING RUNTIME ANSWERS None RATHER THAN RAISING, AND THE GUARD IS HERE ON PURPOSE. Every + caller's contract is already "the tenant's credential, or None", and `None` means exactly + "not configured" everywhere it is consumed — but `rt=None` used to reach `rt.get(KEY)` and + raise `AttributeError`, so the three resolvers had a documented contract their shared + implementation did not keep. Found by `verify_amazon` asserting the contract rather than the + behaviour. ⛔ FIXED HERE AND NOT IN ONE RESOLVER, because this module's own rule is that + "which stored credential serves this connector" has ONE implementation: patching `amazon_creds` + alone would have created the second normalizer this file warns about. ⭐⭐ WAVE 32 (R4/C1) — A PERSONAL ENTRY OF A TENANT-WIDE TYPE IS SKIPPED, NOT SERVED. `routes_keychain` REFUSES to create one, and that closes the door from the only side B owns; @@ -293,12 +609,31 @@ def _first_creds(rt, etype): into an outage for a workspace that has a perfectly good business entry two rows down. ⚠ AND THE SKIP IS SCOPED TO `TENANT_WIDE_TYPES`. A personal entry of any OTHER type is a normal personal credential and this resolver is not how it is reached. + ⚠ A LOCKED KEYCHAIN STILL RAISES `KeychainLocked`, from `read_fields`. "No key on this + deployment" is an operator fact and silently empty creds would read as "not configured". """ - skip = _personal_ids(rt) if etype in TENANT_WIDE_TYPES else frozenset() - for row in list_entries(rt): - if row['type'] == etype and str(row['id']) not in skip: - return read_fields(rt, row['id']) - return None + row = _row_for(rt, connection_id, etype) + return read_fields(rt, row['id']) if row else None + + +def _creds_for(rt, etype, connection_id=None): + """THE ONE INTERNAL HELPER behind `odoo_creds` / `meta_creds` / `amazon_creds`: the fields of + the `etype` connection named by `connection_id`, or of the tenant's designated default when + no id is named. None when nothing serves. + + ⛔ IT REPLACES `_first_creds`, NAME INCLUDED. "Which stored credential serves this connector" + has ONE implementation in this module, and the old one answered a question C4 forbids. + ⛔ A NAMED CONNECTION THAT DOES NOT RESOLVE IS None, FULL STOP — never the default, never a + sibling, and (at the caller) never the environment. Falling back after an explicit ask is how + one client's Odoo comes to answer another client's question under R7's strict isolation. + """ + if rt is None: + return None + if connection_id is None: + connection_id = default_connection_id(rt, etype) + if connection_id is None: + return None + return connection_creds(rt, connection_id, etype) # --------------------------------------------------------------------------------------------- diff --git a/platform/core/measure_resolve.py b/platform/core/measure_resolve.py index 392db565c3ba3a873e67f311e9d7dac5dae9ae78..53633d1226c3177a498bfec750894b5cd1a6a3b2 100644 --- a/platform/core/measure_resolve.py +++ b/platform/core/measure_resolve.py @@ -19,6 +19,7 @@ Memo discipline (unchanged from the app.py originals): not leaks. """ import hashlib +from core import relations as _relations def offer(team_id=None, on_error=None): @@ -43,7 +44,7 @@ def _pool_key(allowed_pids): def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stamp, memo, - on_error=None): + on_error=None, related=None): """Every measure condition across `view_configs` (+ the optional pending config, which goes LAST and overwrites — the freshest statement of what the user wants) -> `{ruleId: [pid,…]}`. @@ -64,20 +65,24 @@ def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stam if not rules: return {} pk = _pool_key(allowed_pids) - out = {} + out = _relations.MeasureAnswer() if related is not None else {} + rkey = _relations.measure_pin_key(related) for rule in rules: if not _mf.rule_complete(rule): continue - key = (stamp, team_id, pk, _mf.signature(rule)) + key = (stamp, team_id, pk, _mf.signature(rule), rkey) if key not in memo: try: - memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids) + memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids, related=related) except Exception as e: # An unresolvable condition must NOT become "everything" or "nothing" silently. if on_error: on_error('measure-resolve', e) continue out[str(rule['id'])] = sorted(memo[key]) + if related is not None: + out.refusals.extend(dict(marker, rule=str(rule['id'])) + for marker in getattr(memo[key], 'refusals', ())) if len(memo) > 200: # a lifetime cache, not a leak memo.clear() return out @@ -137,13 +142,18 @@ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp, return out -def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None): +def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None, related=None): """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`, handed to `rows_from_pool(derived=…)` beside the cohort cells. A column that cannot be computed (store down, BU scope on a company-level measure, unresolvable window) degrades to BLANK — the value dict simply omits that field key, and the client renders empty cells, never $0: blank is "could not compute", 0 is a real zero. + + `related` is the row-window {table, record, via} pin. The host binds its scoped readers + with relations.measure_pin_context and publishes the returned mapping's `refusals` beside + its cells. A refused pin retains unscoped values plus a per-field named marker, on memo + hits too. Every measure and its conditions use the same query scope and caller's pool. """ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)] if not mfields: @@ -151,16 +161,17 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No from harness import measure_filter as _mf from harness import windows as _wn pk = _pool_key(allowed_pids) - out = {} + out = _relations.MeasureAnswer() if related is not None else {} + rkey = _relations.measure_pin_key(related) for f in mfields: spec = f['measure'] w = _wn.normalize(spec.get('window')) key = (stamp, team_id, pk, spec.get('key'), - None if w is None else tuple(sorted(w.items()))) + None if w is None else tuple(sorted(w.items())), rkey) if key not in memo: try: memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'), - today, team_id, allowed_pids) + today, team_id, allowed_pids, related=related) except Exception as e: if on_error: on_error('measure-column', e) @@ -183,6 +194,9 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No if vals is None: continue fkey = f['key'] + if related is not None: + out.refusals.extend(dict(marker, field=fkey) + for marker in getattr(vals, 'refusals', ())) for pid, v in vals.items(): out.setdefault(pid, {})[fkey] = v if len(memo) > 100: diff --git a/platform/core/monid.py b/platform/core/monid.py index a20331bd6214ef82d108d89003473906d6a27c32..657260a568f7eabf28862cde28c70cfc29cdc198 100644 --- a/platform/core/monid.py +++ b/platform/core/monid.py @@ -25,7 +25,7 @@ transient fault: retrying it spends nothing and tells the operator nothing, so i ⚠ THE ENDPOINT IS RECORDED, NOT REDISCOVERED. `discover("amazon reviews")` chose `apify /axesso_data/amazon-reviews-scraper` on 2026-09-02 and the losing candidates are written -down in `.claude/wiki/waves/wave49/proto/monid-research.md`. Re-running discover at request time +down in `AutoWiki/waves/wave49/proto/monid-research.md`. Re-running discover at request time would let a ranking change silently move which vendor we pay; the constants below are the decision, and `discover` remains available for the next slice. diff --git a/platform/core/odoo.py b/platform/core/odoo.py index 26883e5f513cc766d9012f2d17a07322908f565a..0e709a1b0e7ef30acf467898d8986dabf9330bbf 100644 --- a/platform/core/odoo.py +++ b/platform/core/odoo.py @@ -79,7 +79,7 @@ def _client_for(slug, creds): if slug == _DEFAULT_SLUG: return _global_client() raise TenantOdooUnavailable( - f"tenant {slug!r} has no Odoo credentials — store them under Settings → Keychains. " + f"tenant {slug!r} has no Odoo credentials; store them under Settings → Keychains. " f"(Refusing the environment fallback: that is another tenant's connection.)") fp = (str(creds.get('url') or ''), str(creds.get('db') or ''), str(creds.get('user') or ''), str(creds.get('api_key') or '')) diff --git a/platform/core/perm_migrate.py b/platform/core/perm_migrate.py index a6865d876c37d740a418abf693041a57df1cc4f4..ab0f3e4060227664b23474fa8d5e62891f1d44cb 100644 --- a/platform/core/perm_migrate.py +++ b/platform/core/perm_migrate.py @@ -8,7 +8,7 @@ no-op before anything relies on it. ⛔ WHY THE PROOF IS ABOUT VALUES AND NOT ABOUT ROWS (C-PERM amendment 3). `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, LY, LTM) and into `cust._cadence_bulk`. It therefore decides what `rev`, `ly`, `ltm`, `aov`, -`est_missed` and the derived `status` MEAN — it is not a row filter. Swap it for a post-filter +and the derived `status` MEAN; it is not a row filter. Swap it for a post-filter and a Fisch-only user keeps a plausible row LIST while every number on it quietly becomes Fisch+Royal, worst for `dba = Both` customers, who are precisely the ones a BU filter admits. A pid-set reconciliation reports GREEN on that. So `reconcile()` compares VALUES. @@ -97,7 +97,7 @@ def plan(registry_dict, modules=(MODULE,)): #: The row members whose meaning depends on `team_id` — the ones amendment 3 is about. A #: reconciliation that skipped these would be the pid-only gate that cannot see the leak. -VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'est_missed', 'status', 'orders', 'last_order') +VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'status', 'orders', 'last_order') def reconcile(rec, pool_fn, modules=(MODULE,)): diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index cd76e98fb52b9f954422f2921b1a21325083dfb0..de0a35a7606287201446a71f6e90e21c10b35c06 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -1980,3 +1980,321 @@ def pivot_scope(user, source, target, st=None, path=None): return PivotScope(user, src, tgt, walk, source_scope=derive_pool_scope(user, src), target_scope=derive_pool_scope(user, tgt)) + + +# ── C4 / R4: THE CONNECTOR AXIS — WHO MAY CREATE A CONNECTION (wave 54, W54-T47) ────────────── +#: The per-tenant bucket one administrator's decision lives in: `{username: [, …]}`. +#: +#: ⛔⛔ AN ABSENT ROW AND "NO GRANT" ARE THE SAME THING, AND THAT IS WHY THIS IS A READ DEFAULT +#: RATHER THAN A MIGRATION. `routes_keychain.SCOPE_KEY` carries the identical posture one layer up +#: (*"a business entry writes no row — it IS the default"*), and the reason is +#: [[a-migration-that-runs-on-the-next-write]]: materialising an empty grant for every account in +#: every tenant would rewrite the store on the next read, for a fact nobody decided. +CONNECTOR_GRANT_KEY = 'connector_grants' + + +def connector_grants(username, st=None): + """The connector kinds `username` has been granted, lower-cased, as a frozenset. + + ⚠ THE ANSWER TO AN UNREADABLE STORE IS THE EMPTY SET, AND HERE THAT IS THE SAFE DIRECTION + RATHER THAN THE UNSAFE ONE — the opposite of `user_generated_fields`, which must answer `None`. + That function's answer is PRUNED AGAINST, so a degraded read would delete a live rule; this one + only ever WIDENS what a member may do (see `may_add_connection`), so a degraded read leaves the + member with exactly the rights they had before this ticket existed. Under-granting is visible + and reportable; over-granting is a credential elevation. + """ + if st is None: + return frozenset() + uname = str(username or '').strip().lower() + if not uname: + return frozenset() + try: + rows = st.get(CONNECTOR_GRANT_KEY) or {} + except Exception: # noqa: BLE001 + return frozenset() + if not isinstance(rows, dict): + return frozenset() + got = rows.get(uname) + if not isinstance(got, (list, tuple, set, frozenset)): + return frozenset() + return frozenset(str(k).strip().lower() for k in got if str(k or '').strip()) + + +def may_add_connection(user, etype, st=None): + """May this account create a WORKSPACE-WIDE connection of kind `etype`? (W54-T47 / R4 / C4.) + + ⭐⭐ THE THIRD GRANT AXIS. Owner, verbatim: *"Under Settings, an Admin should grant a user + access to add connectors, so access becomes Database / Module / Connector. This allows many + connectors per tenant … depending on the admin's configuration per user."* Database and Module + are the two groups `PermsEditor.tsx` already renders off `perms`; this is the third, and it is + a per-KIND answer rather than a single boolean because R4's agency tenant is the whole case: + one member may be the person who wires up Shopify and not the person who wires up Odoo. + + ⛔⛔ ABSENCE WIDENS NOTHING AND REVOKES NOTHING, AND THAT IS THE MOST IMPORTANT PROPERTY OF + THIS FUNCTION. No grant was STORABLE before this ticket, so every account in every tenant + carries none — reading that absence as DENY at the connection door would revoke personal + connections for everybody on the day this shipped, with nobody having decided anything. That + is `may_metrics`' argument one file over, in the same words, and the resolution is the same: + this predicate is consulted ONLY on the path that was already refused + (`routes_keychain.add_key`'s `scope == "business" and not session.admin` leg), so a member with + no grant meets byte-identical behaviour — 403 `not_admin` on a business-wide create, 400 + `scope_not_available` on a tenant-wide personal one, and their own personal connection of an + ordinary type still succeeds. An explicit grant is the ONLY thing this adds, and all it can do + is admit. + + ⛔ AN ADMIN IS ALWAYS TRUE, before the store is touched — break-glass, as every predicate in + this file opens, and it is what keeps a tenant whose grant bucket cannot be read from losing + its own administrator's ability to connect anything. + + ⚠ `st` IS THE TENANT HANDLE AND OMITTING IT DENIES A NON-ADMIN. The grant is per TENANT, so a + caller that cannot lend a handle is asking about no workspace in particular; answering from the + process-default store is the D-612 shape (`nav_may_open`'s leg 3), and there the wrong answer + was reachable in BOTH directions. Here the handle-less answer is the pre-ticket one, so a door + that has not learned to thread `st` refuses rather than admitting somebody else's grant. + + ⚠ IT IS NOT A SECOND SPELLING OF ELIGIBILITY. `routes_keychain.allowed_scopes` answers what the + SOURCE permits (Odoo is workspace-wide by construction); this answers who may act on it. Both + must hold, and a tenant-wide type refused to a personal scope stays refused for everybody, + grant or no grant. + """ + if perms.is_admin(user): + return True + kind = str(etype or '').strip().lower() + if not kind: + return False + return kind in connector_grants((user or {}).get('username'), st=st) + + +# ── C4 / R7: THE CONNECTION AXIS — ONE READ, ONE CONNECTION (wave 54, W54-T49) ───────────────── +#: ⭐⭐ RULING R7, verbatim: *"Multi-connection isolation is STRICT, on the Business Unit pattern. +#: Every Odoo-reading module gains a connection selector; picking one client shows only that +#: client's numbers. ⛔ There is no 'All' option that blends two clients."* +#: +#: ⛔⛔ AND THE FAILURE THIS SECTION EXISTS TO PREVENT IS ON THE **VALUES** AXIS, WHICH IS WHY IT +#: LOOKS FINE. Wave 51's tenancy defect ran for a whole wave with a correct-looking row LIST while +#: `rev`, `ly`, `ltm`, `aov` and `est_missed` were silently two clients' numbers added together — +#: `_PUSHDOWN_OPS`' own note above is that incident's record. A connection scope is therefore not +#: a post-filter over rows: it decides WHICH TABLE is read at all, before a row exists to filter. +#: +#: ⭐ THE PRECEDENT IS `derive_pool_scope`, NOT A PICKER. `pages.py::bu_control` builds a BU +#: control from `perms.allowed_bu_labels`, but `pages.py::_BUILDERS` is `{}` on purpose and +#: `verify_api` asserts that emptiness, so that assembly is unreachable in production. The live BU +#: pattern is the per-request DERIVATION twenty lines up, and this is its shape one axis over. +#: +#: ⛔ IT TAKES NO PRINCIPAL, DELIBERATELY, AND THAT IS NOT AN OVERSIGHT TO TIDY. WHICH connection +#: serves is a property of the workspace and of the request, never of the reader: there is no +#: per-user connection grant in this product (`may_add_connection` governs who may CREATE one, a +#: different question), so a `user` parameter here would read as a wall this function does not +#: have. The reader's wall is `may_read` on the table key the scope resolves to, and it composes +#: AFTER this one, unchanged. + +#: What a LEGACY (bare) table key's owner is called when the workspace has no stored connection at +#: all to name — every tenant before this wave, tenant #0 included. +#: +#: ⛔ RESTATED FROM `odoo_relational.DEFAULT_CONNECTION` RATHER THAN IMPORTED, because `core` never +#: imports up (`platform/ARCHITECTURE.md`). It is the same restatement `_FORMULA_REF` at the top of +#: this file makes, with the same guard: `verify_scopes` asserts the two spell the same word, so a +#: rename in the API layer cannot leave this copy behind [[constant-two-features-share]]. +DEFAULT_CONNECTION = 'default' + +#: The connector kind a scope answers for when a caller names none. `odoo` is the only kind that +#: spawns per-connection tables today (W54-T46); the parameter exists so the second one needs no +#: second function. +DEFAULT_CONNECTION_TYPE = 'odoo' + + +class UnknownConnection(LookupError): + """A NAMED connection does not resolve in this workspace. + + ⛔⛔ RAISED, NEVER FALLEN BACK FROM, and that asymmetry IS R7. `keychain.default_connection_id` + answers *"no id was named"*; it is not consulted when an id WAS named and failed to resolve, + because serving a sibling to a caller that asked for a specific client is exactly the leak this + contract forbids. `odoo_relational.connection_namespace` and `runtime.odoo_source` already hold + that posture, and this is the third place it has to hold or the first two are decoration. + """ + + +class ConnectionScope: + """⭐⭐ CONTRACT C4 / R7 — the ONE connection a read is scoped to. Blending is UNREPRESENTABLE. + + Built by `connection_scope()` and by nothing else worth building: it holds a SINGLE + `connection_id` string, there is no member that can hold two, and the constructor refuses + anything that is not one non-empty id. So "read both clients" is not a value this type can + carry — it is not a flag that happens to be off, which is the difference between a rule and a + default. A list, an empty string and the word "all" all arrive here as a refusal. + + ⚠ IT HOLDS THE TYPE TOO, so an odoo scope can never be handed to a meta_ads reader as though + an id were an address across connectors. `keychain._row_for` makes the same statement one layer + down (*"an id is only an address within the connector that was asked for"*). + """ + __slots__ = ('connection_id', 'etype') + + def __init__(self, connection_id, etype=DEFAULT_CONNECTION_TYPE): + cid = connection_id.strip() if isinstance(connection_id, str) else '' + if not cid: + raise UnknownConnection( + 'a connection scope is exactly one connection. Name a single connection, or ' + 'name none and the workspace default serves.') + self.connection_id = cid + self.etype = str(etype or DEFAULT_CONNECTION_TYPE).strip().lower() + + def __repr__(self): + return f'ConnectionScope({self.connection_id!r}, {self.etype!r})' + + def owner_of(self, table_key, rt=None): + """WHICH connection owns `table_key`, through the registered owner. `None` for a key no + connector claims. + + ⛔ NO REGISTRAR IS `Unresolvable`, NOT `False`. The question "whose table is this" is + answered in the API layer (`odoo_relational.connection_for_table_key`) and `core` never + imports up, so a process that never registered one cannot answer it — and a wall that + cannot be evaluated must say so with its cause and a recommendation rather than quietly + admitting or quietly refusing. That is standing rule 1's second sentence, and it is the + same shape `_read_user_table` uses when no mirror reader is registered. + """ + key = str(table_key or '').strip() + if _CONNECTION_OWNER is None: + raise Unresolvable( + subject='connection', effect='unresolvable', + cause=(f"'{key}' cannot be attributed to a connection in this process because no " + f'connection owner is registered'), + recommendation=('call `perm_scope.register_connection_owner(...)` from the app ' + 'layer before scoping a read, or read through the API')) + owner = _CONNECTION_OWNER(key, rt) + return str(owner) if owner else None + + def covers(self, table_key, rt=None): + """Does `table_key` belong to THIS connection? + + ⛔ FAIL CLOSED. An unanswerable question is a NO here, never a pass: a key no connector + claims, an owner that cannot be resolved and a missing registrar all answer False, because + the only thing "I do not know whose this is" can safely mean under R7 is "not yours". + """ + try: + return self.owner_of(table_key, rt) == self.connection_id + except Exception: # noqa: BLE001 + return False + + def require(self, table_key, rt=None): + """`table_key` itself, or `Denied` when it belongs to another connection. + + ⛔ THE WALL, AND IT IS ONE SENTENCE ON REFUSAL (DESIGN section 4). `covers` is the + predicate a caller may branch on; this is the door a reader passes through, so that a + route which forgets to branch fails CLOSED rather than reading the other client's table. + """ + key = str(table_key or '').strip() + owner = self.owner_of(key, rt) + if owner != self.connection_id: + raise Denied(f"'{key}' belongs to another connection. Switch the connection selector " + f'to the one that owns it.') + return key + + +#: WHO answers "which connection owns this table key", declared by the app layer. +#: `reader(table_key, rt) -> connection_id | None`. +#: +#: ⛔ A REGISTRATION SLOT AND NOT AN IMPORT, for the reason `_ROW_SOURCES` and `_MIRROR_READER` +#: above are, and `keychain.register_prober` one file over: the inverse lives in +#: `aios-web/api/odoo_relational.py` (`connection_for_table_key`), `core` never imports up, and +#: `verify_meta` asserts that fence. The app tells this layer rather than being interrogated by it. +_CONNECTION_OWNER = None + + +def register_connection_owner(reader): + """Declare THE reader for "whose table is this" — one reader, because a table has one owner. + + Returns the registrar's own answer, exactly as `register_mirror` does, so the read door and + the write door of this fact have one construction site rather than two. + """ + global _CONNECTION_OWNER + _CONNECTION_OWNER = reader + return _CONNECTION_OWNER is not None + + +def connection_scope(rt, etype=DEFAULT_CONNECTION_TYPE, connection_id=None): + """The ONE connection this read is scoped to, as a `ConnectionScope`. R7's derivation. + + ⭐⭐ ABSENCE CHANGES NOTHING, AND THAT IS THE PROPERTY THAT LETS THIS SHIP. `connection_id=None` + means *"no connection was named"* and resolves to the workspace's DESIGNATED DEFAULT — the same + connection `keychain._creds_for`, `odoo_relational.connection_namespace` and every pre-ticket + reader already served, resolved by the same `keychain.default_connection_id`. A tenant with one + connection therefore cannot tell this function exists, which is what every ticket in this lane + has preserved for the same reason (W54-T44's read defaults, T46's legacy keys, T47's grant). + + ⛔ A NAMED CONNECTION THAT DOES NOT RESOLVE IS `UnknownConnection`, NEVER THE DEFAULT. See that + class; the fallback is the leak. + + ⛔⛔ AND THERE IS NO "ALL". `None` is the only absence this function has, and it pins exactly + one connection rather than widening to every connection. An empty string, the word "all", a + LIST of ids and any non-string are refused BY CONSTRUCTION rather than by enumeration: only a + single non-empty string can address a connection, so the vocabulary cannot express a blend for + a future caller to discover. That is the wave-51 lesson in the shape of a type — there the + filter vocabulary widened underneath a reader that could not read it, and the numbers merged. + + ⚠ `rt=None` WITH A NAMED CONNECTION REFUSES rather than answering from the process-default + store, which is D-612's shape (`nav_may_open` leg 3): a caller with no workspace handle is + asking about no workspace in particular. + """ + kc = _keychain() + if connection_id is None: + cid = kc.default_connection_id(rt, etype) if rt is not None else None + # No connection is stored at all: the legacy keys still belong to "whichever connection is + # designated default", which is a real answer rather than an absence, and it is the same + # sentinel `connection_for_table_key` hands back for exactly this case. + return ConnectionScope(cid or DEFAULT_CONNECTION, etype) + named = connection_id.strip() if isinstance(connection_id, str) else '' + if not named: + raise UnknownConnection( + 'name one connection, or name none and the workspace default serves.') + cid = kc.resolve_connection_id(rt, etype, named) if rt is not None else None + if not cid: + raise UnknownConnection( + f"'{named}' does not name a connection in this workspace. Pick a connection in the " + f'Connectors panel and read it again.') + return ConnectionScope(cid, etype) + + +def connection_options(rt, etype=DEFAULT_CONNECTION_TYPE): + """The connections a selector may offer, in slot order: `[{'id', 'label'}, ...]`. + + ⭐ `allowed_bus()` ONE AXIS OVER, and the ticket asks for it to read like it. The list is what + a module's connection selector renders, and the FIRST member is the workspace's designated + default, so a client that shows the first row selected shows what a read with no id named would + actually serve. + + ⛔⛔ THERE IS NO "ALL" MEMBER AND THERE IS NO PLACE TO PUT ONE. R7 is explicit that no option + may blend two clients, so the option list carries only real connections; a caller that wants + "everything" has to ask twice, once per connection, which is exactly what `refresh_all` does. + + ⚠ IDENTITY COMES FROM `keychain.resolve_connection_id`, NEVER FROM A SECOND WALK OF THE ROWS. + Four first-entry walks were deleted in W54-T45 and a fifth would reopen the ceiling C4 exists + to close; the row is read here only for its LABEL and its slot, which no resolver answers. The + resolver is also what SKIPS a personal entry of a tenant-wide type, so a credential the readers + would never serve cannot appear in a picker as though it would. + """ + if rt is None: + return [] + kc = _keychain() + want = str(etype or DEFAULT_CONNECTION_TYPE).strip().lower() + rows, seen = [], set() + try: + entries = list(kc.list_entries(rt)) + except Exception: # noqa: BLE001 + return [] + for row in entries: + if str(row.get('type') or '') != want: + continue + cid = kc.resolve_connection_id(rt, want, + str(row.get('connection_id') or row.get('id') or '')) + if not cid or cid in seen: + continue + seen.add(cid) + label = ' '.join(str(row.get('label') or '').split()) or kc.kind_label(want) + rows.append((int(row.get('slot') or 0), len(rows), cid, label)) + rows.sort() + return [{'id': cid, 'label': label} for _slot, _pos, cid, label in rows] + + +def _keychain(): + import core.keychain as keychain + return keychain diff --git a/platform/core/relations.py b/platform/core/relations.py index 4d8fee534346b077dfee20412986f09c0f008e34..301c18461cfe04f98d3f886d6870b5a31b9a925c 100644 --- a/platform/core/relations.py +++ b/platform/core/relations.py @@ -56,9 +56,95 @@ asserted in a comment), and ANDed into the read ahead of the client predicate. booked in this ticket's handoff), ``pin_sql`` delegates to it and this note is deleted. """ +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +import json + + #: The three ways a database can be related to another. `via.direction` is always one of these. DIRECTIONS = ('forward', 'reverse', 'bridge') +# =============================================================================================== +# W54-T37 / CONTRACT C5 / INSTRUCTION 22 - A DATABASE THAT IS DECLARED BUT ABSENT. +# +# The owner, verbatim: *"a wall of ... this account may not open `ut_odoo_customers` ... THAT +# DATABASE DOES NOT EXIST."* He is right and the sentence was lying. `ut_odoo_customers` and +# `ut_odoo_products` were RETIRED: popped from every tenant document on every sync and replaced by +# the `customer_data` / `product_data` registry modules. The connector spec still DESCRIBES them +# (a spec describes a shape, it does not assert a table is materialised), so the enumeration below +# happily minted edges pointing at them, the route wall failed closed on a key it could not find, +# and the reader was told about a permission problem that does not exist. +# +# TWO REPAIRS, IN THIS ORDER, AND THE FIRST IS THE REAL ONE: +# 1. an edge whose TARGET is retired is NEVER ENUMERATED, in either direction and in BOTH the +# walled and the unwalled pass. It is not a tab, so it is not a refusal either. +# 2. a path that names one ANYWAY earns `retired_target`, which says the database is gone. It is +# a BACKSTOP, and a backstop that fires often means repair 1 has a hole in it. +# +# THE LIST LIVES ABOVE, NOT HERE, AND THAT IS THE WHOLE POINT OF THE REGISTRY. `core` never +# imports up, so it cannot read `odoo_relational.RETIRED_KEYS`; copying the two keys down here +# would make a second declaration of the same fact, which is the defect class this repo already +# names [[one-question-two-normalizers]]. The shape is `core.user_tables.register_connected_prefix` +# verbatim, down to its comment: the engine pushes its list down rather than this module reaching +# for it. So THIS SET STARTS EMPTY AND STAYS EMPTY until an app-layer registrar runs. +# +# A CONSEQUENCE WORTH STATING: in a process where nothing registered anything (a bare platform +# gate) the filter is a no-op and every edge is enumerated. That is the honest answer there, since +# such a process has no way to know which tables a deployment retired. +# =============================================================================================== + +#: The table keys that are DECLARED somewhere and ABSENT from this deployment: retired, or never +#: materialised. Empty by construction. See the banner above before adding a literal to it. +_RETIRED_TABLES = set() + + +def register_retired_tables(*keys): + """Declare table keys that are described somewhere but do not exist as databases. + + Takes bare keys, an iterable of keys, or any mix, so a registrar can push a whole tuple down in + one call. Returns the WHOLE registered set, exactly as `register_connected_prefix` does, so the + caller can assert what it just joined and the write door answers through the read door. + + A KEY REGISTERED HERE STOPS BEING AN EDGE TARGET. Nothing else changes: rows, definitions and + permissions are not this module's business, and a key that is merely unreadable by one account + is a PERMISSION and belongs to the wall, not here. + """ + for key in keys: + named = [key] if isinstance(key, str) else list(key or ()) + for name in named: + text = str(name or '').strip() + if text: + _RETIRED_TABLES.add(text) + return retired_tables() + + +def retired_tables(): + """The registered absent-table keys. The ONE list a caller may read to name the same set.""" + return frozenset(_RETIRED_TABLES) + + +def is_retired_table(key): + """Is `key` a database this deployment declares and does not have?""" + return str(key or '').strip() in _RETIRED_TABLES + + +def _live(entries, include_retired=False): + """Drop every entry whose TARGET is retired. The one filter, applied by every builder. + + ⛔ IT FILTERS THE **TARGET**, WHICH COVERS BOTH DIRECTIONS WITH ONE RULE. Outbound, the target + is the parent the spec names (`ut_odoo_orders` -> `ut_odoo_customers`). Inbound, the target is + the OTHER table, so a retired spec row whose own links point at a live table (the retired + customers row names `ut_odoo_agents`) produces a reverse entry on the agents grid whose target + is the retired table. Both are the same test. + ⚠ ENUMERATING **FROM** A RETIRED TABLE IS STILL ALLOWED and is deliberately not this rule: the + edges of such a table are what a diagnosis walks, and refusing to describe them would replace + one wrong answer with a shorter one. + """ + if include_retired or not _RETIRED_TABLES: + return list(entries or ()) + return [e for e in (entries or ()) if not is_retired_table(e.get('target'))] + #: The identity column of a module grid, as the wire spells it. `customer_data.pid` IS the Odoo #: `res.partner` id and `ut_odoo_orders.pid` IS the `sale_order` id, so a bridge hanging off `pid` #: needs no second lookup. `product_data.pid` is NOT an Odoo id (it is `crc32(default_code)`), @@ -81,29 +167,57 @@ OPEN_INVOICE_WHERE = ("state = 'posted' AND move_type IN ('out_invoice','out_ref #: #: ⚠ `served` RECORDS WHETHER THE PINNED READ CAN ANSWER THIS EDGE TODAY, and it is a measurement #: rather than a wish. The pinned read is built into `routes_odoo_tables._compiled_read`, which -#: serves the READ-THROUGH grids only (`GRID_SOURCES` binds exactly `ut_odoo_orders`, -#: `ut_odoo_order_lines`, `ut_odoo_gl_lines` and `ut_odoo_accounts`). `ut_odoo_invoices` is -#: MATERIALISED, so its rows come from `routes_tables.table_rows`, which takes no predicate at all -#: and is in no lane's fence this wave. The edge is still DECLARED, because a tab that is known to -#: exist and cannot yet be served is a reported limit, while an undeclared one is a silent absence. +#: serves the READ-THROUGH grids only, i.e. the keys `GRID_SOURCES` binds. +#: +#: ⭐⭐ W53-T19 / RULING R5 — ALL FIVE ROWS NOW READ `served: True`, AND THE FLAG STAYS ANYWAY. +#: R5: *"Every `ut_odoo_*` table becomes read-through bound."* `bridge_customer_open_invoices` was +#: the ONE row carrying `False`, because `ut_odoo_invoices` had no `GRID_SOURCES` spec and its +#: stored-read door (`routes_tables.table_rows`) takes no predicate at all. `_odoo_sources` now +#: binds that grain, so the pinned read can push the open-invoice predicate into SQL and the flag +#: is True by measurement rather than by wish. ⛔ THE FLAG IS NOT DELETED WITH THE ONE `False` +#: THAT USED IT: `served` is a per-edge DECLARATION, a bridge onto a grain some future connector +#: has not bound yet is exactly the case it exists for, and `unserved_target_refusal` below is the +#: sentence that turns such an edge into a reported limit instead of a tab that quietly serves +#: nothing (standing rule 1's second sentence). A tab that is known to exist and cannot yet be +#: served is a reported limit; an undeclared one is a silent absence. +#: +#: ⭐⭐ `cardinality` WAS ADDED TO EVERY ROW BY W53-T17 UNDER RULING R12, AND THE BANNER ABOVE IS +#: WHY THAT SENTENCE IS HERE. "NOTHING ELSE MAY BE ADDED WITHOUT A RULING" asks for a ruling before +#: this list changes; R12 (*"the connector spec declares edges; the server enumerates what is +#: reachable within two hops"*) IS that ruling, and it needs every edge to state its own shape. +#: ⛔ NO SIXTH BRIDGE WAS ADDED. The list is still the five C6 names; what is new is one key on each. +#: ⛔⛔ AND THE VALUE IS DECLARED, NEVER INFERRED FROM `direction`. An earlier draft of the depth-2 +#: work mapped `bridge -> one_to_many` off the direction alone and got the right answer here by +#: luck; the same map read `forward -> many_to_one`, which is FALSE of `module_link_fields +#: ('customer_data')`, where `invoices` and `orders` are forward link fields and both are one-to-MANY +#: fan-outs. A mislabelled cardinality does not go red, it goes to the client, which unnests the +#: embedding and repeats values. Each of the five below was read off its own row: one customer has +#: many orders, many order lines and many open invoices; one product has many order lines; one order +#: has many order lines. BRIDGES = ( {'key': 'bridge_customer_orders', 'label': 'Orders', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_orders', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', - 'where': None, 'served': True}, + 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_customer_order_lines', 'label': 'Order lines', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_order_lines', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', - 'where': None, 'served': True}, + 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_customer_open_invoices', 'label': 'Open invoices', + 'source_label': 'Customers', 'source': 'customer_data', 'target': 'ut_odoo_invoices', 'source_col': IDENTITY_COL, 'target_col': 'partner_id', - 'where': OPEN_INVOICE_WHERE, 'served': False}, + # ⭐ W53-T19 / R5: `served` flipped to True when `ut_odoo_invoices` gained its read-through + # binding (`routes_odoo_tables._odoo_sources`). The predicate above is pushed into that + # window's SQL beside the pin, which is the only thing that was ever missing here. + 'where': OPEN_INVOICE_WHERE, 'served': True, 'cardinality': 'one_to_many'}, # ⚠ `product_id`, NOT `pid`, AND THE OBVIOUS CHOICE WOULD HAVE JOINED NOTHING AT ALL. # `product_data.pid` is `crc32(default_code)` (`modules/product_data.sku_pid`), so matching it @@ -117,15 +231,17 @@ BRIDGES = ( # column to join to. {'key': 'bridge_product_order_lines', 'label': 'Order lines', + 'source_label': 'Products', 'source': 'product_data', 'target': 'ut_odoo_order_lines', 'source_col': 'product_id', 'target_col': 'product_id', - 'where': None, 'served': True}, + 'where': None, 'served': True, 'cardinality': 'one_to_many'}, {'key': 'bridge_order_order_lines', 'label': 'Order lines', + 'source_label': 'Orders', 'source': 'ut_odoo_orders', 'target': 'ut_odoo_order_lines', 'source_col': IDENTITY_COL, 'target_col': 'order_id', - 'where': None, 'served': True}, + 'where': None, 'served': True, 'cardinality': 'one_to_many'}, ) @@ -150,7 +266,7 @@ def _link_bag(field): def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True, - where=None): + where=None, cardinality=None): """ONE `/related` entry, in C6's shape. ⚠ TWO ADDITIONS TO C6'S LITERAL `{direction, field?, on?}`, both deliberate, both declared in @@ -180,10 +296,17 @@ def _entry(key, label, target, direction, field=None, on=None, frm=None, served= 'served': bool(served)} if where: out['where'] = str(where) + if cardinality: + # ⚠ OMITTED WHEN ABSENT, ON PURPOSE (W53-T17). `related()` calls the three builders without + # a cardinality, so its entries keep the EXACT shape `routes_odoo_tables._related_pin` + # already resolves against; `one_hop()` is what annotates them, from the declaration each + # edge came with. A key that appeared unconditionally as None would be a new field in a + # contract another ticket's fence already reads. + out['cardinality'] = str(cardinality) return out -def forward_links(table, fields): +def forward_links(table, fields, include_retired=False): """The FORWARD entries for `table`: every `type: "link"` field on it, in field order. `fields` is whatever the caller's permission wall handed back, so a link field this account @@ -192,6 +315,10 @@ def forward_links(table, fields): ⭐ TWO LINKS BETWEEN THE SAME PAIR GIVE TWO ENTRIES, and R2 says so in as many words. The entry `key` is the FIELD key, not the target key, which is what keeps them apart on the wire. + + ⛔ W54-T37 / C5: a field pointing at a RETIRED table is dropped. A tenant document written + before a table was retired can still carry such a column, and the tab it would draw opens a + database this deployment does not have. """ out = [] for field in (fields or ()): @@ -207,10 +334,10 @@ def forward_links(table, fields): target=target, direction='forward', field=field.get('key'), on=bag.get('on'), frm=bag.get('from') or IDENTITY_COL)) - return out + return _live(out, include_retired) -def reverse_links(table, defs): +def reverse_links(table, defs, include_retired=False): """The REVERSE entries for `table`: every OTHER table's link field that points at it. `defs` is `{key: definition}` as `core.user_tables.all_defs` returns it, already narrowed to @@ -220,6 +347,9 @@ def reverse_links(table, defs): ⛔ A TABLE'S LINK TO ITSELF IS NOT A REVERSE LINK. It is already a forward one, and emitting both would give the same edge two tabs pointing at the same grid, one of them labelled from the wrong end. + + ⛔ W54-T37 / C5: a definition left behind for a RETIRED table is not a reverse link either. + Its rows are gone, so the tab would open nothing. """ me = str(table or '') out = [] @@ -244,15 +374,20 @@ def reverse_links(table, defs): # mislabelled `forward` as a negative control for exactly this. on=bag.get('from') or IDENTITY_COL, frm=bag.get('on') or IDENTITY_COL)) - return out + return _live(out, include_retired) -def bridge_entries(table): - """The BRIDGE entries for `table`, from `BRIDGES` and from nowhere else.""" - return [_entry(key=b['key'], label=b['label'], target=b['target'], direction='bridge', - on=b['target_col'], frm=b['source_col'], served=b['served'], - where=b['where']) - for b in bridges_from(table)] +def bridge_entries(table, include_retired=False): + """The BRIDGE entries for `table`, from `BRIDGES` and from nowhere else. + + ⚠ NO BRIDGE TARGETS A RETIRED TABLE TODAY and the filter is applied anyway, because the day one + does the alternative is a tab onto a database that is not there. A declared bridge onto an + absent grain is a DECLARATION BUG, not a served edge, and it should disappear the same way. + """ + return _live([_entry(key=b['key'], label=b['label'], target=b['target'], direction='bridge', + on=b['target_col'], frm=b['source_col'], served=b['served'], + where=b['where']) + for b in bridges_from(table)], include_retired) def edge_identity(entry): @@ -269,7 +404,7 @@ def edge_identity(entry): str(via.get('from') or ''), str(entry.get('where') or '')) -def related(table, fields=None, defs=None): +def related(table, fields=None, defs=None, include_retired=False): """THE `/related` ANSWER for one table: forward, then reverse, then bridge. `[{key, label, via: {direction, key, field?, on?, from?}, target, served, where?}]`. @@ -293,10 +428,11 @@ def related(table, fields=None, defs=None): ⚠ ON A TENANT WITH NO PRE-SET LINKS the bridge is not a duplicate of anything and is offered unchanged, which is the whole reason the suppression is computed rather than declared. """ - forward = forward_links(table, fields or ()) - reverse = reverse_links(table, defs or {}) + forward = forward_links(table, fields or (), include_retired) + reverse = reverse_links(table, defs or {}, include_retired) seen = {edge_identity(e) for e in forward} | {edge_identity(e) for e in reverse} - bridges = [b for b in bridge_entries(table) if edge_identity(b) not in seen] + bridges = [b for b in bridge_entries(table, include_retired) + if edge_identity(b) not in seen] return forward + reverse + bridges @@ -347,23 +483,120 @@ def pick(entries, via): return hits[0] if len(hits) == 1 else None -def unknown_via_refusal(table, via): +def pick_candidates(entries, via): + """THE LIST `pick` COMPUTES AND THROWS AWAY: every entry the bag `via` could have meant. + + ⭐⭐ W53-T18 / C3. `pick` answers None for an AMBIGUOUS bag, which is right, and then discards + the very list that would tell the caller how to disambiguate: `customer_data` really does carry + two bridges reading `{direction: "bridge", on: "partner_id"}`, and "cannot resolve" plus + silence leaves the caller no way to find that out except by listing the whole door again. C3 + asks for *"the ENUMERATED candidates ... not prose"* for an unknown OR AMBIGUOUS `via`, and + this is the ambiguous half of it. + + ⛔⛔ IT MIRRORS `pick`'s MATCHING AND CHANGES NOTHING ABOUT `pick`. That function is resolved + against by `routes_odoo_tables._related_pin`, which is in another ticket's fence, so it is left + byte for byte as it was rather than refactored to share this body. `verify_odoo_relational` + asserts the two agree in both directions (exactly one candidate iff `pick` answers an entry, + and it is that entry) rather than letting the duplication drift unobserved. + + ⛔ IT IS A STRICT MIRROR AND RETURNS `[]` WHEN NOTHING MATCHED. The TEACHING fallback (show the + caller what IS there when their bag matched nothing) lives in `via_candidates` below, and the + two are kept apart on purpose: mixing them would break the very biconditional the gate checks, + because a no-match bag that fell back to a single-entry list would look exactly like the + unambiguous case `pick` answers. + """ + entries = list(entries or ()) + if not isinstance(via, dict): + return [] + want_dir = str(via.get('direction') or '') + if want_dir not in DIRECTIONS: + return [] + want_key = str(via.get('key') or '') + want_field = str(via.get('field') or '') + want_on = str(via.get('on') or '') + if not (want_key or want_field or want_on): + return [] # a direction alone names no edge; `pick` refuses it too + hits = [] + for entry in entries: + got = entry['via'] + if got['direction'] != want_dir: + continue + if want_key: + if entry['key'] == want_key: + hits.append(entry) + continue + if want_field and str(got.get('field') or '') != want_field: + continue + if want_on and str(got.get('on') or '') != want_on: + continue + hits.append(entry) + return hits + + +def via_candidates(entries, via=None): + """WHAT TO OFFER A CALLER WHOSE BAG DID NOT RESOLVE: join columns and a retry bag each. + + `[{key, label, target, direction, on, from, retry}]`, where `retry` is the LITERAL `via` bag + that resolves that one edge and nothing else. `via.key` is what makes a bag unambiguous (see + `_entry`), so that is what the retry carries. + + ⭐ AMBIGUOUS FIRST, THEN THE DIRECTION, THEN EVERYTHING. An ambiguous bag is offered exactly the + entries it was ambiguous BETWEEN, which is the shortest useful answer; a bag that matched + nothing at all is offered its direction's entries, and a malformed one the whole scoped list, + because a caller who named nothing resolvable has to see what exists before it can name it. + ⛔ ALWAYS OUT OF THE SCOPED `entries` IT WAS HANDED, so it can never name an edge this account + would not have been offered. + """ + hits = pick_candidates(entries, via) + if not hits: + want_dir = str((via or {}).get('direction') or '') if isinstance(via, dict) else '' + hits = [e for e in (entries or ()) if e['via']['direction'] == want_dir] or list( + entries or ()) + out = [] + for entry in hits: + got = entry.get('via') or {} + out.append({'key': str(entry.get('key') or ''), 'label': str(entry.get('label') or ''), + 'target': str(entry.get('target') or ''), + 'direction': str(got.get('direction') or ''), + 'on': str(got.get('on') or ''), 'from': str(got.get('from') or ''), + 'retry': {'direction': str(got.get('direction') or ''), + 'key': str(entry.get('key') or '')}}) + return out + + +def unknown_via_refusal(table, via, candidates=None): """The SENTENCE an unknown `via` earns: what was asked, why it cannot be answered, what to do. ⛔ A SENTENCE, NOT A CODE. C6 says an unknown `via` *"refuses with a sentence"*, and the reason is that the alternative to refusing is serving the whole table under a tab that claims to show one record's related rows. A bare 400 leaves the caller unable to tell a typo from a link an administrator has since deleted. + + ⭐ `candidates` IS OPTIONAL AND ADDITIVE (W53-T18 / C3), so `routes_odoo_tables._related_pin` + keeps its exact existing sentence with two positional arguments while a caller that has run + `via_candidates` gets the enumeration appended. ⛔ THE STRUCTURED LIST IS THE ANSWER AND THIS + IS ONLY ITS PROSE: a client renders `via_candidates`' array, not this clause, which is why the + array is returned separately rather than being recoverable by parsing English. ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). """ named = '' if isinstance(via, dict): named = str(via.get('field') or via.get('on') or via.get('direction') or '') - return ("this account cannot resolve a relation " + elif isinstance(via, (list, tuple)): + named = ' then '.join(str(v) for v in via) + base = ("this account cannot resolve a relation " + (f"named '{named}' " if named else "") + f"on '{table}', so there is no edge to pin these rows to. Ask for the relation by " f"one of the entries GET /api/v1/grid/" + str(table) + "/related returns, or open " "the linked database on its own and filter it there") + cands = list(candidates or ()) + if not cands: + return base + shown = '; '.join( + "'{k}' opens '{t}' joined on '{on}' from '{frm}' (retry with key '{k}')".format( + k=c.get('key'), t=c.get('target'), on=c.get('on') or '', frm=c.get('from') or '') + for c in cands) + return base + f". The relations this account is offered on '{table}': {shown}" def unresolvable_source_refusal(table, entry): @@ -393,14 +626,23 @@ def unresolvable_source_refusal(table, entry): def unserved_target_refusal(entry): """The sentence a DECLARED but UNSERVED edge earns. - `served: False` means the target grid stores its rows rather than being read through, so its - window has no predicate to push a pin into. Naming it is the whole difference between a - reported limit and a feature that quietly does nothing. + `served: False` means the target grid has no read-through binding on this deployment, so the + door that answers its rows accepts no row filter and there is nowhere to push a pin. Naming it + is the whole difference between a reported limit and a feature that quietly does nothing. + + ⛔⛔ W53-T19 / R5 — REWRITTEN, NOT DELETED, AND NO EDGE CARRIES `served: False` TODAY. Every + Odoo bridge is now servable, so this function has no live subject in `BRIDGES`; it is kept + because `served` is a per-edge DECLARATION that the next connector's bridges will set from + their own measurement, and deleting the refusal would leave that flag with no consequence + except an empty tab. ⚠ THE WORDING CHANGED FOR A SECOND REASON: the old sentence described + the target by where its rows were kept, and R5 retires that phrasing from the tree along + with the refusal it belonged to. The cause is now named as what it actually is, the ABSENCE + OF A BINDING, which is also what `routes_odoo_tables`' `not_read_through` refusal now says. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). """ - return (f"'{entry.get('target')}' is served from its stored copy rather than read through the " - "live mirror, and that door takes no row filter, so this relation cannot be pinned " - "yet. Open that database directly, or ask for the stored read to accept a relation " - "pin") + return (f"'{entry.get('target')}' has no read-through binding on this deployment, so the door " + "that answers its rows takes no row filter and this relation cannot be pinned yet. " + "Open that database directly, or ask for a read-through binding to be added for it") # --------------------------------------------------------------------------- the pin @@ -444,6 +686,184 @@ def pin_sql(id_sql, member_ids): return member_ids_tail(id_sql, member_ids) +class PinRefusal(ValueError): + """A named refusal that accompanies an explicitly unscoped measure answer.""" + + def __init__(self, code, message, path=()): + super().__init__(message) + self.marker = {'code': code, 'message': message, 'path': list(path), + 'scoped': False} + + +class MeasureAnswer(dict): + """Ordinary value mapping plus `refusals` for the host's response envelope. + + Hosts must serialize `refusals` alongside the values when accepting a related pin. + Keeping metadata out of the pid/field namespace preserves every existing mapping consumer. + """ + + def __init__(self, values=(), refusals=()): + super().__init__(values) + self.refusals = list(refusals) + + +class MeasureMatches(set): + """Condition membership with the same refusal channel as MeasureAnswer.""" + + def __init__(self, values=(), refusals=()): + super().__init__(values) + self.refusals = list(refusals) + + +_MEASURE_PIN_CONTEXT = ContextVar('measure_pin_context', default=None) + + +@contextmanager +def measure_pin_context(*, read, bindings, may_open, fields_for=None, defs=None): + """Bind the host's permission-checked relation reads to this request only. + + `read(table, column, values, where)` returns canonical rows, including pid and join + columns, through the SAME scoped reader used by row windows. It must AND the named + edge's predicate and membership with the account's wall, without a row-window limit. + `bindings[semantic_store_table]` declares `{table, column, sql}` rows mapping those + canonical ids to a metric's SQL. `sql` may contain `{alias}` from the semantic model. + Neither callbacks nor SQL bindings come from the client's related object. + + This adapter is necessary because core has no session and cannot import API readers. + Missing context refuses by name; it never guesses a join or reads an unscoped store. + """ + token = _MEASURE_PIN_CONTEXT.set(dict(read=read, bindings=bindings, may_open=may_open, + fields_for=fields_for, defs=defs, cache_key=object())) + try: + yield + finally: + _MEASURE_PIN_CONTEXT.reset(token) + + +def measure_pin_key(related): + """Wire identity plus the trusted read context; permission contexts never share answers.""" + if related is None: + return None + ctx = _MEASURE_PIN_CONTEXT.get() or {} + return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key') + + +@dataclass +class MeasurePin: + path: list + rows: dict + bindings: dict + refusal: dict = None + + def predicate(self, store): + """AND every declared, traversed member set into this metric's raw rows.""" + if self.refusal is not None: + return None, [] + parts, params = [], [] + for binding in self.bindings.get(store['table'], ()): + table, column = binding['table'], binding['column'] + if table not in self.rows: + continue + values = [] + for row in self.rows[table]: + if row.get(column) in (None, ''): + raise PinRefusal('relation_source_unavailable', + f"The related metric needs '{column}' on '{table}', " + "but that value could not be read. Values are unscoped.", + self.path) + values.append(row[column]) + try: + sql, args = pin_sql(binding['sql'].format(alias=store['alias']), values) + except (TypeError, ValueError) as exc: + raise PinRefusal('relation_binding_unavailable', + f"The metric binding on '{table}' cannot carry this pin. " + "Values are unscoped.", self.path) from exc + parts.append(f'({sql})') + params.extend(args) + if not parts: + raise PinRefusal('relation_binding_unavailable', + f"The metric source '{store['table']}' has no declared binding " + "to this record path. Values are unscoped.", self.path) + return ' AND '.join(parts), params + + +def measure_pin(related): + """Resolve only the requested named edges, retaining intermediate row membership. + + Retaining the intermediate is essential: customer -> lines -> products must not expand + back to every customer's lines for those products. Empty membership remains a real pin. + """ + if isinstance(related, MeasurePin): + return related + if not isinstance(related, dict): + raise PinRefusal('bad_relation', 'A related metric needs {table, record, via}.') + table, record, via = related.get('table'), related.get('record'), related.get('via') + path = via.get('path') if isinstance(via, dict) else via + if isinstance(path, str): + path = [path] + if isinstance(path, (list, tuple)) and len(path) > MAX_DEPTH: + raise PinRefusal('over_depth', over_depth_refusal(table, path), path) + if not isinstance(table, str) or not table or record in (None, ''): + raise PinRefusal('bad_relation', 'A related metric needs {table, record, via}.') + ctx = _MEASURE_PIN_CONTEXT.get() or {} + if ctx and not ctx['may_open'](table): + raise PinRefusal('forbidden_table', + f"'{table}' cannot be opened for this record scope. " + 'Metric values are unscoped.') + fields_for = ctx.get('fields_for') + fields = fields_for(table) if fields_for else None + entries = related_paths(table, fields=fields, defs=ctx.get('defs'), + may_open=ctx.get('may_open'), fields_for=fields_for) + if isinstance(via, dict) and path is None: + # Bag matching still requires an edge name, never just a join-column heuristic. + entry = pick([e for e in entries if e['hops'] == 1], via) if via.get('key') else None + else: + entry = pick_path(entries, path) + if entry is None: + if isinstance(path, (list, tuple)) and path: + diagnosis = diagnose_path(table, path, fields=fields, defs=ctx.get('defs'), + may_open=ctx.get('may_open'), fields_for=fields_for, + entries=entries) + raise PinRefusal(diagnosis['code'], diagnosis['message'], path) + raise PinRefusal('unknown_relation', unknown_via_refusal(table, via)) + path = entry['path'] + # Resolve each leg before reading. A composite's target is not necessarily the target + # whose binding is absent; the refusal must name the actual unserved leg. + hops = [pick_path(entries, path[:1])] + if len(path) == 2: + mid = hops[0]['target'] + hops.append(pick_path(related_paths( + mid, fields=fields_for(mid) if fields_for else None, defs=ctx.get('defs'), + depth=1, may_open=ctx.get('may_open'), fields_for=fields_for), path[1:])) + for hop in hops: + if hop is None: + raise PinRefusal('unknown_relation', unknown_via_refusal(table, path), path) + if not hop.get('served'): + raise PinRefusal('relation_not_served', + unserved_target_refusal(hop) + '. Metric values are unscoped.', path) + if not ctx: + raise PinRefusal('relation_context_required', + 'This request has no scoped relation reader for metrics. ' + 'Metric values are unscoped.', path) + rows = {table: list(ctx['read'](table, IDENTITY_COL, [record], None))} + if not rows[table]: + raise PinRefusal('relation_source_unavailable', + f"The origin record could not be read from '{table}'. " + 'Metric values are unscoped.', path) + source = table + for hop in hops: + frm = hop['via'].get('from') or IDENTITY_COL + values = [r.get(frm) for r in rows[source]] + if any(v in (None, '') for v in values): + raise PinRefusal('relation_source_unavailable', + unresolvable_source_refusal(source, hop), path) + target = hop['target'] + rows[target] = list(ctx['read'](target, hop['via'].get('on') or IDENTITY_COL, + values, hop.get('where'))) if values else [] + source = target + return MeasurePin(path, rows, ctx['bindings']) + + def compiled_pin_matches_filter_sql(id_sql, member_ids, columns=None): """Does `pin_sql` still spell the member tail exactly as `filter_sql` spells it? @@ -473,3 +893,949 @@ def compiled_pin_matches_filter_sql(id_sql, member_ids, columns=None): if theirs.endswith(')'): theirs = theirs[:-1] return ours, theirs + + +# ═══════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W53-T17 / RULING R12 / CONTRACT C3 — DEPTH 2, BY NAMED EDGES, AND NEVER A PATH SEARCH. +# +# R12, verbatim: *"Relation traversal is DEPTH 2, and every hop is NAMED. The connector spec +# declares edges; the server enumerates what is reachable within two hops and offers it; the client +# pins one by naming each edge (`via = ["orders","order_lines"]`). Never a path search. Depth 3 +# REFUSES and names the missing hop. Aggregation stays at depth 1."* +# +# ⛔⛔ ENUMERATE, DO NOT SEARCH, AND THE DIFFERENCE IS THE WHOLE TICKET. A search asks "how do I get +# from A to B" and answers with whichever route it found first; the answer changes when an edge is +# added somewhere else entirely, silently, and the number on the screen changes with it. Enumeration +# asks "what is reachable in at most two named hops" and hands the caller the list; the caller pins +# one BY NAME. `related_paths` below therefore has exactly two levels written out as two loops, does +# not call itself, and `verify_odoo_relational` asserts that mechanically by parsing this source +# rather than by trusting this paragraph. +# +# ⛔ NOTHING HERE AGGREGATES. R12's last sentence keeps rollups at depth 1, so a two-hop entry is a +# ROW SET resolved at read time and this module stores nothing about it. +# +# ⛔ AN EDGE KEY IS UNIQUE PER SOURCE TABLE, NOT ACROSS A CONNECTOR. `connector_spec.edges_for +# ("odoo")` returns ten edges carrying seven distinct keys, because four tables link to +# `ut_odoo_customers` on `partner_id` and the key is `f"{target}_link__{from_col}"`. A `path` is +# unambiguous ONLY because it is anchored at the table it was enumerated from, exactly as +# PostgREST's embed syntax is anchored at the table in the URL. ⇒ NEVER build a dict keyed on edge +# key alone across a connector, and always resolve a path against the entries of ONE source table. +# ═══════════════════════════════════════════════════════════════════════════════════════════════ + +#: ⭐ TWO. R12 fixes it and this constant is the only place it is written down; the route's `depth` +#: default and the over-depth refusal both read it, and the gate asserts the route's default equals +#: it rather than letting two literal 2s drift. +MAX_DEPTH = 2 + +#: The cardinality vocabulary, as a FALLBACK only. `core.connector_spec.CARDINALITIES` is the +#: source; this copy exists because that import is done lazily (see `_spec`) and a caller in a +#: stripped environment must still get a sane answer instead of a NameError. `verify_odoo_relational` +#: asserts the two tuples are equal, which is this file's own idiom for a transcribed constant +#: (`OPEN_INVOICE_WHERE` is held against `odoo_relational._AR_WHERE` the same way). +CARDINALITIES = ('many_to_one', 'one_to_many', 'one_to_one', 'many_to_many') + +#: The inverse of each cardinality, read from the other end of the same edge. ⛔ IT IS A PROPERTY OF +#: THE EDGE, NOT OF THE DIRECTION IT IS READ IN: `one_to_one` and `many_to_many` are symmetric and +#: invert to themselves, which a `forward -> many_to_one / reverse -> one_to_many` map cannot say. +_INVERSE = {'many_to_one': 'one_to_many', 'one_to_many': 'many_to_one', + 'one_to_one': 'one_to_one', 'many_to_many': 'many_to_many'} + + +def _spec(): + """`core.connector_spec`, or None when it cannot be imported. + + ⛔ LAZY AND FORGIVING, like `compiled_pin_matches_filter_sql`'s `harness` import. It is a + SIBLING inside `core` rather than a layer above, so there is no layering objection to it; the + laziness is about import spelling (`core.connector_spec` from the api layer, plain + `connector_spec` from inside `platform/`) and about a caller that has neither. + """ + try: + import core.connector_spec as spec # noqa: PLC0415 + return spec + except Exception: # noqa: BLE001 + pass + try: + import connector_spec as spec # noqa: PLC0415 + return spec + except Exception: # noqa: BLE001 + return None + + +def cardinality_of_link_field(bag): + """The cardinality of a FORWARD link field, from the bag's own `single` flag. + + ⛔⛔ THE INPUT IS THE DECLARATION, NEVER THE DIRECTION, AND THAT IS WHY THIS IS ONE HELPER PER + SOURCE INSTEAD OF ONE MAP KEYED ON `via.direction`. A map saying `forward -> many_to_one` is + FALSE of the grid this ticket is about: `odoo_relational.module_link_fields('customer_data')` + declares `invoices` and `orders` as forward link fields and both are one-to-MANY fan-outs, and + `agent_link` carries its own warning that Odoo's `agent_ids` is a many2many. All three would + have reached the client labelled `many_to_one`. + + `core.user_tables._clean_link` stores `out['single'] = True` when and only when the author asked + for one row, so `single` IS the declaration and its absence is the declaration of the other case. + """ + return 'many_to_one' if (bag or {}).get('single') is True else 'one_to_many' + + +def cardinality_of_reverse_field(bag): + """The cardinality of a link field read FROM THE TARGET END: the inverse of the forward one. + + Same rule as above and the same reason it is not keyed on direction: the answer comes from the + forward declaration, inverted, so a many2many forward stays many2many rather than becoming the + `one_to_many` a direction map would have produced. + """ + return _INVERSE[cardinality_of_link_field(bag)] + + +def _bridge_cardinality(row): + """A bridge's cardinality, off its own `BRIDGES` row. Declared there, never inferred here.""" + return str(row.get('cardinality') or 'one_to_many') + + +def compose_cardinality(a, b): + """The cardinality of a two-hop path, from its two hops. See the vocabulary above. + + ⭐ TWO `many_to_one` HOPS STAY `many_to_one` (a line names its order names its customer: still + one customer). Two `one_to_many` hops stay `one_to_many` (a customer's orders' lines: still a + fan-out). A MIX IS HONESTLY `many_to_many`, and it is NOT rounded toward either side: fanning + out and then folding back reaches many rows from many, and calling that `one_to_many` would + promise the client a nesting the rows do not have. + ⚠ `one_to_one` composes as the identity, because a one-to-one hop changes no multiplicity. + ⚠ ANYTHING UNRECOGNISED COMPOSES TO `many_to_many`, the widest honest answer, rather than to a + guess that happens to look tidy. + """ + a, b = str(a or ''), str(b or '') + if a not in CARDINALITIES or b not in CARDINALITIES: + return 'many_to_many' + if a == 'one_to_one': + return b + if b == 'one_to_one': + return a + if a == b and a in ('many_to_one', 'one_to_many'): + return a + return 'many_to_many' + + +def spec_entries(table, include_retired=False): + """The CONNECTOR-SPEC edges of `table`, in BOTH directions, in `related()`'s entry shape. + + ⛔⛔ BOTH DIRECTIONS, AND AN OUTBOUND-ONLY READ CANNOT ANSWER THE QUESTION THIS TICKET ASKS. + MEASURED after W53-T38 landed: `edges_for('ut_odoo_customers')` is exactly one edge, `agent_id + -> ut_odoo_agents`, because `connector_spec` declares every Odoo edge CHILD -> PARENT (a line + names its order, an order names its customer). The edges TARGETING `ut_odoo_customers` are the + invoices, orders, order lines and GL lines, all on `partner_id`, and none of them is outbound. + So `ut_odoo_customers -> ut_odoo_orders -> ut_odoo_order_lines` is TWO REVERSE HOPS and an + outbound-only reader would enumerate a customer's agent and stop. `verify_odoo_relational` + drives an outbound-only `spec_entries` as a negative control so this stays load-bearing rather + than decorative. + + ⛔ THE COLUMNS SWAP WITH THE DIRECTION, EXACTLY AS `reverse_links` WARNS. Read its note before + changing anything here. Outbound, THIS table's `from_col` is matched against the target's + `to_col`. Inbound, the roles invert: the tab pins the OTHER table on ITS OWN `from_col`, keyed + by the value THIS table's `to_col` carries. Getting it backwards is the silent-wrong-join bug, + which returns a plausible row count and no error. + + ⚠ THE INBOUND KEY IS `f"{source_key}__{edge_key}"`, the shape `reverse_links` already uses + (`f"{key}__{field_key}"`), because an edge key is unique per SOURCE table and four Odoo tables + share `ut_odoo_customers_link__partner_id`. The inbound LABEL names the SOURCE table, since that + is the database the tab opens. + + ⚠ `served` TAKES `_entry`'s DEFAULT, exactly as the forward and reverse link FIELDS do. Only + `BRIDGES` carries a measured `served` today, and inventing one per spec edge here would be a + second, unmeasured source for the same judgement. + + ⛔⛔ W54-T37 / C5 / INSTRUCTION 22 - THIS IS WHERE THE OWNER'S REFUSAL WALL WAS MINTED, AND THE + RETIRED FILTER IS THE PRIMARY REPAIR. Four Odoo specs (invoices, orders, order lines, GL lines) + declare a link at `ut_odoo_customers` and the line grain declares one at `ut_odoo_products`; + both keys are RETIRED, so the outbound half offered tabs onto databases that do not exist. The + inbound half is the mirror image: the retired customers spec row carries its own link at + `ut_odoo_agents`, so the agents grid was offered a reverse tab back onto the absent table. + """ + spec = _spec() + me = str(table or '') + if spec is None or not me: + return [] + out = [] + for edge in spec.edges_for(me): + out.append(_entry( + key=str(edge.get('key') or ''), label=str(edge.get('label') or ''), + target=str(edge.get('target') or ''), direction='forward', + on=edge.get('to_col'), frm=edge.get('from_col'), + cardinality=edge.get('cardinality'))) + for source_key, source_spec in sorted((spec.all_specs() or {}).items()): + if str(source_key) == me: + continue # a table's link to itself is already an outbound edge + for from_col, target, to_col, cardinality in (getattr(source_spec, 'links', ()) or ()): + if str(target) != me: + continue + out.append(_entry( + key=f"{source_key}__{target}_link__{from_col}", + label=str(getattr(source_spec, 'label', '') or source_key), + target=str(source_key), direction='reverse', + on=from_col, # the column on the OTHER table + frm=to_col, # the column on THIS one + cardinality=_INVERSE.get(str(cardinality), 'many_to_many'))) + return _live(out, include_retired) + + +def _link_field_cardinalities(table, fields, defs): + """`{entry_key: cardinality}` for the FORWARD and REVERSE link-field entries of `table`. + + It rescans the same two inputs `forward_links` and `reverse_links` scan, keyed the same way, so + that `related()` keeps its exact current shape and the annotation lives entirely in `one_hop`. + """ + out = {} + for field in (fields or ()): + bag = _link_bag(field) + if bag is None or not str(bag.get('table') or ''): + continue + out[str(field.get('key') or '')] = cardinality_of_link_field(bag) + me = str(table or '') + for key, defn in sorted((defs or {}).items()): + key = str(key) + if key == me or not isinstance(defn, dict): + continue + for field in (defn.get('fields') or ()): + bag = _link_bag(field) + if bag is None or str(bag.get('table') or '') != me: + continue + out[f"{key}__{field.get('key') or ''}"] = cardinality_of_reverse_field(bag) + return out + + +def one_hop(table, fields=None, defs=None, include_retired=False): + """EVERYTHING ONE NAMED HOP FROM `table`: `related()` plus the connector spec, deduplicated. + + Every entry carries a `cardinality`, taken from the declaration it came with: + + * a forward link FIELD -> the bag's `single` flag (`cardinality_of_link_field`) + * a reverse link FIELD -> the inverse of that (`cardinality_of_reverse_field`) + * a BRIDGE -> its own `BRIDGES` row + * a CONNECTOR-SPEC edge -> the 4th element of the link tuple, inverted when read inbound + + ⛔⛔ THE EXISTING SOURCE WINS A COLLISION, which is the same posture `related()` already takes + when it suppresses a bridge that repeats a link field. `bridge_order_order_lines` and the spec's + inbound `ut_odoo_order_lines -> ut_odoo_orders` edge are the SAME join, column for column, and + offering both would give one order two tabs both labelled Order lines over identical rows. R2: + *"one tab per related table"*. The dedupe is on `edge_identity`, which is target plus both join + columns plus the predicate; read its docstring for why `where` is in there. + """ + base = list(related(table, fields, defs, include_retired)) + cards = _link_field_cardinalities(table, fields, defs) + bridge_cards = {b['key']: _bridge_cardinality(b) for b in BRIDGES} + out = [] + for entry in base: + entry = dict(entry) + got = cards.get(entry['key']) or bridge_cards.get(entry['key']) + entry['cardinality'] = str(got or 'many_to_many') + out.append(entry) + seen = {edge_identity(e) for e in out} + # A product bridge is also the declared line -> product hop. Reverse the declared + # columns, including product_id, rather than inventing a pid join or searching a path. + for bridge in bridges_into(table): + entry = _entry(key=f"{bridge['source']}__{bridge['key']}", + label=bridge.get('source_label') or bridge['source'], + target=bridge['source'], direction='reverse', + on=bridge['source_col'], frm=bridge['target_col'], + served=bridge['served'], + cardinality=_INVERSE.get(_bridge_cardinality(bridge), 'many_to_many')) + # A predicate belongs to the forward target; reversing it needs an explicit binding. + # Keep such reverse edges absent instead of applying the predicate on the wrong table. + if bridge.get('where') or edge_identity(entry) in seen: + continue + if not _live([entry], include_retired): + continue + seen.add(edge_identity(entry)) + out.append(entry) + for entry in spec_entries(table, include_retired=include_retired): + if edge_identity(entry) in seen: + continue + seen.add(edge_identity(entry)) + out.append(dict(entry)) + return out + + +def _two_hop_entry(first, second): + """ONE two-hop entry, composed from the two one-hop entries it is made of. + + ⭐⭐ `via.direction` IS `'path'`, WHICH IS DELIBERATELY **NOT** IN `DIRECTIONS`, AND THAT IS A + FAIL-CLOSED DESIGN RATHER THAN AN OVERSIGHT. `pick()` returns None for any direction outside + `DIRECTIONS`, so every existing resolver — including `routes_odoo_tables._related_pin`, which + is in another ticket's fence — refuses a two-hop pin instead of resolving it as if it were one + hop. `pick_path` is the only thing that can answer it. The day a resolver learns to serve two + hops, it opts in by calling `pick_path`; until then nothing serves half a traversal. + + ⚠ NO `where` IS FLATTENED ONTO THE COMPOSITE. Each hop keeps its own predicate on its own entry + and `path` names the hops in order, so a resolver applies them in order. Copying one hop's + predicate onto the pair would claim ONE predicate where there may be two, and the open-invoice + bridge is exactly an edge whose predicate is half of what it means. + + ⛔ THE LABEL IS DISPLAY COPY: sentence case, no em dash, no en dash (standing rule 2). + """ + path = [first['key'], second['key']] + key = f"{first['key']}>{second['key']}" + return { + 'key': key, + 'label': f"{second['label']} via {first['label']}", + 'via': {'direction': 'path', 'key': key, 'path': list(path)}, + 'target': second['target'], + 'served': bool(first.get('served')) and bool(second.get('served')), + 'path': path, + 'hops': 2, + 'cardinality': compose_cardinality(first.get('cardinality'), second.get('cardinality')), + } + + +def related_paths(table, fields=None, defs=None, depth=MAX_DEPTH, may_open=None, + fields_for=None, include_retired=False): + """C3'S `/related` ANSWER: everything reachable from `table` in at most `depth` NAMED hops. + + `[{key, label, via, target, served, where?, cardinality, path, hops}]`, one-hop entries first + and in `related()`'s existing order, then the two-hop ones. + + ⛔⛔ THIS FUNCTION DOES NOT CALL ITSELF AND MUST NEVER LEARN TO. R12 says *"never a path + search"*, and the two levels are written out as two loops precisely so that the claim is + checkable: `verify_odoo_relational` parses this source and asserts there is no recursive call + and that `MAX_DEPTH` is 2. A `depth` argument that recursed would make depth 3 a one-character + change and R12 a comment. + + `may_open(key) -> bool` is an optional permission wall, applied to EVERY table on the path + INCLUDING the intermediate: hop 1 is filtered on its target, and the intermediate of a two-hop + path IS a hop-1 target, so closing a table removes both the tab and everything reachable + through it. ⚠ W53-T18 turns that silent disappearance into a NAMED refusal; this ticket only + makes the filtering correct. + + `fields_for(key) -> fields` supplies the INTERMEDIATE table's field list, since the caller's + permission wall is the only thing that knows it. ⛔ `defs` IS PASSED DOWN TO THE SECOND HOP. + A first cut passed neither, and the intermediate's reverse links were then invisible: the + entire second half of the answer was missing with nothing red beside it. + + ⛔ A HOP-2 EDGE POINTING BACK AT `table` IS DROPPED. `customer -> orders -> customer` is a round + trip, not a relation, and offering it would put a tab on the customers grid labelled Customers. + + ⛔⛔ `include_retired` IS FOR DIAGNOSIS ONLY AND NOTHING SERVES IT. Left False, no hop lands on + a table this deployment declares and does not have; set True, the absent hops come back so a + caller can say WHICH one a request named. Both passes of the permission diff leave it False, + which is what stops an absent edge being reported as a wall removal. + """ + if depth not in (1, MAX_DEPTH): + raise ValueError( + "relation depth is 1 or %d; %r was asked for. The route turns this into a named " + "refusal rather than a 500" % (MAX_DEPTH, depth)) + table = str(table or '') + hop1 = [e for e in one_hop(table, fields, defs, include_retired) + if may_open is None or may_open(e['target'])] + out = [] + for entry in hop1: + entry = dict(entry) + entry['path'] = [entry['key']] + entry['hops'] = 1 + out.append(entry) + if depth < MAX_DEPTH: + return out + seen = set() + for first in hop1: + mid = first['target'] + mid_fields = fields_for(mid) if fields_for is not None else None + for second in one_hop(mid, mid_fields, defs, include_retired): + if second['target'] == table: + continue + if may_open is not None and not may_open(second['target']): + continue + path = (first['key'], second['key']) + if path in seen: + continue + seen.add(path) + out.append(_two_hop_entry(first, second)) + return out + + +def pick_path(entries, via): + """The ONE entry of an ALREADY-BUILT list whose `path` is EXACTLY `via`, or None. + + ⛔⛔ EXACT. NOT A PREFIX, NOT A SUFFIX, AND THE REASON IS THE ONE `pick()` ALREADY STATES FOR + THE DICT SHAPE. A prefix match would let `via=["orders"]` be answered by the two-hop + `orders > order lines` entry, which shares its first key: the caller asked for a customer's + orders and would be handed their order LINES, correctly shaped and wrong. A suffix match has the + mirror problem, because an edge key is unique per SOURCE table and the same key appears under + several sources across a connector. Refusing sends the caller back with a sentence; guessing + sends it a number. + + ⚠ A BARE STRING IS ONE HOP, so a client that sends `via=orders` is asking for the one-hop entry + and gets it. That is the back-compat leg of C3, and it is a one-element PATH rather than a + second matching rule. + + ⚠ THE LIST IS AN ARGUMENT, exactly as it is for `pick()`, so the route resolves a pin against + the SCOPED enumeration and an edge a reader cannot be offered is an edge they cannot pin. + """ + if isinstance(via, str): + want = [via] + elif isinstance(via, (list, tuple)): + want = [str(v) for v in via] + else: + return None + if not want or not all(want): + return None + for entry in (entries or ()): + if [str(k) for k in (entry.get('path') or ())] == want: + return entry + return None + + +def over_depth_refusal(table, via): + """The sentence a `via` naming MORE than `MAX_DEPTH` hops earns, naming the hops beyond the limit. + + ⛔ IT NAMES THEM. R12 says depth 3 *"REFUSES and names the missing hop"*, and the naming is the + difference between a limit a caller can work around in one more request and a wall. The caller + is told which hops were dropped and how to reach them: ask for the first two, open that database + on its own, pin the next hop from there. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + hops = [str(v) for v in (via or ())] + beyond = hops[MAX_DEPTH:] + named = ', '.join(f"'{h}'" for h in beyond) or 'none' + kept = ', '.join(f"'{h}'" for h in hops[:MAX_DEPTH]) or 'none' + plural = 'hops' if len(beyond) != 1 else 'hop' + return (f"a relation on '{table}' is resolved through at most {MAX_DEPTH} named hops, and this " + f"request names {len(hops)}. The {plural} beyond the limit: {named}. Ask for {kept}, " + f"open the database that answer lands on, and pin the next hop from there") + + +# ═══════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W53-T18 / RULING R13 / CONTRACT C3 — EVERY HOP IS PERMISSION CHECKED, AND A FILTERED HOP +# SAYS SO. THREE DISTINGUISHABLE ANSWERS, NEVER ONE EMPTY LIST. +# +# R13, verbatim: *"Permission on a traversal requires read on EVERY table in the path, intermediate +# included. A permission-filtered traversal must be distinguishable on the wire from an empty one: +# 'no related rows', 'you may not see this table' and 'this hop was filtered' are three different +# answers."* +# +# W53-T17 made the FILTERING correct: `related_paths` applies `may_open` to every table on the +# path, intermediate included, so a closed table removes the tab and everything reachable through +# it. What it could not do was SAY SO. A path that vanished under the wall and a path that never +# existed came back as the same `None` from `pick_path`, and a listing shortened by the wall came +# back as a 200 with a shorter array. That is the exact defect R13 exists to prevent, and it is not +# hypothetical anywhere: PostgREST silently EMPTIES an embed under RLS and Odoo omits forbidden +# One2many rows with no error at all, so a reader of either cannot tell "nothing here" from "not +# for you". +# +# ⛔⛔ THE MECHANISM IS A DIFF, NOT AN INFERENCE FROM WHERE A WALK STOPPED. An earlier attempt +# walked the path one key at a time and named the code after the step that failed, which conflates +# "this edge does not exist" with "there are no rows" and cannot tell a permission-filtered +# traversal from an empty one at all. The reachable set is instead enumerated TWICE, once WITH the +# wall and once WITHOUT it, and the two are compared: +# +# * present UNFILTERED, absent FILTERED, and a table on the path is closed -> `forbidden_table`, +# NAMING that table +# * present UNFILTERED, absent FILTERED, and every table on it is open -> `hop_filtered`, +# NAMING the hop (the wall that bit is the FIELD wall: the link column the hop is built on is +# not visible to this account, so the edge is gone while both of its tables remain open) +# * absent from BOTH -> `unknown_via`, +# WITH the enumerated candidates, their join columns and the exact retry strings +# * present in BOTH, permitted end to end, reaching nothing -> `no_related_rows` +# +# ⛔⛔ THE UNFILTERED ENUMERATION CHOOSES THE CODE AND NAMES THE TABLE OR HOP. IT NEVER WIDENS WHAT +# IS SERVED. Nothing in this block returns a row, an entry or a tab; `wall_refusals` takes the +# SERVED list as an argument and only ever reports what is NOT in it, and `verify_odoo_relational` +# proves byte for byte that `related_paths`' answer is unchanged by any of it. +# +# ⚠ NAMING THE FORBIDDEN TABLE IS A DELIBERATE, RULED DISCLOSURE. R13's own words are *"you may +# not see this table"*, which cannot be said without saying WHICH. Do not "fix" this as a leak. +# ⛔ NOTHING ELSE ABOUT IT IS DISCLOSED: no label, no row count, no column list, no hint about +# whether it holds rows for this record. The name, and the fact that the door is shut. +# ═══════════════════════════════════════════════════════════════════════════════════════════════ + +#: ⭐ THE THREE R13 ANSWERS, PLUS THE ONE THAT IS NOT A PERMISSION ANSWER AT ALL, PLUS C5'S. +#: ⚠ `unknown_via` KEEPS THE NAME `routes_grid.grid_related` ALREADY RAISES rather than gaining a +#: synonym: a fourth spelling of the same refusal would be a client-visible rename dressed as a +#: feature. `routes_odoo_tables` spells its depth-1 cousin `unknown_relation` and is in another +#: ticket's fence, so the two names survive side by side and this constant says which is which. +REFUSAL_NO_ROWS = 'no_related_rows' +REFUSAL_FORBIDDEN_TABLE = 'forbidden_table' +REFUSAL_HOP_FILTERED = 'hop_filtered' +REFUSAL_UNKNOWN_VIA = 'unknown_via' + +#: ⭐⭐ W54-T37 / C5's FOURTH ANSWER: the edge names a database this deployment does not have. +#: ⛔ IT IS NOT A PERMISSION ANSWER AND MUST NEVER BE CONFUSED WITH ONE. That confusion IS +#: instruction 22: the route wall fails closed on any key it cannot resolve, so an absent table +#: came back as `forbidden_table` and the reader was told to ask an administrator for access to a +#: database nobody can be granted. +REFUSAL_RETIRED_TARGET = 'retired_target' + +#: ⛔⛔ FIVE ENTRIES, AND C5's *"four wire answers"* IS STILL SATISFIED BY FOUR OF THEM. +#: C5 names `no_related_rows`, `forbidden_table`, `hop_filtered` and the new `retired_target`. +#: `unknown_via` PREDATES C5 (it is the name `routes_grid.grid_related` has always raised for an +#: edge that does not exist at all) and is neither removed nor renamed by it: a fifth spelling of +#: an existing refusal would be a client-visible rename dressed as a feature. So the tuple holds +#: five and the four C5 names are the ones that are new-or-ruled. A gate reads this rather than +#: five literals scattered through two files. +REFUSAL_CODES = (REFUSAL_NO_ROWS, REFUSAL_FORBIDDEN_TABLE, REFUSAL_HOP_FILTERED, + REFUSAL_RETIRED_TARGET, REFUSAL_UNKNOWN_VIA) + + +def _named_path(path): + """`'a' then 'b'` — how a path is spelled inside a sentence. Empty for an empty path.""" + return ' then '.join(f"'{k}'" for k in (str(p) for p in (path or ()))) + + +# ═══════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W54-T38 / INSTRUCTION 22 / DESIGN.md SECTION 4 — TWO STRINGS PER REFUSAL: THE SENTENCE AND +# THE DIAGNOSIS. THE SPLIT IS AUTHORED HERE BECAUSE THE CLIENT MAY NOT AUTHOR IT. +# +# The owner pasted one of these refusals back at us, four times over, off one screen. Every word of +# it was TRUE after W54-T37 fixed the cause, and it was still the wrong thing to put on a screen: +# DESIGN.md section 4 says *"never over-explain"*, and a sixty-word paragraph repeated once per +# removed edge is a wall a reader scrolls past to reach the rows they asked for. +# +# ⛔⛔ THE FIX IS NOT TO SHORTEN THE DIAGNOSIS. W53-T18 wrote those paragraphs deliberately, so a +# traversal is never served short and silent, and deleting them would trade one defect for the one +# they were written against. The fix is that the refusal now carries TWO server-authored strings: +# +# * `message` — ONE SHORT SENTENCE, the thing a reader sees first. It still names whatever R13 +# and C5 require it to name (the closed table, the filtered hop, the absent database). +# * `detail` — the FULL diagnosis, word for word what `message` used to be, for the disclosure +# the client puts it behind. +# +# ⛔ WHY THE SERVER SPLITS IT AND NOT THE CLIENT. `RecordDetail.tsx` already carries the rule in as +# many words: *"THE SENTENCE IS STILL THE SERVER'S, WHOLE. The code chooses the treatment; it never +# rewrites the words, which would fork one wire contract into two opinions."* A client that cut a +# paragraph at its first full stop would be authoring copy out of a regular expression, and the day +# a sentence gains a comma the summary line changes with nobody having decided it should. +# +# ⚠ `message` STAYS POPULATED AND STAYS FIRST, so every existing reader keeps working: the route's +# `_refusal_error` copies it into the X2 envelope, and a client that has never heard of `detail` +# shows the short sentence and loses only the explanation. +# ⛔ NO EM DASH AND NO EN DASH IN EITHER STRING (standing rule 2). Both reach a screen, and D-653 +# records that no gate scans a Python string for them, so this is checked by eye and by the +# snippet in this ticket's handoff. +# ═══════════════════════════════════════════════════════════════════════════════════════════════ + + +def no_related_rows_refusal(table, path=None): + """R13's FIRST answer: everything was open, and there is genuinely nothing on the other side. + + ⛔ IT IS STILL AN ANSWER, NOT AN ABSENCE. C3: *"never one empty list"*. A caller handed `[]` + with no code beside it cannot tell this apart from the two refusals below, which is the whole + of R13, so the empty case is spelled out loud and says in as many words that nothing was + hidden from it. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + ⭐ W54-T38: THE SENTENCE, and only the sentence. `no_related_rows_detail` carries what this + used to say in full. Read the banner above `_named_path` before shortening either of them. + """ + named = _named_path(path) + if named: + return f"Nothing is linked through {named} on '{table}'." + return f"'{table}' is not linked to any database this account can open." + + +def no_related_rows_detail(table, path=None): + """The DIAGNOSIS behind `no_related_rows_refusal`, for the client's disclosure. + + ⭐ WORD FOR WORD WHAT THE SENTENCE USED TO BE. Nothing was cut: an empty side that says only + "nothing is linked" leaves a reader unable to tell it from a wall, which is the whole of R13, + so the paragraph that rules that out still exists and is one click away. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + named = _named_path(path) + if named: + return (f"The relation reached by {named} on '{table}' is open to this account at every " + f"hop, and it reaches no rows. Nothing was hidden by permission and no hop was " + f"filtered, so there is simply nothing on the other side of it.") + return (f"'{table}' reaches no other database within {MAX_DEPTH} named hops that this account " + f"may open, and nothing was removed by permission on the way. Ask an administrator to " + f"link it to another database, or open the database you want on its own.") + + +def forbidden_table_refusal(table, path, blocked, hop=None): + """R13's SECOND answer: a table ON the path, intermediate included, is shut to this account. + + ⭐ IT NAMES `blocked`, AND THAT IS RULED RATHER THAN CARELESS. R13 asks for *"you may not see + this table"* as a distinct answer, and an answer that will not say which table is the empty + list again wearing a code. Nothing else about that table is disclosed. + ⚠ THE CAUSE IS NAMED AS A PAIR, the posture `routes_odoo_tables._related_pin` already takes: + the wall is fail-closed and answers False both for a genuine refusal and for a database it + could not read at all, so claiming only the first would tell a reader their access was revoked + when the store had merely hiccupped. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + ⭐ W54-T38: THE SENTENCE, and only the sentence. It still NAMES the blocked table, because + R13's disclosure is the one thing that may not move behind a disclosure: a reader who cannot + see which door is shut has been handed the empty list again. `forbidden_table_detail` carries + the pair of causes, the path and the recommendation. + ⚠ NO DIGITS AND NO PATH IN THIS ONE. `verify_odoo_relational` asserts that this sentence + discloses the table's NAME and nothing else about it, digits included, so the hop keys and the + path (which may carry either) stay in the detail where the same leg does not reach. + """ + return (f"This relation passes through '{blocked}', and this account may not open it.") + + +def forbidden_table_detail(table, path, blocked, hop=None): + """The DIAGNOSIS behind `forbidden_table_refusal`, for the client's disclosure. + + ⚠ THE CAUSE IS STILL NAMED AS A PAIR, the posture `routes_odoo_tables._related_pin` takes: the + wall is fail-closed and answers False both for a genuine refusal and for a database it could + not read at all, so claiming only the first would tell a reader their access was revoked when + the store had merely hiccupped. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + named = _named_path(path) or 'this relation' + where = f" at the hop named '{hop}'" if hop else '' + return (f"The relation reached by {named} on '{table}' passes through '{blocked}'{where}, and " + f"this account may not open '{blocked}', either because it is not granted to you or " + f"because it could not be read just now. The traversal is refused rather than served " + f"short, so no part of it is silently missing. Ask an administrator for access to " + f"'{blocked}', or ask for a relation that does not pass through it.") + + +def retired_target_refusal(table, path, missing): + """C5's FOURTH answer: the relation opens a database this workspace does not have any more. + + ⛔⛔ IT SAYS THE TRUE CAUSE AND NAMES NO PERMISSION. Instruction 22 is a wall of *"this account + may not open `ut_odoo_customers`"* about a table that was retired, so this sentence may not say + "not granted to you" and may not send the reader to an administrator for access: there is no + access to grant. It names the database and says it is gone. + ⛔ SHORT, ON PURPOSE. One sentence. The long diagnosis belongs behind a disclosure, not in a + second wall. + ⭐ W54-T38 FINISHED THAT SENTENCE'S OWN NOTE. T37 wrote "short, on purpose" and then shipped a + sentence carrying the path and an API pointer, because there was nowhere else to put them yet. + There is now: `retired_target_detail`. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + return f"'{missing}' is no longer part of this workspace." + + +def retired_target_detail(table, path, missing): + """The DIAGNOSIS behind `retired_target_refusal`, for the client's disclosure. + + ⛔⛔ IT NAMES NO PERMISSION EITHER, and that rule does not weaken one level down. Instruction + 22 is a reader being sent to an administrator for access to a database nobody can grant; the + same sentence hidden behind a caret would still be that. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + named = _named_path(path) or 'this relation' + return (f"The relation reached by {named} on '{table}' opens '{missing}', which this " + f"deployment does not have any more. It was retired and its rows are gone, so there " + f"is nothing to show here and nothing to ask anyone for. Ask GET /api/v1/grid/{table}" + f"/related for the relations '{table}' still has.") + + +def first_retired_on_path(table, keys, fields=None, defs=None, depth=MAX_DEPTH, fields_for=None, + retired_entries=None): + """`(absent_target, hop_key)` for the FIRST hop of `keys` that opens a retired table. + + `('', '')` when the path names none, which is every ordinary path: the enumeration already + dropped these edges, so this walk is the BACKSTOP and answering nothing is its normal day. + + ⚠ IT COSTS NOTHING WHERE NOTHING IS REGISTERED. A process with an empty registry cannot have + an absent target, so it returns before enumerating anything at all. + ⛔ THE PREFIX WALK IS WHY AN INTERMEDIATE IS COVERED WITHOUT A SPECIAL CASE, exactly as the + permission diff below covers it: the intermediate of a two-hop path is the target of its + one-hop prefix, so a retired intermediate is found at `idx` 0 and named there. + """ + keys = [str(k) for k in (keys or ())] + if not keys or not _RETIRED_TABLES: + return ('', '') + if retired_entries is None: + retired_entries = related_paths(table, fields=fields, defs=defs, depth=depth, + may_open=None, fields_for=fields_for, + include_retired=True) + for idx in range(len(keys)): + step = pick_path(retired_entries, keys[:idx + 1]) + if step is not None and is_retired_table(step.get('target')): + return (str(step.get('target')), keys[idx]) + return ('', '') + + +def hop_filtered_refusal(table, path, hop): + """R13's THIRD answer: every table on the path is open, and ONE HOP was removed anyway. + + ⭐ THIS IS THE ANSWER THE OTHER TWO CANNOT GIVE AND THE REASON THE DIFF EXISTS. The wall has + two halves: the TABLE wall (`may_open`) and the FIELD wall (the column list this account may + see). When the second bites, the hop's own link column is invisible while both of its tables + stay open, so the traversal is neither forbidden nor empty. Served as `[]` it would read as + "this record has nothing here", which is a claim about the business made out of a permission. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + ⭐ W54-T38: THE SENTENCE, and only the sentence. It still NAMES the hop for the same reason + the forbidden sentence still names the table. `hop_filtered_detail` carries the rest. + """ + return f"The hop named '{hop}' was filtered out for this account." + + +def hop_filtered_detail(table, path, hop): + """The DIAGNOSIS behind `hop_filtered_refusal`, for the client's disclosure. + + ⭐ IT IS THE ONE THAT MOST NEEDS ITS PARAGRAPH KEPT: "the databases are open to you and the + link column is not" is not a thing a reader can guess from a short sentence, and served as an + empty grid it would read as a claim about the business. + ⛔ NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2). + """ + named = _named_path(path) or 'this relation' + return (f"The hop named '{hop}' of the relation reached by {named} on '{table}' was filtered " + f"out for this account. The databases it crosses are open to you. The link column " + f"this hop is built on is not one you may see, so the hop was removed rather than " + f"answered empty. Ask an administrator for that column, or ask for a relation that " + f"does not use it.") + + +def candidate_paths(table, entries, defs=None, fields_for=None): + """EVERY path this account is offered, with its JOIN COLUMNS and the EXACT string to retry. + + `[{key, label, target, path, hops, served, columns: [{key, target, on, from}], via, retry}]`. + + ⭐⭐ PostgREST's `PGRST201` IS THE SHAPE, AND THE REASON IS THAT A REFUSAL SHOULD TEACH. That + error does not say "the embed is ambiguous"; it lists every relationship it could have meant, + each with the columns it joins on, plus a hint naming the exact strings to retry with. A caller + told "resolve this yourself among these N things" without the N things has to go and list them + in a second request, and a UI cannot offer them at all. + + ⛔⛔ IT TAKES THE ALREADY-BUILT, ALREADY-SCOPED `entries` AND WIDENS NOTHING. A candidate list + computed from the unfiltered enumeration would hand an account the names of edges it may not be + offered, which is the opposite of every wall in this file. `pick_path(entries, c['via'])` + resolves for every candidate returned, and the gate proves that rather than trusting it. + + ⚠ `columns` CARRIES ONE ROW PER HOP, so a two-hop candidate names BOTH joins. The first hop's + columns are on the entry itself; the second hop's are an edge of the INTERMEDIATE table, which + is not in `entries` at all, so it is re-read from `one_hop(mid, fields_for(mid), defs)` under + the caller's own field wall (memoised per intermediate) rather than invented here. + + ⚠ `retry` IS THE LITERAL QUERY STRING (`via=orders&via=order_lines`), not a description of one. + """ + entries = list(entries or ()) + by_path = {} + for entry in entries: + by_path[tuple(str(k) for k in (entry.get('path') or ()))] = entry + memo = {} + out = [] + for entry in entries: + path = [str(k) for k in (entry.get('path') or ())] + if not path: + continue + columns = [] + first = by_path.get((path[0],)) + if first is not None: + fv = first.get('via') or {} + columns.append({'key': path[0], 'target': str(first.get('target') or ''), + 'on': str(fv.get('on') or ''), 'from': str(fv.get('from') or '')}) + if len(path) > 1 and first is not None: + mid = str(first.get('target') or '') + if mid not in memo: + memo[mid] = one_hop(mid, fields_for(mid) if fields_for is not None else None, defs) + second = next((e for e in memo[mid] if str(e.get('key')) == path[1]), None) + if second is not None: + sv = second.get('via') or {} + columns.append({'key': path[1], 'target': str(second.get('target') or ''), + 'on': str(sv.get('on') or ''), 'from': str(sv.get('from') or '')}) + out.append({ + 'key': str(entry.get('key') or ''), 'label': str(entry.get('label') or ''), + 'target': str(entry.get('target') or ''), 'path': path, + 'hops': int(entry.get('hops') or len(path)), 'served': bool(entry.get('served')), + 'columns': columns, 'via': path, + 'retry': '&'.join(f"via={k}" for k in path), + }) + return out + + +def unknown_via_answer(table, keys, candidates): + """The whole `unknown_via` refusal as a BAG: `{code, message, path, candidates}`. + + ⛔ THE CANDIDATES ARE STRUCTURED, NOT PROSE, AND C3 SAYS SO IN AS MANY WORDS. The sentence is + still there because a human reads it; the array is there because a client renders it. Spelling + them only into the sentence is what `unknown_via_refusal` used to do, and lane E cannot draw a + retry button out of an English clause. + ⛔ NO EM DASH AND NO EN DASH anywhere in `message` (standing rule 2). + + ⭐⭐ W54-T38 — AND THIS IS THE LONGEST WALL OF THE FIVE. Measured before this ticket: 1,921 + characters on `customer_data`, because the enumeration of every candidate with its join + columns and retry string is inside the sentence. It splits like the other four: a short + `message`, and the whole of what it used to say in `detail`. ⛔ THE `candidates` ARRAY IS + UNTOUCHED and is still the answer a client renders; the prose was never the machine-readable + half and C3 says so. + """ + keys = [str(k) for k in (keys or ())] + cands = list(candidates or ()) + named = _named_path(keys) or 'that relation' + tail = '' + if cands: + shown = '; '.join( + "'{k}' opens '{t}' joined on '{on}' from '{frm}' (retry with {r})".format( + k=c['key'], t=c['target'], + on=(c['columns'][-1]['on'] if c.get('columns') else ''), + frm=(c['columns'][-1]['from'] if c.get('columns') else ''), + r=c['retry']) + for c in cands) + tail = (f" The relations this account is offered on '{table}', with the exact string to " + f"retry with: {shown}") + else: + tail = (f" Ask GET /api/v1/grid/{table}/related with no 'via' to see every relation this " + f"account is offered, then send the 'path' of the one you want, one edge key per " + f"hop") + return {'code': REFUSAL_UNKNOWN_VIA, 'path': keys, 'candidates': cands, + 'message': f"'{table}' offers no relation reached by {named} to this account.", + 'detail': (f"'{table}' offers no relation reached by {named} to this account, and " + f"nothing was removed by permission on the way, so this is an edge that " + f"does not exist rather than one you may not see." + tail)} + + +def _paths_of(entries): + """The set of `path` tuples in an enumeration. The unit the diff is computed in.""" + return {tuple(str(k) for k in (e.get('path') or ())) for e in (entries or ())} + + +def diagnose_path(table, keys, fields=None, defs=None, may_open=None, fields_for=None, + depth=MAX_DEPTH, entries=None, open_entries=None, + open_fields=None, open_fields_for=None, retired_entries=None): + """WHY `pick_path` came back None, as one of the named answers. + + `{code, message, detail, table?, hop?}`. ⭐ W54-T38 — `message` is ONE SHORT SENTENCE and + `detail` is the full diagnosis it used to be; see the banner above `no_related_rows_refusal` + for why the split is authored here rather than cut out of a paragraph by the client. + + ⛔⛔ NEVER RAISES AND NEVER RETURNS None. A diagnosis that can itself fail is the fail-open bug + `routes_grid._may_open` warns about: every branch below ends in a real bag carrying a code from + `REFUSAL_CODES` and a sentence. + + THE DIFF, and it is the whole function: + + 1. the FILTERED enumeration (`may_open` + the caller's field wall). If `keys` resolves in it, + the path is permitted end to end and reaches nothing: `no_related_rows`. + 1b. ⛔⛔ W54-T37 / C5 - DOES ANY HOP OPEN A DATABASE THIS DEPLOYMENT DOES NOT HAVE? Then + `retired_target`, NAMING it. THIS STEP IS ABOVE BOTH OF THE ONES BELOW AND THE ORDER IS + THE TICKET: an absent table is absent from both enumerations, so step 2 would have called + it an edge that never existed, and the ROUTE's fail-closed wall (which answers False for + a key it cannot resolve) would have made step 3 call it a permission. Instruction 22 is + the second of those two wrong answers, printed at a reader who was told to ask for access + to a database nobody can grant. + 2. the UNFILTERED enumeration (no `may_open`, and the widest field list the caller can offer). + If `keys` does not resolve THERE either, the edge genuinely does not exist and no wall was + involved: `unknown_via`, with the candidates, their join columns and the retry strings. + 3. otherwise the wall removed it, and WHICH hop it removed is read off the PREFIXES: the + shortest prefix of `keys` present unfiltered and absent filtered is the hop that went. If + a table on that hop is closed, `forbidden_table` NAMING it; if every table is open, the + field wall took the edge and it is `hop_filtered` NAMING the hop. + + ⭐ STEP 3 IS WHY THE INTERMEDIATE IS COVERED WITHOUT A SPECIAL CASE. The intermediate of a + two-hop path IS the target of the one-hop prefix, so a closed intermediate makes prefix `[k1]` + the first absent one and its target is the intermediate. R13's *"read on EVERY table in the + path, intermediate included"* falls out of the prefix walk rather than being asserted beside it. + + ⛔ `open_fields` / `open_fields_for` MUST BE A SUPERSET of `fields` / `fields_for`, or step 2 + can call a wall-removed path unknown. The route builds them as a UNION for exactly that reason; + when they are omitted the diff runs on the table wall alone, which is correct and narrower. + ⛔ `entries` / `open_entries` / `retired_entries` ARE AN OPTIMISATION ONLY. The route has + already paid for the first two; a gate hands none and gets the same answer computed from + scratch. The third is the absent-target pass, which is skipped outright where no table is + registered as absent. + """ + keys = [str(k) for k in (keys or ())] + table = str(table or '') + if entries is None: + entries = related_paths(table, fields=fields, defs=defs, depth=depth, + may_open=may_open, fields_for=fields_for) + if pick_path(entries, keys) is not None: + return {'code': REFUSAL_NO_ROWS, 'path': keys, + 'message': no_related_rows_refusal(table, keys), + 'detail': no_related_rows_detail(table, keys)} + # ⛔⛔ BEFORE THE TWO PERMISSION ANSWERS AND BEFORE `unknown_via`. See step 1b above: a table + # that is not there is not a permission and not a typo, and saying either was the defect. + missing, missing_hop = first_retired_on_path( + table, keys, fields=(open_fields if open_fields is not None else fields), defs=defs, + depth=depth, fields_for=(open_fields_for if open_fields_for is not None else fields_for), + retired_entries=retired_entries) + if missing: + return {'code': REFUSAL_RETIRED_TARGET, 'path': keys, 'table': missing, + 'hop': missing_hop, + 'message': retired_target_refusal(table, keys, missing), + 'detail': retired_target_detail(table, keys, missing)} + if open_entries is None: + open_entries = related_paths( + table, fields=(open_fields if open_fields is not None else fields), defs=defs, + depth=depth, may_open=None, + fields_for=(open_fields_for if open_fields_for is not None else fields_for)) + if pick_path(open_entries, keys) is None: + return unknown_via_answer(table, keys, candidate_paths(table, entries, defs, fields_for)) + + have = _paths_of(entries) + for idx in range(len(keys)): + prefix = tuple(keys[:idx + 1]) + if prefix in have: + continue + step = pick_path(open_entries, list(prefix)) + blocked = str((step or {}).get('target') or '') + if blocked and may_open is not None and not may_open(blocked): + return {'code': REFUSAL_FORBIDDEN_TABLE, 'path': keys, 'table': blocked, + 'hop': keys[idx], + 'message': forbidden_table_refusal(table, keys, blocked, keys[idx]), + 'detail': forbidden_table_detail(table, keys, blocked, keys[idx])} + return {'code': REFUSAL_HOP_FILTERED, 'path': keys, 'hop': keys[idx], + 'message': hop_filtered_refusal(table, keys, keys[idx]), + 'detail': hop_filtered_detail(table, keys, keys[idx])} + # ⚠ EVERY PREFIX SURVIVED AND THE WHOLE DID NOT. Only reachable if two enumerations disagree + # about a path they agree about every prefix of; the last hop is the honest thing to name. + return {'code': REFUSAL_HOP_FILTERED, 'path': keys, 'hop': (keys[-1] if keys else ''), + 'message': hop_filtered_refusal(table, keys, keys[-1] if keys else ''), + 'detail': hop_filtered_detail(table, keys, keys[-1] if keys else '')} + + +def wall_refusals(table, fields=None, defs=None, depth=MAX_DEPTH, may_open=None, fields_for=None, + entries=None, open_entries=None, open_fields=None, open_fields_for=None): + """THE LISTING ANNOTATION: what the wall removed from this table's enumeration, BY NAME. + + `[{code, message, detail, path, table?, hop?}]`, a SIBLING of the served list and never a + replacement for it. ⭐ W54-T38 — `message` is the short sentence, `detail` the diagnosis behind + the client's disclosure. + + ⛔⛔ AN EMPTY LIST IS NEVER AN ACCEPTABLE ANSWER TO A PERMISSION-FILTERED TRAVERSAL, AND THIS + IS THE HALF OF C3 THAT IS EASIEST TO MISS. A 400 on a pin covers the request that named ONE + path; the LISTING answers 200 with a shorter array, and a shorter array is exactly the silent + empty PostgREST and Odoo both hand out. So a path the wall removed is reported here rather than + omitted. ⛔ AND A 403 FOR THE WHOLE STRIP WOULD BE EQUALLY WRONG: the other tabs are + legitimately visible, and refusing them all to explain one is a wall with a bug in it. + + ⭐ COLLAPSED ON `(code, table, hop)`, ON PURPOSE. One closed table can remove a dozen paths; a + dozen identical refusals naming it would be a longer answer carrying no more information, and + on a heavily restricted account it would enumerate most of the tenant on every request. R13 + asks for the table and the hop to be NAMED, which each distinct row does exactly once. + + ⭐ AND WHEN NOTHING WAS REMOVED AND NOTHING IS THERE, `no_related_rows` IS SAID OUT LOUD. That + is the case `{related: []}` alone cannot express, and it is R13's first answer. + """ + table = str(table or '') + if entries is None: + entries = related_paths(table, fields=fields, defs=defs, depth=depth, + may_open=may_open, fields_for=fields_for) + if open_entries is None: + open_entries = related_paths( + table, fields=(open_fields if open_fields is not None else fields), defs=defs, + depth=depth, may_open=None, + fields_for=(open_fields_for if open_fields_for is not None else fields_for)) + # ⛔ ONE absent-target pass for the WHOLE listing, not one per removed path. Every path below + # comes out of `open_entries`, which never carries an absent target, so this walk is expected + # to find nothing; computing it per path would pay for a full enumeration per refusal to learn + # that. ⚠ Skipped entirely where nothing is registered as absent. + retired_entries = None + if _RETIRED_TABLES: + retired_entries = related_paths( + table, fields=(open_fields if open_fields is not None else fields), defs=defs, + depth=depth, may_open=None, + fields_for=(open_fields_for if open_fields_for is not None else fields_for), + include_retired=True) + have = _paths_of(entries) + out, seen = [], set() + for entry in open_entries: + path = [str(k) for k in (entry.get('path') or ())] + if not path or tuple(path) in have: + continue + got = diagnose_path(table, path, fields=fields, defs=defs, may_open=may_open, + fields_for=fields_for, depth=depth, entries=entries, + open_entries=open_entries, retired_entries=retired_entries) + if got.get('code') == REFUSAL_UNKNOWN_VIA: + continue # cannot happen: `path` came out of `open_entries` itself + ident = (got.get('code'), got.get('table') or '', got.get('hop') or '') + if ident in seen: + continue + seen.add(ident) + out.append(got) + if not entries and not out: + out.append({'code': REFUSAL_NO_ROWS, 'path': [], + 'message': no_related_rows_refusal(table), + 'detail': no_related_rows_detail(table)}) + return out diff --git a/platform/core/schema_overlay.py b/platform/core/schema_overlay.py index d1804fb8f8090fc41560ba92d1b69e7c575be068..38e2e7286a4a67593aa692c8374520855e459d23 100644 --- a/platform/core/schema_overlay.py +++ b/platform/core/schema_overlay.py @@ -15,6 +15,14 @@ empty overlay must reproduce the contract byte-for-byte, and nothing here may ev `order` for every field, because a UI that PATCHes back what it read would then pin every column forever and "absent means inherit" would be gone permanently. +⭐ W53-T27 (R9 / C2) — THE SAME IDEA ONE LEVEL UP, FOR A DATABASE'S OWN NAME, AND WITHOUT A +SECOND DOCUMENT. `database_label()` / `merge_database_labels()` at the bottom of this file answer +"what is this database called for this tenant" with the rule the whole module already runs on: +shipped default, per-tenant override, ABSENT MEANS INHERIT. They are PURE and they touch no +store, because the database-name override does NOT live in this document — see the section header +above them for where it does live and why putting it here would have built the very thing R9 +forbids. + R12 — v1 is SHOW/HIDE, RELABEL, REORDER ONLY. No field typing, no hard delete, no endpoint browser. That wall lives in `clean_patch()` and NOWHERE else, so there is exactly one place to read when the question is "what may an overlay do". @@ -329,6 +337,101 @@ def ignored_hides(database_key, fields, overlay_doc, locked=None): if k in locked_set and isinstance(ov.get(k), dict) and ov[k].get("hidden") is True] +# ================================================================================================= +# DATABASE LABELS — the same rule, one level up (W53-T27; R9, contract C2) +# +# ⛔⛔ THE TRAP FIRST, BECAUSE THE RULING'S OWN PROSE WALKS INTO IT. R9 says a tenant's override of +# a DATABASE's name "lives in core/schema_overlay.py". It does not, and building it here would +# have created the second override store R9's own sentence forbids. This document is FIELD scoped +# — `{database_key: {field_key: {hidden,label,order}}}` — and has no slot for a database's own +# name; meanwhile this app has renamed a `ut_*` database from the navigation rail since wave 19 +# through the tenant-wide `nav_meta` bucket (`routes_nav.save_nav_meta`). Two documents would mean +# one workspace holding two names for one database, with the rail and the connector's Databases +# tab each correct about their own. +# +# ⭐ WHAT THIS MODULE OWNS OF THE QUESTION IS THE IDEA, NOT THE STORAGE: a shipped default, a +# per-tenant override, and ABSENT MEANS INHERIT. These three functions are the ONE place that idea +# is spelled for a database's NAME, so a route, a gate and a future second connector cannot each +# decide separately what a blank override means. They are pure: no store, no tenant, no mutation. +# `routes_connectors` holds the tenant's store handle and calls them. +# +# ⛔ AND THAT PURITY IS WHY A RE-SYNC CANNOT CLOBBER AN OVERRIDE (R9's second clause). A connector +# re-sync (`odoo_relational.apply_plan`) rewrites table DEFINITIONS; it never reaches `nav_meta` +# and it never calls these. The guarantee is structural rather than remembered. +# ================================================================================================= + +def database_label(default_label, override): + """What ONE database is called for this tenant: its override, else our shipped default. + + ⛔ BLANK IS ABSENT. `""`, `" "`, `None` and a non-string all mean INHERIT, exactly as an + absent field key does in `merge()`. Treating an empty string as a name is how "clearing the + override falls back to the default" quietly becomes "clearing the override empties the row", + and a nameless database is a row nobody can talk about. + """ + if isinstance(override, str) and override.strip(): + return override.strip() + return default_label if isinstance(default_label, str) else "" + + +def override_name(entry): + """The override carried by one stored entry, normalised to a string. `""` means none. + + Accepts BOTH shapes a caller has: the stored `{"name": ..., "icon": ...}` record (this module + has no opinion about the icon and never touches it) and a plain string from a flattened map. + Anything else is absent, because a malformed record must read as "no override" rather than as + a name nobody can see the source of. + """ + if isinstance(entry, str): + return entry.strip() + if isinstance(entry, dict): + name = entry.get("name") + if isinstance(name, str): + return name.strip() + return "" + + +def merge_database_labels(rows, overrides): + """C2's shipped defaults plus this tenant's overrides, over a list of database rows. + + Returns NEW row dicts in the caller's order, each carrying THREE separate facts rather than + one blended one:: + + defaultLabel our shipped default (C2's `label`), never overwritten by an override + override this tenant's own name, or "" when none is set + label what to CALL it today: `database_label` of the two + + ⛔ THE THREE ARE KEPT APART ON PURPOSE. A payload serving only the effective name cannot tell + a reader whether they are looking at our default or at their own rename, and a screen that + cannot tell those apart cannot honestly offer to go back to the default. + + ⛔ NOTHING IS MUTATED, for the reason in this file's header one level down: the rows a route + hands in are built from MODULE GLOBALS (`odoo_relational.TABLES`, the Amazon label map), so + relabelling one in place would leak one tenant's rename into the next tenant's response with + no error anywhere. + + ⛔ AND `native` IS NEVER TOUCHED. It is the vendor's own identifier for the table; no tenant + may edit it (C2), so no function here writes it. A row that arrives without one keeps + whatever it had. + """ + ov = overrides if isinstance(overrides, dict) else {} + out = [] + for row in rows or (): + if not isinstance(row, dict): + continue + new = dict(row) + default = new.get("defaultLabel") + if not (isinstance(default, str) and default.strip()): + default = new.get("label") + default = default if isinstance(default, str) else "" + key = new.get("key") + override = override_name(ov.get(key) if isinstance(key, str) else None) + new["defaultLabel"] = default + new["override"] = override + new["label"] = database_label(default, override) + out.append(new) + return out + + # ================================================================================================= # THE STORE SEAM — injectable, so nothing here has to reach a real store # ================================================================================================= diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py index b9b349048dc04751ecbf4422af7dd0e0285d08dc..502ba719f5b6c8ee6c37127c9281f188f715d62b 100644 --- a/platform/core/sheet_import.py +++ b/platform/core/sheet_import.py @@ -41,6 +41,10 @@ hands back a workbook that looks complete and is not. from __future__ import annotations import re +import zipfile +import hashlib +import json +import xml.etree.ElementTree as ET from datetime import date, datetime, time, timezone from core import sheets @@ -119,7 +123,9 @@ _RULE_OPS = { #: does not import four chart classes it would then have to keep in step. _CHART_KINDS = {"BarChart": "bar", "BarChart3D": "bar", "LineChart": "line", "LineChart3D": "line", "AreaChart": "area", "AreaChart3D": "area", - "PieChart": "donut", "PieChart3D": "donut", "DoughnutChart": "donut"} + "PieChart": "pie", "PieChart3D": "pie", "DoughnutChart": "donut", + "ScatterChart": "scatter", "StockChart": "stock", + "BubbleChart": "bubble", "RadarChart": "radar"} #: The currency symbols a number format may wear, mapped to the ISO code `CellFormat.currency` #: carries. ⚠ AN UNKNOWN SYMBOL FALLS TO `code`, which renders it EXACTLY as the file asked; @@ -491,7 +497,7 @@ def freeze_of(anchor): def width_px(chars): - """Excel's character width to pixels, clamped into `MIN_COL_WIDTH..MAX_COL_WIDTH`. + """Excel's character width to pixels without applying the grid's drag limits. ⚠ THE CONVERSION IS EXCEL'S OWN APPROXIMATION (7 px per character of the default face plus 5 px of padding), not a measurement of Inter. It cannot be exact across two typefaces, and a @@ -500,7 +506,7 @@ def width_px(chars): """ if not isinstance(chars, (int, float)) or chars <= 0: return 0 - return max(sheets.MIN_COL_WIDTH, min(sheets.MAX_COL_WIDTH, int(round(chars * 7.0 + 5)))) + return int(round(chars * 7.0 + 5)) def _spread(sqref, limit=_MAX_SPREAD): @@ -556,19 +562,17 @@ def _painting(style): return {k: v for k, v in (style or {}).items() if k in _PAINT_KEYS} -def _cells_of(ws, cached, notes, counts): +def _cells_of(ws, cached, notes, counts, excel=None): """One tab's cell map, keyed A1, from the formula pass plus the cached-value pass.""" cells = {} style_only = {} - counts["blank_styles"] = 0 - max_row = min(int(ws.max_row or 1), sheets.MAX_SHEET_ROWS) - max_col = min(int(ws.max_column or 1), sheets.MAX_SHEET_COLS) - if int(ws.max_row or 1) > sheets.MAX_SHEET_ROWS or int(ws.max_column or 1) > \ - sheets.MAX_SHEET_COLS: - notes.add("this tab is larger than one sheet holds, so its far rows or columns were " - "not read") - for row in ws.iter_rows(min_row=1, max_row=max_row, min_col=1, max_col=max_col): - for cell in row: + # openpyxl's normal worksheet already holds a sparse map. iter_rows would materialize + # billions of empty cells when only A1 and XFD1048576 were authored. + if excel is None: + excel = {} + blank_styles = excel.setdefault("blankStyles", {}) + for cell in list(ws._cells.values()): + if cell.__class__.__name__ != "MergedCell": addr = cell.coordinate raw = cell.value has_style = bool(getattr(cell, "has_style", False)) @@ -576,27 +580,20 @@ def _cells_of(ws, cached, notes, counts): continue style = style_of(cell, notes) if has_style else {} if raw is None: - #: ⛔⛔ A STYLE-ONLY CELL IS STORED ONLY IF ITS STYLE PAINTS, AND THAT ONE TEST IS - #: WORTH 26 MiB ON THE OWNER'S MODEL (MEASURED: 41.9 MiB before it, 15.6 after). - #: Excel stamps a number format down whole columns, so 148,327 blank cells - #: arrived carrying an accounting code and nothing else. A number format on an - #: empty cell formats NOTHING: there is no value to render, so the cell is - #: byte for byte invisible, and storing it is pure weight against a 16 MiB cap. - #: A FILL, A BORDER, AN INK OR A MARK is different: that is paint somebody can - #: see on an empty cell, and `sheets.CELL_KINDS`' `empty` exists to carry it. + # Keep paint in a native empty cell; retain other blank styles by reference. paint = _painting(style) if paint: - style_only[addr] = {"kind": "empty", "style": paint} - elif style: - #: ⛔ COUNTED AND SAID, NEVER SILENT (standing rule 1's second sentence). This - #: is a deliberate lossy decision, so it belongs in the DOCUMENT the person - #: keeps, not only in the notes of whoever wrote the importer. - counts["blank_styles"] += 1 + style_only[addr] = {"kind": "empty", "style": paint, + "excel": {"styleId": cell.style_id}} + elif has_style: + # A compact reference retains original formats on blank cells without + # expanding them into 100,000 value-bearing cells or duplicating styles. + blank_styles[addr] = cell.style_id continue if isinstance(raw, str) and raw.startswith("="): counts["formulas"] += 1 cell_out = {"kind": "formula", "state": "stale", - "formula": raw[:sheets._MAX_FORMULA]} + "formula": raw} #: The cached reading, for the differential. Dropped before the save; see the #: module note for why it is not parked in `source`. cell_out["imported_value"] = json_value(cached.get(addr)) @@ -604,10 +601,12 @@ def _cells_of(ws, cached, notes, counts): elif type(raw).__name__ in ("ArrayFormula", "DataTableFormula"): counts["formulas"] += 1 text = str(getattr(raw, "text", "") or "") - notes.add("an array formula was imported as an ordinary formula") cell_out = {"kind": "formula", "state": "stale", "formula": (text if text.startswith("=") else "=" + text - )[:sheets._MAX_FORMULA]} + ), + "formulaMeta": {**dict(raw), + "kind": "array" if type(raw).__name__ == "ArrayFormula" + else "dataTable"}} cell_out["imported_value"] = json_value(cached.get(addr)) cell_out["value"] = cell_out["imported_value"] else: @@ -617,24 +616,24 @@ def _cells_of(ws, cached, notes, counts): #: what a person TYPED; for a number the value is the content and a second copy #: of it is about 0.5 MiB of duplication on the owner's model. if isinstance(value, str): - cell_out["text"] = value[:sheets._MAX_TEXT] + cell_out["text"] = value + if has_style: + cell_out["excel"] = {"styleId": cell.style_id} + if cell.data_type in ("d", "e"): + cell_out.setdefault("excel", {})["dataType"] = cell.data_type + if cell.data_type == "e": + cell_out["error"] = {"code": str(raw)} if style: cell_out["style"] = style cells[addr] = cell_out counts["cells"] += len(cells) - #: ⛔ THE CONTENT CELLS WIN THE CAP. A tab at the ceiling drops its decoration rather than - #: refusing the import, and says so, because a style-only cell is paint and a value is data. + # Styles beyond the populated-cell budget still survive in the compact style-reference map. room = sheets._MAX_CELLS_PER_SHEET - len(cells) if room < len(style_only): - notes.add("this tab holds more cells than one sheet does, so " + - str(len(style_only) - max(0, room)) + " cells that carried only colour were " - "left out") + for addr, blank in list(style_only.items())[max(0, room):]: + blank_styles[addr] = blank["excel"]["styleId"] for addr, blank in list(style_only.items())[:max(0, room)]: cells[addr] = blank - if counts["blank_styles"]: - notes.add("on " + str(counts["blank_styles"]) + " cells that hold no value, a number " - "format or a text style was not kept, because it shapes a value that is not " - "there") return cells @@ -739,30 +738,39 @@ def _rules_of(ws, notes): def _charts_of(ws, tab_name, notes): - """C2's charts for one tab, with the rectangle their series READ (R15).""" + """Retain sources, settings and document geometry separately from the source rectangle.""" + from openpyxl.xml.functions import tostring + out = [] for chart in list(getattr(ws, "_charts", []) or []): kind = _CHART_KINDS.get(type(chart).__name__) if not kind: - notes.add("a " + type(chart).__name__ + " chart has no equivalent here, so it was " - "not imported") - continue + raise ImportRefusal("sheet_chart_kind_unknown", "A " + type(chart).__name__ + + " chart cannot be imported without losing its chart kind.") + if kind == "bar" and getattr(chart, "type", "bar") == "col": + kind = "column" + if len(getattr(chart, "_charts", [])) > 1: + kind = "combo" rect = None - for series in list(getattr(chart, "series", []) or []): - refs = [] - for holder in (getattr(series, "val", None), getattr(series, "cat", None)): + sources = [] + for series in [s for part in (getattr(chart, "_charts", []) or [chart]) + for s in (getattr(part, "series", []) or [])]: + source = {} + for role in ("val", "cat", "xVal", "yVal", "zVal", "tx"): + holder = getattr(series, role, None) for attr in ("numRef", "strRef", "multiLvlStrRef"): node = getattr(holder, attr, None) if holder is not None else None if node is not None and getattr(node, "f", None): - refs.append(str(node.f)) - for ref in refs: - named = sheet_of_ref(ref) - if named and named != tab_name: - #: ⛔ A SERIES THAT READS ANOTHER TAB CANNOT BE DRAWN AS A RECTANGLE ON THIS - #: ONE. Folding it in would give the chart a range that points at this tab's - #: cells, which is a picture of the wrong numbers. - notes.add("a chart series that reads another tab was not imported") - continue + ref = str(node.f) + source[role] = {"ref": ref, "sheet": sheet_of_ref(ref) or tab_name, + "range": rect_of(ref)} + if getattr(series, "tx", None) is not None and getattr(series.tx, "v", None): + source["title"] = str(series.tx.v) + if getattr(series, "graphicalProperties", None) is not None: + source["settings"] = {"xml": tostring(series.graphicalProperties.to_tree()).decode("utf-8")} + sources.append(source) + for data in [v for v in source.values() if isinstance(v, dict) and "ref" in v]: + ref = data["ref"] piece = rect_of(ref) if piece is None: continue @@ -774,13 +782,65 @@ def _charts_of(ws, tab_name, notes): if rect is None: notes.add("a chart whose series range could not be read was not imported") continue - out.append({"range": rect, "kind": kind}) - notes.add("a chart keeps its data range but not its title, colours or placement") + title = getattr(chart, "title", None) + title_text = [] + if title is not None: + for paragraph in getattr(getattr(getattr(title, "tx", None), "rich", None), "p", []) or []: + title_text.extend(str(run.t) for run in (getattr(paragraph, "r", []) or []) + if getattr(run, "t", None) is not None) + out.append({"range": rect, "kind": kind, "series": sources, + "title": "".join(title_text), "geometry": _chart_geometry(chart, ws), + "settings": {"style": getattr(chart, "style", None), + "legend": getattr(getattr(chart, "legend", None), "legendPos", None)}, + "excel": {"xml": tostring(chart._write()).decode("utf-8")}}) + if kind == "stock": + # Native series references exclude their heading row. Keep every price observation. + out[-1]["presentation"] = {"headers": "none", "categoryLabels": "firstColumn", + "title": "".join(title_text), "series": [ + {"key": "column:" + _addr(item["val"]["range"]["col0"], 0).rstrip("0123456789"), + "label": item.get("title", "")} + for item in sources if isinstance(item.get("val", {}).get("range"), dict)]} return out +def _chart_geometry(chart, ws): + """Excel drawing markers in CSS pixels, always relative to the sheet document.""" + anchor = getattr(chart, "anchor", None) + marker = getattr(anchor, "_from", None) + col, row = (int(marker.col), int(marker.row)) if marker is not None else (0, 0) + if isinstance(anchor, str): + col, row = _corner(anchor) or (0, 0) + dx = getattr(marker, "colOff", 0) / 9525 + dy = getattr(marker, "rowOff", 0) / 9525 + default_height = float(getattr(ws.sheet_format, "defaultRowHeight", None) or 15) * 4 / 3 + + def col_width(index): + letters = _addr(index, 0).rstrip("0123456789") + dim = ws.column_dimensions.get(letters) + return width_px(dim.width) if dim is not None else width_px(ws.sheet_format.defaultColWidth or 8.43) + + def row_height(index): + dim = ws.row_dimensions.get(index + 1) + return float(dim.height) * 4 / 3 if dim is not None and dim.height else default_height + + x, y = sum(col_width(c) for c in range(col)) + dx, sum(row_height(r) for r in range(row)) + dy + ext, end = getattr(anchor, "ext", None), getattr(anchor, "to", None) + if ext is not None: + width, height = ext.cx / 9525, ext.cy / 9525 + elif end is not None: + width = sum(col_width(c) for c in range(col, end.col)) + end.colOff / 9525 - dx + height = sum(row_height(r) for r in range(row, end.row)) + end.rowOff / 9525 - dy + else: + width, height = chart.width * 96 / 2.54, chart.height * 96 / 2.54 + position = getattr(anchor, "pos", None) + if position is not None: + x, y = position.x / 9525, position.y / 9525 + return {"col": col, "row": row, "offsetX": dx, "offsetY": dy, + "x": x, "y": y, "width": width, "height": height} + + def _geometry_of(ws, notes): - """`(cols, rows, colWidths, freeze, hiddenRows, hiddenCols)` for one tab. + """`(cols, rows, colWidths, freeze, hiddenRows, hiddenCols, rowHeights)` for one tab. ⛔⛔ HIDDEN LINES ARE STORED ONE-BASED, THE WAY A PERSON READS THEM: row 1 is `1` and column A is `1`. Nothing in `sheet/types.ts` pinned a basis when this was written (`_clean_hidden` @@ -789,33 +849,97 @@ def _geometry_of(ws, notes): ZERO-BASED, because `SheetChartSpec` says so; the two are different quantities and the difference is stated here rather than discovered. """ - cols = max(sheets.MIN_SHEET_COLS, min(sheets.MAX_SHEET_COLS, int(ws.max_column or 1))) - rows = max(sheets.MIN_SHEET_ROWS, min(sheets.MAX_SHEET_ROWS, int(ws.max_row or 1))) + cols = max(sheets.MIN_SHEET_COLS, int(ws.max_column or 1)) + rows = max(sheets.MIN_SHEET_ROWS, int(ws.max_row or 1)) widths, hidden_cols = {}, [] for letter, dim in (ws.column_dimensions or {}).items(): first, last = col_index(letter), col_index(letter) if getattr(dim, "min", None) and getattr(dim, "max", None): first, last = int(dim.min) - 1, int(dim.max) - 1 - if first < 0 or last - first > sheets.MAX_SHEET_COLS: + if first < 0 or last - first > sheets.DOCUMENT_MAX_COLS: continue px = width_px(getattr(dim, "width", None)) for index in range(first, last + 1): - if index >= sheets.MAX_SHEET_COLS: + if index >= sheets.DOCUMENT_MAX_COLS: break if px: widths[_addr(index, 0).rstrip("0123456789")] = px if getattr(dim, "hidden", False): hidden_cols.append(index + 1) - hidden_rows = [] + hidden_rows, heights = [], {} for number, dim in (ws.row_dimensions or {}).items(): if getattr(dim, "hidden", False) and isinstance(number, int) and number > 0: hidden_rows.append(number) if getattr(dim, "customHeight", False): - notes.add("a row height was not imported, because a row is one height here") + heights[str(number)] = float(dim.height) * 4 / 3 freeze = freeze_of(getattr(ws, "freeze_panes", "") or "") if getattr(ws, "merged_cells", None) and list(ws.merged_cells.ranges): - notes.add("a merged cell was imported as its top left cell alone") - return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)) + notes.add("merged cells display their top left value; original merge ranges are kept for export") + return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights + + +def _print_of(ws): + """Store print setup and inclusive print areas; export executes these settings later.""" + setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__ + if getattr(ws.page_setup, key) is not None and key != "id"} + if "paperSize" in setup: + setup["paper"] = setup["paperSize"] + setup["area"] = str(ws.print_area or "") + setup["titleRows"] = ws.print_title_rows + setup["titleCols"] = ws.print_title_cols + setup["margins"] = {key: getattr(ws.page_margins, key) for key in ws.page_margins.__attrs__} + setup["options"] = {key: getattr(ws.print_options, key) for key in ws.print_options.__attrs__} + if ws.sheet_properties.pageSetUpPr is not None: + setup["fitToPage"] = ws.sheet_properties.pageSetUpPr.fitToPage + return setup + + +def _authored_chart_document(path, notes): + """Restore chart-only metadata after validating the untrusted document and its native parts. + + The checksum detects edits made by another editor; it never establishes trust. Values and + formulas always come from the native worksheets, and the standard cleaner owns all refusal + rules. No custom XML field becomes an instruction, a store target, or an external request. + """ + with zipfile.ZipFile(path) as archive: + member = "customXml/item1.xml" + if member not in archive.namelist(): + return None + if archive.getinfo(member).file_size > sheets.MAX_WORKBOOK_BYTES: + raise ImportRefusal("workbook_too_large", "The embedded workbook document is too large.") + raw = archive.read(member) + if b"urn:loopable:sheet" not in raw: + return None + try: + text = raw.decode("utf-8") + if re.search(r" sheets._MAX_SHEETS: - skipped = len(formulas.sheetnames) - sheets._MAX_SHEETS - names_note = str(skipped) + " tabs past the first " + str(sheets._MAX_SHEETS) + \ - " were not imported" - else: - names_note = "" + formulas.close() + values.close() + raise ImportRefusal("workbook_too_many_sheets", "That workbook has more tabs than can be imported without losing sheets.") counts = {"cells": 0, "formulas": 0} book_notes = set() - if names_note: - book_notes.add(names_note) + authored = _authored_chart_document(path, book_notes) + chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {} + if authored and len(chart_tabs) != len(authored["sheets"]): + raise ImportRefusal("import_chart_sheets_changed", "Embedded chart worksheets must have unique names.") + if chart_tabs and set(chart_tabs) != set(names): + raise ImportRefusal("import_chart_sheets_changed", "The embedded charts do not match the workbook's worksheets.") + authored_ids = {tab["id"] for tab in chart_tabs.values()} + for tab in chart_tabs.values(): + for chart in tab.get("charts", []): + source_spec = chart.get("source") if chart.get("source") is not None else {} + if not isinstance(source_spec, dict): + raise ImportRefusal("sheet_chart_source_invalid", "A chart source must name worksheet ranges.") + source_id = source_spec.get("sheetId", tab["id"]) + if not isinstance(source_id, str) or source_id not in authored_ids: + raise ImportRefusal("sheet_chart_source_unknown", "A chart refers to a worksheet outside this workbook.") + if "ranges" in source_spec and (not isinstance(source_spec["ranges"], list) + or not source_spec["ranges"] + or any(sheets._clean_range(item) is None for item in source_spec["ranges"])): + raise ImportRefusal("sheet_chart_range_invalid", "A chart needs valid worksheet source ranges.") stamp = datetime.now(timezone.utc).replace(microsecond=0).isoformat() tabs = [] + calculation = {key: getattr(formulas.calculation, key) for key in formulas.calculation.__attrs__ + if getattr(formulas.calculation, key) is not None} if formulas.calculation else {} + if calculation.get("iterate"): + formulas.close() + values.close() + raise ImportRefusal("workbook_iteration_unsupported", + "Iterative calculation is not supported. Circular formulas return an error.") + names_out = [] + for scope, holder in [(None, formulas.defined_names)] + [ + (name, getattr(formulas[name], "defined_names", {})) for name in names]: + entries = holder.values() if hasattr(holder, "values") else getattr(holder, "definedName", []) + for entry in entries: + item = {"name": entry.name, "ref": entry.attr_text} + if scope is not None: + item["sheet"] = scope + elif entry.localSheetId is not None and entry.localSheetId < len(names): + item["sheet"] = names[entry.localSheetId] + if entry.hidden is not None: + item["hidden"] = entry.hidden + names_out.append(item) total = max(1, len(names)) for index, name in enumerate(names): ws = formulas[name] notes = set() - if str(getattr(ws, "sheet_state", "visible")) != "visible": - #: ⛔ A HIDDEN TAB HAS NO HOME IN `_clean_sheet`, which builds a fresh dict from the - #: keys it names, so a `hidden` key would be dropped on the first save with nothing - #: red anywhere. It is SHOWN and SAID rather than silently lost. - notes.add("this tab was hidden in the original file and is shown here") _say("cells", 0.3 + 0.4 * (index / total)) cached = {} try: source_ws = values[name] - for row in source_ws.iter_rows(): - for cell in row: - if cell.value is not None: - cached[cell.coordinate] = cell.value + for cell in source_ws._cells.values(): + if cell.value is not None: + cached[cell.coordinate] = cell.value except Exception: # noqa: BLE001 notes.add("this tab's stored answers could not be read, so its formulas start unfilled") - cells = _cells_of(ws, cached, notes, counts) + excel = {"blankStyles": {}, "mergedCells": [str(r) for r in ws.merged_cells.ranges], + "rowDimensions": {str(k): dict(v) for k, v in ws.row_dimensions.items()}, + "columnDimensions": {str(k): dict(v) for k, v in ws.column_dimensions.items()}, + "sheetFormat": dict(ws.sheet_format)} + cells = _cells_of(ws, cached, notes, counts, excel) _say("formats", 0.7 + 0.25 * (index / total)) _validations_of(ws, cells, notes) - cols, rows, widths, freeze, hidden_rows, hidden_cols = _geometry_of(ws, notes) + cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes) tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows, - "cells": cells} + "cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]), + "visibility": ws.sheet_state, "print": _print_of(ws), "excel": excel} + if heights: + tab["rowHeights"] = heights if widths: tab["colWidths"] = widths if freeze: @@ -905,6 +1065,9 @@ def import_workbook(path, *, source="", progress=None): if hidden_cols: tab["hiddenCols"] = hidden_cols charts = _charts_of(ws, name, notes) + if name in chart_tabs: + tab["id"] = chart_tabs[name]["id"] + charts = chart_tabs[name].get("charts", []) if charts: tab["charts"] = charts tab["imported"] = {"at": stamp, "source": source[:sheets._MAX_NAME], @@ -919,11 +1082,24 @@ def import_workbook(path, *, source="", progress=None): #: ⭐ THE FONT NOTE IS WORKBOOK-WIDE AND UNCONDITIONAL, because the answer is the same on #: every file: the app is one typeface, so no imported font survives and the reader is told #: once rather than per tab. - book_notes.add("every font was replaced by the app's own, which is the one typeface it uses") + book_notes.add("every font was replaced for display; original fonts and formats are kept for export") + + # Original styles are shared once, with integer references on cells, so exact formats, + # theme colours, border weights and blank-cell styles survive without repeating XML per cell. + original = {} + with zipfile.ZipFile(path) as archive: + for key, member in (("stylesXml", "xl/styles.xml"), ("themeXml", "xl/theme/theme1.xml")): + if member in archive.namelist(): + original[key] = archive.read(member).decode("utf-8") + date_system = "1904" if formulas.epoch.year == 1904 else "1900" + formulas.close() name = str(source or "").rsplit(".", 1)[0] or "Imported workbook" return { - "workbook": {"name": name[:sheets._MAX_NAME], "sheets": tabs}, + "workbook": {"schemaVersion": sheets.SHEET_SCHEMA_VERSION, + "name": name[:sheets._MAX_NAME], "sheets": tabs, + "namedRanges": names_out, "dateSystem": date_system, + "calculation": calculation, "excel": original}, "report": {"cells": counts["cells"], "formulas": counts["formulas"], "matched": 0, "mismatched": [], "unsupported": _note_list(book_notes)}, } diff --git a/platform/core/sheets.py b/platform/core/sheets.py index 6532f433ae4e25c8d47d287566bd58006ea07618..c80931acb68d3fdd2763f09116f14a64006dcdba 100644 --- a/platform/core/sheets.py +++ b/platform/core/sheets.py @@ -47,10 +47,18 @@ read-modify-write under its lock. There is no file writer here and there must ne recomputes them. A cell goes stale by being marked stale, not by being read. """ +#: ⛔ C5 POINTER ONLY: a bound cell's PERIOD is not validated in this workbook record; +#: `modules/sheet.py::clean_period` and `period_window` own it. They accept +#: `{unit, start, locked, ref?}` and the wave-49 `{window: }` spelling; the latter is resolved +#: by `harness/windows.py::normalize`, which accepts exactly `WINDOW_KINDS`. C5 named this file as +#: the acceptance point, but the vocabulary deliberately does not live here. + from __future__ import annotations import copy +import hashlib import json +import math import re import uuid from datetime import datetime, timezone @@ -61,6 +69,19 @@ WORKBOOK_KEY_PREFIX = "sheet_workbook_" #: The companion pointer list. See the module note for why a per-document key needs one. INDEX_KEY = "sheet_workbooks" +#: DPP 1.8's durable origin vocabulary for Amazon-derived Sheet documents. The analytical +#: database key is retained because one workbook may read several Amazon reports; a bare +#: ``amazon_sp`` tag would identify the connector but lose which materialised source supplied the +#: values. This deliberately lives in the document layer rather than in ``amazon_relational``: +#: Sheet persistence is shared core code and must not import the API's connector implementation. +AMAZON_DATABASE_PREFIX = "ut_amzn_" +AMAZON_DATA_ORIGIN_KEY = "amazon_sp" +AMAZON_DATA_ORIGIN_NAME = "Amazon Services API" +# Document format version, independent of the optimistic save revision (`rev`). +SHEET_SCHEMA_VERSION = 1 +DOCUMENT_MAX_COLS = 16384 +DOCUMENT_MAX_ROWS = 1048576 + #: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py` #: assert `web/src/sheet/types.ts` carries exactly these, name for name. #: @@ -97,6 +118,24 @@ CELL_ALIGNS = ("left", "center", "right") CELL_VALIGNS = ("top", "middle", "bottom") CELL_SIZES = ("s", "m", "l", "xl") +#: ⭐⭐ WAVE 54 / R10 + R11 - THE TYPEFACE AND THE POINT SIZE. Mirrored in +#: `types.ts::CELL_FONTS` / `DEFAULT_CELL_FONT` / `DEFAULT_FONT_PT` / `MIN_FONT_PT` / `MAX_FONT_PT`. +#: +#: R10: a cell's typeface is USER DATA, exactly as its fill colour is; the app's own chrome stays +#: Inter and that is a different decision. The list is closed and every face in it is one a machine +#: already has, because the client imports no webfont; the bare NAME is what is stored and the +#: client resolves it to a full stack at paint time, for the reason a colour is stored as a token. +#: +#: R11: the `size` slot carries EITHER one of the four step names above OR a point size, and never +#: a second field. `DEFAULT_FONT_PT` is `m`, in points, and both are stored as an ABSENCE. +CELL_FONTS = ( + "Inter", "Arial", "Georgia", "Times New Roman", "Courier New", "Verdana", "Trebuchet MS", +) +DEFAULT_CELL_FONT = "Inter" +DEFAULT_FONT_PT = 10 +MIN_FONT_PT = 6 +MAX_FONT_PT = 96 + #: The style keys A1 and C2 name. Anything else is DROPPED, in silence and on purpose; see #: `_clean_style` for why that is not the same decision as an unknown `format.kind`. _STYLE_MARKS = ("bold", "italic", "underline") @@ -106,6 +145,28 @@ _BORDER_SIDES = ("top", "right", "bottom", "left") #: outlives every layout change, exactly as a stored hex outlives a palette. _MAX_INDENT = 8 +#: Wave 54: how far a cell's value may be turned, in DEGREES. Excel's own range, and it is signed +#: because turning a header up and turning it down are two different requests. Zero is upright and +#: is stored as an ABSENCE, for `_clean_style`'s falsey-mark reason. Mirrored in +#: `types.ts::MIN_TEXT_ROTATION` / `MAX_TEXT_ROTATION`. +MIN_TEXT_ROTATION = -90 +MAX_TEXT_ROTATION = 90 + +#: Wave 54: a cell may carry a LINK and a NOTE, and neither is a value. A note is never counted as +#: hand typed, never reaches a formula and never becomes a cell's display; a link says what its +#: target IS, because a web address, an A1 address and a tab name are opened by three different +#: doors. Mirrored in `types.ts::Cell.link` / `Cell.note`. +CELL_LINK_KINDS = ("url", "cell", "sheet") +_MAX_LINK_HREF = 512 +_MAX_CELL_NOTE = 1000 + +#: Wave 54: a cell may also carry a SPARKLINE, a picture of a range drawn inside the cell itself. +#: It is CELL level and not tab level, unlike a chart, because that is what it is: the renderer is +#: `web/src/sheet/sparkline.ts`, called from `SheetGrid.tsx::paintCell` with the cell in hand, so +#: a tab-level array would have to be searched per cell on every paint. The three kinds are +#: Excel's own. Mirrored in `types.ts::SPARKLINE_KINDS` / `CellSparkline`. +SPARKLINE_KINDS = ("line", "column", "winloss") + #: C2's custom format code. ⚠ THE CAPS ARE PART OF THE VOCABULARY, not a guess: a format code is #: interpreted by the client's formatter, so an unbounded one is an unbounded parse, and Excel's #: own grammar holds at most four sections (positive; negative; zero; text). @@ -122,13 +183,60 @@ _MAX_VALIDATION_ENTRY = 200 SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains", "empty", "notEmpty") +#: ⛔⛔ W54-T05's THREE RULE KINDS, and the SAME refusal discipline the ten operators get. A +#: `cell` rule is a condition and a flat style; a `scale` bands its value across a stored low..high +#: range; a `bar` draws a proportional rectangle inside the cell. A kind nothing can paint is +#: refused by name rather than dropped, because dropping it would silently turn a colour scale into +#: a highlight rule reading `between`, which is a DIFFERENT paint than the one somebody asked for. +#: +#: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what +#: every rule written before this ticket already is, so writing the word onto them would make a +#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`. +SHEET_RULE_KINDS = ("cell", "scale", "bar") + #: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it #: cannot change how a number reads, so it carries no `format`. _RULE_STYLE_MARKS = ("bold",) _RULE_STYLE_COLOURS = ("fill", "color") -#: The chart kinds W49-T08 draws. Mirrored in `types.ts::SheetChartSpec`. -SHEET_CHART_KINDS = ("bar", "line", "area", "donut") +#: ⛔⛔ THE CHART KINDS A DOCUMENT MAY HOLD, which is NOT the same list as the chart kinds a +#: renderer can draw, and the two were one list until wave 54 made the difference matter. +#: `_clean_charts` drops a chart whose kind is outside this tuple, so a kind missing from here is +#: a picture somebody made that is gone on the next read, in silence: exactly D-640's mechanism, +#: one level down. This tuple is therefore the STORAGE vocabulary and it is mirrored, name for +#: name and in this order, by `types.ts::SheetChartSpec.kind`. +#: +#: ⚠ `types.ts::SHEET_CHART_DRAWN_KINDS` IS A DIFFERENT AND NARROWER THING: a CAPABILITY list, +#: naming the kinds the client can put on a canvas today. A kind stored here and not drawn there +#: is a chart the document keeps safely while its renderer is built. The reverse, a kind drawn +#: but not stored here, is the defect, and it is what widening this tuple prevents. +SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo", + "waterfall", "histogram", "bubble", "bullet", "funnel", "gauge", + "polar", "radar", "treemap", "stock", "gantt") + +#: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is +#: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet` +#: returns a FRESH dict rather than an amended copy of what it was handed, so the document saves, +#: the tab is valid, and whatever the client put in the unnamed slot is simply not in it. That is +#: how `charts` was eaten between W49-T08 and W51-T21, and it is how the NEXT slot added to a tab +#: gets eaten unless its name is added to this tuple in the same change that adds it. +#: +#: ⭐ THE SET IS THE FUNCTION'S OWN RULER. `_clean_sheet` emits its optional strata by walking +#: this tuple, so the two cannot drift apart by an edit to one of them, and `verify_sheet.py` +#: section 1 drives the real cleaner with a tab carrying every stratum and asserts it returns +#: nothing outside this set. Removing `charts` from here reds that leg, which is the negative +#: control D-640 is closed with. +#: +#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata. +SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze", + "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights", + "visibility", "print", "excel") + +#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of +#: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab +#: with no rules and a tab with `rules: []` read identically, so only one of them is stored and a +#: reload compares equal to the thing it saved. +SHEET_TAB_ALWAYS_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent") _MAX_RULES = 100 _MAX_HIDDEN = 5000 @@ -151,8 +259,8 @@ _MAX_DECIMALS = 10 #: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter #: column (ZZ) and 100000 rows is what glide's virtualised scroller stays comfortable inside. A #: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first. -DEFAULT_SHEET_COLS = 26 -DEFAULT_SHEET_ROWS = 200 +DEFAULT_SHEET_COLS = 200 +DEFAULT_SHEET_ROWS = 10000 DEFAULT_COL_WIDTH = 120 MIN_SHEET_COLS = 1 MIN_SHEET_ROWS = 1 @@ -168,7 +276,7 @@ _COL_LETTERS = re.compile(r"^[A-Z]{1,3}$") _MAX_NAME = 120 _MAX_ID = 80 _MAX_TEXT = 4000 -_MAX_FORMULA = 4000 +_MAX_FORMULA = 8192 _MAX_DISPLAY = 400 _MAX_SHEETS = 50 @@ -185,16 +293,14 @@ _MAX_CELLS_PER_SHEET = 100_000 #: A client may not park an unbounded blob in the tenant store. A BYTE CAP is not interpretation: #: it never looks at a key, only at the serialized length. Mirrors `chart_system.MAX_CONFIG_BYTES`. #: -#: ⭐⭐ 16 MiB IS THE BINDING CAP (A1), and it is the reason the grill's 500,000-cell figure is -#: unreachable: the recalc prototype measured 89 B for a BARE cell, so 16 MiB is about 150,000 -#: styled cells and the byte cap runs out first on any real workbook. The wave-49 figure of 4 MiB -#: refused the owner's 4.92 MB model on every save, so the module could not hold its own use case. +#: W58 uses the corrected W51 census: 32.275 MiB including 36,275 array formulas. +#: The 64 MiB document ceiling gives that measured workbook room for round-trip metadata. #: #: ⚠ IT IS NOT ONLY THE DOCUMENT CAP. `clean_cell` runs `_json_safe` over a cell's `value`, #: `binding`, `source` and `refusal` too, so raising this raises the per-value ceiling with it. #: That is harmless (a single value at the cap would fail the document check one line later) but #: it is not obvious from the name. -MAX_WORKBOOK_BYTES = 16 * 1024 * 1024 +MAX_WORKBOOK_BYTES = 64 * 1024 * 1024 #: A stored address is plain A1 with no `$`. Anchoring is a property of a FORMULA's text, not of #: the cell it lands in, so a `$` here would be a second way to say the same thing. @@ -313,7 +419,7 @@ def _json_safe(payload, what): write that fails later, somewhere else. """ try: - blob = json.dumps(payload) + blob = json.dumps(payload, allow_nan=False) except (TypeError, ValueError): raise SheetRefusal("workbook_not_serializable", "That " + what + " holds a value the store cannot save.") @@ -452,9 +558,38 @@ def _clean_style(raw): if steps: out["indent"] = steps - size = _text(raw.get("size"), 4) - if size in CELL_SIZES and size != "m": - out["size"] = size + #: ⭐⭐ R11: THE SAME SLOT, TWO SHAPES, AND THE ORDER OF THE BRANCHES IS THE WHOLE OF IT. + #: `_text` stringifies anything it is handed, so reading a point size through it would turn 12 + #: into "12", find it outside the four step names and drop it in silence: a size the person + #: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step + #: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for + #: the rotation's reason below, and the default is stored as an absence just as `m` is. + size = raw.get("size") + if isinstance(size, str): + step = _text(size, 4) + if step in CELL_SIZES and step != "m": + out["size"] = step + elif not isinstance(size, bool) and isinstance(size, (int, float)): + points = max(MIN_FONT_PT, min(MAX_FONT_PT, int(round(size)))) + if points != DEFAULT_FONT_PT: + out["size"] = points + + #: R10's typeface. A name outside the closed list is DROPPED in silence, which is this + #: cleaner's house rule for a decoration the client cannot promise to paint; the default face + #: is an absence, so a cell set back to it is the cell it was. + font = _text(raw.get("font"), 40) + if font in CELL_FONTS and font != DEFAULT_CELL_FONT: + out["font"] = font + + #: Wave 54's turn, clamped rather than refused, exactly as the indent is: a rotation is a + #: presentation, so 400 degrees is somebody asking for the most turn there is, not a document + #: that has to be rejected. Upright is stored as an absence, so an unrotated cell is byte for + #: byte the cell it was before somebody turned it and turned it back. + rotation = raw.get("textRotation") + if not isinstance(rotation, bool) and isinstance(rotation, (int, float)): + degrees = max(MIN_TEXT_ROTATION, min(MAX_TEXT_ROTATION, int(rotation))) + if degrees: + out["textRotation"] = degrees fmt = _clean_format(raw.get("format")) if fmt: @@ -498,6 +633,56 @@ def _clean_validation(raw): return {"list": out} if out else None +def _clean_link(raw): + """Wave 54's cell link `{href, kind}`, or None when nothing survives. + + ⚠ BOTH HALVES OR NEITHER. A link with a target and no kind cannot be opened, because the kind + is what decides WHICH door opens it: a web address, an A1 address on this workbook, and a tab + by name are three different acts. Storing half of one would leave a cell wearing an + affordance that does nothing when it is clicked. + + ⚠ THE HREF IS CLAMPED, NOT PARSED. This module stores documents; deciding whether a URL is + reachable, or whether `Sheet2!B7` exists, belongs to the door that opens it, which can say so + at the moment somebody clicks. + """ + if not isinstance(raw, dict): + return None + kind = _text(raw.get("kind"), 12) + href = _text(raw.get("href"), _MAX_LINK_HREF) + if kind not in CELL_LINK_KINDS or not href: + return None + return {"href": href, "kind": kind} + + +def _clean_sparkline(raw): + """Wave 54's in-cell sparkline `{kind, range}`, or None when nothing survives. + + ⚠ BOTH HALVES OR NEITHER, for `_clean_link`'s reason. A sparkline with a range and no kind + cannot be drawn, because the kind is what decides WHICH picture is drawn over those numbers: a + line, a column run and a win/loss bar say three different things about the same range. Storing + half of one would leave a cell that claims to hold a picture nothing can put on a canvas. + + ⚠ IT IS DROPPED WHEN MALFORMED, NEVER REFUSED, which is the link's decision and not the format + code's: a sparkline does not change what any cell READS AS, so a broken one is a missing + picture and never a wrong number on a screen. Refusing would 400 a whole workbook save over a + decoration. + + ⭐ THE RANGE IS `_clean_range`'s, so a sparkline's rectangle is spelled the same way a chart's + and a rule's are. It takes `"B2:D40"`, the single cell `"B2"` and the rectangle dict, and it + stores the rectangle: a document with two spellings of one rectangle is a document every later + reader has to branch on. + """ + if not isinstance(raw, dict): + return None + kind = _text(raw.get("kind"), 20) + if kind not in SPARKLINE_KINDS: + return None + rect = _clean_range(raw.get("range")) + if rect is None: + return None + return {"kind": kind, "range": rect} + + def clean_cell(raw, addr=""): """Normalize ONE cell, or raise `SheetRefusal` with a named code. @@ -527,7 +712,35 @@ def clean_cell(raw, addr=""): if kind in _TYPED_KINDS: state = LITERAL_STATE - cell = {"kind": kind, "state": state} + known = {"kind", "state", "text", "formula", "display", "value", "style", "validation", + "binding", "source", "refusal", "refreshed_at", "link", "note", "sparkline", + "formulaMeta", "error", "excel", "imported_value"} + cell = {key: copy.deepcopy(value) for key, value in raw.items() if key not in known} + _json_safe(cell, "cell extensions") + cell.update({"kind": kind, "state": state}) + + # These are document values, never trusted as a server calculation. In particular an + # array formula's extent and an Excel error are not reconstructible from its display. + for key in ("formulaMeta", "error", "excel"): + if raw.get(key) is not None: + cell[key] = _document_object(raw[key], "cell_" + key) + meta = cell.get("formulaMeta") or {} + if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"): + raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.") + binding = raw.get("binding") + if isinstance(binding, dict): + for tag in ("kind", "mode"): + if tag in binding and binding[tag] not in ("number", "column"): + raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.") + aggregate = binding.get("aggregate") + if isinstance(aggregate, dict) and aggregate.get("metric_key") \ + and (aggregate.get("fn") or aggregate.get("field")): + raise SheetRefusal("cell_binding_paths_conflict", "Choose either a Metric or a physical Column for this cell.") + for key, kinds in (("link", CELL_LINK_KINDS), ("sparkline", SPARKLINE_KINDS)): + spec = raw.get(key) + if isinstance(spec, dict) and "kind" in spec and spec["kind"] not in kinds: + raise SheetRefusal("cell_" + key + "_kind_unknown", + "That cell " + key + " kind is not supported.") #: ⭐ C2's STYLE ONLY CELL. `empty` says the cell holds no content, so nothing that would BE #: content is carried out of here, whatever arrived. It is DROPPED rather than refused on @@ -540,14 +753,24 @@ def clean_cell(raw, addr=""): validation = _clean_validation(raw.get("validation")) if validation: cell["validation"] = validation + #: ⛔ THE SPARKLINE SURVIVES THIS BRANCH, AND LEAVING IT OUT WOULD BE THE COMMON CASE + #: RATHER THAN THE EDGE ONE. A sparkline cell usually holds no text at all: somebody + #: paints a picture of the row beside it and types nothing, which is precisely an + #: `empty` cell. A picture is not content in the sense this branch strips, because it + #: has no value, is never hand typed and never reaches a formula, so it belongs with + #: the style rather than with the text that is dropped here. + sparkline = _clean_sparkline(raw.get("sparkline")) + if sparkline: + cell["sparkline"] = sparkline return cell - if raw.get("text") is not None: - cell["text"] = _text(raw.get("text"), _MAX_TEXT) - if raw.get("formula") is not None: - cell["formula"] = _text(raw.get("formula"), _MAX_FORMULA) - if raw.get("display") is not None: - cell["display"] = _text(raw.get("display"), _MAX_DISPLAY) + for key, limit in (("text", _MAX_TEXT), ("formula", _MAX_FORMULA), ("display", _MAX_DISPLAY)): + if raw.get(key) is not None: + value = str(raw[key]) + if len(value) > limit: + raise SheetRefusal("cell_" + key + "_too_long", + "That cell's " + key + " is too long to save without losing text.") + cell[key] = value #: R10: stored as last computed. This module does not look inside `value` and does not #: recompute it; it only refuses what the store could not persist. @@ -566,6 +789,21 @@ def clean_cell(raw, addr=""): if raw.get("refreshed_at") is not None: cell["refreshed_at"] = _text(raw.get("refreshed_at"), 120) + #: Wave 54's link, note and sparkline. All three are DROPPED when malformed rather than + #: refused, which is the validation list's decision and not the format code's: none of them + #: changes what the cell READS AS, so a broken link is a missing affordance and a broken + #: sparkline is a missing picture, never a wrong number on a screen. + link = _clean_link(raw.get("link")) + if link: + cell["link"] = link + if raw.get("note") is not None: + note = _text(raw.get("note"), _MAX_CELL_NOTE) + if note: + cell["note"] = note + sparkline = _clean_sparkline(raw.get("sparkline")) + if sparkline: + cell["sparkline"] = sparkline + #: C2: the in-cell list. Normalized before the style so a refusing style still reports first. validation = _clean_validation(raw.get("validation")) if validation: @@ -612,7 +850,8 @@ def _clean_extent(raw, default, low, high, what): """One axis of a sheet's extent: a whole number inside its range, or a named refusal.""" if raw is None: return default - if isinstance(raw, bool) or not isinstance(raw, (int, float)): + if isinstance(raw, bool) or not isinstance(raw, (int, float)) \ + or not math.isfinite(raw) or raw != int(raw): raise SheetRefusal("sheet_extent_invalid", "A sheet's " + what + " count must be a whole number.") value = int(raw) @@ -624,7 +863,7 @@ def _clean_extent(raw, default, low, high, what): return value -def _clean_col_widths(raw, cols): +def _clean_col_widths(raw, cols, *, lossless=False): """`{"A": 160}` for the columns that were dragged, dropped for the ones that were not. ⚠ A DEFAULT WIDTH IS STORED AS AN ABSENCE, for `_clean_style`'s reason: two sheets that read @@ -641,14 +880,20 @@ def _clean_col_widths(raw, cols): raise SheetRefusal("col_width_column_invalid", "Column " + (letters or str(key)) + " is not a column letter.") if isinstance(value, bool) or not isinstance(value, (int, float)): + if lossless: + raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.") continue - width = max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value))) - if width != DEFAULT_COL_WIDTH: + if not math.isfinite(value): + raise SheetRefusal("col_width_invalid", "A column width must be a finite size.") + if lossless and value <= 0: + raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.") + width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value))) + if lossless or width != DEFAULT_COL_WIDTH: out[letters] = width return out -def _clean_freeze(raw, cols, rows): +def _clean_freeze(raw, cols, rows, *, lossless=False): """`{"cols": 1}` / `{"rows": 1}`, clamped into the sheet, or `{}` when nothing is frozen.""" if raw is None: return {} @@ -656,7 +901,16 @@ def _clean_freeze(raw, cols, rows): raise SheetRefusal("sheet_freeze_invalid", "A sheet's freeze must be an object.") out = {} for key, limit in (("cols", cols), ("rows", rows)): + if key not in raw: + continue value = raw.get(key) + if lossless: + ceiling = DOCUMENT_MAX_COLS if key == "cols" else DOCUMENT_MAX_ROWS + if isinstance(value, bool) or not isinstance(value, (int, float)) \ + or not math.isfinite(value) or value != int(value) or not 0 <= value <= ceiling: + raise SheetRefusal("sheet_freeze_invalid", "Frozen rows and columns need whole counts within the sheet dimensions.") + out[key] = value + continue if isinstance(value, bool) or not isinstance(value, (int, float)): continue count = max(0, min(int(limit), int(value))) @@ -733,22 +987,52 @@ def _clean_rule_style(raw): return out -def _clean_rules(raw): +def _clean_rule_scale(raw): + """A COLOUR SCALE's two end TOKENS, or None. + + ⛔ BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would + leave the client looking up a pair that cannot match and falling back to a default scale the + person never chose. The tokens go through the same `_COLOUR` guard `_clean_rule_style` uses, + so nothing free form reaches a canvas `fillStyle` from a stored document. + + ⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which + `_clean_rules` already persists for every op that takes operands, so a colour scale needed no + new numeric slot at all. + """ + if not isinstance(raw, dict): + return None + out = {} + for key in ("low", "high"): + token = _text(raw.get(key), 40) + if not token or not _COLOUR.match(token): + return None + out[key] = token + return out + + +def _clean_rules(raw, *, lossless=False): """C2's conditional format rules for one tab, at most `_MAX_RULES` of them. ⛔ AN OP OUTSIDE `SHEET_RULE_OPS` IS REFUSED BY NAME, for `_clean_format`'s reason: a rule whose operator nothing can evaluate never fires, so dropping it would leave a person looking - at a sheet they believe is painted by a rule that is not there. - - ⚠ THE COUNT IS CLAMPED rather than refused: the rules past the cap are ones nobody could see - the effect of, and refusing would make a sheet that can never be saved again. + at a sheet they believe is painted by a rule that is not there. A KIND outside + `SHEET_RULE_KINDS` is refused the same way, and for a sharper version of the same reason: a + dropped kind does not stop the rule firing, it makes it fire as a DIFFERENT paint. + + ⛔⛔ AND THIS IS THE D-640 SHAPE, SO READ IT BEFORE ADDING A FIELD. Each rule below is built + EXPLICITLY, key by key, which means a field the client starts writing and this function does + not learn is dropped on the very next read: the feature would delete itself one save later, + with the editor still listing what it wrote. W54-T05's `kind` and `scale` are here because + that ticket added them to both halves in one commit. """ if raw is None: return [] if not isinstance(raw, list): raise SheetRefusal("sheet_rules_not_a_list", "A sheet's rules must be a list.") + if lossless and len(raw) > _MAX_RULES: + raise SheetRefusal("sheet_too_many_rules", "That sheet has too many format rules to save without losing rules.") out = [] - for row in raw[:_MAX_RULES]: + for index, row in enumerate(raw[:_MAX_RULES]): if not isinstance(row, dict): continue op = _text(row.get("op"), 20) @@ -756,11 +1040,26 @@ def _clean_rules(raw): raise SheetRefusal( "sheet_rule_op_unknown", "A sheet rule condition must be one of " + ", ".join(SHEET_RULE_OPS) + ".") + kind = _text(row.get("kind"), 20) or "cell" + if kind not in SHEET_RULE_KINDS: + raise SheetRefusal( + "sheet_rule_kind_unknown", + "A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".") rect = _clean_range(row.get("range")) if rect is None: continue - rule = {"id": _text(row.get("id"), _MAX_ID) or uuid.uuid4().hex[:12], + rule = {"id": _text(row.get("id"), _MAX_ID) or _migration_id("rule", index, row), "range": rect, "op": op, "style": _clean_rule_style(row.get("style"))} + if lossless: + rule.update({key: copy.deepcopy(value) for key, value in row.items() + if key not in {"id", "range", "op", "style", "kind", "scale", "value", "value2"}}) + #: ⭐ THE DEFAULT KIND IS AN ABSENCE (see `SHEET_RULE_KINDS`), so a rule that was turned + #: into a colour scale and back is byte for byte the rule it started as. + if kind != "cell": + rule["kind"] = kind + scale = _clean_rule_scale(row.get("scale")) + if scale: + rule["scale"] = scale #: ⚠ `empty` and `notEmpty` take NO operand, and carrying one would store a rule whose #: text says it compares against something it can never read. if op not in ("empty", "notEmpty"): @@ -772,7 +1071,7 @@ def _clean_rules(raw): def _clean_hidden(raw, what): - """`hiddenRows` / `hiddenCols`: whole numbers, deduped and sorted, at most `_MAX_HIDDEN`. + """`hiddenRows` / `hiddenCols`: valid whole line numbers, deduped and sorted without slicing. ⚠ THE BASIS IS THE CLIENT'S AND THIS FUNCTION DOES NOT SECOND GUESS IT. Nothing in the tree pinned whether a hidden line is counted from 0 or from 1 when this was written, and a server @@ -786,36 +1085,115 @@ def _clean_hidden(raw, what): raise SheetRefusal("sheet_hidden_not_a_list", "A sheet's hidden " + what + " must be a list.") seen = set() - for item in raw[:_MAX_HIDDEN]: - if isinstance(item, bool) or not isinstance(item, (int, float)): - continue + for item in raw: + if isinstance(item, bool) or not isinstance(item, (int, float)) or not math.isfinite(item) \ + or item != int(item): + raise SheetRefusal("sheet_hidden_invalid", "Hidden rows and columns need whole line numbers.") line = int(item) - if line >= 0: - seen.add(line) + ceiling = DOCUMENT_MAX_ROWS if what == "rows" else DOCUMENT_MAX_COLS + if not 0 <= line <= ceiling: + raise SheetRefusal("sheet_hidden_invalid", "That hidden line is outside the sheet dimensions.") + seen.add(line) return sorted(seen) def _clean_charts(raw): - """D-640: the allow-list that lets a chart SURVIVE a save. - - ⛔⛔ WITHOUT THIS FUNCTION A CHART IS GONE ON THE NEXT RELOAD, and nothing goes red: the sheet - saves, the document is valid, and the picture the person made is simply not in it. - `_clean_sheet` returns a fresh dict, so a tab key that is not named here is dropped. - """ + """Validate chart discriminants and geometry, retaining all source and settings metadata.""" if raw is None: return [] if not isinstance(raw, list): raise SheetRefusal("sheet_charts_not_a_list", "A sheet's charts must be a list.") out = [] - for row in raw[:_MAX_CHARTS]: + if len(raw) > _MAX_CHARTS: + raise SheetRefusal("sheet_too_many_charts", "That sheet holds more charts than can be saved.") + for index, row in enumerate(raw): if not isinstance(row, dict): - continue + raise SheetRefusal("sheet_chart_invalid", "A sheet chart must be an object.") kind = _text(row.get("kind"), 20) rect = _clean_range(row.get("range")) - if kind not in SHEET_CHART_KINDS or rect is None: - continue - out.append({"id": _text(row.get("id"), _MAX_ID) or uuid.uuid4().hex[:12], - "range": rect, "kind": kind}) + if kind not in SHEET_CHART_KINDS: + raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.") + if rect is None: + raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.") + # Settings and source/series bags are authored data. Validate the discriminants we + # interpret; retain the remaining JSON rather than deleting a peer's additive fields. + chart = _document_object(row, "sheet_chart") + chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row), + "range": rect, "kind": kind}) + for key in ("geometry", "anchor"): + if chart.get(key) is not None: + box = _document_object(chart[key], "sheet_chart_" + key) + for axis in ("x", "y", "col", "row", "offsetX", "offsetY", "width", "height"): + if axis in box and (isinstance(box[axis], bool) + or not isinstance(box[axis], (int, float)) + or not math.isfinite(box[axis]) + or box[axis] < (1 if axis in ("width", "height") else 0)): + raise SheetRefusal("sheet_chart_geometry_invalid", + "A chart needs finite document coordinates and a positive size.") + chart[key] = box + out.append(chart) + return out + + +def _document_object(raw, what): + """A lossless JSON metadata bag; schema migrations must not stringify numeric values.""" + if not isinstance(raw, dict): + raise SheetRefusal(what + "_invalid", "That " + what.replace("_", " ") + " must be an object.") + return _json_safe(copy.deepcopy(raw), what.replace("_", " ")) + + +def _migration_id(kind, index, raw): + """A missing legacy identity must be stable even when migration is a read-only operation.""" + payload = json.dumps(_json_safe(raw, kind), sort_keys=True, ensure_ascii=True) + return hashlib.sha256((kind + str(index) + payload).encode("utf-8")).hexdigest()[:12] + + +def _used_extent(cells, blank_styles=None): + """True inclusive used rectangle, independent of the grid's default scroll dimensions.""" + rect = None + for addr in (*cells, *(blank_styles or {})): + piece = _clean_range(addr) + if piece is None: + raise SheetRefusal("cell_address_invalid", "A stored cell needs a valid A1 address.") + if rect is None: + rect = piece + else: + for axis in ("col0", "row0"): + rect[axis] = min(rect[axis], piece[axis]) + for axis in ("col1", "row1"): + rect[axis] = max(rect[axis], piece[axis]) + return rect + + +def _clean_base_row_height(raw): + if raw is None: + return None + if isinstance(raw, bool) or not isinstance(raw, (int, float)) \ + or not math.isfinite(raw) or not 18 <= raw <= 200: + raise SheetRefusal("sheet_row_height_invalid", "Choose a row height between 18 and 200 pixels.") + return raw + + +def _clean_row_heights(raw): + if raw is None: + return {} + out = _document_object(raw, "sheet_row_heights") + for row, height in out.items(): + if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \ + or isinstance(height, bool) or not isinstance(height, (int, float)) \ + or not math.isfinite(height) or height <= 0: + raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.") + return out + + +def _clean_print(raw): + if raw is None: + return None + out = _document_object(raw, "sheet_print") + if "orientation" in out and out["orientation"] not in ("portrait", "landscape", "default"): + raise SheetRefusal("sheet_print_orientation_unknown", "Choose portrait or landscape for printing.") + # Paper identifiers include Excel's numeric codes; preserve them and inclusive page + # selections/fit settings exactly. The export door owns their execution and validation. return out @@ -845,54 +1223,79 @@ def _clean_imported(raw): return out if (out["at"] or out["source"] or notes) else None -def _clean_sheet(raw, index=0): - """ONE tab: identity, cells, A2's geometry and C2's strata. +def _clean_sheet(raw, index=0, *, lossless=False): + """Normalize a tab's known strata and retain additive JSON metadata without interpretation. - ⛔ EVERY KEY THIS FUNCTION DOES NOT NAME IS DROPPED, and that is worth knowing before adding - one: the return below is a fresh dict, not an amended copy of `raw`. A2's four geometry keys - and C2's five strata are named here for exactly that reason, and `charts` is named because it - was the one this rule silently ate until W51-T21 (D-640). + SHEET_TAB_KEYS enumerates the normalized fields. Its gate proves each cleaner emits its + stratum, while unknown metadata remains intact for later document consumers. """ if not isinstance(raw, dict): raise SheetRefusal("sheet_not_an_object", "A sheet must be an object.") name = _text(raw.get("name"), _MAX_NAME) or ("Sheet" + str(index + 1)) - sid = _text(raw.get("id"), _MAX_ID) or uuid.uuid4().hex[:12] - cols = _clean_extent(raw.get("cols"), DEFAULT_SHEET_COLS, MIN_SHEET_COLS, MAX_SHEET_COLS, + sid = _text(raw.get("id"), _MAX_ID) or _migration_id("sheet", index, {"name": name}) + cols = _clean_extent(raw.get("cols"), DEFAULT_SHEET_COLS, MIN_SHEET_COLS, DOCUMENT_MAX_COLS, "columns") - rows = _clean_extent(raw.get("rows"), DEFAULT_SHEET_ROWS, MIN_SHEET_ROWS, MAX_SHEET_ROWS, + rows = _clean_extent(raw.get("rows"), DEFAULT_SHEET_ROWS, MIN_SHEET_ROWS, DOCUMENT_MAX_ROWS, "rows") #: ⭐ WRITTEN EVEN WHEN THEY ARE THE DEFAULTS. A sheet that states its own extent needs no #: client-side fallback, so the grid cannot disagree with the document about how big it is, #: and a sheet written before A2 acquires the defaults on its first save rather than staying #: silently un-described. - sheet = {"id": sid, "name": name, "cols": cols, "rows": rows, - "cells": _clean_cells(raw.get("cells"), sheet_name=name)} - widths = _clean_col_widths(raw.get("colWidths"), cols) - if widths: - sheet["colWidths"] = widths - freeze = _clean_freeze(raw.get("freeze"), cols, rows) - if freeze: - sheet["freeze"] = freeze - #: C2's five strata. Each is written only when it holds something, for `_clean_style`'s - #: reason: a tab with no rules and a tab with `rules: []` read identically, so one of them is - #: what gets stored and a reload compares equal to the thing it saved. - rules = _clean_rules(raw.get("rules")) - if rules: - sheet["rules"] = rules - for key, what in (("hiddenRows", "rows"), ("hiddenCols", "columns")): - hidden = _clean_hidden(raw.get(key), what) - if hidden: - sheet[key] = hidden - charts = _clean_charts(raw.get("charts")) - if charts: - sheet["charts"] = charts - imported = _clean_imported(raw.get("imported")) - if imported: - sheet["imported"] = imported + cells = _clean_cells(raw.get("cells"), sheet_name=name) + excel = _document_object(raw["excel"], "sheet_excel") if raw.get("excel") is not None else None + used = _used_extent(cells, (excel or {}).get("blankStyles")) + if used: + cols, rows = max(cols, used["col1"] + 1), max(rows, used["row1"] + 1) + if cols > DOCUMENT_MAX_COLS or rows > DOCUMENT_MAX_ROWS: + raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.") + sheet = {"id": sid, "name": name, "cols": cols, "rows": rows, "cells": cells, + "usedExtent": used} + visibility = raw.get("visibility") + if visibility is not None and visibility not in ("visible", "hidden", "veryHidden"): + raise SheetRefusal("sheet_visibility_unknown", "A sheet must be visible, hidden or very hidden.") + + #: ⚠ THE ORDER OF THIS DICT IS THE ORDER THE CLEANERS RUN IN, and it is load bearing: two of + #: them REFUSE by name (`_clean_rules` on an unknown op, `_clean_imported` on a record that is + #: not an object), so a reordering here would change which refusal a caller sees first when a + #: tab is wrong in two places at once. + optional = { + "colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless), + "freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless), + "rules": _clean_rules(raw.get("rules"), lossless=lossless), + "hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"), + "hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"), + "charts": _clean_charts(raw.get("charts")), + "imported": _clean_imported(raw.get("imported")), + "rowHeight": _clean_base_row_height(raw.get("rowHeight")), + "rowHeights": _clean_row_heights(raw.get("rowHeights")), + "visibility": visibility, + "print": _clean_print(raw.get("print")), + "excel": excel, + } + + #: ⛔ THE WALK IS THROUGH `SHEET_TAB_KEYS`, NOT THROUGH `optional`, AND THAT IS THE WHOLE + #: POINT OF D-640'S FIX. A stratum cleaned above but never added to the tuple is dropped here + #: rather than written, which is the same silent loss as before EXCEPT that the gate now sees + #: it: `verify_sheet.py` drives a tab carrying every stratum and compares what comes back + #: against the tuple, so the disagreement reds instead of shipping. + #: + #: Each is written only when it holds something, for `_clean_style`'s reason: a tab with no + #: rules and a tab with `rules: []` read identically, so only one of them is stored and a + #: reload compares equal to the thing it saved. + for key in SHEET_TAB_KEYS: + value = optional.get(key) + if value: + sheet[key] = value + extensions = {key: copy.deepcopy(value) for key, value in raw.items() + if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount"}} + sheet.update(_json_safe(extensions, "sheet extensions")) + if lossless: + sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0)) + sheet["rows"] = max(rows, (sheet.get("freeze") or {}).get("rows", 0)) return sheet -def _clean_sheets(raw): +def _clean_sheets(raw, *, lossless=False): if raw is None: return [] if not isinstance(raw, list): @@ -900,20 +1303,50 @@ def _clean_sheets(raw): if len(raw) > _MAX_SHEETS: raise SheetRefusal("workbook_too_many_sheets", "That workbook holds more tabs than one workbook accepts.") - return [_clean_sheet(s, i) for i, s in enumerate(raw)] + return [_clean_sheet(s, i, lossless=lossless) for i, s in enumerate(raw)] # ── the workbook ───────────────────────────────────────────────────────────────────────────── +def migrate_workbook(raw): + """Pure unversioned/v0 -> v1 migration; validate current documents without recalculating. + + A read never writes or changes `rev`, timestamps or cached results. Unknown versions + refuse before cleaning so a newer document cannot be downgraded by a save. + """ + if not isinstance(raw, dict): + raise SheetRefusal("workbook_not_an_object", "A workbook must be an object.") + version = raw.get("schemaVersion", 0) + if isinstance(version, bool) or not isinstance(version, int) or version not in (0, SHEET_SCHEMA_VERSION): + raise SheetRefusal("workbook_schema_unknown", "This workbook uses a document version this app cannot open or save.") + book = {key: copy.deepcopy(value) for key, value in raw.items() if key != "sheets"} + if "dateSystem" in book and book["dateSystem"] not in ("1900", "1904"): + raise SheetRefusal("workbook_date_system_unknown", "A workbook date system must be 1900 or 1904.") + if book.get("calculation") is not None: + calc = _document_object(book["calculation"], "workbook_calculation") + if calc.get("iterate") or calc.get("iterative") or calc.get("iterativeCalculation"): + raise SheetRefusal("workbook_iteration_unsupported", "Iterative calculation is not supported. Circular formulas return an error.") + book["calculation"] = calc + if book.get("namedRanges") is not None and not isinstance(book["namedRanges"], (dict, list)): + raise SheetRefusal("workbook_names_invalid", "Workbook names must be a list or an object.") + if book.get("print") is not None: + book["print"] = _clean_print(book["print"]) + book["schemaVersion"] = SHEET_SCHEMA_VERSION + book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=True) + return _json_safe(book, "workbook") + + def new_workbook(name="", created_by="", workbook_id=""): """The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs).""" wid = _slug_id(workbook_id) if workbook_id else uuid.uuid4().hex[:12] stamp = _now() return { "id": wid, + "schemaVersion": SHEET_SCHEMA_VERSION, "name": _text(name, _MAX_NAME) or "Untitled workbook", "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1", - "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {}}], + "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {}, + "usedExtent": None}], "rev": 1, "refreshed_at": None, "created_by": _text(created_by, _MAX_ID), @@ -922,6 +1355,67 @@ def new_workbook(name="", created_by="", workbook_id=""): } +def data_origin_for_database(database): + """Canonical durable origin tag for a known connector database, or ``None``. + + DPP 1.8 permits separation or a mechanism that tags the source of Amazon Information. A + workbook and its row-scope cache are shared documents, so they use the tagging arm: every + ``ut_amzn_*`` source is named explicitly and retained after the binding itself is removed. + + The helper is intentionally conservative. It recognises the reserved Amazon database + namespace only; it never guesses that an unrelated database is Amazon from a display label. + """ + database = str(database or "").strip() + if not database.startswith(AMAZON_DATABASE_PREFIX): + return None + return { + "key": AMAZON_DATA_ORIGIN_KEY, + "origin": AMAZON_DATA_ORIGIN_NAME, + "database": database, + } + + +def _merge_data_origins(doc, *, sheets=(), databases=()): + """Canonicalise and monotonically extend one document's ``dataOrigins`` inventory. + + ``dataOrigins`` is SERVER-MAINTAINED: PUT bodies never supply it directly. Only a prior + canonical tag, a stored binding/source, or an explicit database from a server-side resolve can + add an origin. Existing origins are retained deliberately, even when a user later deletes the + binding, because the workbook can still hold pasted values and formulas derived from it. + """ + found = set() + for raw in (doc.get("dataOrigins") or ()) if isinstance(doc, dict) else (): + if not isinstance(raw, dict): + continue + tag = data_origin_for_database(raw.get("database")) + if (tag is not None + and raw.get("key") == tag["key"] + and raw.get("origin") == tag["origin"]): + found.add(tag["database"]) + for database in databases or (): + tag = data_origin_for_database(database) + if tag is not None: + found.add(tag["database"]) + for tab in sheets or (): + if not isinstance(tab, dict): + continue + for cell in (tab.get("cells") or {}).values(): + if not isinstance(cell, dict): + continue + for member in ("binding", "source"): + ref = cell.get(member) + tag = data_origin_for_database( + ref.get("database") if isinstance(ref, dict) else None) + if tag is not None: + found.add(tag["database"]) + origins = [data_origin_for_database(database) for database in sorted(found)] + if origins: + doc["dataOrigins"] = origins + else: + doc.pop("dataOrigins", None) + return origins + + def create_workbook(st, name="", created_by=""): """Write a new `sheet_workbook_` document and register it in the index.""" book = new_workbook(name=name, created_by=created_by) @@ -942,7 +1436,7 @@ def get_workbook(st, workbook_id): if not doc or not doc.get("id"): raise SheetRefusal("workbook_missing", "That workbook no longer exists.") - return doc + return migrate_workbook(doc) def put_workbook(st, workbook_id, raw, expect_rev=None): @@ -967,7 +1461,8 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): if expect_rev is False: expect_rev = None - sheets = _clean_sheets(raw.get("sheets")) + incoming = migrate_workbook(raw) + sheets = incoming["sheets"] name = _text(raw.get("name"), _MAX_NAME) refreshed_at = raw.get("refreshed_at") saved = {} @@ -980,9 +1475,23 @@ def put_workbook(st, workbook_id, raw, expect_rev=None): raise SheetRefusal( "workbook_revision_stale", "That workbook changed somewhere else, so this save was not applied.") - book = dict(prior) + book = migrate_workbook(prior) + # Immutable ownership and server read projections cannot be reassigned by a PUT. + # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may + # neither supply it nor clear it. It is listed here because this blanket copy of the + # incoming document would otherwise hand the client both halves of the control — sending + # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an + # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned + # cells by ``_merge_data_origins`` immediately below. + protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at", + "sheets", "name", "refreshedFor", "shares", "role", "permissions", + "dataOrigins"} + book.update({key: value for key, value in incoming.items() if key not in protected}) book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook" book["sheets"] = sheets + # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is + # ignored, so a client cannot erase or replace the server-maintained attribution list. + _merge_data_origins(book, sheets=sheets) book["rev"] = int(prior.get("rev") or 0) + 1 if refreshed_at is not None: book["refreshed_at"] = _text(refreshed_at, 120) or None @@ -1052,6 +1561,7 @@ def workbook_meta(book): to say how close the workbook is to `_MAX_CELLS_PER_SHEET` short of loading every tab, which is the exact fetch this door exists to avoid. """ + book = migrate_workbook(book) meta = {k: v for k, v in (book or {}).items() if k != "sheets"} tabs = [] for tab in (book or {}).get("sheets") or (): @@ -1136,7 +1646,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): "This sheet changed somewhere else while you had it open, so this save was not " "applied. Reload the tab and try again.", rev_current=current) - book = copy.deepcopy(prior) + book = migrate_workbook(prior) tabs = book.get("sheets") or [] at = _tab_index(tabs, tab_ref) if at is None: @@ -1151,13 +1661,24 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): if len(cells) > _MAX_CELLS_PER_SHEET: raise SheetRefusal("sheet_too_many_cells", _too_many_cells_message(len(cells))) tab["cells"] = cells + tab["usedExtent"] = _used_extent(cells, (tab.get("excel") or {}).get("blankStyles")) + if tab["usedExtent"]: + tab["cols"] = max(tab["cols"], tab["usedExtent"]["col1"] + 1) + tab["rows"] = max(tab["rows"], tab["usedExtent"]["row1"] + 1) + if tab["cols"] > DOCUMENT_MAX_COLS or tab["rows"] > DOCUMENT_MAX_ROWS: + raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.") book["sheets"] = tabs + _merge_data_origins(book, sheets=tabs) book["rev"] = current + 1 book["updated_at"] = _now() _json_safe(book, "workbook") out["rev"] = book["rev"] out["tab"] = str(tab.get("id") or tab.get("name") or "") out["cells"] = len(cells) + out["schemaVersion"] = SHEET_SCHEMA_VERSION + out["usedExtent"] = tab["usedExtent"] + # Server-cleaned keys only. The route consumes this delta for calculation context. + out["accepted"] = list(dict.fromkeys([*merged, *deleted])) saved.update(book) return book, book @@ -1167,6 +1688,118 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None): return out +# ── W52-T12: the per-scope bound-cell cache (instr 9, ruling R3) ──────────────────────────── +# +# ⭐⭐ A SEPARATE DOCUMENT, NEVER A KEY ON THE CELL, AND THAT IS THE WHOLE DESIGN. The obvious +# spelling is a `cachedByScope` map carried ON the cell, but a cell round-trips through +# `clean_cell` on every PUT and PATCH, and `clean_cell` builds a cell from a fixed list of named +# keys taken from RAW CLIENT input. Naming `cachedByScope` there would make an editor's ordinary +# save the one door able to plant a value under a scope key that is not theirs - a forged cache +# entry served, uninspected, to whichever viewer's wall happens to hash to that key. That is the +# wave-51 leak class again, this time planted on purpose instead of merely gone stale. Keeping the +# cache in its own document means `clean_cell` never sees the key at all, so there is no PUT/PATCH +# body shaped to reach it, and it never rides a wire response either. +# +# ⭐ KEYED BY THE FILTER ITSELF, NEVER BY A USER ID OR ROLE NAME. Two viewers who carry the SAME +# row restriction share a slot, correctly - nothing about what they may see differs. The same +# viewer gets a DIFFERENT slot on a database where the restriction differs, or where it is lifted, +# because the wall is part of what `scope_hash` hashes. +# +# ⛔⛔ ONLY TWO CALLERS EXIST, AND THAT IS WHAT MAKES R3 TRUE WITHOUT A SPECIAL CASE. A reader's own +# in-memory re-resolve (`routes_sheet.py::_reresolved_for_reader`) is the only place that ever +# calls `fill_scope_cache`, right after `_resolve_binding` answers under THAT reader's own scope. +# `refresh_sheet` is the only place that ever calls `clear_scope_cache`. A connector sync never +# touches a Sheet cell at all, so it needs no call here to leave the cache alone - there is +# nothing wired for it to reach. + +SCOPE_CACHE_KEY_PREFIX = "sheet_scope_cache_" + +#: A cell that has been read under more distinct scopes than this evicts its oldest slot first. +#: Generous for what a real tenant's permission scopes look like (a handful of walled roles per +#: database), and bounded so a workbook nobody ever refreshes cannot grow this document forever. +_MAX_SCOPE_ENTRIES_PER_CELL = 64 + + +def _scope_cache_key(workbook_id): + """The store key for one workbook's per-scope bound-cell cache.""" + return SCOPE_CACHE_KEY_PREFIX + _slug_id(workbook_id) + + +def scope_hash(database, wall_tree, binding_scope): + """A short, stable key for the row-scope a bound cell resolves under, right now (R3). + + ⚠ THE ONE FUNCTION EITHER SIDE CALLS. `routes_sheet.py` hashes the wall it is ABOUT to resolve + under before deciding whether to recompute, and hashes it again (implicitly, via the same + inputs) when it fills the slot it just computed. Both go through this function so a change to + what the wall looks like cannot make the read side and the write side drift apart silently. + """ + blob = json.dumps({"database": str(database or ""), "wall": wall_tree, + "scope": binding_scope or {}}, sort_keys=True, default=str) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:24] + + +def cached_scope_value(st, workbook_id, cell_key, scope_key): + """`{value, refreshedAt}` for one cell's already-computed slot under `scope_key`, or None. + + ⛔ NEVER FALLS BACK TO ANOTHER SCOPE'S ENTRY. A miss answers None so the caller recomputes; it + does not, under any circumstance, hand back a neighbouring slot. + """ + try: + doc = _read(st, _scope_cache_key(workbook_id)) + except SheetRefusal: + return None + hit = (doc.get("cells") or {}).get(cell_key) + if not isinstance(hit, dict): + return None + slot = hit.get(scope_key) + if not isinstance(slot, dict): + return None + return {"value": slot.get("value"), "refreshedAt": slot.get("refreshedAt")} + + +def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at, database): + """Write ONE scope's slot for ONE cell. Best effort: a bad workbook id is a no-op, not a raise. + + ⛔⛔ THIS IS THE ONLY DOOR THAT WRITES A SLOT, and its callers are server-side resolve paths: + a reader's in-memory re-resolve and the one-cell compute door, each immediately after + `_resolve_binding` answers under that caller's own scope. `value`, `refreshed_at` and + `database` are always what THIS server just computed - never client-supplied, because nothing + here parses a request body. + + DPP 1.8: this cache is a SECOND durable document containing the resolved value, not an + implementation detail inside the workbook. Its own top-level origin inventory therefore has + to name Amazon independently; relying on the opaque scope hash would not identify a source. + """ + try: + wid = _slug_id(workbook_id) + except SheetRefusal: + return + + def _apply(doc): + cells = dict(doc.get("cells") or {}) + slots = dict(cells.get(cell_key) or {}) + if scope_key not in slots and len(slots) >= _MAX_SCOPE_ENTRIES_PER_CELL: + slots.pop(next(iter(slots)), None) + slots[scope_key] = {"value": value, "refreshedAt": refreshed_at} + cells[cell_key] = slots + doc["cells"] = cells + _merge_data_origins(doc, databases=(database,)) + return doc, None + + _persist(st, SCOPE_CACHE_KEY_PREFIX + wid, _apply) + + +def clear_scope_cache(st, workbook_id): + """Drop a workbook's whole per-scope cache. The ONLY caller is `refresh_sheet` (R3): + pressing Refresh is what R3 names as the one event allowed to invalidate it. + """ + try: + wid = _slug_id(workbook_id) + except SheetRefusal: + return + st.update(SCOPE_CACHE_KEY_PREFIX + wid, lambda _current: {}, flush="sync") + + def delete_workbook(st, workbook_id): """Empty the document and drop its index row. Returns True when something was there.""" wid = _slug_id(workbook_id) @@ -1178,6 +1811,7 @@ def delete_workbook(st, workbook_id): _persist(st, workbook_key(wid), _apply) _index_drop(st, wid) + clear_scope_cache(st, wid) # ⛔ C5 / W51-T25: THE GRANTS DIE WITH THE OBJECT, and this is `core.shares.drop_objects`'s own # stated contract rather than an extra: "a deleted object's grants must die with it — # `shared_with` would otherwise serve ghost ids into every receiver's Shared with me forever, diff --git a/platform/core/store.py b/platform/core/store.py index 2284c558fc19ccb79aec94a94c012b8b6029871c..7a3f0532404c04a2b0ae1028b50237ddc792ff83 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -143,7 +143,7 @@ class _Projected(dict): def _refuse(self, key): raise KeyError( - f'{key!r} was DROPPED by a projected store read and is not absent — this document came ' + f'{key!r} was DROPPED by a projected store read and is not absent; this document came ' f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would ' f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if ' f'you need {key!r}.') @@ -657,7 +657,7 @@ class Store: """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents). Bytes are never cached in-process: large, cold, fetched on explicit user action.""" if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + raise RuntimeError('No HF_TOKEN configured; persistence is unavailable.') # D-315: a blob is still this tenant's data. ⚠ No `parent_commit` here and that is correct, # not an omission: a blob is written at its own path and read whole, so two writers cannot # silently merge into one damaged document the way `{name}.json` can. The concurrency @@ -730,7 +730,7 @@ class Store: being uploaded over an hour of unseen work. """ if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + raise RuntimeError('No HF_TOKEN configured; persistence is unavailable.') data_binding.check_write(self.repo, f'put {name!r}') with self._key_lock(name): def _replace(_current): @@ -986,7 +986,7 @@ class Store: if flush == 'async': if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + raise RuntimeError('No HF_TOKEN configured; persistence is unavailable.') with self._lock: self._cache[name] = json.loads(json.dumps(data)) self._owned[name] = True @@ -1002,7 +1002,7 @@ class Store: # place that resolves a parent, rebases and clears the journal. `put` still keeps its own # body (its op REPLACES rather than merges), but neither can upload unconditionally. if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + raise RuntimeError('No HF_TOKEN configured; persistence is unavailable.') landed, _oid = self._commit(name, data, replay) # network, no cache lock held with self._lock: # ⚠ `landed`, NOT `data`. A rebase rebuilt the document from the hub's newer copy, so diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index 81c5abcf550328653180dc7f50e588f1a2e93357..2368a3d45df9dbb98d44d8ec55cb4eea21108f47 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -228,7 +228,7 @@ def _pool(): url = _url() if not url: raise RuntimeError( - f'{_URL_ENV} is not set — the Postgres store backend has nothing to connect to.') + f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.') if _POOL['pool'] is not None and _POOL['url'] == url: return _POOL['pool'] try: diff --git a/platform/core/table_store.py b/platform/core/table_store.py index 19aed664b8ecc5e9808ea6bc8186afe52832d4a5..f70e2130c4cc7e1ae15acffd775f20d81e1f5779 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -2133,6 +2133,34 @@ class TableStore: self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None)) + def _shared_stratum_key(self): + """WHICH document holds this table's tenant-wide stratum — D-643. + + ⛔⛔ THE BUG THIS CLOSES, AND THE DIRECTION IT IS CLOSED IN. Every call below used to hand + `self.table_key` straight to `core.shared_overlay`, whose `bucket()` appends `__shared`. + For a user database that key is `ut__table_workspace`, so the three sites addressed + `ut__table_workspace__shared` while the CREATE door + (`core.grid_events._field_share_keys`, the authority) mints and every one of the seven + readers reads the BARE `ut___shared`. `_tenant_wide_keys` therefore answered the empty + set for every `ut_*` table that has a shared column, and `patch_overlay` routed a + tenant-wide value into the caller's own per-user member. + + ⛔ THIS IS THE OPPOSITE OF THE CHANGE WAVE 50 REFUSED, and the difference matters because + the two look alike. Its lane D was asked to make `_field_storage_keys` resolve the + WORKSPACE spelling, and refused: that would have pointed the share dialog at a bucket + nothing writes and orphaned every `ut_*` shared column with every gate green + ([[the-ut-shared-bucket-is-bare-on-purpose]]). This aligns `TableStore` DOWN to the bare + spelling the create door and the readers already use, which is the same rule stated once. + + ⚠ ONLY THE `ut_` BRANCH MOVES. `product_table_workspace` and `customer_table_workspace` + are correct exactly as they stand (a gate asserts them), so every other key is returned + unchanged. + """ + raw = str(self.table_key or '') + if raw.startswith('ut_') and raw.endswith('_table_workspace'): + return raw[:-len('_table_workspace')] + return raw + def _tenant_wide_keys(self): """The columns of THIS table whose values live in the tenant-wide stratum. @@ -2148,7 +2176,7 @@ class TableStore: """ try: import core.shared_overlay as shared_overlay - return set(shared_overlay.fields(self.table_key, st=self._st) or ()) + return set(shared_overlay.fields(self._shared_stratum_key(), st=self._st) or ()) except Exception: # noqa: BLE001 return set() @@ -2186,7 +2214,7 @@ class TableStore: # ⚠ `st=self._st`, NEVER the module default. The two strata must resolve to the SAME # tenant handle, or a value written by one is invisible to the other and the user's # edit vanishes the moment they save it (`_ProductTableStore` records the same rule). - shared_overlay.put_cells(self.table_key, pid, shared, st=self._st) + shared_overlay.put_cells(self._shared_stratum_key(), pid, shared, st=self._st) if not personal: return @@ -2223,7 +2251,7 @@ class TableStore: if shared: import core.shared_overlay as shared_overlay shared_overlay.put_cells_and_choice_options( - self.table_key, pid, shared, shared_additions, st=self._st) + self._shared_stratum_key(), pid, shared, shared_additions, st=self._st) if not personal: return diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index b2e96441019fa01f3a8c0f8b923a7400032baa56..897cc75b80971f01362bbaefb841b26b3b1626e7 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -377,6 +377,169 @@ def limit_report(table_key, st=None): } +# W57-T37 / instruction 30 / C13: revocation policy belongs to a CREDENTIAL TYPE, not to +# every connected table. W32 R10 keeps Odoo data for reconnect; Meta has no purge ruling. +# The API declares its families here because core must never import the connector layer. +REVOKE_PURGE = 'purge' +REVOKE_KEEP = 'keep' +_CREDENTIAL_TABLES = {} + + +def register_credential_tables(entry_type, prefix, *, on_revoke=REVOKE_KEEP, why='', owner=None): + """Declare a credential's table family and revocation rule; repeat declarations overwrite. + + Empty types/prefixes and unknown policies raise ValueError: a typo must not silently KEEP + data that revocation promised to remove. Without an owner resolver, fixed-key families + belong to the tenant's designated default connection, as Amazon and Meta do today. + """ + etype = str(entry_type or '').strip().lower() + prefix = str(prefix or '').strip().lower() + if not etype or not prefix: + raise ValueError('credential type and table prefix must not be empty') + if on_revoke not in (REVOKE_PURGE, REVOKE_KEEP): + raise ValueError('credential revocation must be purge or keep') + if owner is not None and not callable(owner): + raise ValueError('credential table owner must be callable') + _CREDENTIAL_TABLES[etype] = { + 'prefix': prefix, 'on_revoke': on_revoke, 'why': str(why or ''), 'owner': owner, + } + return dict(_CREDENTIAL_TABLES[etype]) + + +def credential_table_types(): + """Copy the declarations; a disclosure caller must not be able to rewrite purge policy.""" + return {etype: dict(spec) for etype, spec in _CREDENTIAL_TABLES.items()} + + +def connection_of_table(table_key, rt=None, st=None): + """Return (credential type, connection id), or (None, None) outside declared families. + + The longest prefix wins, never registration order. A broken owner resolver falls back to + the designated default: calling its tables ownerless would quietly turn a purge into nothing. + """ + specs = sorted(_CREDENTIAL_TABLES.items(), key=lambda item: -len(item[1]['prefix'])) + for etype, spec in specs: + if not str(table_key).startswith(spec['prefix']): + continue + if rt is None: + return etype, None + if spec['owner'] is not None: + try: + return etype, spec['owner'](table_key, rt) + except Exception: + pass + # keychain imports the store too; importing sideways here avoids a boot-time cycle. + from core import keychain + return etype, keychain.default_connection_id(rt, etype) + return None, None + + +def connector_database_identity(table_key, rt=None, st=None, definition=None): + """Read a connected database's provider identity without importing its API implementation.""" + if definition is None: + definition = all_defs(st=st if st is not None else rt).get(str(table_key)) or {} + if not definition: + return None + declared = definition.get('connector') or {} + provider, connection_id = connection_of_table(table_key, rt=rt, st=st) + from core import connector_spec + spec = connector_spec.spec_for(table_key) + provider = provider or connector_spec.connector_of(table_key) + provider = declared.get('provider') or declared.get('key') or provider + if not provider: + return None + from core import connector_sync + return connector_sync.database_identity( + provider, declared.get('connectionId') or connection_id, table_key, + declared.get('model') or (getattr(spec, 'native', '') if spec else '')) + + +def credential_footprint(entry_type, connection_id, rt=None, st=None): + """Disclose what revoking this ONE credential purges and what its policy keeps. + + A read-through table's row count is UNKNOWN, not zero: its rows live in the mirror. + That distinction is the point of [[no-unverifiable-aggregates]] in a deletion dialog; + a fabricated zero invites consent under the false promise that no records will be lost. + """ + etype = str(entry_type or '').strip().lower() + cid = str(connection_id or '') + spec = credential_table_types().get(etype) + out = {'type': etype, 'connectionId': cid, 'action': 'none', 'purge': [], 'keep': [], + 'rows': 0, 'rowsKnown': True, 'why': 'This credential type spawns no databases.'} + if spec is None: + return out + out.update(action=spec['on_revoke'], why=spec['why']) + if not cid: + return out + + class _Reads: + # all_defs/all_tables serve display fallbacks on store errors. A revocation cannot + # mistake that empty fallback for proof there is nothing to delete. Projection failure + # may still recover through the whole read; only failure of that read is fatal here. + error = None + + def get_projection(self, name, drop=()): + return _st(st).get_projection(name, drop=drop) + + def get(self, name): + try: + return _st(st).get(name) + except Exception as exc: + self.error = exc + raise + + reads = _Reads() + defs = all_defs(reads) + if reads.error is not None: + raise reads.error + candidates = [] + for key, defn in defs.items(): + if (str(key).startswith(spec['prefix']) + and connection_of_table(key, rt, st)[1] == cid): + candidates.append((key, defn, materialises(key, st=st, defn=defn))) + # all_tables is a 28.6 MB deep copy on tenant #0. No materialised candidate means no row + # count needs that document, so the disclosure must not pay for it merely to list labels. + tables = all_tables(reads) if any(known for _, _, known in candidates) else {} + if reads.error is not None: + raise reads.error + for key, defn, known in candidates: + if known and key not in tables: + raise RuntimeError('credential table rows could not be read') + rows = len(tables[key].get('rows') or {}) if known else 0 + out[spec['on_revoke']].append({ + 'key': key, 'label': defn.get('label') or key, 'rows': rows, 'rowsKnown': known, + }) + for action in (REVOKE_PURGE, REVOKE_KEEP): + out[action].sort(key=lambda table: table['key']) + out['rows'] = sum(table['rows'] for table in out['purge']) + out['rowsKnown'] = all(table['rowsKnown'] for table in out['purge']) + return out + + +def purge_credential_tables(entry_type, connection_id, rt=None, st=None): + """Recompute this credential's footprint, then use the existing artifact-family sweep. + + A refused or failed delete is residue, never a success count. Exception text may contain + a store path, so the report carries only its type. A second sweep finds no deleted tables. + """ + footprint = credential_footprint(entry_type, connection_id, rt=rt, st=st) + out = {'purged': [], 'refused': [], 'failed': [], 'rows': 0, 'rowsKnown': True, + 'footprint': footprint} + for table in footprint['purge']: + key = table['key'] + try: + if delete(key, st=st) is False: + out['refused'].append(key) + continue + except Exception as exc: + out['failed'].append({'key': key, 'error': type(exc).__name__}) + continue + out['purged'].append(key) + out['rows'] += table['rows'] + out['rowsKnown'] = out['rowsKnown'] and table['rowsKnown'] + return out + + MAX_FIELDS = 60 #: the badge the nav paints for a table with no connector behind it BLANK_SOURCE = 'Blank' @@ -1841,6 +2004,30 @@ def _clean_metric(raw): #: ~3 KB — comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway. LINK_MAX_IDS = 500 +#: ⭐⭐ W53-T40 / STANDING RULE 1 — HOW MANY DISTINCT TARGET RECORDS ONE **TWO-HOP** LINK MAY +#: RESOLVE FOR ONE SOURCE ROW, and it is a DIFFERENT QUANTITY from `LINK_MAX_IDS` above. +#: +#: ⛔ THE TWO ARE NOT INTERCHANGEABLE AND CONFLATING THEM WOULD BE A SILENT TRUNCATION. +#: `LINK_MAX_IDS` is a DISPLAY cap: `relation_cells` slices the CELL to it and every rollup keeps +#: reading the uncapped `resolved` set, so a 900-record relation shows 500 ids and still averages +#: over 900. This one bounds `resolved` ITSELF, because the second hop is where the fan-out is: +#: `ut_odoo_order_lines` holds 256,593 in-scope rows (256,502 with a `product_id`), and a walk +#: with no ceiling is a cross product of one customer's basket against the catalogue. +#: +#: ⛔ SO IT MUST BE REPORTED, NEVER SILENTLY APPLIED — standing rule 1's SECOND sentence, which is +#: the half that gets dropped. `automation_engine.relation_cells` counts the rows that reach this +#: ceiling and emits a `limit_report`-shaped entry naming the route, the count and the fix. A +#: rollup folded over a capped set is a number about a subset, and nothing on screen would say so. +#: +#: ⚠ 5,000 IS CHOSEN AGAINST THE POPULATION, NOT PICKED FOR ROUNDNESS. The shipped two-hop +#: (`customer_data -> ut_odoo_order_lines -> product_data`) can never exceed the catalogue, which +#: is 5,875 active products, so in production this ceiling is a backstop rather than a filter and +#: the report should stay silent. It bounds the pathological case: a bridge grain and a target +#: grain that are both large would otherwise build one python list per source row with no bound +#: at all. It is deliberately ABOVE `LINK_MAX_IDS`, so on any ordinary relation the display cap +#: bites first and the rollups still fold everything. +LINK_THROUGH_MAX_IDS = 5000 + # Link picker filters are schema, not a View selection: the leaf below constrains candidate rows # in the linked table. `sourceField` is explicitly a field on the current record; absence means # this is the static/literal form. Keeping the two shapes disjoint prevents a dynamic picker @@ -2157,6 +2344,50 @@ def _clean_link(raw): # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]] # with the control still on screen. return None + # ⭐⭐ W53-T40 (a) — `through`: THE SECOND TABLE HOP, AND THE THREE KEYS THAT NAME EVERY LEG. + # + # ⛔ IT IS NOT `via`, AND READING IT AS ONE IS THE EASY MISTAKE HERE. `_clean_via` above + # demands topic/source_dim/target_dim/target_key/scope_key/measure and refuses unless the dims + # belong to a semantic TOPIC; it is a governed measure route, `automation_engine` reads a link + # bag's `via` ZERO times, and it is not a table hop at all. `through` names a DATABASE the + # join travels across, and it is validated the way `table` is — by `is_linkable_target`, the + # one predicate that decides whether anything can resolve rows under a key. + # + # ⛔ A `through` WITH NO `on` CONFIGURES NOTHING, and it is refused for the identical reason + # the bare `from` two lines up is: there is no second hop to make when the far side of it is + # unnamed, and a bag half of which is silently ignored is the worst of the three outcomes. + # + # ⭐⭐ WHY THREE KEYS AND NOT ONE. A two-hop join has FOUR join columns, and the existing + # vocabulary already names two of them: `from` (on the SOURCE) and `on` (on the TARGET). The + # missing two are the bridge's own ends, and they are genuinely different columns — on the + # shipped route `customer_data.pid` meets `ut_odoo_order_lines.partner_id` (Odoo's + # `order_partner_id`) while `ut_odoo_order_lines.product_id` meets `product_data.product_id`. + # ⛔ NEITHER IS DEFAULTED FROM THE OTHER SIDE'S SPELLING. A bridge whose inbound column + # happened to share the source's name would make the convention look right and would join two + # unrelated columns the first time it did not, with a fully populated cell and nothing red. + # Both are REQUIRED, so an incomplete route is refused at this door rather than guessed at the + # compute one. + # + # ⚠ THE BRIDGE MAY NOT BE THE TARGET. `through == table` is a ONE-hop join wearing two names: + # it would resolve, it would look configured, and its second hop would be the identity. + if raw.get('through') is not None: + if 'on' not in out: + return None + through = _field_key(raw.get('through')) + if not is_linkable_target(through) or through == table: + return None + hop_in = _field_key(raw.get('throughOn')) + hop_out = _field_key(raw.get('throughTo')) + if not hop_in or not hop_out: + return None + out['through'] = through + out['throughOn'] = hop_in + out['throughTo'] = hop_out + elif raw.get('throughOn') is not None or raw.get('throughTo') is not None: + # ⛔ HALF A ROUTE IS NOT A ROUTE. A bridge column with no bridge names a hop across a + # database nobody declared; dropping the two keys would store a bag that reads as an + # ordinary one-hop link while its author believed they had configured two. + return None # A semantic hop is meaningful only on a derived link with a first join to travel from. # Its absence preserves the historical one-hop bag byte-for-byte. if raw.get('via') is not None: @@ -4484,15 +4715,27 @@ def enabled_odoo_schema_fields(st=None, model=None): return rows -def record_odoo_sync_status(*, phase, errors=(), st=None): +def record_odoo_sync_status(*, phase, errors=(), st=None, connection_id=None): """Lifecycle writer seam for the out-of-fence sync runner.""" clean_phase = str(phase or 'unknown').strip().lower() if clean_phase not in {'syncing', 'ok', 'error', 'unknown'}: clean_phase = 'unknown' + all_errors = tuple(errors or ()) row = {'phase': clean_phase, 'lastRun': _utc_now().isoformat(), - 'errors': [str(error)[:500] for error in errors][:50]} + 'errors': [str(error)[:500] for error in all_errors][:50]} _st(st).update(ODOO_SYNC_STATUS_KEY, lambda _current: row, flush='sync') + # The measured legacy vocabulary is syncing/ok/error against C6's six states, and its one + # tenant document cannot distinguish two Odoo connections. Keep that document byte-for-byte + # while forwarding to the provider/connection shape. This remains best-effort because a + # status forward must never take down the boot-time resync loop that produced the actual data. + try: + from core import connector_sync + mapped = {'syncing': 'syncing', 'ok': 'ready', 'error': 'error'}[clean_phase] + connector_sync.record(_st(st), provider='odoo', connection_id=connection_id, + state=mapped, errors=all_errors) + except Exception: # noqa: BLE001 - additive status telemetry cannot fail a sync + pass return row diff --git a/platform/core/users.py b/platform/core/users.py index f532cc96bf45ec3005eb25fe54151da45e511127..8eead67bd78b84d763c2a337246dabaac06b7cdc 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -1,8 +1,9 @@ """Per-user accounts for the platform, persisted in the HF Dataset store (users.json). Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext. -A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create -users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable. +A bootstrap 'admin' account is seeded from APP_PASSWORD only when the registry is genuinely empty. +After that seed, APP_PASSWORD has no bypass semantics: every production sign-in resolves a stored, +active account and is subject to password age, durable lockout, and MFA at the API boundary. Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch). @@ -10,7 +11,14 @@ allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only us import os import hmac import hashlib +import math import secrets +import re +import base64 +import binascii +import struct +import time +from urllib.parse import quote import core.store as store @@ -70,10 +78,340 @@ def _hash(pw, salt): return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex() +# ------------------------------------------------------------------ password policy (W57-T35 / C12) +# ONE policy for account creation and both rotation doors. Login only reads the age: checking +# history there would spend eleven PBKDF2 passes on every sign-in without improving authentication. +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength +PASSWORD_HISTORY = 10 +PASSWORD_MIN_AGE_DAYS = 1 +PASSWORD_MAX_AGE_DAYS = 90 + +# Amazon SP-API's current account-control baseline: no more than ten failed attempts, followed by +# a thirty-minute suspension. This state is a STORE document rather than a module dict so several +# workers and a deploy see one budget. The live PostgreSQL backend serialises update() with +# SELECT ... FOR UPDATE; the HF compatibility backend retains its existing sync-update contract. +LOGIN_LOCKOUT_AFTER = 10 +LOGIN_LOCKOUT_SECONDS = 30 * 60 +LOGIN_LOCKOUT_BUCKET = 'auth_login_failures' +_LOGIN_LOCKOUT_MAX_TRACKED = 2048 +_LOGIN_LOCKOUT_RETENTION_SECONDS = 7 * 24 * 60 * 60 + + +class PasswordPolicyError(ValueError): + """Raised with FAILED RULES attached, never a bare message. + + `.failures` is a list of {"rule": , "message": }. + """ + def __init__(self, failures): + self.failures = [dict(failure) for failure in failures] + super().__init__('; '.join(failure['message'] for failure in self.failures)) + + +def password_rules(): + """The public checklist, including the rules enforced over an account's lifetime.""" + return [ + {'rule': 'length', 'message': f'it must be at least {PASSWORD_MIN_LENGTH} characters'}, + {'rule': 'max_length', + 'message': f'it must be no longer than {PASSWORD_MAX_LENGTH} characters'}, + {'rule': 'uppercase', 'message': 'it must contain at least one capital letter'}, + {'rule': 'lowercase', 'message': 'it must contain at least one small letter'}, + {'rule': 'digit', 'message': 'it must contain at least one number'}, + {'rule': 'symbol', + 'message': 'it must contain at least one character that is not a letter or a number'}, + {'rule': 'identifier', + 'message': 'it must not contain the username, the email local part, or any word of ' + 'the display name that is 3 characters or longer'}, + {'rule': 'minimum_age', + 'message': f'it can normally be changed only after {PASSWORD_MIN_AGE_DAYS} day'}, + {'rule': 'reuse', + 'message': f'it must not repeat the current password or any of the last ' + f'{PASSWORD_HISTORY} passwords'}, + {'rule': 'expiry', 'message': f'it must be changed every {PASSWORD_MAX_AGE_DAYS} days'}, + ] + + +def password_failures(pw, username=None, name=None, email=None, record=None, + allow_early_rotation=False, now=None): + """Every rule this password breaks, in policy order. Empty list means it passes.""" + pw = str(pw or '') + # Case-fold identifiers, but never the credential being hashed. Name punctuation separates + # words too, so a hyphenated display name cannot slip through a whitespace-only split. + identifiers = [str(username or '').strip().casefold(), + str(email or '').strip().split('@', 1)[0].casefold()] + identifiers.extend(word.casefold() for word in re.findall(r'[^\W_]+', str(name or '')) + if len(word) >= 3) + folded = pw.casefold() + broken = { + 'length': len(pw) < PASSWORD_MIN_LENGTH, + 'max_length': len(pw) > PASSWORD_MAX_LENGTH, + 'uppercase': not any(c.isupper() for c in pw), + 'lowercase': not any(c.islower() for c in pw), + 'digit': not any(c.isdigit() for c in pw), + 'symbol': not any(not c.isalnum() for c in pw), + 'identifier': any(identifier and identifier in folded for identifier in identifiers), + } + # Amazon DPP 1.4.1's one-day minimum age prevents rapid rotations from walking a password out + # of the ten-entry history. The first activation/legacy rotation has no history and stays + # available; an administrator-forced reset marks must_change_password and likewise remains an + # emergency recovery path. The separate admin password route passes allow_early_rotation only + # after its role and tenant walls, for a direct suspected-compromise reset. + age = password_age_days(record, now=now) if record is not None else None + broken['minimum_age'] = ( + record is not None + and not allow_early_rotation + and (record or {}).get('must_change_password') is not True + and bool((record or {}).get('password_history')) + and age is not None + and age < PASSWORD_MIN_AGE_DAYS + ) + # An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse. + # Every history entry owns its own salt, so comparing against only today's salt proves nothing. + if record is not None and len(pw) <= PASSWORD_MAX_LENGTH: + for credential in [record, *(record.get('password_history') or [])]: + if (credential.get('salt') and credential.get('hash') + and hmac.compare_digest(_hash(pw, credential['salt']), credential['hash'])): + broken['reuse'] = True + break + return [rule for rule in password_rules() if broken.get(rule['rule'])] + + +def password_changed_at(record): + """The stored ISO timestamp, or None when absent or unreadable.""" + import datetime as _dt + stamp = (record or {}).get('password_changed_at') + try: + parsed = _dt.datetime.fromisoformat(stamp) + # Our writer always includes an offset. A partial legacy/manual stamp must never raise + # while comparing it with UTC on the login path; treat it like an absent stamp instead. + if parsed.utcoffset() is None: + return None + except (TypeError, ValueError, OverflowError): + return None + return stamp + + +def password_age_days(record, now=None): + """Fractional days since the last password change, or None without a readable stamp.""" + import datetime as _dt + stamp = password_changed_at(record) + if stamp is None: + return None + current = now or _dt.datetime.now(_dt.timezone.utc) + if current.utcoffset() is None: + current = current.replace(tzinfo=_dt.timezone.utc) + return (current - _dt.datetime.fromisoformat(stamp)).total_seconds() / 86400 + + +def password_expired(record): + """Whether a forced rotation or the password's age requires the change door.""" + if (record or {}).get('must_change_password') is True: + return True + age = password_age_days(record) + # ABSENT STAMP IS NOT EXPIRED: pre-wave accounts have none, and deploying this policy must + # not force the whole tenant to reset. Their first successful sign-in starts the clock. + return age is not None and age > PASSWORD_MAX_AGE_DAYS + + +def generate_password(): + """A random credential that SATISFIES the policy above, for a caller that has none to offer. + + ⛔ IT EXISTS BECAUSE A GENERATOR AND A VALIDATOR THAT DISAGREE FAIL AT PROVISIONING TIME. + `provision_web_tenant.py` and `ops/provision_user.py` both defaulted to + `secrets.token_urlsafe(12)`, whose alphabet contains no symbol and need not contain an + uppercase letter or a digit in any given draw. Under the policy that is a tenant that + provisions on Monday and refuses on Tuesday, for no reason anyone could see. So the default + is built to the rules rather than checked against them, and asserted before it is returned. + + ⚠ NOT a password anybody types: it is handed to the operator to hand on, and the account is + expected to rotate it. Entropy is what matters here, which is why the bulk of it is + `token_urlsafe`. + """ + for _ in range(8): + candidate = f"A{secrets.token_urlsafe(16)}9!" + if not password_failures(candidate): + return candidate + # Unreachable in practice; a loop that can silently fall out of the bottom and return a + # policy-breaking string is exactly the defect this function exists to remove. + raise RuntimeError('could not generate a password that satisfies the policy') + + +# ------------------------------------------------------------------ second factor (W57-T36 / C12) +# RFC 6238: SHA-1, six digits and a 30-second step are what authenticator apps assume. +# They are constants, not account options: two parameter sets would make a scanned secret +# produce codes the server never accepts. Twenty random bytes is RFC 4226's reference length. +MFA_DIGITS = 6 +MFA_PERIOD = 30 +MFA_DRIFT = 1 # accept the previous and next window: phones drift +MFA_ISSUER = 'Loopable' +RECOVERY_CODE_COUNT = 10 + + +def new_mfa_secret() -> str: + return base64.b32encode(secrets.token_bytes(20)).decode('ascii').rstrip('=').upper() + + +def totp_code(secret, at=None, offset=0) -> str: + key = base64.b32decode(secret + '=' * (-len(secret) % 8), casefold=True) + if not key: + # Padding alone decodes to empty bytes, which is not an authenticator secret. + raise ValueError('missing authenticator secret') + counter = int(time.time() if at is None else at) // MFA_PERIOD + offset + digest = hmac.new(key, struct.pack('>Q', counter), hashlib.sha1).digest() + start = digest[-1] & 0x0f + number = struct.unpack('>I', digest[start:start + 4])[0] & 0x7fffffff + return str(number % (10 ** MFA_DIGITS)).zfill(MFA_DIGITS) + + +def totp_valid(secret, code, at=None) -> bool: + if not isinstance(secret, str) or not secret: + return False + code = str(code or '').replace(' ', '') + if len(code) != MFA_DIGITS or not code.isascii() or not code.isdigit(): + return False + try: + # Capture the time once: crossing a step boundary mid-loop must not change the window. + at = time.time() if at is None else at + valid = False + for offset in range(-MFA_DRIFT, MFA_DRIFT + 1): + # There is no counter before the Unix epoch; this only matters for boundary tests. + expected = (totp_code(secret, at=at, offset=offset) + if int(at) // MFA_PERIOD + offset >= 0 else '') + # Compare EVERY window, even after a match. Early return leaks which window matched. + valid |= hmac.compare_digest(expected, code) + return valid + except (ValueError, TypeError, binascii.Error, struct.error, OverflowError): + # A malformed stored/candidate secret is an invalid factor, never a login-path 500. + return False + + +def otpauth_uri(secret, username, issuer=MFA_ISSUER) -> str: + label = quote(f'{issuer}:{username}', safe='') + return (f'otpauth://totp/{label}?secret={quote(secret, safe="")}' + f'&issuer={quote(issuer, safe="")}&algorithm=SHA1' + f'&digits={MFA_DIGITS}&period={MFA_PERIOD}') + + +def new_recovery_codes(n=RECOVERY_CODE_COUNT) -> list[str]: + codes = [] + while len(codes) < n: + raw = secrets.token_hex(5).upper() + code = f'{raw[:5]}-{raw[5:]}' + # Even the unlikely duplicate must not make one printed code redeemable twice. + if code not in codes: + codes.append(code) + return codes + + +def _recovery_key(code) -> str: + # NOT PBKDF2: a recovery code is 40 bits of CSPRNG output with no user-chosen structure, + # so there is no password dictionary to run against it. Stretching adds no entropy; the + # codes use SHA-256 here. A PASSWORD is different and stays on PBKDF2-200k. + normal = str(code or '').strip().upper().replace('-', '').replace(' ', '') + return hashlib.sha256(normal.encode('utf-8')).hexdigest() + + +def mfa_enrolled(record) -> bool: + secret = (record or {}).get('mfa_secret') + return isinstance(secret, str) and bool(secret.strip()) + + +class MFAStateError(ValueError): + """A concurrent enrolment or credential change refused an MFA write, before mutation.""" + + +def set_mfa(username, secret, recovery_codes, *, require_unenrolled=False, expected_epoch=None): + """Store the factor and hashed codes together, returning the LANDED revocation epoch. + + The optional guards bind the HTTP door's decision to the actual write. A route-only + enrolment check lets two workers both issue a first set of codes; an epoch check prevents + regeneration racing an admin reset from silently reinstalling the lost factor. + """ + username = (username or '').strip().lower() + # The replay closure captures only hashes, never printable recovery codes: the HF store + # journals closures for conflict replay, and that journal must not retain clear codes. + hashes = [_recovery_key(c) for c in recovery_codes] + bumped = {} + + def _set(reg): + bumped.clear() # a replay may find the account gone + u = reg.get(username) + if u: + if require_unenrolled and mfa_enrolled(u): + raise MFAStateError('already_enrolled') + if expected_epoch is not None and int(u.get('epoch') or 0) != expected_epoch: + raise MFAStateError('changed') + u['mfa_secret'] = secret + u['recovery_codes'] = list(hashes) + u['mfa_enrolled_at'] = _now_iso() + # C12: enrolment and code rotation revoke old cookies in the SAME update as the + # secret. A separate bump could fail after enrolment and leave every old cookie live. + u['epoch'] = int(u.get('epoch') or 0) + 1 + bumped['epoch'] = u['epoch'] + return reg + landed = store.update('users', _set) + if 'epoch' not in bumped: + return None + # As with bump_epoch, a rebased document can land at a newer epoch than our first attempt. + try: + rec = (landed or {}).get(username) or {} + if rec.get('epoch') is not None: + return int(rec['epoch']) + except Exception: + pass + return int(bumped['epoch']) + + +def clear_mfa(username): + """Admin lost-factor reset: remove enrolment and revoke every session in one write.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + for key in ('mfa_secret', 'recovery_codes', 'mfa_enrolled_at'): + u.pop(key, None) + # Absence sends the next login through phone enrolment; the bump kills old cookies. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def consume_recovery_code(username, code) -> bool: + username = (username or '').strip().lower() + wanted = _recovery_key(code) + consumed = [False] + + def _set(reg): + consumed[0] = False # report the final replay, not an earlier attempt + u = reg.get(username) + if u: + hashes = u.get('recovery_codes') or [] + matched = None + for i, stored in enumerate(hashes): + equal = hmac.compare_digest(stored, wanted) + # Compare EVERY entry, retaining only the first match; never leak its position. + if equal and matched is None: + matched = i + if matched is not None: + u['recovery_codes'] = hashes[:matched] + hashes[matched + 1:] + consumed[0] = True + # NO EPOCH BUMP: this runs during sign-in. Login owns the bump and the cookie it + # is about to mint; consuming a code must not independently invalidate that cookie. + return reg + store.update('users', _set) + return consumed[0] + + +def recovery_codes_remaining(record) -> int: + return len((record or {}).get('recovery_codes') or []) + + def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None, perms=None, tenant='royal-imports', platform_admin=False): salt = secrets.token_hex(16) rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role, + 'password_changed_at': _now_iso(), 'bus': bus, 'active': active, 'modules': modules, 'agent': agent or None, 'email': email or None, # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every @@ -96,14 +434,132 @@ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=N return rec +def _resolve_identifier(reg, identifier): + """Return ``(canonical_username, record_or_none)`` using login's key-first rule.""" + identifier = (identifier or '').strip().lower() + record = (reg or {}).get(identifier) + if record is None and '@' in identifier: + for username, candidate in (reg or {}).items(): + if (isinstance(candidate, dict) + and str(candidate.get('email') or '').strip().lower() == identifier): + return str(username).strip().lower(), candidate + return identifier, record + + +def canonical_login_identifier(identifier): + """Resolve a username/email alias to the one durable lockout bucket for that account. + + The value is used only as input to a one-way digest before persistence. Unknown identifiers + retain their normalised spelling, so their responses remain indistinguishable from a bad + password for a real account. + """ + reg = store.get('users', fresh=True) or {} + canonical, _record_or_none = _resolve_identifier(reg, identifier) + return canonical + + +def _login_lockout_key(identifier): + normalised = (identifier or '').strip().lower() + return hashlib.sha256(('aios-login-v1\0' + normalised).encode('utf-8')).hexdigest() + + +def login_lockout_remaining(identifier, now=None): + """Seconds left in the shared account lock, or zero. + + Store failures deliberately propagate. A login route that cannot read the shared budget must + fail closed rather than silently grant each worker a fresh set of guesses. + """ + now = time.time() if now is None else float(now) + doc = store.get(LOGIN_LOCKOUT_BUCKET, fresh=True) or {} + hit = doc.get(_login_lockout_key(identifier)) if isinstance(doc, dict) else None + if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER: + return 0 + try: + remaining = float(hit.get('locked_until') or 0) - now + except (TypeError, ValueError, OverflowError): + return 0 + return max(0, int(math.ceil(remaining))) + + +def note_login_failure(identifier, now=None): + """Atomically count one failed password/MFA proof and return lock seconds remaining.""" + now = time.time() if now is None else float(now) + key = _login_lockout_key(identifier) + + def _note(doc): + doc = dict(doc) if isinstance(doc, dict) else {} + # Keep the security document bounded under random-identifier abuse. An active lock is + # never pruned, even if a malformed/old ``last_failed_at`` accompanies it. + retained = {} + for stored_key, value in doc.items(): + if not isinstance(value, dict): + continue + try: + last = float(value.get('last_failed_at') or 0) + until = float(value.get('locked_until') or 0) + except (TypeError, ValueError, OverflowError): + continue + if until > now or last >= now - _LOGIN_LOCKOUT_RETENTION_SECONDS: + retained[stored_key] = value + doc = retained + + prior = doc.get(key) or {} + try: + count = int(prior.get('count') or 0) + last = float(prior.get('last_failed_at') or 0) + until = float(prior.get('locked_until') or 0) + except (TypeError, ValueError, OverflowError): + count, last, until = 0, 0.0, 0.0 + # Once a lock expires, or an incomplete failure window ages out, the next miss starts a + # new budget. Attempts rejected while locked never reach this writer and cannot extend it. + if until <= now and (count >= LOGIN_LOCKOUT_AFTER or now - last >= LOGIN_LOCKOUT_SECONDS): + count, until = 0, 0.0 + count += 1 + if count >= LOGIN_LOCKOUT_AFTER: + until = now + LOGIN_LOCKOUT_SECONDS + doc[key] = {'count': count, 'last_failed_at': now, 'locked_until': until} + + if len(doc) > _LOGIN_LOCKOUT_MAX_TRACKED: + # Discard expired/incomplete budgets before active locks. A pathological set of more + # than 2,048 simultaneous active identifiers still evicts oldest-first: the document + # stays bounded rather than becoming an attacker-controlled memory allocation. + oldest = sorted(doc, key=lambda stored_key: ( + float((doc.get(stored_key) or {}).get('locked_until') or 0) > now, + float((doc.get(stored_key) or {}).get('last_failed_at') or 0))) + excess = len(doc) - _LOGIN_LOCKOUT_MAX_TRACKED + for stored_key in [candidate for candidate in oldest if candidate != key][:excess]: + doc.pop(stored_key, None) + return doc + + landed = store.update(LOGIN_LOCKOUT_BUCKET, _note) + hit = (landed or {}).get(key) if isinstance(landed, dict) else None + if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER: + return 0 + return max(0, int(math.ceil(float(hit.get('locked_until') or 0) - now))) + + +def clear_login_failures(identifier): + """Clear the account's shared guessing budget after a complete successful proof.""" + key = _login_lockout_key(identifier) + + def _clear(doc): + doc = dict(doc) if isinstance(doc, dict) else {} + doc.pop(key, None) + return doc + + return store.update(LOGIN_LOCKOUT_BUCKET, _clear) + + def registry(): return store.get('users') def ensure_bootstrap(): """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet). - Idempotent; no-op if the store is unavailable (the app then falls back to the master-password - path in verify()). + + Idempotent and a no-op if the store is unavailable. There is intentionally no authentication + fallback: after the seed, the stored account is the sole production identity and must enrol + MFA like every other account. Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users') is definitively False. A transient read failure at startup used to return {} and make this @@ -121,6 +577,8 @@ def ensure_bootstrap(): master = os.environ.get('APP_PASSWORD', '') if not master: return + # Keep the fresh-store bootstrap seed exempt from the password policy. Rejecting whatever + # APP_PASSWORD holds here could leave a new deployment with nobody able to begin enrolment. try: store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')}) except Exception: @@ -173,8 +631,8 @@ def epoch(username): None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" — which the session verifier must treat as a reason to refuse, not as a default to compare - against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles - that case explicitly rather than inventing an epoch for it here.) + against. A username with no stored account is always refused; a signed cookie is not an + identity record. """ username = (username or '').strip().lower() try: @@ -226,18 +684,20 @@ def bump_epoch(username): def verify(username, pw): - """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the - 'admin' login even if the store is unreachable, so the owner is never locked out.""" + """Return a public stored user on success, else ``None``. + + ``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A + missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity. + """ username = (username or '').strip().lower() if not username or not pw: return None - master = os.environ.get('APP_PASSWORD', '') try: # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once reg = store.get('users', fresh=True) except Exception: reg = {} - u = reg.get(username) + username, u = _resolve_identifier(reg, username) # ⛔ KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT. # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a # different account's `email` field at the same time. The key wins, here, by construction: this @@ -245,40 +705,8 @@ def verify(username, pw): # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the # ambiguity cannot be introduced through the product; this line is what decides it for any # record that arrived some other way. - if u is None and '@' in username: - # Wave 18 (R1): the login box takes a username OR an email — admin@nurilab.id signs in - # without knowing the slug an admin chose. First case-insensitive email match wins; - # ambiguity is an admin data problem, not a login feature. - for k, r in reg.items(): - if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username: - username, u = k, r - break if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']): return _public(username, u) - # emergency master: admin + APP_PASSWORD always works (covers first run / store outage) - if username == 'admin' and master and hmac.compare_digest(str(pw), master): - # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie - # the API mints from this dict AGREES with the stored account. - # - # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does - # that, and a negative control confirmed the lockout is gone with or without this line. - # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the - # record falls through to that master fallback, which hands back a SYNTHETIC identity - # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would - # therefore be silently WIDENED to consolidated, all-module access for the life of that - # session. Matching the epoch means the record branch wins and the account's real scope - # applies, leaving the master fallback as the true last resort it is meant to be. - # 0 when the store is unreachable, which is the case this branch was written for. - # - # ⚠ WAVE 40 (R2) — `emergency_master` MARKS THIS DICT SO THE LOGIN CAN SKIP THE EPOCH BUMP. - # Every ordinary sign-in now bumps the account's epoch to sign its other sessions out, but - # this identity is deliberately NOT epoch-revocable: `deps._user_for`'s master fallback - # admits it whatever the record says. A bump here would therefore revoke nothing, while - # forcing a store WRITE on the one code path that exists for a store outage. Internal to - # the server: `routes_auth._public_user` is a whitelist projection and does not name it. - return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all', - 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0), - 'emergency_master': True} return None @@ -297,8 +725,12 @@ def create_user(username, pw, name, role='user', bus='all', modules='all', session-invalidating as a password change, and it usually IS one. """ username = (username or '').strip().lower() - if not username or not pw: + if not username: raise ValueError('username and password are required') + # Refuse before the update so even an empty password names its broken rules and writes nothing. + failures = password_failures(pw, username=username, name=name, email=email) + if failures: + raise PasswordPolicyError(failures) def _add(reg): prior = reg.get(username) or {} @@ -321,14 +753,36 @@ def create_user(username, pw, name, role='user', bus='all', modules='all', store.update('users', _add) -def set_password(username, pw): +def set_password(username, pw, enforce=True): + """Rotate the credential, bounded history, age and epoch in one write. + + `enforce=False` is only for a caller that already validated (or a future migration) to rotate + without re-reading. The admin password route is that one caller; all others enforce here. + """ username = (username or '').strip().lower() + if enforce: + try: + reg = store.get('users', fresh=True) or {} + except Exception: + reg = {} + rec = reg.get(username) or {} + # This must happen BEFORE store.update: refusing a rule may never leave half a rotation. + failures = password_failures(pw, username=username, name=rec.get('name'), + email=rec.get('email'), record=rec) + if failures: + raise PasswordPolicyError(failures) def _set(reg): u = reg.get(username) if u: + # Merge into the fresh record rather than rebuilding it: access and concurrent + # activity stamps must survive, and each retired credential keeps the salt it used. + u['password_history'] = ([{'salt': u['salt'], 'hash': u['hash'], 'at': _now_iso()}] + + (u.get('password_history') or []))[:PASSWORD_HISTORY] u['salt'] = secrets.token_hex(16) u['hash'] = _hash(pw, u['salt']) + u['password_changed_at'] = _now_iso() + u.pop('must_change_password', None) # X3: a password change revokes every outstanding API session for the account. Bumped # INSIDE the same read-modify-write as the hash so the two can never disagree — a # separate update() could rotate the password and leave old cookies live if the second @@ -338,6 +792,24 @@ def set_password(username, pw): store.update('users', _set) +def set_must_change_password(username, on): + """Require a new password at next login, revoking existing sessions in the same write.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if on: + u['must_change_password'] = True + else: + u.pop('must_change_password', None) + # The flag blocks FUTURE logins; the epoch kills sessions already issued. Clearing + # the flag also bumps so it can never resurrect a cookie from before the forced reset. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + def set_active(username, active): username = (username or '').strip().lower() @@ -454,6 +926,11 @@ def flush_stamps(timeout=10.0): return all(not t.is_alive() for t in pending) +def stamp_password_age(username, when=None): + """Backfill a legacy account's clock through the same fail-silent merge as login activity.""" + return _stamp(username, {'password_changed_at': when or _now_iso()}) + + def touch_login(username, when=None): """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login. diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py new file mode 100644 index 0000000000000000000000000000000000000000..8e3c6175d661e6be3bba645ac9363a16b380b374 --- /dev/null +++ b/platform/harness/amazon_client.py @@ -0,0 +1,837 @@ +"""amazon_client.py — the SP-API transport, in the DATA LAYER where the loaders can reach it. + +⛔⛔ **WHY THIS FILE MOVED DOWN OUT OF `aios-web/api/`, AND THE GATE THAT MADE IT.** The transport +started life inside `connectors_amazon.py`, and `harness/amazon_store.py` reached UP to import it. +`verify_meta`'s architecture leg caught it in one line -- `amazon_store.py -> connectors_amazon` -- +and it was right: layers point ONE way (`modules -> core`, `api -> modules`), and nothing under +`platform/core` or `platform/harness` may import an API-layer module. That check exists because the +same shortcut was taken once before, with `sys.path.insert` plus an import from inside `core`. + +⭐ **THE ALTERNATIVE WAS WORSE, WHICH IS WHY THIS AND NOT A SECOND READER.** `meta_store` avoids the +inversion by bringing its own reader, and Graph is simple enough that a second one costs little. +SP-API is not: the LWA token exchange, the read-only guard, the report request-poll-download loop +and the rate-limit reporting would all have existed twice, and "which stored credential serves this +connector" having two answers is a defect class this codebase already names. So the TRANSPORT lives +here, in the layer both callers can see, and `connectors_amazon.py` stays the API-layer face that +registers the liveness probe. + +⭐ **WHY THIS FILE IS SMALL, AND THE STALE ASSUMPTION IT KILLS.** Since 2 October 2023 SP-API +disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a plain +`Authorization: Bearer` header. Any library, blog post or memory that has us minting IAM users and +signing requests is describing a dead API. +· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4 + +⛔ **READ-ONLY, ENFORCED HERE, WITH EXACTLY TWO NAMED EXCEPTIONS.** `_request` refuses any method but +GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings; +none of that is reachable from this module, and the refusal sits below every route and payload so no +caller can opt out. + +⚠ **The exceptions are `_create_report` and `_competitive_summary_batch`, and both are declared +rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's +`getCompetitiveSummary` is a POST-wrapped batch of read-only GET subrequests. Both functions have +their path as a module constant rather than a parameter, so no caller can redirect either one. They +cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full +reasoning sits on their path constants. + +⛔ **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per +operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong +call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's +records, and its second half is what applies: a limit that genuinely cannot be removed must be +REPORTED with cause and a recommended fix. `REPORT_TYPES` below is that fix, written down. +· https://developer-docs.amazon.com/sp-api/docs/usage-plans-and-rate-limits + +Ideation record, with every owner ruling: `AutoWiki/research/ideation/amazon-connector.md` +""" +from __future__ import annotations + +import contextvars +import json +import random +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +KEY = "amazon_sp" +LABEL = "Amazon" + +#: W57-T38/C13: five total attempts, including the first, bound one throttled operation. +RETRY_MAX_ATTEMPTS = 5 +#: W57-T38/C13: exponential backoff starts at one second after the first 429. +RETRY_BASE_SECONDS = 1.0 +#: W57-T38/C13: no individual retry sleep exceeds one minute, including Retry-After/jitter. +RETRY_MAX_SECONDS = 60.0 +#: W57-T38/C13: randomize each wait by up to half its value to separate competing clients. +RETRY_JITTER = 0.5 +#: W57-T38/C13: separate consecutive table report creations by at least five seconds. +REPORT_STAGGER_SECONDS = 5.0 +#: Amazon createReport usage plan: 15 initial tokens, shared by callers in this process. +#: https://developer-docs.amazon.com/sp-api/reference/createreport +REPORT_BURST = 15 +#: Amazon createReport usage plan: 0.0167 requests/second, approximately one token/minute. +#: https://developer-docs.amazon.com/sp-api/reference/createreport +REPORT_SUSTAINED_SECONDS = 60.0 + +#: Product Pricing v2022-05-01 `getCompetitiveSummary`, official default usage plan. +#: https://developer-docs.amazon.com/sp-api/reference/getcompetitivesummary +COMPETITIVE_SUMMARY_RATE = 0.033 +COMPETITIVE_SUMMARY_INTERVAL = 1.0 / COMPETITIVE_SUMMARY_RATE +COMPETITIVE_SUMMARY_BATCH_SIZE = 20 + +#: AWD v2024-05-09 `listInventory`, official default usage plan and maximum page size. +#: https://developer-docs.amazon.com/sp-api/reference/listinventory +AWD_INVENTORY_RATE = 2.0 +AWD_INVENTORY_INTERVAL = 1.0 / AWD_INVENTORY_RATE +AWD_INVENTORY_PAGE_SIZE = 200 + +# ⭐ One process-wide bucket and lock prevent concurrent syncs from spending the same token. +# The contextual wait belongs to the current report_document logger, never another sync's log. +_report_pacer = { + "tokens": float(REPORT_BURST), + "updated": None, + "last_create": None, + "lock": threading.Lock(), + "waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0), +} + +#: One process-wide reservation per entity operation. Holding the lock through the short wait keeps +#: concurrent tenant syncs from spending the same operation token. +_entity_pacers = { + "competitive_summary": {"last": None, "lock": threading.Lock()}, + "awd_inventory": {"last": None, "lock": threading.Lock()}, +} + +#: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data +#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the +#: comments in `keychain.py` warn about it in as many words. +ENTRY_TYPE = "amazon_sp" + +#: Login With Amazon: the ONLY place a refresh token is exchanged. Not a regional endpoint. +LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token" + +#: Regional SP-API hosts. Royal Imports is US, so North America is the default. +ENDPOINTS = { + "na": "https://sellingpartnerapi-na.amazon.com", + "eu": "https://sellingpartnerapi-eu.amazon.com", + "fe": "https://sellingpartnerapi-fe.amazon.com", +} + +#: ⭐ THE SANDBOX HOSTS, so the `Loopable Sandbox` app that ALREADY EXISTS is usable today. +#: +#: Owner ruling R9 created that app precisely so the connector could be built and proven while +#: identity verification is pending. Without these hosts its credentials would be sent to the +#: PRODUCTION endpoint and fail with an authorization error that looks like a bad credential rather +#: than a wrong host -- the most expensive kind of wrong answer. +#: +#: ⛔⛔ A SANDBOX CREDENTIAL IS FENCED OUT OF `sync`, AND THAT FENCE IS THE POINT. The sandbox +#: returns STATIC MOCK responses. Standing rule 8 forbids an unverifiable aggregate on a screen and +#: R3 requires reconciliation to Amazon's own summary; a mock figure satisfies neither, and mock +#: rows written into `ut_amzn_*` would be indistinguishable from real ones once they are in a grid. +#: So sandbox is allowed for the LIVENESS PROBE and for `--check`, and `amazon_store.sync` REFUSES +#: it by name. That is what makes the sandbox app useful without making it dangerous. +SANDBOX_ENDPOINTS = { + "na": "https://sandbox.sellingpartnerapi-na.amazon.com", + "eu": "https://sandbox.sellingpartnerapi-eu.amazon.com", + "fe": "https://sandbox.sellingpartnerapi-fe.amazon.com", +} +DEFAULT_REGION = "na" + +#: These paths are constants, not caller input. The outer Product Pricing request is POST, but +#: every request inside its batch is a fixed GET to the read-only item URI. +_COMPETITIVE_SUMMARY_BATCH_PATH = ( + "batches/products/pricing/2022-05-01/items/competitiveSummary") +_COMPETITIVE_SUMMARY_ITEM_URI = "/products/pricing/2022-05-01/items/competitiveSummary" +_COMPETITIVE_SUMMARY_INCLUDED = ("featuredBuyingOptions", "referencePrices") +_AWD_INVENTORY_PATH = "awd/2024-05-09/inventory" + + +def is_sandbox(fields): + """Does this stored credential belong to a SANDBOX app? -> bool. + + ⚠ ONE READING, IN ONE PLACE. The probe, the request builder and the loader's refusal all ask + this function rather than each testing the field themselves, because three spellings of + "is this sandbox" is three chances for one of them to answer differently -- and the one that + matters is the refusal. + """ + v = (fields or {}).get("sandbox") + return str(v).strip().lower() in ("1", "true", "yes", "on") if v is not None else False + + +def base_for(fields, region=None): + """The SP-API host this credential should talk to. Sandbox or production, never a guess.""" + r = str(region or (fields or {}).get("region") or DEFAULT_REGION).lower() + table = SANDBOX_ENDPOINTS if is_sandbox(fields) else ENDPOINTS + return table.get(r) + +#: US marketplace. Stored per credential; this is only the fallback for a probe. +US_MARKETPLACE_ID = "ATVPDKIKX0DER" + +#: The five values an admin pastes. `refresh_token` is the secret; the other four are identifiers. +#: ⚠ `refresh_token` MUST also join `keychain._PREVIEW_FIELDS`, or the entry stores correctly and +#: renders a BLANK preview: nothing raises, and the only symptom is an empty cell on the screen the +#: feature exists for. Exactly the wave-31 `access_token` defect. +CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "seller_id", "marketplace_id") + +#: ⚠ OPTIONAL, AND NOT ONE OF THE FIVE. `sandbox` is a DECLARATION about which app minted the +#: credential, not a secret and not something an admin must supply. Absent means production, which +#: is the safe default: a sandbox credential mistakenly left undeclared fails loudly against the +#: production host, whereas a production credential wrongly marked sandbox would quietly read mocks. +OPTIONAL_FIELDS = ("sandbox", "region") + +#: ⭐ THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names +#: the table, the bulk report +#: type that fills it, and the summary aggregate its `validate()` reconciles against. +#: +#: ⭐⭐ R3 — `validate()` RECONCILES TO AMAZON'S OWN SUMMARY, not to Odoo. Standing rule 8 says a +#: number that does not tie to Odoo does not ship; settlements, FBA inventory and Brand Analytics +#: have no Odoo counterpart at all. The owner ruled the reconciliation is source-internal +#: (summary vs detail), because the bug class rule 8 exists to catch is OUR transformation, not +#: Amazon's arithmetic. That answer governs every API-only connector, not just this one. +#: +#: ⛔⛔ R12 — THESE ARE STANDALONE DATABASES. Owner, verbatim: *"We're not building a cross +#: source integration. Its all different. Assume that its a whole new databases which they +#: are."* So NOTHING here joins to Odoo: `listings` is not a bridge to `product_data`'s +#: `default_code`, no seller-SKU mapping table is built, and the hand-maintained ASIN +#: mastersheet is NOT retired by this connector. Amazon numbers are read, reconciled and +#: presented on Amazon's own terms. +#: ⭐ The consequence worth stating: this REMOVES the whole class of defect that +#: [[amazon-conversion-k-is-inverted]] describes, because no unit conversion between an Odoo +#: unit and an Amazon unit is ever performed. +#: +#: ⛔⛔ R14 — NO DASHBOARD, AND NO MODULE. Owner: *"this is just database from Amazon, all +#: its fields mapped into our app so it can be seen under Data module and later be pulled +#: into Sheet. All database connector has this exact flow."* We ship the data, not the +#: conclusion. ⭐ `sheet/BindingEditor.tsx:61` binds on `database.startsWith("ut_")`, so +#: `ut_amzn_*` is Sheet-bindable the moment it exists; that path needs no work. +#: +#: ⚠ `report` is `None` where no bulk report flavor is confirmed. That is O5 of the ideation +#: record and it is NOT a licence to page an entity endpoint quietly: an unconfirmed row must +#: either find its report or REPORT the limit with cause, per standing rule 1. +REPORT_TYPES = { + # ⭐⭐ R13 — BOTH ORDER REPORTS, BY ROLE. `report` seeds the one-time historical + # backfill because BY_ORDER_DATE is stable and pages cleanly by period; `report_incremental` + # drives every sync after, because BY_LAST_UPDATE is the ONLY one that returns orders whose + # status changed after they were placed. ⛔ Refunds, cancellations and settlement + # restatements all arrive as updates to OLD orders, so an order-date-only sync drifts away + # from Amazon in the direction that flatters us, and standing rule 8 would only catch it later + # against the settlement summary. + "orders": { + "report": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + "report_incremental": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", + "validates_against": "order report totals vs our order rows", + "role": "Inventory and Order Tracking", + }, + "order_items": { + "report": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + "report_incremental": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", + "validates_against": "order report line totals vs our item rows", + "role": "Inventory and Order Tracking", + }, + "listings": { + "report": "GET_MERCHANT_LISTINGS_ALL_DATA", + "validates_against": "listing report row count vs our listing rows", + "role": "Product Listing", + }, + # ⛔⛔ THE REPORT THIS ROW ORIGINALLY NAMED IS BEING **REMOVED ON 2026-11-11**. + # `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` and `GET_V2_SETTLEMENT_REPORT_DATA_XML` are both + # deprecated and deleted on that date; Amazon's named replacement is + # `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2`, which "offers the same information in fewer + # columns and streamlined formatting" and is available NOW. + # ⭐ CAUGHT BEFORE THE FIRST SYNC, WHICH IS THE ONLY REASON IT COST NOTHING: had we shipped on + # the deprecated report, settlements would have worked, been reconciled, been trusted, and then + # gone dark in November with no code change on our side to explain it. + # ⚠ FEWER COLUMNS MEANS A DIFFERENT HEADER. This table is `measured_at_sync`, so the parser + # reads whatever V2 actually sends and nothing here needs to know the column list -- which is + # exactly the property that makes swapping the report type a one-line change. + # · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement + "settlements": { + "report": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2", + "validates_against": "settlement summary vs settlement lines", + "role": "Finance and Accounting", + }, + "inventory": { + "report": "GET_LEDGER_SUMMARY_VIEW_DATA", + "entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)", + "validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows", + "role": "Amazon Fulfillment / Amazon Warehousing and Distribution", + }, + # ⛔⛔ O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were + # enumerated and ZERO contain BUYBOX, PRICING, PRICE or COMPETITIVE. So this table SPLITS: + # Buy Box SHARE -> GET_SALES_AND_TRAFFIC_REPORT.salesAndTrafficByAsin.buyBoxPercentage, + # which IS bulk. ⚠ needs the Brand Analytics role, the same one R5 marks + # UNPROVEN, so this row inherits that risk. + # Buy Box PRICE -> Product Pricing entity endpoints ONLY. No bulk path exists. + # ⛔ STANDING RULE 1'S SECOND HALF: the ceiling is Amazon's and cannot be removed, so it is + # REPORTED here with its cause and its fix. Best path is Product Pricing v2022-05-01 + # `getCompetitiveSummary` (featured buying options + competitive price threshold): batch 20, + # rate 0.033/sec, burst 1 — about 40 ASINs per minute. + # ⭐ So the ticket ships SHARE from the bulk report as this table's spine, and PRICE as an + # explicitly rate-limited enrichment that names its own ceiling on screen. A silent per-ASIN + # page loop over a full catalogue is precisely what rule 1 forbids. + "pricing": { + "report": "GET_SALES_AND_TRAFFIC_REPORT", + "report_covers": "buy box SHARE only; PRICE has no bulk path", + "price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)", + "validates_against": "sales and traffic report totals and competitive ASIN responses", + "role": "Pricing + Brand Analytics", + }, + "brand_analytics": { + # ⚠ Returns data only for Brand Registry brand owners (owner ruling R5). Carried as the + # wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows + # for any registered brand, else demote to a declared-but-unsynced stub. + "report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", + "validates_against": "report period totals vs our search-term rows", + "role": "Brand Analytics", + }, +} + +#: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation +#: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those +#: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an +#: architecture review over screen share at developer registration. +#: · https://developer-docs.amazon.com/sp-api/docs/register-as-a-public-developer +ROLES_TAKEN = ( + "Inventory and Order Tracking", "Amazon Fulfillment", + "Amazon Warehousing and Distribution", "Finance and Accounting", + "Pricing", "Product Listing", "Brand Analytics", "Selling Partner Insights", +) +ROLES_REFUSED = ("Buyer Communication", "Buyer Solicitation") + +_UA = "AIOS-Loopable/1.0 (+read-only)" +_TIMEOUT = 20 + + +class AmazonError(RuntimeError): + """Any refusal from LWA or SP-API, already stripped of anything secret.""" + + +class AmazonThrottled(AmazonError): + """An SP-API 429, with an optional Retry-After delay in seconds.""" + + def __init__(self, message, retry_after=None): + super().__init__(message) + try: + self.retry_after = max(0.0, float(retry_after)) if retry_after is not None else None + except (TypeError, ValueError): + self.retry_after = None + + +def _safe(url): + """A URL fit to appear in an error message: query string dropped entirely. + + Same idiom as `connectors_meta._safe`. SP-API puts marketplace ids, seller ids and date + windows in query parameters, and an error string is the one place a caller reliably copies + into a ticket or a screenshot. + """ + try: + p = urllib.parse.urlsplit(str(url)) + return urllib.parse.urlunsplit((p.scheme, p.netloc, p.path, "", "")) + except Exception: # noqa: BLE001 + return "" + + +def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random, exhausted_fix=None): + """Retry only SP-API throttling; return successes and propagate other refusals unchanged.""" + for n in range(1, RETRY_MAX_ATTEMPTS + 1): + try: + return attempt_fn() + except AmazonThrottled as e: + if n == RETRY_MAX_ATTEMPTS: + # ⛔ probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160]. + # Throttle, cause AND the complete fix must fit in the first 160 characters; + # this head is 152 with the named cap. Operation/URL detail must stay AFTER it. + fix = exhausted_fix or "sync fewer tables at once and widen the report window" + raise AmazonThrottled( + f"Amazon throttled (429); {n - 1} retries spent. " + "Cause: SP-API limits per operation, app and seller. " + f"Fix: {fix}. " + f"Operation: {_safe(what)}", retry_after=e.retry_after) from None + wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1)) + floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0) + wait = max(wait, floor) + wait *= 1 - RETRY_JITTER + RETRY_JITTER * rand() * 2 + # ⭐ Negative jitter must not undo Amazon's Retry-After floor; the named cap wins. + sleep(min(RETRY_MAX_SECONDS, max(0.0, floor, wait))) + + +def _pace_report_create(*, sleep=time.sleep, now=time.monotonic): + """Reserve one report creation under the burst, refill rate and minimum gap; return its wait.""" + with _report_pacer["lock"]: + stamp = now() + updated = _report_pacer["updated"] + elapsed = max(0.0, stamp - updated) if updated is not None else 0.0 + tokens = min(REPORT_BURST, _report_pacer["tokens"] + elapsed / REPORT_SUSTAINED_SECONDS) + last = _report_pacer["last_create"] + gap = last + REPORT_STAGGER_SECONDS - stamp if last is not None else 0.0 + wait = max(0.0, gap, (1.0 - tokens) * REPORT_SUSTAINED_SECONDS) + if wait: + sleep(wait) + # ⭐ Count refill during the wait too. A late wake may refill up to the burst, never more. + issued = max(stamp + wait, now()) + _report_pacer.update( + tokens=min(REPORT_BURST, tokens + (issued - stamp) / REPORT_SUSTAINED_SECONDS) - 1.0, + updated=issued, last_create=issued) + return wait + + +def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic): + """Reserve one call for a fixed entity operation under Amazon's default usage plan.""" + pacer = _entity_pacers[name] + with pacer["lock"]: + stamp = now() + last = pacer["last"] + wait = max(0.0, last + interval - stamp) if last is not None else 0.0 + if wait: + sleep(wait) + pacer["last"] = max(stamp + wait, now()) + return wait + + +def _post_form(url, data, timeout=_TIMEOUT): + body = urllib.parse.urlencode(data).encode("ascii") + req = urllib.request.Request(url, data=body, method="POST") + req.add_header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") + req.add_header("User-Agent", _UA) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: + detail = "" + try: + payload = json.loads((e.read() or b"").decode("utf-8") or "{}") + # LWA answers {error, error_description}. Neither echoes the secret. + detail = str(payload.get("error_description") or payload.get("error") or "")[:200] + except Exception: # noqa: BLE001 + detail = "" + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + + +def access_token(fields, timeout=_TIMEOUT): + """Exchange a stored refresh token for a ~1 hour LWA access token. + + ⛔ The returned value is a live bearer credential. It is never logged, never written to disk, + and never returned to a client: callers hold it for the length of one sync and drop it. + """ + f = fields or {} + missing = [k for k in ("client_id", "client_secret", "refresh_token") if not f.get(k)] + if missing: + raise AmazonError("credential is missing " + ", ".join(f"`{m}`" for m in missing)) + got = _post_form(LWA_TOKEN_URL, { + "grant_type": "refresh_token", + "refresh_token": str(f["refresh_token"]), + "client_id": str(f["client_id"]), + "client_secret": str(f["client_secret"]), + }, timeout=timeout) + tok = str(got.get("access_token") or "") + if not tok: + raise AmazonError("LWA returned no access token") + return tok + + +def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT, + sandbox=False, throttle_fix=None, **params): + """One read-only SP-API call. Returns the decoded JSON body. + + ⛔ NON-GET IS REFUSED HERE, BEFORE THE REQUEST IS BUILT. There is no flag, no keyword and no + caller that can turn this off, which is the point: the guard sits below every route, payload + and UI, so nothing above it has to be trusted to stay read-only. + """ + if str(method).upper() != "GET": + raise AmazonError("this connector is read-only: only GET is permitted") + + def attempt(): + """One raw read-only SP-API attempt, with no retry or pacing policy.""" + # ⚠ AN EXPLICIT FLAG, NOT A `fields` DICT. This function is handed a TOKEN, never the + # credential -- deliberately, so a secret cannot reach the request builder at all -- so the + # sandbox decision is made by the caller that does hold the credential and passed in as one + # boolean. A first draft reached for `fields` here, which does not exist in this scope. + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower()) + if not base: + raise AmazonError(f"unknown region `{region}`") + clean = {k: v for k, v in (params or {}).items() if v not in (None, "")} + url = f"{base}/{str(path).lstrip('/')}" + if clean: + url += "?" + urllib.parse.urlencode(clean, doseq=True) + req = urllib.request.Request(url, method="GET") + req.add_header("x-amz-access-token", token) + req.add_header("Accept", "application/json") + req.add_header("User-Agent", _UA) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: + detail = "" + try: + payload = json.loads((e.read() or b"").decode("utf-8") or "{}") + errs = payload.get("errors") or [] + if errs: + detail = str((errs[0] or {}).get("message") or "")[:200] + except Exception: # noqa: BLE001 + detail = "" + # 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed. + if e.code == 429: + fix = throttle_fix or "use the Reports API bulk path instead of a slower page loop" + raise AmazonThrottled( + f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; " + f"the fix is to {fix}", + retry_after=(e.headers or {}).get("Retry-After")) from None + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower(), "") + return _retry( + attempt, f"{base}/{str(path).lstrip('/')}", + exhausted_fix=(throttle_fix + or "use the Reports API bulk path instead of a slower page loop")) + + +#: ⛔ THE SECOND NON-GET SP-API PATH. `getCompetitiveSummary` is an Amazon-defined POST wrapper, +#: but every item in its body is a GET to `_COMPETITIVE_SUMMARY_ITEM_URI`. The path, item method, +#: item URI, included data and 20-item maximum are all fixed below; no caller can turn this into a +#: write-shaped Product Pricing request. +def _competitive_summary_batch(token, asins, marketplace_id, region=DEFAULT_REGION, + timeout=_TIMEOUT, sandbox=False): + """Read one fixed `getCompetitiveSummary` batch. -> its response list.""" + items = [str(asin or "").strip() for asin in (asins or ()) if str(asin or "").strip()] + if not 1 <= len(items) <= COMPETITIVE_SUMMARY_BATCH_SIZE: + raise AmazonError( + f"competitive summary needs 1 to {COMPETITIVE_SUMMARY_BATCH_SIZE} ASINs per batch") + marketplace = str(marketplace_id or "").strip() + if not marketplace: + raise AmazonError("competitive summary needs a marketplace_id") + + body = {"requests": [{ + "asin": asin, + "marketplaceId": marketplace, + "includedData": list(_COMPETITIVE_SUMMARY_INCLUDED), + "method": "GET", + "uri": _COMPETITIVE_SUMMARY_ITEM_URI, + } for asin in items]} + + def attempt(): + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower()) + if not base: + raise AmazonError(f"unknown region `{region}`") + url = f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}" + req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST") + req.add_header("x-amz-access-token", token) + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + req.add_header("User-Agent", _UA) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + got = json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: + detail = "" + try: + errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or []) + detail = str((errs[0] or {}).get("message") or "")[:200] if errs else "" + except Exception: # noqa: BLE001 + detail = "" + if e.code == 429: + raise AmazonThrottled( + f"{_safe(url)} was throttled (429). getCompetitiveSummary defaults to " + "0.033 requests/second with a burst of 1; keep 20 ASINs per batch", + retry_after=(e.headers or {}).get("Retry-After")) from None + if e.code == 403: + raise AmazonError( + f"{_safe(url)} refused getCompetitiveSummary (403). The application needs " + "the Pricing role") from None + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + responses = got.get("responses") + if not isinstance(responses, list): + raise AmazonError("getCompetitiveSummary returned no response list") + return responses + + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower(), "") + return _retry( + attempt, f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}", + exhausted_fix="retain 20 ASINs per batch and lower the batch request rate") + + +def competitive_summaries(fields, asins, timeout=_TIMEOUT, *, sleep=time.sleep, + now=time.monotonic): + """Read competitive summaries for every distinct ASIN, in complete fixed-size batches.""" + unique = [] + seen = set() + for raw in asins or (): + asin = str(raw or "").strip() + if asin and asin not in seen: + unique.append(asin) + seen.add(asin) + if not unique: + return [] + + f = fields or {} + region = str(f.get("region") or DEFAULT_REGION) + marketplace = str(f.get("marketplace_id") or US_MARKETPLACE_ID) + sandbox = is_sandbox(f) + token = access_token(f, timeout=timeout) + token_issued = now() + out = [] + for offset in range(0, len(unique), COMPETITIVE_SUMMARY_BATCH_SIZE): + _pace_entity("competitive_summary", COMPETITIVE_SUMMARY_INTERVAL, + sleep=sleep, now=now) + if now() - token_issued >= 3000: + token = access_token(f, timeout=timeout) + token_issued = now() + batch = unique[offset:offset + COMPETITIVE_SUMMARY_BATCH_SIZE] + responses = _competitive_summary_batch( + token, batch, marketplace, region=region, timeout=timeout, sandbox=sandbox) + returned = [str(((r or {}).get("body") or {}).get("asin") or "") for r in responses] + if len(responses) != len(batch) or sorted(returned) != sorted(batch): + raise AmazonError( + "getCompetitiveSummary returned an incomplete or mismatched ASIN batch; refusing " + "to treat a partial response as complete") + out.extend(responses) + return out + + +def awd_inventory(fields, timeout=_TIMEOUT, *, sleep=time.sleep, now=time.monotonic): + """Read every AWD inventory page, including through empty pages. -> inventory summaries.""" + f = fields or {} + region = str(f.get("region") or DEFAULT_REGION) + sandbox = is_sandbox(f) + token = access_token(f, timeout=timeout) + token_issued = now() + next_token = None + seen_tokens = set() + rows = [] + while True: + _pace_entity("awd_inventory", AWD_INVENTORY_INTERVAL, sleep=sleep, now=now) + if now() - token_issued >= 3000: + token = access_token(f, timeout=timeout) + token_issued = now() + try: + got = _request( + _AWD_INVENTORY_PATH, token, region=region, timeout=timeout, sandbox=sandbox, + throttle_fix="resume the same listInventory pagination later without dropping pages", + details="SHOW", maxResults=AWD_INVENTORY_PAGE_SIZE, nextToken=next_token) + except AmazonError as e: + if "answered 403" in str(e): + raise AmazonError( + f"{_AWD_INVENTORY_PATH} was refused (403). The application needs the Amazon " + "Warehousing and Distribution role") from None + raise + page = got.get("inventory") + if not isinstance(page, list): + raise AmazonError("listInventory returned no inventory list") + rows.extend(page) + following = str(got.get("nextToken") or "").strip() + if not following: + break + if following in seen_tokens: + raise AmazonError( + "listInventory repeated a nextToken; refusing an infinite or duplicate page loop") + seen_tokens.add(following) + next_token = following + return rows + + +#: ⛔⛔ THE REPORT-JOB NON-GET PATH, AND IT IS A CONSTANT RATHER THAN A PARAMETER. +#: +#: The Reports API bulk path cannot be reached with GET alone: asking Amazon to BUILD a report is +#: `POST /reports/2021-06-30/reports`. Standing rule 1 forbids a silent cap on a connector's +#: records and names the Reports API as the fix, so refusing this one call would mean either no +#: bulk path at all or a quiet page loop over entity endpoints. Both are worse. +#: +#: ⭐ **THE EXCEPTION IS SHAPED LIKE `modules/collections_send.py`'s, deliberately** — that is the +#: codebase's single standing precedent for a connector that writes, and its discipline is: ONE +#: named operation, whitelisted in the data layer where no route, payload or UI can widen it. +#: Here the path is a module constant and `_create_report` takes NO path argument, so there is no +#: caller-supplied value to smuggle another operation through. +#: +#: ⚠ AND THE BLAST RADIUS IS WORTH STATING PLAINLY, because "read-only" is a promise to the owner +#: and this is one of two places the HTTP verb bends: a report request creates a JOB, not seller +#: data. The other is Amazon's POST-wrapped batch of fixed competitive-summary GETs. Neither can +#: alter a listing, a price, an order or an inventory level, and neither can send a buyer anything. +#: The two roles that could do those things were REFUSED at registration (R7). +_REPORTS_CREATE_PATH = "reports/2021-06-30/reports" + + +def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION, + start=None, end=None, options=None, timeout=_TIMEOUT, sandbox=False): + """Ask Amazon to build one report. -> its `reportId`. + + ⛔ NO `path` PARAMETER, ON PURPOSE. See `_REPORTS_CREATE_PATH` above: the whole safety argument + is that this function can reach exactly one operation and a caller cannot redirect it. + """ + def attempt(): + """One raw attempt at the declared report-creation POST.""" + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower()) + if not base: + raise AmazonError(f"unknown region `{region}`") + body = {"reportType": str(report_type), + "marketplaceIds": list(marketplace_ids or [US_MARKETPLACE_ID])} + if start: + body["dataStartTime"] = str(start) + if end: + body["dataEndTime"] = str(end) + if options: + body["reportOptions"] = dict(options) + url = f"{base}/{_REPORTS_CREATE_PATH}" + req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST") + req.add_header("x-amz-access-token", token) + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + req.add_header("User-Agent", _UA) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + got = json.loads(r.read().decode("utf-8") or "{}") + except urllib.error.HTTPError as e: + detail = "" + try: + errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or []) + detail = str((errs[0] or {}).get("message") or "")[:200] if errs else "" + except Exception: # noqa: BLE001 + detail = "" + if e.code == 429: + raise AmazonThrottled( + f"{_safe(url)} was throttled (429) creating a {report_type} report. Report " + "creation is limited to about one call every 60 seconds with a burst of 15; the " + "fix is fewer, wider report windows, not more frequent requests", + retry_after=(e.headers or {}).get("Retry-After")) from None + if e.code == 403: + raise AmazonError( + f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this " + "report needs a role the application was not granted") from None + raise AmazonError(f"{_safe(url)} answered {e.code}" + + (f": {detail}" if detail else "")) from None + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + rid = str(got.get("reportId") or "") + if not rid: + raise AmazonError(f"Amazon accepted the {report_type} request but returned no reportId") + return rid + + base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get( + str(region or DEFAULT_REGION).lower(), "") + # ⭐ Pace once per new report. Retrying that same report pays only the shared backoff. + _report_pacer["waited"].set(_pace_report_create()) + return _retry( + attempt, f"{base}/{_REPORTS_CREATE_PATH}", + exhausted_fix="create fewer reports with wider report windows") + + +#: Terminal processing states. `DONE` is the only one that yields a document. +_REPORT_DONE = "DONE" +_REPORT_DEAD = {"CANCELLED", "FATAL"} + + +def report_document(fields, report_type, start=None, end=None, options=None, + poll_seconds=15, max_wait=900, log=None, timeout=_TIMEOUT): + """The whole bulk path for one report: request, poll, resolve the document. -> `(text, meta)`. + + ⭐ THIS IS STANDING RULE 1'S "RECOMMENDED FIX", IMPLEMENTED RATHER THAN CITED. Amazon's own + guidance is that entity endpoints are for deltas and lookups and the Reports API is for a set: + one report document carries every row for its window, so a backfill costs one rate-limited + request instead of one per page. + + ⚠ THE WAIT IS BOUNDED AND THE TIMEOUT IS REPORTED, NOT SWALLOWED. A report that is still + `IN_PROGRESS` after `max_wait` raises with the reportId in the message, so the next call can + resume from it rather than paying for the whole build again. + """ + say = log or (lambda *_a, **_k: None) + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + mkts = [str((fields or {}).get("marketplace_id") or US_MARKETPLACE_ID)] + sbx = is_sandbox(fields) + pace_context = _report_pacer["waited"].set(0.0) + try: + rid = _create_report(token, report_type, mkts, region=region, start=start, end=end, + options=options, timeout=timeout, sandbox=sbx) + finally: + paced = _report_pacer["waited"].get() + _report_pacer["waited"].reset(pace_context) + if paced: + say(f" {report_type}: waited {paced:.2f}s to stagger report creation") + say(f" {report_type}: reportId {rid}") + + waited, state, doc_id = 0, "", "" + while waited <= max_wait: + got = _request(f"reports/2021-06-30/reports/{rid}", token, region=region, timeout=timeout, + sandbox=sbx) + state = str(got.get("processingStatus") or "") + if state == _REPORT_DONE: + doc_id = str(got.get("reportDocumentId") or "") + break + if state in _REPORT_DEAD: + raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})") + time.sleep(poll_seconds) + waited += poll_seconds + # ⛔ The token is ~1 hour and a long report can outlive it. Re-mint rather than 403. + if waited and waited % 1800 == 0: + token = access_token(fields, timeout=timeout) + if not doc_id: + raise AmazonError( + f"the {report_type} report was still {state or 'pending'} after {max_wait}s " + f"(reportId {rid}). Resume from that id rather than requesting it again") + + meta = _request(f"reports/2021-06-30/documents/{doc_id}", token, region=region, + timeout=timeout, sandbox=sbx) + url = str(meta.get("url") or "") + if not url: + raise AmazonError(f"the {report_type} report document carried no download URL") + text = _download_document(url, str(meta.get("compressionAlgorithm") or "")) + say(f" {report_type}: {len(text):,} chars") + return text, {"reportId": rid, "reportDocumentId": doc_id, + "compression": meta.get("compressionAlgorithm")} + + +def _download_document(url, compression="", timeout=120): + """Fetch one presigned report document and decode it. -> text. + + ⚠ THE URL IS PRESIGNED AND SHORT-LIVED, AND IT IS NEVER LOGGED. `_safe` strips the query + string, which for an S3 presigned URL is the entire credential. + ⚠ Amazon flat files are frequently `cp1252`, not UTF-8, and a hard decode drops whole rows on + one smart quote. Decoded with replacement, and that is a deliberate lossy choice on a + presentation field rather than on a number. + """ + req = urllib.request.Request(url, method="GET") + req.add_header("User-Agent", _UA) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + raw = r.read() + except Exception as e: # noqa: BLE001 + raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None + if str(compression).upper() == "GZIP": + import gzip + try: + raw = gzip.decompress(raw) + except Exception as e: # noqa: BLE001 + raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None + for enc in ("utf-8", "cp1252"): + try: + return raw.decode(enc) + except UnicodeDecodeError: + continue + return raw.decode("utf-8", errors="replace") + + +def marketplace_participations(fields, timeout=_TIMEOUT): + """The cheapest honest liveness call: which marketplaces this seller participates in. + + Chosen over an orders or inventory read on purpose. It needs no role beyond the basic grant, it + returns no PII, it is not a bulk endpoint so it cannot burn a report quota, and a credential + that authenticates but participates in NO marketplace is a real and reportable state rather + than an empty result. + """ + token = access_token(fields, timeout=timeout) + region = str((fields or {}).get("region") or DEFAULT_REGION) + got = _request("sellers/v1/marketplaceParticipations", token, region=region, timeout=timeout, + sandbox=is_sandbox(fields)) + return got.get("payload") or [] diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..00f9e149b4e115b011296ab17c2c0cf22642d7da --- /dev/null +++ b/platform/harness/amazon_schema.py @@ -0,0 +1,401 @@ +"""amazon_schema.py — the pinned field catalogue for the Amazon connector, and the honest map of +which tables actually have one. + + python platform/harness/amazon_schema.py --refresh re-pin from amzn/selling-partner-api-models + python platform/harness/amazon_schema.py --report what each table's columns come from + python platform/harness/amazon_schema.py --selftest prove the flattener and the provenance map + +⭐⭐ **WHY THIS FILE EXISTS: R6, AND THE TEST IT CHANGED.** Owner ruling R6 (wave 52) gives Amazon +the onboarding disclosure screen fed by Amazon's PUBLISHED report schemas, and amends +[[connector-onboarding-schema]]'s R4 so the test is *"is there a real schema artifact"* rather than +*"is there a live `fields_get`"*. SP-API has no introspection endpoint at all, so a probe is not +available at any price; a versioned, vendor-authored schema file is the artifact. + +⛔⛔ **AND THE ENUMERATION SAYS ONLY TWO OF SEVEN TABLES HAVE ONE.** Measured 2026-09-04 against +`schemas/reports` at commit `3659f96`: 22 artifacts exist and they are all JSON-format analytics +reports. **Every flat-file (TSV) report we depend on — orders, listings, settlements, the FBA +ledger — has NO schema artifact anywhere in the repository.** That is not a gap to paper over: a +transcribed column list is exactly what W31's R8 forbids, and `meta_store.py`'s header spends three +paragraphs on why (Graph's 285 columns were ASKED-AND-ACCEPTED, never read off a doc). + +⭐ **THE RESOLUTION, AND IT NEEDS NO CREDENTIAL TO BE CORRECT: A FLAT-FILE REPORT CARRIES ITS OWN +HEADER ROW.** Line 1 of the downloaded document IS the column list, straight from Amazon. So those +tables are declared with **no columns at all** until the first sync, and `amazon_store._measure` +reads the header. That is genuinely measured rather than transcribed, it is the same discipline as +Meta's, and it means this connector ships zero invented columns. + +**Every table therefore declares its PROVENANCE, and the disclosure screen prints it:** + +| provenance | meaning | tables | +|---|---|---| +| `schema_artifact` | columns derived mechanically from a pinned vendor JSON Schema | pricing, brand_analytics | +| `measured_at_sync` | no artifact exists; columns are read from the report document's own header | orders, order_items, listings, settlements, inventory | + +⛔ **THERE IS NO THIRD STATE, AND `transcribed` IS NOT ONE OF THEM.** It was considered and refused: +a column list copied out of documentation prose looks identical on the screen to one that was +measured, which is [[a-seeds-upload-date-is-not-its-cut-date]] wearing a different hat. A table +with no artifact shows an empty column list and says why. + +⚠ **THE CATALOGUE IS A FILE, NOT A LIVE CALL, so it carries `fetched_utc` and the blob sha of every +artifact it was built from** — R6's own warning, and the screen must show that date. + +⛔ NO NETWORK AT IMPORT. `--refresh` is the only path that reaches GitHub; everything else reads +the pinned JSON off disk, so a gate, a screen and a sync all work with no connectivity. +""" +from __future__ import annotations + +import argparse +import json +import sys +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json" + +#: The vendor repository, pinned. ⚠ `main` is what `--refresh` reads, and the resulting file +#: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column +#: list is VISIBLE as a sha change rather than a silent redefinition. +REPO = "amzn/selling-partner-api-models" +RAW = "https://raw.githubusercontent.com/" + REPO + "/{ref}/{path}" +API = "https://api.github.com/repos/" + REPO + +#: table -> (artifact path in the vendor repo, the root property holding the row array). +#: ⛔ ONLY TABLES WITH A REAL ARTIFACT APPEAR HERE. The others are declared in PROVENANCE below +#: with an empty column list on purpose; adding a row here without an artifact is the one edit +#: this file exists to prevent. +ARTIFACTS = { + "pricing": ("schemas/reports/sellerSalesAndTrafficReport.json", "salesAndTrafficByAsin"), + "brand_analytics": ("schemas/reports/sellingPartnerSearchTermsReport.json", + "dataByDepartmentAndSearchTerm"), +} + +#: Every table the connector ships, and where its columns come from. The seven are owner ruling +#: R4; the provenance split is the enumeration above. +PROVENANCE = { + "orders": "measured_at_sync", + "order_items": "measured_at_sync", + "listings": "measured_at_sync", + "settlements": "measured_at_sync", + "inventory": "measured_at_sync", + "pricing": "schema_artifact", + "brand_analytics": "schema_artifact", +} + +#: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a +#: screen and standing rule 2 governs it. +PROVENANCE_COPY = { + "schema_artifact": "Columns come from Amazon's published report schema, pinned below.", + "measured_at_sync": ("Amazon publishes no schema for this report. Its columns are read from " + "the report's own header row the first time it syncs."), +} + +_JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"} + + +def _snake(path): + """`salesByAsin.orderedProductSales.amount` -> `sales_by_asin_ordered_product_sales_amount`. + + A dotted JSON path becomes one flat snake_case column, because the mirror is a table and the + grid that reads it has no notion of nesting. Collisions are impossible by construction: the + full path is preserved, only its punctuation changes. + """ + out = [] + for part in str(path).split("."): + buf = "" + for i, ch in enumerate(part): + if ch.isupper() and i and not part[i - 1].isupper(): + buf += "_" + buf += ch.lower() + out.append(buf) + return "_".join(x for x in out if x) + + +def _resolve(node, defs, seen=()): + """Follow a local `$ref` chain to its definition. -> `(node, names_followed)`. + + ⚠ CYCLE-GUARDED, AND THE GUARD HAD TO MOVE ONCE. A first version tracked visited names inside + this function alone, which is useless: `walk` calls it afresh at every level, so `A.a -> A` + resolved cleanly each time and recursed 1000 deep before Python stopped it. **NC-AS4 caught it + on the first run** — the check discriminates, which is the whole reason it was written before + the code was trusted. The names followed are now RETURNED so the caller can carry them down + its own recursion. + """ + names = [] + n = node or {} + while isinstance(n, dict) and n.get("$ref"): + name = str(n["$ref"]).rsplit("/", 1)[-1] + if name in seen or name in names: + return {}, names + names.append(name) + n = defs.get(name) or {} + return n or {}, names + + +def flatten(schema, root): + """One vendor JSON Schema -> an ordered list of `(column, duckdb_type)`. + + ⭐ MECHANICAL, WITH NO HAND-WRITTEN COLUMN ANYWHERE. That is the whole point of preferring an + artifact to a doc: the list cannot drift from what Amazon published, and a refresh that changes + it changes the pinned sha too. + + ⚠ An array of objects nested INSIDE a row is stored as compact JSON text, the same call + `meta_store._cell` makes. Dropping it would violate "every field the source returns"; exploding + it would silently multiply the row count. + """ + defs = (schema or {}).get("definitions") or {} + top = ((schema or {}).get("properties") or {}).get(root) or {} + node, first = _resolve(top.get("items") or top, defs) + cols = [] + + def walk(n, prefix, seen): + n, names = _resolve(n, defs, seen) + seen = seen + tuple(names) + kind = n.get("type") + props = n.get("properties") or {} + if kind == "object" or props: + for name, child in props.items(): + walk(child, f"{prefix}.{name}" if prefix else name, seen) + return + if kind == "array": + cols.append((_snake(prefix), "VARCHAR")) # nested rows -> JSON text + return + if not prefix: + return # an unresolvable root yields nothing + cols.append((_snake(prefix), _JSON_SCALAR.get(str(kind), "VARCHAR"))) + + walk(node, "", tuple(first)) + seen, out = set(), [] + for c, t in cols: + if c and c not in seen: + seen.add(c) + out.append((c, t)) + return out + + +def vendor_examples(table): + """AMAZON'S OWN example rows for a `schema_artifact` table, from the pinned artifact. -> [row] + + ⭐⭐ THIS IS THE ONLY FIXTURE THIS CONNECTOR IS ALLOWED TO HAVE, AND THAT IS THE POINT. A + hand-authored fixture agrees with the parser that reads it by construction, which makes the + gate's ruler the module under test [[a-gate-whose-ruler-is-the-module-under-test]]. These rows + are Amazon's: they ship inside the vendor schema's own `examples` blocks, pinned by the same + blob sha as the column list, so a change to either is visible as a sha change. + + ⚠ RETURNS [] FOR A `measured_at_sync` TABLE, and that is correct rather than a gap. Those + reports publish no schema and therefore no examples, and inventing a flat-file header here + would be the transcription the whole design refuses. + """ + spec = (catalogue().get("tables") or {}).get(str(table)) or {} + if spec.get("provenance") != "schema_artifact": + return [] + return [dict(r) for r in (spec.get("examples") or [])] + + +def _fetch(url, timeout=45): + req = urllib.request.Request(url, headers={"User-Agent": "AIOS-Loopable/1.0 (+read-only)", + "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8")) + + +def refresh(log=print): + """Re-pin the catalogue from the vendor repository. The ONLY path here that uses the network.""" + head = _fetch(f"{API}/commits/main") + ref = str(head["sha"]) + log(f"{REPO} @ {ref[:7]} ({head['commit']['committer']['date']})") + + listing = {e["path"]: e for e in _fetch(f"{API}/contents/schemas/reports")} + tables = {} + for table, (path, root) in ARTIFACTS.items(): + schema = _fetch(RAW.format(ref=ref, path=path)) + cols = flatten(schema, root) + blob = (listing.get(path) or {}).get("sha") + # ⭐ THE VENDOR'S OWN EXAMPLE ROWS, pinned beside the columns they belong to. They are + # what `verify_amazon`'s pipeline section drives, so the end-to-end proof runs on Amazon's + # data rather than on a fixture written by the person who wrote the parser. + defs = schema.get("definitions") or {} + top = ((schema.get("properties") or {}).get(root) or {}) + node, _refs = _resolve(top.get("items") or top, defs) + examples = [e for e in (node.get("examples") or []) if isinstance(e, dict)] + tables[table] = { + "provenance": "schema_artifact", + "artifact": path, + "artifact_sha": blob, + "root": root, + "columns": [{"name": c, "type": t} for c, t in cols], + "examples": examples, + } + log(f" {table:<16} {len(cols):>3} columns {path.rsplit('/', 1)[-1]}") + + for table, prov in PROVENANCE.items(): + if prov == "measured_at_sync": + tables[table] = {"provenance": prov, "artifact": None, "artifact_sha": None, + "root": None, "columns": []} + log(f" {table:<16} 0 columns NO ARTIFACT PUBLISHED: measured at first sync") + + out = { + "_comment": ("Pinned Amazon report field catalogue. Rebuild with " + "`python platform/harness/amazon_schema.py --refresh`. " + "Never hand-edit: every column here is derived from a vendor schema."), + "repo": REPO, + "commit": ref, + "fetched_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "artifacts_in_repo": len(listing), + "tables": tables, + } + CATALOGUE.parent.mkdir(parents=True, exist_ok=True) + CATALOGUE.write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8") + log(f"\nwrote {CATALOGUE}") + return out + + +def catalogue(): + """The pinned catalogue off disk, or an empty shell when it has never been refreshed.""" + try: + return json.loads(CATALOGUE.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 + return {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}} + + +def columns_for(table): + """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync.""" + spec = (catalogue().get("tables") or {}).get(str(table)) or {} + return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])] + + +def disclosure(): + """What the onboarding screen shows: one row per table, with its provenance and sync date. + + ⚠ `synced` IS THE CATALOGUE'S FETCH DATE, NOT TODAY. R6 is explicit that an unstamped + catalogue is [[a-seeds-upload-date-is-not-its-cut-date]] waiting to happen, and the screen is + the only place a stale pin becomes visible. + """ + cat = catalogue() + rows = [] + for table in PROVENANCE: + spec = (cat.get("tables") or {}).get(table) or {} + prov = spec.get("provenance") or PROVENANCE[table] + rows.append({ + "table": f"ut_amzn_{table}", + "provenance": prov, + "note": PROVENANCE_COPY[prov], + "columns": len(spec.get("columns") or []), + "artifact": spec.get("artifact"), + }) + return {"source": "Amazon", "synced": cat.get("fetched_utc"), "commit": cat.get("commit"), + "repo": cat.get("repo"), "tables": rows} + + +# --------------------------------------------------------------------------- selftest + +def selftest(log=print): + """Prove the flattener, the provenance map and the disclosure shape. No network. + + ⛔ EVERY CHECK BELOW WAS DRIVEN RED BEFORE IT WAS TRUSTED. Green on a first run proves a check + exists, never that it discriminates [[revert-one-file-to-see-if-a-gate-discriminates]]. + """ + fails = [] + + def ck(name, cond, detail=""): + """Detail is ASCII-escaped: a failure message carrying a hostile byte must not kill the + run. Same reasoning, and the same incident, as `amazon_store.selftest`'s reporter.""" + if cond: + log(f" ok {name}") + return + try: + shown = str(detail).encode("ascii", "backslashreplace").decode("ascii") + except Exception: # noqa: BLE001 + shown = "" + log(f" FAIL {name} {shown}") + fails.append(name) + + log("amazon_schema selftest") + + # NC-AS1..3 — the flattener, against a miniature of the real artifact shape. + mini = { + "properties": {"rows": {"type": "array", "items": {"$ref": "#/definitions/Row"}}}, + "definitions": { + "Row": {"type": "object", "properties": { + "parentAsin": {"type": "string"}, + "salesByAsin": {"$ref": "#/definitions/Sales"}, + "tags": {"type": "array", "items": {"type": "string"}}}}, + "Sales": {"type": "object", "properties": { + "unitsOrdered": {"type": "integer"}, + "orderedProductSales": {"$ref": "#/definitions/Amount"}}}, + "Amount": {"type": "object", "properties": { + "amount": {"type": "number"}, "currencyCode": {"type": "string"}}}, + }, + } + got = dict(flatten(mini, "rows")) + ck("NC-AS1 refs resolve through two levels", + "sales_by_asin_ordered_product_sales_amount" in got, sorted(got)[:6]) + ck("NC-AS2 scalar types map to DuckDB types", + got.get("sales_by_asin_units_ordered") == "BIGINT" + and got.get("sales_by_asin_ordered_product_sales_amount") == "DOUBLE" + and got.get("parent_asin") == "VARCHAR", + {k: got.get(k) for k in ("parent_asin", "sales_by_asin_units_ordered")}) + ck("NC-AS3 a nested array becomes one JSON text column", + got.get("tags") == "VARCHAR" and len(got) == 5, sorted(got)) + + # NC-AS4 — a cycle must terminate rather than hang. + cyc = {"properties": {"rows": {"$ref": "#/definitions/A"}}, + "definitions": {"A": {"type": "object", "properties": {"a": {"$ref": "#/definitions/A"}, + "x": {"type": "string"}}}}} + ck("NC-AS4 a self-referential schema terminates", dict(flatten(cyc, "rows")).get("x") == "VARCHAR") + + # NC-AS5 — ARTIFACTS may never claim a table whose provenance is measured-at-sync. This is the + # check that stops the one edit this file exists to prevent. + bad = [t for t in ARTIFACTS if PROVENANCE.get(t) != "schema_artifact"] + ck("NC-AS5 every ARTIFACTS row is declared schema_artifact", not bad, bad) + + # NC-AS6 — and the converse: a schema_artifact table with no artifact is a lie on the screen. + orphan = [t for t, p in PROVENANCE.items() if p == "schema_artifact" and t not in ARTIFACTS] + ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan) + + # NC-AS7 — the seven tables of R4, no more and no fewer. + ck("NC-AS7 seven tables are declared", len(PROVENANCE) == 7, sorted(PROVENANCE)) + + # NC-AS8 — standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen. + dashed = [k for k, v in PROVENANCE_COPY.items() if "—" in v or "–" in v] + ck("NC-AS8 no em dash or en dash in screen copy", not dashed, dashed) + + # NC-AS9 — disclosure() names every table and carries the sync stamp key even when unpinned. + d = disclosure() + ck("NC-AS9 disclosure covers all seven and carries a synced field", + len(d["tables"]) == 7 and "synced" in d and all(r["table"].startswith("ut_amzn_") + for r in d["tables"])) + + # NC-AS10 — a measured_at_sync table must ship ZERO columns. An invented column is the defect. + cat = catalogue() + invented = [t for t, s in (cat.get("tables") or {}).items() + if s.get("provenance") == "measured_at_sync" and (s.get("columns") or [])] + ck("NC-AS10 no measured_at_sync table carries invented columns", not invented, invented) + + log(f"\n{'GREEN' if not fails else 'RED'}: {10 - len(fails)}/10") + return 0 if not fails else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--refresh", action="store_true", help="re-pin from the vendor repository") + ap.add_argument("--report", action="store_true", help="print the provenance map") + ap.add_argument("--selftest", action="store_true") + a = ap.parse_args(argv) + if a.selftest: + return selftest() + if a.refresh: + refresh() + return 0 + d = disclosure() + print(f"Amazon field catalogue repo {d['repo']} commit {str(d['commit'])[:7]} " + f"synced {d['synced']}") + for r in d["tables"]: + print(f" {r['table']:<24} {r['provenance']:<18} {r['columns']:>3} columns " + f"{r['artifact'] or ''}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py new file mode 100644 index 0000000000000000000000000000000000000000..24f4ba7a0306a75a2b4e53fb13e300c1ea73a791 --- /dev/null +++ b/platform/harness/amazon_store.py @@ -0,0 +1,1267 @@ +"""amazon_store.py — the Reports-API loader for the tenant mirror, and the only place Amazon rows +are written. Reports remain the bulk spine; Pricing and AWD use their exact read-only operations. + + python platform/harness/amazon_store.py --sync pull into tenant #0's mirror + python platform/harness/amazon_store.py --sync --tenant gtmlab + python platform/harness/amazon_store.py --status what is in the mirror now + python platform/harness/amazon_store.py --purge --tenant gtmlab --confirm-tenant gtmlab + python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate + +⭐ THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one: +`Odoo XML-RPC -> mirror -> odoo_relational -> ut_odoo_*`, `Meta Graph -> mirror -> meta_relational +-> ut_meta_*`, and now `SP-API Reports -> mirror -> amazon_relational -> ut_amzn_*`. Same middle, +same end, one different first hop. The mirror is ONE store with three loaders, never three stores. + +⛔⛔ **NOT ONE COLUMN IN THIS FILE WAS TRANSCRIBED FROM A DOCUMENT, AND THAT IS THE WHOLE DESIGN.** +`meta_store.py` measured Meta's 285 columns by asking the API what it accepted. SP-API cannot be +asked: it has no introspection endpoint at all. So the columns come from one of two places, per +`amazon_schema.PROVENANCE`: + · **`schema_artifact`** — Amazon's own published JSON Schema, flattened mechanically and pinned + with its blob sha (`pricing`, `brand_analytics`). + · **`measured_at_sync`** — the report document's OWN HEADER ROW, read at sync time by `_measure` + below. Amazon publishes no schema for any flat-file report, and line 1 of the document is the + column list straight from the source. +⭐ The consequence worth stating: **a table with no artifact ships with NO columns until it first +syncs.** That is not a limitation to apologise for, it is the alternative to inventing them. + +⛔⛔ **ENTITY KEYS ARE `amzn_`-PREFIXED, AND THIS IS A NAMING RULE STANDING IN FOR A SCHEMA CHANGE.** +`datastore._sync_state`'s primary key is `entity VARCHAR` with NO source column, so two sources +syncing "orders" would overwrite each other's progress marker. Amazon orders is exactly that +collision. Meta already shares the table and avoids it the same way. **Never write a bare entity +name here.** + +⛔ **STANDING RULE 1: NO CAP ON A CONNECTOR'S RECORDS.** `records_mutable('ut_amzn_*')` is False, so +there is no `MAX_ROWS` on this path and there must not be. Where Amazon imposes a ceiling we cannot +remove, it is REPORTED with its cause and a recommended fix rather than silently enforced: see +`pricing`'s Buy Box PRICE split in `connectors_amazon.REPORT_TYPES`. + +⛔ **NO DASHBOARD AND NO MODULE (R14).** This file ends at the mirror. `amazon_relational` spawns +the grids, Data lists them, and Sheet binds them on the `ut_` prefix with no further work. +""" +from __future__ import annotations + +import argparse +import csv +import io +import json +import os +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE.parent) not in sys.path: + sys.path.insert(0, str(_HERE.parent)) + +from harness import amazon_schema # noqa: E402 +from harness import datastore # noqa: E402 + +#: ⛔ THE ENTITY-KEY PREFIX THAT KEEPS `_sync_state` COLLISION-FREE. See the header. +PREFIX = "amzn_" + +#: The seven tables of owner ruling R4, in sync order. ⚠ `order_items` is derived from the SAME +#: report document as `orders` rather than costing a second report: the flat-file order report is +#: already one row per order ITEM, so orders is the deduplicated head and order_items the lines. +TABLES = ("orders", "order_items", "listings", "settlements", "inventory", + "pricing", "brand_analytics") + +#: Durable proof that the two non-report role operations actually ran. A report-backed table can +#: contain rows while one of these role grants is still missing, so table presence alone is not +#: evidence that Pricing or AWD was authorized. +ENTITY_STATE_KEYS = { + "pricing": "amzn_pricing_competitive_summary", + "inventory": "amzn_inventory_awd", +} + +#: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`. +def table_name(table): + """`orders` -> `amzn_orders`. One place, so a rename cannot half-land.""" + return PREFIX + str(table) + + +def entity_key(table): + """The `_sync_state` key. ⛔ ALWAYS prefixed: see the header's collision note.""" + return PREFIX + str(table) + + +#: How far back a first sync reaches when the caller names no window. Amazon's flat-file order +#: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`). +DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730) +#: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one. +MAX_WINDOW_DAYS = 30 + +#: Columns that must be text whatever they look like. An Amazon order id is `111-1234567-1234567` +#: and an ASIN is `B0…`; a settlement id is a long decimal that exceeds 2^53. Same ruling as Meta's +#: ids and `ig_id` before it, for the same reason: a float path corrupts them silently. +_FORCE_TEXT = {"amazon_order_id", "merchant_order_id", "order_id", "asin", "parent_asin", + "child_asin", "sku", "seller_sku", "settlement_id", "shipment_id", "order_item_id", + "merchant_order_item_id", "fnsku", "product_id", "marketplace_id", "seller_id"} + + +def _snake(name): + """A report's own header cell -> a safe DuckDB column name. + + ⚠ MEASURED HEADERS ARE HOSTILE. Amazon flat files carry `purchase-date`, `Units Ordered`, + `item-price`, a UTF-8 BOM on cell one, and occasionally a blank trailing column. Every one of + those has to become a legal identifier without colliding with its neighbour. + + ⭐ THE BOM NEEDS NO SPECIAL CASE, AND PROVING THAT DELETED A LINE. A first version opened with + `.replace("\\ufeff", "")`; perturbing it changed nothing, because the sanitizer below already + maps every non-alphanumeric character to `_` and the trailing `strip("_")` removes it. The + explicit replace was dead code that made NC-AZ2 a TAUTOLOGY: the check could not fail whatever + the code did. The guarantee is kept and now asserted against the real mechanism. + """ + s = str(name or "").strip().lower() + out = [] + for ch in s: + out.append(ch if (ch.isalnum() or ch == "_") else "_") + col = "".join(out) + while "__" in col: + col = col.replace("__", "_") + col = col.strip("_") + if not col: + return "" + return ("c_" + col) if col[0].isdigit() else col + + +def _dedupe(cols): + """Make a measured header list unique, preserving order. A duplicate becomes `name_2`.""" + seen, out = {}, [] + for c in cols: + if not c: + continue + if c in seen: + seen[c] += 1 + out.append(f"{c}_{seen[c]}") + else: + seen[c] = 1 + out.append(c) + return out + + +def parse_flat_file(text): + """One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header. + + ⭐ THIS FUNCTION IS WHY NO ORDER, LISTING OR SETTLEMENT COLUMN IS TRANSCRIBED ANYWHERE. Line 1 + of the document is Amazon's own column list; anything else would be a guess that looks like a + measurement on the disclosure screen. + + ⚠ TAB-SEPARATED, AND `QUOTE_NONE` IS LOAD-BEARING. Amazon flat files are TSV in which a plain + `"` inside a product title is a literal character, not a quote. Python's default dialect would + swallow the rest of the row and silently merge it with the next one, so a catalogue with one + inch-mark in a title would lose rows with nothing raising. + """ + if not str(text or "").strip(): + return [], [] + buf = io.StringIO(str(text).replace("\r\n", "\n").replace("\r", "\n")) + reader = csv.reader(buf, delimiter="\t", quoting=csv.QUOTE_NONE) + try: + header = next(reader) + except StopIteration: + return [], [] + cols = _dedupe([_snake(h) for h in header]) + width = len(cols) + rows = [] + for raw in reader: + if not raw or (len(raw) == 1 and not str(raw[0]).strip()): + continue + vals = list(raw[:width]) + [None] * max(0, width - len(raw)) + rows.append({c: (v if (v is None or str(v) != "") else None) + for c, v in zip(cols, vals)}) + return cols, rows + + +def parse_json_report(text, root): + """One JSON report document -> `(columns, rows)` flattened onto the pinned artifact's names. + + ⚠ THE COLUMN LIST IS THE PINNED SCHEMA'S, NOT THE PAYLOAD'S. Amazon omits null fields from a + JSON report exactly as Graph does, so building the list from what came back would drop columns + with nothing going red — the defect `meta_store.py`'s header describes in as many words. The + artifact is the authority; a field the payload omits is NULL. + """ + try: + doc = json.loads(str(text or "") or "{}") + except Exception: # noqa: BLE001 + return [], [] + body = doc.get(root) or [] + if isinstance(body, dict): + body = [body] + rows = [] + for item in body: + rows.append(_flatten_row(item or {})) + return sorted({k for r in rows for k in r}), rows + + +def _flatten_row(obj, prefix=""): + """`{"salesByAsin": {"unitsOrdered": 3}}` -> `{"sales_by_asin_units_ordered": 3}`. + + Uses `amazon_schema._snake` so a measured row and a pinned column agree on spelling. A nested + LIST becomes compact JSON text rather than being dropped or exploded, the same call + `meta_store._cell` makes. + """ + out = {} + for k, v in (obj or {}).items(): + path = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + out.update(_flatten_row(v, path)) + elif isinstance(v, list): + out[amazon_schema._snake(path)] = json.dumps(v, separators=(",", ":"), + ensure_ascii=False) + else: + out[amazon_schema._snake(path)] = v + return out + + +def _competitive_rows(responses): + """Amazon batch responses -> complete flattened enrichments keyed by returned ASIN.""" + out = [] + for response in responses or (): + body = (response or {}).get("body") or {} + asin = str(body.get("asin") or "").strip() + if not asin: + continue + row = _flatten_row(response or {}, "competitive") + row["competitive_body_asin"] = asin + out.append(row) + return out + + +def _awd_rows(inventory): + """AWD inventory summaries -> flattened rows that remain distinguishable from FBA ledger.""" + rows = [] + for item in inventory or (): + row = _flatten_row(item or {}) + row["inventory_source"] = "AWD" + rows.append(row) + return rows + + +def _tag_fba_inventory(rows): + """Mark report rows so one combined inventory table never conflates FBA with AWD.""" + for row in rows or (): + row["inventory_source"] = "FBA" + return rows + + +def _pricing_asins(rows): + """Distinct child ASINs from Sales and Traffic rows, preserving source order.""" + seen, out = set(), [] + for row in rows or (): + asin = str(row.get("child_asin") or row.get("parent_asin") or "").strip() + if asin and asin not in seen: + seen.add(asin) + out.append(asin) + return out + + +def _apply_competitive(con, table, responses): + """Add every competitive response field to all matching Sales and Traffic ASIN rows.""" + rows = _competitive_rows(responses) + if not rows: + return 0, 0 + cols = sorted({key for row in rows for key in row}) + types = _ensure(con, table, cols, rows) + assignments = ", ".join(f'"{c}"=?' for c in cols) + updated = 0 + for row in rows: + asin = row["competitive_body_asin"] + values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols] + result = con.execute( + f'UPDATE {table} SET {assignments} WHERE child_asin=? OR parent_asin=?', + values + [asin, asin]) + try: + updated += int(result.fetchone()[0]) + except Exception: # noqa: BLE001 + pass + return len(rows), updated + + +def merge_columns(pinned, parsed): + """The column set for a `schema_artifact` table: the pin, plus anything the payload really had. + + ⛔⛔ AMAZON'S OWN ARTIFACT CONTRADICTS ITSELF, WHICH IS WHY THIS IS A UNION AND NOT THE PIN. + `sellerSalesAndTrafficReport.json` declares `mobileAppPageViews` in `properties` while its own + `examples` carry `mobilePageViews`, four columns' worth. `properties` is the right authority for + a COLUMN LIST; the payload is the only authority on what actually arrived. Taking the pin alone + would DROP a field Amazon really returned, silently, which is the one thing R2's *"do not drop + any column"* forbids and which nothing would go red for. + [[a-published-fixture-corpus-can-answer-wrongly]] + + ⭐ PINNED ORDER FIRST, so the grid still follows the schema and the surprises land at the end + where a reader notices them. + + ⚠ IT IS A NAMED FUNCTION RATHER THAN THREE LINES INSIDE `sync` BECAUSE THE GATE HAS TO DRIVE + THE REAL ONE. A first version inlined it, so `verify_amazon`'s pipeline check computed the same + union itself and stayed green when the production line was broken -- the check's ruler was its + own copy of the logic. One implementation, two callers. + """ + have = set(pinned or ()) + return list(pinned or ()) + [c for c in (parsed or ()) if c not in have] + + +def _coltype(name, rows=None): + """The DuckDB type for one column, decided by the DATA when there is data. + + ⛔ A FLAT FILE HAS NO TYPES AT ALL: every cell is text, so an order report's `quantity` arrives + as `"3"` and its `item-price` as `"27.30"`. Sniffing them is what makes the grid sortable and + the `validate()` sums possible. But the sniff is CONSERVATIVE and text wins ties, because a + column that is numeric for 500 rows and then carries `N/A` must not make the insert fail after + five tables are already written — the exact failure `meta_store._coltype` was rewritten for. + """ + if name in _FORCE_TEXT: + return "VARCHAR" + if not rows: + return "VARCHAR" + seen, kinds = 0, set() + for r in rows: + v = r.get(name) + if v is None or v == "": + continue + if isinstance(v, bool): + kinds.add("str") + elif isinstance(v, int): + kinds.add("int") + elif isinstance(v, float): + kinds.add("float") + elif isinstance(v, (list, dict)): + kinds.add("str") + else: + s = str(v).strip() + try: + int(s) + kinds.add("int") + except ValueError: + try: + float(s) + kinds.add("float") + except ValueError: + kinds.add("str") + seen += 1 + if seen >= 500: + break + if not kinds or "str" in kinds: + return "VARCHAR" + return "DOUBLE" if "float" in kinds else "BIGINT" + + +def _cell(value, kind): + """One parsed value -> one DuckDB cell of the declared type. Never raises on bad input.""" + if value is None or value == "": + return None + if kind == "VARCHAR": + return value if isinstance(value, str) else json.dumps( + value, separators=(",", ":"), ensure_ascii=False) if isinstance( + value, (list, dict)) else str(value) + try: + return int(str(value).strip()) if kind == "BIGINT" else float(str(value).strip()) + except (TypeError, ValueError): + return None + + +def _ensure(con, table, cols, rows=None, declared=None): + """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched. + + ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data + that can be re-pulled. A type disagreement drops and recreates rather than leaving a mirror + permanently unwritable over one bad guess. + """ + # ⛔⛔ A DECLARED TYPE BEATS A SNIFFED ONE, AND THE PIPELINE CHECK PROVED WHY WITH ONE ROW. + # Amazon's own example has `buyBoxPercentage: 100` -- a whole number -- so sniffing typed the + # column BIGINT, and the very next real value (91.2) would have been truncated to 91 or refused + # outright. The schema DECLARES it `number`. **Sniffing is the right answer only where there is + # no schema to ask**, which is exactly the `measured_at_sync` tables; where Amazon publishes a + # type, that type is the authority and one sample cannot overrule it. + # ⚠ A column the schema does not declare still sniffs -- that is how the union's extra fields + # (the vendor's own examples/properties disagreement) get a usable type at all. + declared = declared or {} + want = {c: (declared.get(c) or _coltype(c, rows)) for c in cols} + # ⛔⛔ GUARDED, AND IT WAS UNGUARDED FIRST, WHICH WOULD HAVE BROKEN THE VERY FIRST SYNC. + # DuckDB RAISES `CatalogException` from `PRAGMA table_info` on a table that does not exist yet + # -- it does not return an empty list -- and "does not exist yet" is exactly the state every + # table is in on its first ever run. The end-to-end pipeline check found this the first time it + # drove a real document through; no unit-level check could, because each one worked on a table + # some earlier step had already created. + try: + have = {r[1]: str(r[2]).upper() + for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()} + except Exception: # noqa: BLE001 + have = {} + if have: + clash = [c for c, ty in want.items() + if c in have and have[c] != ty + and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")] + if clash: + con.execute(f"DROP TABLE {table}") + have = {} + if not have: + body = ", ".join(f'"{c}" {want[c]}' for c in cols) + con.execute(f"CREATE TABLE IF NOT EXISTS {table} ({body})") + return want + for c in cols: + if c not in have: + con.execute(f'ALTER TABLE {table} ADD COLUMN "{c}" {want[c]}') + return want + + +def _replace(con, table, cols, types, rows): + """Delete-then-insert for one report window. -> rows written. + + ⚠ A REPORT DOCUMENT IS THE TRUTH FOR ITS WINDOW, so the window is replaced wholesale rather + than merged row by row. That is what makes a re-sync idempotent when Amazon RESTATES a row + (refunds, cancellations and settlement adjustments all arrive as changes to old rows), and it + is why `orders` syncs incrementally by LAST UPDATE rather than by order date. + """ + if not rows: + return 0 + ph = ", ".join("?" for _ in cols) + con.executemany( + f'INSERT INTO {table} ({", ".join(chr(34) + c + chr(34) for c in cols)}) VALUES ({ph})', + [[_cell(r.get(c), types.get(c, "VARCHAR")) for c in cols] for r in rows]) + return len(rows) + + +def _windows(start, end, size=MAX_WINDOW_DAYS): + """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden.""" + out, cur = [], start + while cur < end: + nxt = min(cur + timedelta(days=size), end) + out.append((cur, nxt)) + cur = nxt + return out + + +def _iso(dt): + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _runtime_for(tenant_key): + """The tenant runtime, or None when this process cannot resolve one. Lazy and guarded, so + `--status` works with no store reachable.""" + try: + from harness import runtime as _rt + return _rt.get_runtime(str(tenant_key or "")) + except Exception: # noqa: BLE001 + return None + + +def credential(tenant_key="royal-imports", rt=None): + """The tenant's stored Amazon credential, or None. + + ⛔ THE KEYCHAIN IS THE ONLY RUNTIME SOURCE, AND THERE IS NO ENVIRONMENT FALLBACK. `AMAZON_SP_*` + exists for `connectors_amazon.py --check`, which is an operator proving a freshly minted token + before pasting it. Serving it here would hand one tenant another's Amazon seller account, which + is precisely the elevation `keychain.TENANT_WIDE_TYPES` was written to stop. + """ + try: + import core.keychain as kc + return kc.amazon_creds(rt if rt is not None else _runtime_for(tenant_key)) + except Exception: # noqa: BLE001 + return None + + +def _tenant_key(value): + """A non-empty canonical tenant slug for a destructive or store-binding operation.""" + key = str(value or "").strip().lower() + if not key: + raise ValueError("an explicit tenant slug is required") + return key + + +def _tenant_mirror_path(tenant_key, rt=None): + """Resolve the exact tenant mirror path without opening or creating it.""" + key = _tenant_key(tenant_key) + if rt is not None: + runtime_key = str(getattr(rt, "key", "") or "").strip().lower() + if runtime_key != key: + raise RuntimeError( + f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}") + return Path(rt.datastore_path) + return Path(datastore.path_for(key)) + + +def _connect_tenant_mirror(tenant_key, rt=None): + """Bind this single-tenant process to the requested mirror, then open a writer cursor. + + `datastore.connect()` deliberately takes no path argument. The safe path switch is + `datastore.use_path()`, which closes the cached writer and invalidates every read cursor before + changing the process-wide file. Calling `connect(path)` looks plausible but raises before a + credentialed Amazon sync can write its first row. + """ + path = _tenant_mirror_path(tenant_key, rt) + if Path(datastore.DB_PATH).resolve() != path.resolve(): + datastore.use_path(path) + if rt is not None: + rt.assert_datastore_matches() + return datastore.connect(), path + + +def _require_purge_confirmation(tenant_key, confirmation): + """Require the operator to repeat the exact canonical tenant slug.""" + key = _tenant_key(tenant_key) + if str(confirmation or "").strip() != key: + raise RuntimeError( + f"refusing permanent deletion: --confirm-tenant must be exactly {key!r}") + return key + + +def _assert_amazon_credential_revoked(rt): + """Refuse mirror deletion while any Amazon credential metadata still exists. + + This checks metadata, not decrypted fields. It therefore still catches a credential when the + keychain encryption key is unavailable, and an unreadable tenant store raises rather than + masquerading as an empty keychain. + """ + from core import keychain as kc + if kc.first_entry_of_type(rt, "amazon_sp") is not None: + raise RuntimeError( + "refusing permanent deletion: this tenant still has a stored Amazon credential") + return True + + +def _quoted_identifier(value): + """Quote one database-discovered identifier for a DROP statement.""" + return '"' + str(value).replace('"', '""') + '"' + + +def _amazon_objects(con): + """Every live mirror object reserved to Amazon by the `amzn_` naming contract.""" + rows = con.execute( + "SELECT table_name, table_type FROM information_schema.tables " + "WHERE table_schema='main'").fetchall() + out = [] + for name, kind in rows: + name = str(name) + if not name.startswith(PREFIX): + continue + out.append((name, str(kind or "BASE TABLE").upper())) + return sorted(out, key=lambda row: ("VIEW" not in row[1], row[0])) + + +def _sync_state_rows(con): + """The complete state table, used to prove non-Amazon markers survive a purge unchanged.""" + return [tuple(row) for row in con.execute( + "SELECT entity, phase, last_id, cursor_wd, rows, updated_at " + "FROM _sync_state ORDER BY entity").fetchall()] + + +def _purge_mirror(con): + """Transactionally delete every `amzn_` mirror object and state marker, then read it back. + + The prefix is the connector's reserved namespace, so this also catches a future Amazon table + that has not yet been added to `TABLES`. Non-Amazon tables and sync markers are snapshotted and + compared inside the transaction and again after commit. No caller supplies a table name. + """ + objects = _amazon_objects(con) + before_tables = {str(row[0]) for row in con.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='main'").fetchall()} + before_other_tables = sorted(name for name in before_tables if not name.startswith(PREFIX)) + before_state = _sync_state_rows(con) + amazon_state = [str(row[0]) for row in before_state if str(row[0]).startswith(PREFIX)] + other_state = [row for row in before_state if not str(row[0]).startswith(PREFIX)] + row_counts = {} + for name, _kind in objects: + row_counts[name] = int(con.execute( + f"SELECT COUNT(*) FROM {_quoted_identifier(name)}").fetchone()[0]) + + try: + con.execute("BEGIN TRANSACTION") + for name, kind in objects: + noun = "VIEW" if "VIEW" in kind else "TABLE" + con.execute(f"DROP {noun} {_quoted_identifier(name)}") + for entity in amazon_state: + con.execute("DELETE FROM _sync_state WHERE entity=?", [entity]) + + if _amazon_objects(con): + raise RuntimeError("Amazon mirror objects remain inside the purge transaction") + inside_other_tables = sorted(str(row[0]) for row in con.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() + if not str(row[0]).startswith(PREFIX)) + if inside_other_tables != before_other_tables: + raise RuntimeError("a non-Amazon mirror object changed during the purge") + inside_state = _sync_state_rows(con) + if inside_state != other_state: + raise RuntimeError("a non-Amazon sync marker changed during the purge") + con.execute("COMMIT") + except Exception: + try: + con.execute("ROLLBACK") + except Exception: # noqa: BLE001 + pass + raise + + residue_objects = _amazon_objects(con) + residue_state = [str(row[0]) for row in _sync_state_rows(con) + if str(row[0]).startswith(PREFIX)] + after_other_tables = sorted(str(row[0]) for row in con.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall() + if not str(row[0]).startswith(PREFIX)) + after_other_state = [row for row in _sync_state_rows(con) + if not str(row[0]).startswith(PREFIX)] + if residue_objects or residue_state: + raise RuntimeError( + "Amazon mirror purge committed but independent readback found residue") + if after_other_tables != before_other_tables or after_other_state != other_state: + raise RuntimeError( + "Amazon mirror purge committed but independent readback found collateral change") + return { + "tables": [name for name, _kind in objects], + "rows": sum(row_counts.values()), + "syncMarkers": len(amazon_state), + "residueTables": [], + "residueSyncMarkers": [], + } + + +def purge(tenant_key, confirmation, rt=None, log=print): + """Permanently remove one tenant's live Amazon mirror after credential revocation. + + This is an operator-only lifecycle control. It never revokes a credential itself and refuses + to run until the tenant runtime resolves, the exact tenant slug is repeated, the mirror already + exists, and every Amazon credential metadata row is gone. + """ + key = _require_purge_confirmation(tenant_key, confirmation) + rt = rt if rt is not None else _runtime_for(key) + if rt is None: + raise RuntimeError( + f"tenant {key!r} could not be resolved; refusing to guess a database target") + path = _tenant_mirror_path(key, rt) + if not path.is_file(): + raise RuntimeError( + f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof") + _assert_amazon_credential_revoked(rt) + con, bound_path = _connect_tenant_mirror(key, rt) + try: + result = _purge_mirror(con) + finally: + con.close() + report = {"tenant": key, "database": bound_path.name, **result} + log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from " + f"{len(result['tables']):,} tables and {result['syncMarkers']:,} sync markers") + return report + + +def _client(): + """The SP-API transport, from the DATA LAYER. + + ⛔⛔ THIS USED TO REACH UP INTO `aios-web/api/connectors_amazon.py`, AND `verify_meta`'S + ARCHITECTURE LEG CAUGHT IT: `amazon_store.py -> connectors_amazon`. Nothing under + `platform/core` or `platform/harness` may import an API-layer module, because layers point one + way. The transport moved DOWN to `harness/amazon_client.py` rather than being duplicated here, + since the probe and this loader need the same LWA exchange, the same read-only guard and the + same report loop -- and two copies of that is two answers to one question. + + ⚠ Still a function rather than a module-level import, so `--status` and `--selftest` work in a + process that has no network stack configured and no credential at all. + """ + from harness import amazon_client + return amazon_client + + +def _record_state(con, key, phase, rows=0, detail=None): + """Write `_sync_state` in its declared column order, never by a misleading positional guess.""" + con.execute( + "INSERT OR REPLACE INTO _sync_state " + "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", + [str(key), str(phase), None, (str(detail)[:300] if detail else None), int(rows or 0), + _iso(datetime.now(timezone.utc))]) + + +def _read_state(con, key): + """One durable operation marker, or None when that operation has never completed.""" + try: + row = con.execute( + "SELECT phase, rows, updated_at, cursor_wd FROM _sync_state WHERE entity=?", + [str(key)]).fetchone() + except Exception: # noqa: BLE001 + return None + if not row: + return None + return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]} + + +def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None, + max_wait=900): + """Pull every declared table into the tenant's mirror. -> a report dict. + + Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the + reason, when the tenant has stored no credential. + """ + rt = rt if rt is not None else _runtime_for(tenant_key) + fields = credential(tenant_key, rt) + if not fields: + raise RuntimeError( + f"tenant `{tenant_key}` has no stored `amazon_sp` credential. Add one under " + "Settings, Connectors. There is deliberately no environment fallback: " + "AMAZON_SP_* is an operator proving credential, not a tenant's") + + cx = _client() + + # ⛔⛔ A SANDBOX CREDENTIAL IS REFUSED HERE, AND THIS IS THE FENCE THAT MAKES THE SANDBOX APP + # SAFE TO OWN. Amazon's sandbox returns STATIC MOCK responses. Standing rule 8 forbids an + # unverifiable aggregate reaching a screen and R3 requires reconciliation to Amazon's OWN + # summary -- a mock figure satisfies neither. Worse, once mock rows are written into the mirror + # they spawn into `ut_amzn_*` and become indistinguishable from real ones in a grid; there is no + # marker a person would see. So the sandbox is allowed for the LIVENESS PROBE and for + # `connectors_amazon.py --check`, which prove the client end to end, and it stops there. + # ⚠ REFUSED BY NAME, WITH THE REASON, rather than silently syncing nothing -- an empty table is + # exactly what a caller would misread as "Amazon has no data". + if cx.is_sandbox(fields): + raise RuntimeError( + "this is a SANDBOX credential and sync refuses it. The sandbox returns static mock " + "responses, and mock rows in ut_amzn_* are indistinguishable from real ones once they " + "reach a grid (standing rule 8, and R3's reconciliation). Use it to prove the client " + "(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.") + + want = [t for t in (tables or TABLES) if t in TABLES] + end = datetime.now(timezone.utc) + start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS)) + con, _path = _connect_tenant_mirror(tenant_key, rt) + report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)} + + for table in want: + spec = cx.REPORT_TYPES.get(table) or {} + rtype = spec.get("report") + key, mirror = entity_key(table), table_name(table) + if not rtype: + report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"} + log(f" {table:<16} SKIPPED: no confirmed bulk report flavor") + continue + prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync") + root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1] + try: + written, cols, cleared = 0, [], False + pricing_asins = [] + for w0, w1 in _windows(start, end): + text, meta = cx.report_document( + fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log) + if prov == "schema_artifact" and root: + parsed_cols, rows = parse_json_report(text, root) + pinned = [c for c, _t in amazon_schema.columns_for(table)] + # ⛔⛔ THE UNION, NOT THE PIN, AND AMAZON'S OWN ARTIFACT IS WHY. + # + # `sellerSalesAndTrafficReport.json` CONTRADICTS ITSELF: its `properties` declare + # `mobileAppPageViews` while its own `examples` carry `mobilePageViews`, four + # columns' worth. So the pinned list -- which is built from `properties`, and is + # the right authority for a COLUMN LIST -- can be missing a field Amazon + # actually returns. + # + # ⭐ `pinned or parsed_cols` would then have DROPPED that field silently, which + # is the one thing R2's *"do not drop any column"* forbids and which nothing + # would have gone red for. The union keeps every field the payload really + # carried AND every field the schema promises, so a disagreement costs a NULL + # column rather than lost data. [[a-published-fixture-corpus-can-answer-wrongly]] + # + # ⚠ Pinned order first, so the grid's column order still follows the schema and + # only the surprises land at the end where they are visible. + extra = [c for c in parsed_cols if c not in set(pinned)] + if extra: + log(f" {table}: {len(extra)} field(s) returned that the pinned schema " + f"does not declare, KEPT: {', '.join(extra[:6])}") + cols = merge_columns(pinned, parsed_cols) if pinned else parsed_cols + declared = dict(amazon_schema.columns_for(table)) + else: + # ⚠ NO DECLARED TYPES HERE ON PURPOSE: a flat-file report publishes no schema, + # so sniffing the data is the ONLY evidence there is. That is the whole + # provenance split, arriving one layer down. + cols, rows = parse_flat_file(text) + declared = {} + if not cols: + continue + if table == "orders": + rows = _dedupe_orders(rows) + if table == "inventory": + rows = _tag_fba_inventory(rows) + if "inventory_source" not in cols: + cols.append("inventory_source") + if table == "pricing": + pricing_asins.extend(_pricing_asins(rows)) + types = _ensure(con, mirror, cols, rows, declared=declared) + # Clear once, after the table exists. The old multi-window branch deleted nothing, + # so every backfill appended a second copy of every historical row. + if not cleared: + con.execute(f"DELETE FROM {mirror} WHERE 1=1") + cleared = True + written += _replace(con, mirror, cols, types, rows) + + operations = {} + if table == "pricing": + state_key = ENTITY_STATE_KEYS[table] + if not pricing_asins: + _record_state(con, state_key, "unproven", 0, + "Sales and Traffic returned no ASIN to test") + operations["getCompetitiveSummary"] = { + "state": "unproven", "responses": 0} + log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned") + else: + try: + responses = cx.competitive_summaries(fields, pricing_asins) + response_rows, updated = _apply_competitive(con, mirror, responses) + _record_state(con, state_key, "done", response_rows) + operations["getCompetitiveSummary"] = { + "state": "done", "responses": response_rows, + "mirror_rows_enriched": updated} + log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, " + f"{updated:,} mirror rows enriched") + except Exception as e: # noqa: BLE001 + detail = f"{type(e).__name__}: {str(e)[:160]}" + _record_state(con, state_key, "refused", 0, detail) + operations["getCompetitiveSummary"] = { + "state": "refused", "reason": detail} + log(f" pricing: getCompetitiveSummary REFUSED: {str(e)[:160]}") + + if table == "inventory": + state_key = ENTITY_STATE_KEYS[table] + try: + inventory = cx.awd_inventory(fields) + awd_rows = _awd_rows(inventory) + if awd_rows: + awd_cols = sorted({c for row in awd_rows for c in row}) + awd_types = _ensure(con, mirror, awd_cols, awd_rows) + if not cleared: + con.execute(f"DELETE FROM {mirror} WHERE 1=1") + cleared = True + written += _replace(con, mirror, awd_cols, awd_types, awd_rows) + _record_state(con, state_key, "done", len(awd_rows)) + operations["listInventory"] = { + "state": "done", "rows": len(awd_rows)} + log(f" inventory: AWD listInventory {len(awd_rows):,} rows") + except Exception as e: # noqa: BLE001 + detail = f"{type(e).__name__}: {str(e)[:160]}" + _record_state(con, state_key, "refused", 0, detail) + operations["listInventory"] = {"state": "refused", "reason": detail} + log(f" inventory: AWD listInventory REFUSED: {str(e)[:160]}") + + try: + column_count = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall()) + total_rows = int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]) + except Exception: # noqa: BLE001 + column_count, total_rows = len(cols), written + report["tables"][table] = { + "rows": total_rows, "columns": column_count, "provenance": prov, + "report": rtype, "operations": operations} + _record_state(con, key, "done", total_rows) + log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}") + except Exception as e: # noqa: BLE001 + # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN + # ticket answering itself, and it must not cost the six tables that did work. + report["tables"][table] = {"rows": 0, "state": f"{type(e).__name__}: {str(e)[:160]}"} + log(f" {table:<16} REFUSED: {str(e)[:160]}") + report["finished"] = _iso(datetime.now(timezone.utc)) + return report + + +def _dedupe_orders(rows): + """The flat-file order report is one row per ITEM. `orders` is its deduplicated head. + + ⚠ FIRST ROW WINS PER ORDER ID, and the order-level columns are identical across an order's + lines by construction, so this drops line detail rather than losing an order. `order_items` + keeps every row, which is why both tables read the same document instead of costing two. + """ + seen, out = set(), [] + for r in rows: + oid = str(r.get("amazon_order_id") or r.get("order_id") or "") + if oid and oid in seen: + continue + if oid: + seen.add(oid) + out.append(r) + return out + + +def validate(tenant_key="royal-imports", rt=None, log=print): + """⭐⭐ R3 — RECONCILE EACH TABLE TO AMAZON'S OWN SUMMARY, NOT TO ODOO. -> a report dict. + + Standing rule 8 says a number that does not tie to an independent aggregate does not ship. + Settlements, FBA inventory and Brand Analytics have NO Odoo counterpart at all, and the owner + ruled (wave 52, R3) that for an API-only source the reconciliation is SOURCE-INTERNAL: summary + versus detail, the report's own period totals versus our rows. + + ⛔ **THE REASON IS THE RULE, and it is what keeps rule 8's teeth:** the bug class rule 8 exists + to catch is OUR transformation, not Amazon's arithmetic. A summary-versus-detail check catches + exactly that and nothing less. Rule 8 is NARROWED here, never loosened to "any independent + aggregate" — that broader version was offered and refused, because a loosened rule is far + harder to tighten again than a narrow one is to widen. + + ⚠ A TABLE THAT HAS NEVER SYNCED IS `unproven`, NOT `ok`. An empty table trivially reconciles + with an empty summary, and a check that cannot fail is broken + [[a-check-that-cannot-reach-green-is-broken]]. + """ + rt = rt if rt is not None else _runtime_for(tenant_key) + con, _path = _connect_tenant_mirror(tenant_key, rt) + cx = _client() + out = {} + for table in TABLES: + mirror, spec = table_name(table), (cx.REPORT_TYPES.get(table) or {}) + try: + n = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] + except Exception: # noqa: BLE001 + out[table] = {"state": "unproven", "reason": "table has never synced", + "reconciles": spec.get("validates_against")} + continue + if not n: + out[table] = {"state": "unproven", "reason": "table is empty", + "reconciles": spec.get("validates_against")} + continue + if table in ENTITY_STATE_KEYS: + operation = _read_state(con, ENTITY_STATE_KEYS[table]) + if not operation or operation.get("state") != "done": + out[table] = { + "state": "unproven", + "reason": (operation or {}).get("detail") + or "required role operation has not completed", + "reconciles": spec.get("validates_against")} + continue + expected = int(operation.get("rows") or 0) + if table == "inventory": + measured = int(con.execute( + f"SELECT COUNT(*) FROM {mirror} WHERE inventory_source='AWD'").fetchone()[0]) + else: + measured = int(con.execute( + f"SELECT COUNT(DISTINCT competitive_body_asin) FROM {mirror} " + "WHERE competitive_body_asin IS NOT NULL").fetchone()[0]) + if measured != expected: + out[table] = { + "state": "refused", "rows": n, + "reason": f"role operation returned {expected} but mirror carries {measured}", + "reconciles": spec.get("validates_against")} + continue + out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")} + for t, r in out.items(): + log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}") + return out + + +def status(tenant_key="royal-imports", rt=None): + """What is in the mirror now. Needs no credential and no network.""" + rt = rt if rt is not None else _runtime_for(tenant_key) + try: + con, _path = _connect_tenant_mirror(tenant_key, rt) + except Exception as e: # noqa: BLE001 + return {"error": f"{type(e).__name__}: {e}"} + out = {} + for table in TABLES: + mirror = table_name(table) + try: + n = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] + cols = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall()) + except Exception: # noqa: BLE001 + n, cols = None, 0 + out[table] = {"rows": n, "columns": cols, + "provenance": amazon_schema.PROVENANCE.get(table)} + if table in ENTITY_STATE_KEYS: + out[table]["operation"] = _read_state(con, ENTITY_STATE_KEYS[table]) + return out + + +# --------------------------------------------------------------------------- selftest + +def selftest(log=print): + """Prove the parsers, the typing, the naming rule and validate's refusal. No network, no store. + + ⛔ EVERY CHECK HERE WAS DRIVEN RED BEFORE BEING TRUSTED. Green on a first run proves a check + exists, not that it discriminates [[revert-one-file-to-see-if-a-gate-discriminates]]. + """ + fails = [] + + def ck(name, cond, detail=""): + """⛔ THE DETAIL IS ASCII-ESCAPED, AND THAT IS NOT COSMETIC. + + A perturbation run printed a measured header still carrying its BOM into `detail`, and the + Windows console encoder raised on it: the selftest DIED at check 1 of 15 instead of + reporting 14 more. A gate that crashes reports a SMALLER total, and a shrinking total reads + as a passing run to anything counting reds [[a-shrinking-gate-total-is-a-red-result]]. + This is a connector whose whole job is ingesting other people's text, so hostile bytes in a + failure message are the normal case, not the exotic one. + """ + if cond: + log(f" ok {name}") + return + try: + shown = str(detail).encode("ascii", "backslashreplace").decode("ascii") + except Exception: # noqa: BLE001 + shown = "" + log(f" FAIL {name} {shown}") + fails.append(name) + + log("amazon_store selftest") + + # NC-AZ1 — the header IS the column list, and hostile headers survive it. + tsv = ("amazon-order-id\tpurchase-date\tItem Price\t2nd Ref\tsku\n" + "111-2223334-4445556\t2026-08-01T00:00:00Z\t27.30\tx\tRI-001\n" + "111-2223334-4445557\t2026-08-02T00:00:00Z\t14.00\ty\tRI-002\n") + cols, rows = parse_flat_file(tsv) + ck("NC-AZ1 flat-file columns are measured from the header", + cols == ["amazon_order_id", "purchase_date", "item_price", "c_2nd_ref", "sku"], cols) + # ⚠ ASSERTED AGAINST THE SANITIZER, NOT AGAINST A DEAD `.replace`. The first version of this + # check asked "is there a BOM left in the output", which is true no matter what the code does + # once every non-alphanumeric becomes `_`. It survived its own perturbation. It now asserts + # that a BOM'd header and a clean one produce the SAME column, which goes red the moment the + # sanitizer stops normalising. + ck("NC-AZ2 a BOM'd header yields the same column as a clean one", + _snake("amazon-order-id") == "amazon_order_id" == _snake("amazon-order-id"), + _snake("amazon-order-id")) + ck("NC-AZ3 rows parse to the measured width", len(rows) == 2 + and rows[0]["amazon_order_id"] == "111-2223334-4445556", rows[:1]) + + # NC-AZ4 — QUOTE_NONE, and the FIXTURE had to be fixed before the check could fail. + # ⛔ A first version used `12" Vase`, where the quote sits mid-field. `csv` only treats `"` + # specially at the START of a field, so QUOTE_MINIMAL parsed it identically and the check + # survived swapping the dialect. The hazard is a field that OPENS with a quote: minimal + # quoting then treats the rest of the document as one quoted string and silently returns a + # single merged row. That is the real Amazon case, a title like `"Special" Vase`. + q = 'sku\ttitle\nRI-9\t"12 inch Vase\nRI-10\tPlain\n' + qc, qr = parse_flat_file(q) + ck("NC-AZ4 a field opening with a quote does not swallow the rows after it", + len(qr) == 2 and qr[0]["title"] == '"12 inch Vase' and qr[1]["sku"] == "RI-10", qr) + + # NC-AZ5 — a duplicate header cell must not collide. + dc, _ = parse_flat_file("sku\tsku\tqty\nA\tB\t1\n") + ck("NC-AZ5 duplicate headers are disambiguated", dc == ["sku", "sku_2", "qty"], dc) + + # NC-AZ6..7 — typing sniffs the DATA, and ids stay text however numeric they look. + numeric = [{"qty": "3", "price": "27.30", "settlement_id": "9007199254740993"}] + ck("NC-AZ6 a numeric text column is typed from its values", + _coltype("qty", numeric) == "BIGINT" and _coltype("price", numeric) == "DOUBLE", + (_coltype("qty", numeric), _coltype("price", numeric))) + ck("NC-AZ7 an id stays VARCHAR past 2^53", + _coltype("settlement_id", numeric) == "VARCHAR" + and _cell("9007199254740993", "VARCHAR") == "9007199254740993") + + # NC-AZ8 — text wins ties, so one 'N/A' cannot break an insert after five tables are written. + mixed = [{"q": "3"}, {"q": "3"}, {"q": "N/A"}] + ck("NC-AZ8 a mixed column falls back to VARCHAR", _coltype("q", mixed) == "VARCHAR") + + # NC-AZ9 — the JSON path flattens onto the PINNED names, not the payload's. + doc = json.dumps({"salesAndTrafficByAsin": [ + {"parentAsin": "B01", "salesByAsin": {"unitsOrdered": 3, + "orderedProductSales": {"amount": 27.3}}, + "trafficByAsin": {"buyBoxPercentage": 91.2}}]}) + jc, jr = parse_json_report(doc, "salesAndTrafficByAsin") + ck("NC-AZ9 nested JSON flattens to the pinned column spelling", + jr and jr[0].get("sales_by_asin_ordered_product_sales_amount") == 27.3 + and jr[0].get("traffic_by_asin_buy_box_percentage") == 91.2, jr[:1]) + + # NC-AZ10 — ⛔ THE COLLISION GUARD, ASSERTED AGAINST THE LITERAL AND NOT AGAINST `PREFIX`. + # A first version wrote `entity_key(t).startswith(PREFIX)`, which is vacuously TRUE when + # `PREFIX` is the empty string: setting `PREFIX = ""` — the exact regression this guards — + # left it green. A check whose ruler is the constant under test cannot fail + # [[a-source-check-matches-its-own-assertion]]. The literal is spelled out here on purpose. + bad = [t for t in TABLES if not entity_key(t).startswith("amzn_") + or not table_name(t).startswith("amzn_")] + ck("NC-AZ10 every entity key and mirror table is amzn_-prefixed", not bad, bad) + + # NC-AZ11 — and the concrete collision: `orders` must never be the bare key Odoo could own. + ck("NC-AZ11 the orders key is not the bare name", + entity_key("orders") == "amzn_orders" and entity_key("orders") != "orders") + + # NC-AZ12 — a backfill is sliced to Amazon's ceiling, and the ceiling is not silently exceeded. + w = _windows(datetime(2026, 1, 1, tzinfo=timezone.utc), + datetime(2026, 4, 1, tzinfo=timezone.utc)) + ck("NC-AZ12 a backfill slices to Amazon's window ceiling", + len(w) == 3 and all((b - a).days <= MAX_WINDOW_DAYS for a, b in w), len(w)) + + # NC-AZ13 — orders dedupes to the order head; order_items keeps every line. + lines = [{"amazon_order_id": "1", "sku": "A"}, {"amazon_order_id": "1", "sku": "B"}, + {"amazon_order_id": "2", "sku": "C"}] + ck("NC-AZ13 orders dedupes by order id and items do not", + len(_dedupe_orders(lines)) == 2 and len(lines) == 3) + + # NC-AZ14 — the seven tables of R4 agree with the provenance map. Two lists, one truth. + ck("NC-AZ14 TABLES agrees with the schema provenance map", + set(TABLES) == set(amazon_schema.PROVENANCE), set(TABLES) ^ set(amazon_schema.PROVENANCE)) + + # NC-AZ15 — an empty parse is empty, never a phantom row. + ck("NC-AZ15 an empty document yields no columns and no rows", + parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], [])) + + # NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label. + price_input = [ + {"child_asin": "B01", "parent_asin": "P01"}, + {"child_asin": "B01", "parent_asin": "P01"}, + {"parent_asin": "P02"}, + ] + ck("NC-AZ16 pricing ASINs are complete and de-duplicated", + _pricing_asins(price_input) == ["B01", "P02"], _pricing_asins(price_input)) + competitive = [{ + "status": {"statusCode": 200, "reasonPhrase": "Success"}, + "body": {"asin": "B01", "marketplaceId": "ATVPDKIKX0DER", + "featuredBuyingOptions": [{"listingPrice": {"amount": 12.5}}], + "referencePrices": [{"name": "CompetitivePrice", "price": {"amount": 11.0}}]}, + }] + cr = _competitive_rows(competitive) + ck("NC-AZ17 competitive summary keeps status and every returned data family", + bool(cr) and cr[0].get("competitive_status_status_code") == 200 + and "competitive_body_featured_buying_options" in cr[0] + and "competitive_body_reference_prices" in cr[0], cr) + awd = _awd_rows([{"sku": "RI-1", "totalOnhandQuantity": 9, + "inventoryDetails": {"replenishmentQuantity": 2}}]) + ck("NC-AZ18 AWD inventory remains source-labelled and nested fields survive", + bool(awd) and awd[0].get("inventory_source") == "AWD" + and awd[0].get("inventory_details_replenishment_quantity") == 2, awd) + + import duckdb + mem = duckdb.connect(":memory:") + mem.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)") + mem.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'),('B01','P01')") + response_count, updated = _apply_competitive(mem, "amzn_pricing", competitive) + enriched = mem.execute( + "SELECT COUNT(*) FROM amzn_pricing WHERE competitive_body_asin='B01'").fetchone()[0] + ck("NC-AZ19 competitive summary enriches every matching report row without appending", + response_count == 1 and updated == 2 and enriched == 2, + (response_count, updated, enriched)) + + mem.execute("""CREATE TABLE _sync_state ( + entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, + cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") + _record_state(mem, "amzn_inventory_awd", "done", 7) + marker = mem.execute( + "SELECT phase, last_id, cursor_wd, rows, updated_at FROM _sync_state").fetchone() + ck("NC-AZ20 sync state writes rows and timestamp into their declared columns", + marker[0] == "done" and marker[1] is None and marker[2] is None + and marker[3] == 7 and bool(marker[4]), marker) + + # NC-AZ21..24 - the destructive path is hermetic, prefix-complete, and collateral-free. + mem.execute("CREATE TABLE amzn_orders (id BIGINT)") + mem.execute("INSERT INTO amzn_orders VALUES (1), (2)") + mem.execute("CREATE TABLE amzn_future_table (id BIGINT)") + mem.execute("INSERT INTO amzn_future_table VALUES (3)") + mem.execute("CREATE TABLE keep_unrelated (id BIGINT)") + mem.execute("INSERT INTO keep_unrelated VALUES (99)") + _record_state(mem, "amzn_orders", "done", 2) + _record_state(mem, "odoo_orders", "live", 11) + purged = _purge_mirror(mem) + ck("NC-AZ21 purge removes every amzn_ table, including a future one", + purged["tables"] == ["amzn_future_table", "amzn_orders", "amzn_pricing"] + and purged["rows"] == 5, purged) + ck("NC-AZ22 purge removes every Amazon sync marker", + purged["syncMarkers"] == 2 + and not [r for r in _sync_state_rows(mem) if str(r[0]).startswith("amzn_")], + _sync_state_rows(mem)) + kept = mem.execute("SELECT id FROM keep_unrelated").fetchall() + kept_state = mem.execute( + "SELECT phase, rows FROM _sync_state WHERE entity='odoo_orders'").fetchone() + ck("NC-AZ23 purge preserves unrelated tables and sync markers byte for byte", + kept == [(99,)] and kept_state == ("live", 11), (kept, kept_state)) + second = _purge_mirror(mem) + ck("NC-AZ24 purge readback is residue-free and a retry is idempotent", + second == {"tables": [], "rows": 0, "syncMarkers": 0, + "residueTables": [], "residueSyncMarkers": []}, second) + + # NC-AZ25..26 - neither a typo nor a still-live credential can reach the DROP transaction. + refused_confirmation = False + try: + _require_purge_confirmation("royal-imports", "royal_imports") + except RuntimeError: + refused_confirmation = True + ck("NC-AZ25 purge refuses when the exact tenant confirmation mismatches", + refused_confirmation) + + class _FakeRuntime: + def __init__(self, entries): + self.entries = entries + + def get(self, name): + from core import keychain as _kc + return {"entries": self.entries} if name == _kc.KEY else None + + absent_ok, present_refused = False, False + try: + absent_ok = _assert_amazon_credential_revoked(_FakeRuntime({})) + _assert_amazon_credential_revoked(_FakeRuntime({ + "k_probe": {"type": "amazon_sp", "label": "Amazon", + "enc": "not-decrypted-by-this-check"}, + })) + except RuntimeError: + present_refused = True + ck("NC-AZ26 purge refuses while any Amazon credential metadata remains", + absent_ok and present_refused, (absent_ok, present_refused)) + mem.close() + + # NC-AZ27 - drive the public operator wrapper against an isolated file, including its readback. + import tempfile + prior_path = Path(datastore.DB_PATH) + public_ok, public_detail = False, None + try: + with tempfile.TemporaryDirectory() as tmp: + purge_path = Path(tmp) / "qa-purge.duckdb" + seed = duckdb.connect(str(purge_path)) + seed.execute("""CREATE TABLE _sync_state ( + entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, + cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") + seed.execute("CREATE TABLE amzn_orders (id BIGINT)") + seed.execute("INSERT INTO amzn_orders VALUES (1)") + _record_state(seed, "amzn_orders", "done", 1) + seed.close() + + class _PurgeRuntime(_FakeRuntime): + key = "qa-purge" + datastore_path = purge_path + + def assert_datastore_matches(self): + if Path(datastore.DB_PATH).resolve() != purge_path.resolve(): + raise RuntimeError("wrong test mirror") + + public_detail = purge( + "qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None) + datastore.use_path(prior_path) + readback = duckdb.connect(str(purge_path), read_only=True) + tables_after = sorted(str(row[0]) for row in readback.execute("SHOW TABLES").fetchall()) + state_after = readback.execute("SELECT entity FROM _sync_state").fetchall() + readback.close() + public_ok = (public_detail["tenant"] == "qa-purge" + and public_detail["database"] == "qa-purge.duckdb" + and public_detail["rows"] == 1 + and tables_after == ["_sync_state"] and state_after == []) + finally: + datastore.use_path(prior_path) + ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file", + public_ok, public_detail) + + log(f"\n{'GREEN' if not fails else 'RED'}: {27 - len(fails)}/27") + return 0 if not fails else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--sync", action="store_true") + ap.add_argument("--status", action="store_true") + ap.add_argument("--validate", action="store_true") + ap.add_argument("--purge", action="store_true", + help="permanently delete every Amazon mirror table after credential revocation") + ap.add_argument("--selftest", action="store_true") + ap.add_argument("--tenant", default=None) + ap.add_argument("--confirm-tenant", default=None, + help="for --purge, repeat the exact tenant slug") + ap.add_argument("--table", action="append", help="limit to one table (repeatable)") + a = ap.parse_args(argv) + tenant = a.tenant or "royal-imports" + if a.purge: + if not a.tenant: + ap.error("--purge requires an explicit --tenant") + if a.table or a.sync or a.status or a.validate or a.selftest: + ap.error("--purge is all-or-nothing and cannot be combined with another action") + rep = purge(a.tenant, a.confirm_tenant) + print(json.dumps(rep, indent=2)) + return 0 + if a.selftest: + return selftest() + if a.status: + print(json.dumps(status(tenant), indent=2)) + return 0 + if a.validate: + validate(tenant) + return 0 + if a.sync: + t0 = time.time() + rep = sync(tenant, tables=a.table) + print(json.dumps(rep, indent=2)) + print(f"\n{time.time() - t0:.1f}s") + return 0 + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platform/harness/analyst.py b/platform/harness/analyst.py index bed05f0ce66dc39db98a56bbca207109492597b2..619e6e2df6958bda7c1f04d39f2727d1cf4b198b 100644 --- a/platform/harness/analyst.py +++ b/platform/harness/analyst.py @@ -71,15 +71,15 @@ def system_prompt(): for t in topics) return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by calling TOOLS over a governed semantic layer. THE RULES (non-negotiable): -1. You NEVER write SQL or invent field names — only registry keys from the tools' schemas. +1. You NEVER write SQL or invent field names; only registry keys from the tools' schemas. 2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products) - with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 — never mix them in + with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6; never mix them in one series unless explicitly comparing. 3. Every number you present must come from a tool result and keep its result_id (drillable). - If a tool errors, read the error and correct your call — do not make numbers up, ever. + If a tool errors, read the error and correct your call; do not make numbers up, ever. 4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the user's explicit confirmation first. -5. Answer plainly and lead with the finding. State the window and scope you used — READ the +5. Answer plainly and lead with the finding. State the window and scope you used; READ the window from the tool result's 'window' field, never guess it: a query without dates covers ALL recorded history (there is no hidden default window). Time-bound asks ("this year", "past 4 weeks") must pass explicit date_from/date_to. @@ -89,25 +89,25 @@ calling TOOLS over a governed semantic layer. THE RULES (non-negotiable): is consolidated, query WITHOUT group_by (or sum before answering). 7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a total across the rows you fetched, either run the scalar query for it or ADD UP those exact - rows — a round "approximately" figure is a fabrication and forbidden. When in doubt, present + rows; a round "approximately" figure is a fabrication and forbidden. When in doubt, present the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last - year per customer) is equally forbidden — that is transform_result's job ({{"op":"yoy"}} adds + year per customer) is equally forbidden; that is transform_result's job ({{"op":"yoy"}} adds _ly, _delta, _yoy_pct per row); if no transform can compute it, say you cannot. -8. THE GAP PROTOCOL. If — and only if — after checking the schema (list_topics/describe_topic) +8. THE GAP PROTOCOL. If, and only if, after checking the schema (list_topics/describe_topic) no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer in one honest sentence what is missing and offer the nearest ask that IS answerable. Before concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy), rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via - transform_result — a derived-analytics question is never a gap. Both directions are + transform_result; a derived-analytics question is never a gap. Both directions are failures: refusing without report_gap, and calling report_gap for something the tools support. -9. NEVER emit markdown images, data: URIs, or links to charts — you cannot draw. Charts exist +9. NEVER emit markdown images, data: URIs, or links to charts; you cannot draw. Charts exist ONLY through make_chart / make_table; the platform renders them below your answer. Refer to a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving - averages, ranks, bins) exist ONLY through transform_result — never arithmetic in your head. + averages, ranks, bins) exist ONLY through transform_result; never arithmetic in your head. -CHART PICKER (the form follows the comparison — pick by what the question compares): +CHART PICKER (the form follows the comparison; pick by what the question compares): - trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap - this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars - ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled) @@ -147,7 +147,7 @@ def _live_chat(messages, tools, model, tool_choice="auto"): import requests provs = available_providers() if not provs: - raise RuntimeError("no LLM provider configured — set GROQ_API_KEY / CEREBRAS_API_KEY / " + raise RuntimeError("no LLM provider configured; set GROQ_API_KEY / CEREBRAS_API_KEY / " "OPENROUTER_API_KEY in platform/.env") last_err = None # Two patience profiles: CHAT (default) fails over fast — a user is waiting; BATCH @@ -310,7 +310,7 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX # A model occasionally returns NO content and NO tool calls (observed live # 2026-07-16: the reply bubble was simply empty and the next turn hallucinated). # Never surface silence — report the failure honestly instead. - answer = ("(the model returned an empty answer — nothing was computed; " + answer = ("(the model returned an empty answer; nothing was computed; " "please rephrase or ask again)") import harness.telemetry as _tel # Wave 19 (R4): ATTRIBUTED. The token count was previously logged against nobody, so @@ -341,7 +341,7 @@ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX messages.append({"role": "tool", "tool_call_id": tc.get("id", name), "content": json.dumps(res, default=str)[:8000]}) # budget exhausted — report honestly (never as success) - return {"answer": "(stopped: tool-call budget exhausted before a final answer — " + return {"answer": "(stopped: tool-call budget exhausted before a final answer; " "narrow the question or raise max_iters)", "artifacts": artifacts, "tool_trace": trace, "iterations": max_iters, "exhausted": True, "usage": usage_total} diff --git a/platform/harness/connectors/quickbooks.py b/platform/harness/connectors/quickbooks.py index c6f771fd27321b4e8196395d3c7015dbf0d45617..0ddd5a23785c3ce5a91d98503ab822207874cf3f 100644 --- a/platform/harness/connectors/quickbooks.py +++ b/platform/harness/connectors/quickbooks.py @@ -31,7 +31,7 @@ class QuickBooksConnector: def aggregate(self, q: Query) -> list[dict]: raise NotImplementedError( - "QuickBooksConnector.aggregate: TODO — translate the canonical Query to the QBO " + "QuickBooksConnector.aggregate: TODO: translate the canonical Query to the QBO " "Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. " "Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, " "GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.") diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 2297e0ea815393303530097e1245556ef37fac2d..3f40da2c341be8040cd179a14aa40abd949ed230 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -3,7 +3,7 @@ Incremental Odoo → DuckDB mirror: the local analytical store that saved views/dashboards (OM-3) and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC per question. -Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1. +Plan: AutoWiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1. Design rules: - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion, @@ -195,7 +195,7 @@ ENTITIES = { # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors' # etc."* He was right, and the census - # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how + # (`AutoWiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76 # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here # was measured non-empty on the live population first — none is added on the strength of @@ -221,6 +221,9 @@ ENTITIES = { "street", "street2", "zip", # added 2026-08-12 (W30-T33) "email", "phone", "mobile", # added 2026-08-14 (W33-T48) "website", "vat", "ref"], + # Rank drift behind the write_date cursor requires recurring observation; + # schema backfill alone cannot maintain the customer population predicate. + "refresh_fields": ["customer_rank"], # m2m fields land as _id (the FIRST linked id — the Customers-module convention: # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be @@ -509,7 +512,7 @@ def ro_con(): _RO_TLS.cur = None # stale → reopen below if not ready(): raise RuntimeError("the tenant data store is completing its FIRST sync on this " - "deployment — dashboards work meanwhile (live reads); retry " + "deployment; dashboards work meanwhile (live reads); retry " "store queries in a few minutes") cur = ro_cursor() _RO_TLS.cur = cur @@ -598,7 +601,7 @@ def window(table=None, select=None, where=None, params=(), order_by=None, raise ValueError(f"datastore.window: {table!r} is not a table identifier") proj = str(select or "").strip() if not proj: - raise ValueError("datastore.window: a projection is required — there is no implicit *") + raise ValueError("datastore.window: a projection is required; there is no implicit *") try: offset = max(0, int(offset or 0)) except (TypeError, ValueError): @@ -952,7 +955,7 @@ def _field_cols(f): COL_BACKFILL_MAX_BATCHES = 20 -def _backfill_columns(con, key, spec, log=print, max_batches=None): +def _backfill_columns(con, key, spec, log=print, max_batches=None, *, repeat=False): """Scalar/m2o schema evolution — the twin of _backfill_m2m, and for the same reason. `max_batches=None` keeps the historical unbounded walk for a deliberate one-shot migration; @@ -969,15 +972,25 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None): already happened and the list comes back empty while the data is still missing. Declaring it makes the migration re-entrant and reviewable. Leave the entry in place after it completes; the durable marker, not the absence of the declaration, is what stops it re-running. + + `repeat=True` observes declared `refresh_fields` independently of write_date. + Its separate marker resumes incomplete cycles and restarts completed ones; + it never resets a schema migration or advances the entity's delta cursor. + Returns the number of existing rows changed by a recurring refresh, so the + resident rebuild policy cannot mistake a rank-only change for a no-op. """ - fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"] + fields = [f for f in (spec.get("refresh_fields" if repeat else "backfill_fields") or []) + if f != "write_date"] if not fields: - return - marker = f"{key}.cols.{','.join(sorted(fields))}" + return 0 + marker = f"{key}.{'refresh' if repeat else 'cols'}.{','.join(sorted(fields))}" + operation = "column refresh" if repeat else "column backfill" st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", [marker]).fetchone() if st and st[0] == "done": - return + if not repeat: + return 0 + st = None # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses # an hour of work to one dropped connection gets skipped by the next person under time # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an @@ -985,32 +998,48 @@ def _backfill_columns(con, key, spec, log=print, max_batches=None): # completion it never reached. last, n = (st[1] or 0, st[2] or 0) if st else (0, 0) if last: - log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)") + log(f" {key}: {operation} resuming at id {last:,} ({n:,} rows already done)") cols = [c for f in fields for c in _field_cols(f)] setter = ", ".join(f"{c}=?" for c in cols) - pages = 0 + pages, changed = 0, 0 while True: recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], fields, limit=5000, order="id asc") if not recs: break rows = [_flatten(r, fields) for r in recs] - con.executemany(f"UPDATE {key} SET {setter} WHERE id=?", - [[r.get(c) for c in cols] + [r["id"]] for r in rows]) + if repeat: + # UPDATE ... RETURNING counts actual changed mirror rows. Fetched, + # identical and absent rows must not trigger a product-store rebuild. + values = ','.join('(' + ','.join('?' for _ in range(len(cols) + 1)) + ')' + for _ in rows) + assignments = ', '.join(f'{c}=src.{c}' for c in cols) + differs = ' OR '.join(f'dest.{c} IS DISTINCT FROM src.{c}' for c in cols) + updated = con.execute( + f'UPDATE {key} AS dest SET {assignments} ' + f'FROM (VALUES {values}) AS src(id, {", ".join(cols)}) ' + f'WHERE dest.id=src.id AND ({differs}) RETURNING dest.id', + [value for row in rows for value in [row['id']] + [row.get(c) for c in cols]] + ).fetchall() + changed += len(updated) + else: + con.executemany(f"UPDATE {key} SET {setter} WHERE id=?", + [[r.get(c) for c in cols] + [r["id"]] for r in rows]) last, n = recs[-1]["id"], n + len(recs) pages += 1 con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)") + log(f" {key}: {operation} {'+'.join(fields)} -> id {last:,} ({n:,} rows)") # ⛔ RETURN, NOT `break`: falling out of the loop writes 'done', and declaring completion # a bounded pass never reached is the one failure this whole helper's checkpointing # exists to prevent. The marker stays 'running' and the next pass resumes at `last`. if max_batches is not None and pages >= max_batches: - log(f" {key}: column backfill PAUSED at {n:,} rows this run (bounded; resumes)") - return + log(f" {key}: {operation} PAUSED at {n:,} rows this run (bounded; resumes)") + return changed con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: column backfill complete -> {n:,} rows") + log(f" {key}: {operation} complete -> {n:,} rows") + return changed def _backfill_m2m(con, key, spec, log=print): @@ -1075,7 +1104,8 @@ def _base_domain(spec): def sync_entity(key, max_batches=200, batch=BATCH, log=print): """One bounded, resumable sync pass for an entity. Backfill (by id) → live (by write_date). - Returns (phase, pulled_rows). + Returns (phase, pulled_rows). The count includes existing rows changed by + recurring scalar refreshes as well as rows pulled by the delta/backfill. ⭐ D-10: refuses BEFORE opening a cursor or touching Odoo while the connector is paused. The guard is here — the innermost function that reads from `O` — rather than only in `sync_all`, @@ -1095,8 +1125,11 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print): _backfill_m2m(con, key, spec, log=log) st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} fields = spec["fields"] + list(spec.get("m2m") or []) - pulled = 0 + pulled, refreshed = 0, 0 try: + if st["phase"] == "live": + refreshed = _backfill_columns(con, key, spec, log=log, + max_batches=COL_BACKFILL_MAX_BATCHES, repeat=True) if st["phase"] == "backfill": last_id = st["last_id"] or 0 for i in range(max_batches): @@ -1184,7 +1217,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print): if pulled: log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}" + (f" walking id>{tie_id}" if walking else "") + ")") - return "live", pulled + return "live", pulled + refreshed finally: con.close() diff --git a/platform/harness/evals.py b/platform/harness/evals.py index d64172f90f9c2c9dc104254e328290b92d778b2f..ca40b00e677e54c847743eccb05adfaf30fdf521 100644 --- a/platform/harness/evals.py +++ b/platform/harness/evals.py @@ -120,9 +120,9 @@ def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True): if aborted: ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")] if verbose: - print(f"\nEVAL ABORTED — LLM providers exhausted after {len(results)} item(s); " + print(f"\nEVAL ABORTED; LLM providers exhausted after {len(results)} item(s); " f"{passed}/{len(ran) or 1} of the items that actually ran passed. " - "Gate is INCONCLUSIVE (not a model failure) — re-run when quota recovers.") + "Gate is INCONCLUSIVE (not a model failure); re-run when quota recovers.") return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False, "inconclusive": True, "results": results} rate = passed / len(items) if items else 0.0 diff --git a/platform/harness/filter_sql.py b/platform/harness/filter_sql.py index 09ba6b30a586aee99465b7b150e30af461d46118..0eb7b72310953dd8693499788991ea0fd57bd4d4 100644 --- a/platform/harness/filter_sql.py +++ b/platform/harness/filter_sql.py @@ -359,7 +359,7 @@ def _window_bounds(spec, today, probe=False): return w.normalize(spec) is not None if today is None: raise ValueError( - 'a relative-date condition needs `today` — compile_filter_tree(today=...). ' + 'a relative-date condition needs `today`; compile_filter_tree(today=...). ' 'Refusing rather than dropping the condition, which would widen the result while ' 'the count beside it still looked authoritative.') return w.resolve(spec, today) @@ -521,7 +521,7 @@ def _resolve_anchor(mode, value, today): if today is None: if mode != 'exact': raise ValueError( - f"the date anchor {mode!r} is relative and needs `today` — " + f"the date anchor {mode!r} is relative and needs `today`; " f"compile_filter_tree(today=...). Refusing rather than dropping the condition, " f"which would widen the result under an authoritative-looking count.") # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and @@ -598,7 +598,7 @@ def _compile_cohort(rule, state): sets = state.get('cohort_sets') if sets is None: raise ValueError( - 'a cohort condition needs cohort_sets — compile_filter_tree(cohort_sets=...). ' + 'a cohort condition needs cohort_sets; compile_filter_tree(cohort_sets=...). ' 'Refusing rather than dropping it: a dropped condition widens the result while the ' 'count beside it still looks authoritative.') id_sql = state.get('id_sql') diff --git a/platform/harness/measure_filter.py b/platform/harness/measure_filter.py index 7377dbb32670a4725d68e034da11ce63dbea8dd8..06389b14bb79590da1d076eccccbd7b4062ad1d9 100644 --- a/platform/harness/measure_filter.py +++ b/platform/harness/measure_filter.py @@ -46,6 +46,7 @@ count that looks authoritative. `ModelError` is the only safe failure. """ from harness import semantic as _sem from harness import windows as _win +from core import relations as _relations #: metric key -> how it reads in the condition builder. ONLY metrics with an independent #: validation contract (semantic.store_parity / a metric `validate:` block) appear here. @@ -299,7 +300,29 @@ def _base_topic(key): return topic -def _query_parts(key, window, today, team_id): +def _related_scope(related, keys): + """Prepare all bases before querying: a refused composite must be wholly unscoped.""" + if related is None or isinstance(related, _relations.MeasurePin): + return related + try: + pin = _relations.measure_pin(related) + bases, _ = _sem._expand_measures(list(keys)) + for key in bases: + _, _, store = _entity_sql(_base_topic(key)) + pin.predicate(store) + return pin + except _relations.PinRefusal as exc: + return _relations.MeasurePin(exc.marker['path'], {}, {}, exc.marker) + + +def _related_answer(values, related): + if related is None: + return values + cls = _relations.MeasureMatches if isinstance(values, set) else _relations.MeasureAnswer + return cls(values, [related.refusal] if related.refusal else ()) + + +def _query_parts(key, window, today, team_id, related=None): """The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the one-sided and the two-sided resolver so there is one definition of the scope.""" topic = _base_topic(key) @@ -325,6 +348,12 @@ def _query_parts(key, window, today, team_id): agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)" where, params = [_sem.render_scope(s['scope_sql'].strip())], [] + related = _related_scope(related, [key]) + if related is not None: + pin_sql, pin_params = related.predicate(s) + if pin_sql is not None: + where.append(f'({pin_sql})') + params.extend(pin_params) if date_from: where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") params.append(f"{date_from} 00:00:00") @@ -352,7 +381,7 @@ def _scaled(key, value): return (value / scale) if scale else value -def resolve_stat(rule, today, team_id=None, allowed_pids=None): +def resolve_stat(rule, today, team_id=None, allowed_pids=None, related=None): """`Where [Sales] [this year] [>] [the 75th percentile]` — the top quartile of the pool. One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS @@ -363,6 +392,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}") key = rule.get('colId') topic_for(key) + related = _related_scope(related, [key]) op = rule.get('op') if op not in PAIR_OPS: raise _sem.ModelError( @@ -376,7 +406,7 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): "which would be a different question with an authoritative-looking answer") con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id) + totals = _totals(con, key, rule.get('window'), today, team_id, related) finally: con.close() # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL — owner's decision, @@ -398,8 +428,8 @@ def resolve_stat(rule, today, team_id=None, allowed_pids=None): population = {pid: v for pid, v in totals.items() if pid in pool and v} cut = percentile(list(population.values()), stat) if cut is None: # nobody has this measure -> nobody qualifies - return set() - return {pid for pid, v in population.items() if _CMP[op](v, cut)} + return _related_answer(set(), related) + return _related_answer({pid for pid, v in population.items() if _CMP[op](v, cut)}, related) def _pool_of(allowed_pids): @@ -445,7 +475,7 @@ def rule_complete(rule): return v is not None and str(v).strip() != '' -def resolve_rule(rule, today, team_id=None, allowed_pids=None): +def resolve_rule(rule, today, team_id=None, allowed_pids=None, related=None): """One measure condition -> the SET OF CUSTOMER IDS satisfying it. `rule` = {colId: , window: , op: , value: }, @@ -467,11 +497,12 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): 'refusing to answer a question that has not been fully asked') rhs = rule.get('rhs') if isinstance(rhs, dict) and rhs.get('kind') == 'measure': - return resolve_pair(rule, today, team_id, allowed_pids) + return resolve_pair(rule, today, team_id, allowed_pids, related) if isinstance(rhs, dict) and rhs.get('kind') == 'stat': - return resolve_stat(rule, today, team_id, allowed_pids) + return resolve_stat(rule, today, team_id, allowed_pids, related) key = rule.get('colId') topic_for(key) # raises for anything not admitted + related = _related_scope(related, [key]) op = rule.get('op') if op not in OPS: raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})") @@ -492,17 +523,18 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): # compare in Python — the same shape `resolve_pair` uses, and for the same reason. con = _sem._store_con() try: - totals = _totals(con, key, rule.get('window'), today, team_id) + totals = _totals(con, key, rule.get('window'), today, team_id, related) finally: con.close() # No pool means we can only speak about customers a base has seen. WITH a pool, everyone # else scores 0 through the ratio's guard — which is what makes the zero-group work for a # ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing). universe = pool if pool is not None else set(totals) - return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)} + return _related_answer( + {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}, related) frm, id_col, agg, where_sql, params = _query_parts( - key, rule.get('window'), today, team_id) + key, rule.get('window'), today, team_id, related) sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} " f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?") @@ -524,10 +556,10 @@ def resolve_rule(rule, today, team_id=None, allowed_pids=None): con.close() if pool is not None: matched &= pool - return matched + return _related_answer(matched, related) -def _totals(con, key, window, today, team_id): +def _totals(con, key, window, today, team_id, related=None): """`{pid: value}` for one (measure, window), COMPOSITES included. For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived @@ -541,13 +573,13 @@ def _totals(con, key, window, today, team_id): """ if is_composite(key): bases, _ = _sem._expand_measures([key]) - parts = {b: _totals(con, b, window, today, team_id) for b in bases} + parts = {b: _totals(con, b, window, today, team_id, related) for b in bases} pids = set() for got in parts.values(): pids |= set(got) return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key) for pid in pids} - frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id) + frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id, related) sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}") # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in # COALESCE(..., 0), so a NULL cannot arrive here — a second guard would be unreachable, and @@ -558,7 +590,7 @@ def _totals(con, key, window, today, team_id): for r in con.execute(sql, params).fetchall() if r[0] is not None} -def resolve_pair(rule, today, team_id=None, allowed_pids=None): +def resolve_pair(rule, today, team_id=None, allowed_pids=None, related=None): """CG-9 — `Where [Sales] [this year] [>] [Sales] [year to date, last year]`. TWO QUERIES, compared in Python. The obvious alternative — one query with two conditional @@ -576,6 +608,7 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None): rhs = rule.get('rhs') or {} lkey, rkey = rule.get('colId'), rhs.get('colId') topic_for(lkey), topic_for(rkey) # raises for anything not admitted + related = _related_scope(related, [lkey, rkey]) op = rule.get('op') if op not in PAIR_OPS: raise _sem.ModelError( @@ -587,14 +620,15 @@ def resolve_pair(rule, today, team_id=None, allowed_pids=None): pool = _pool_of(allowed_pids) con = _sem._store_con() try: - left = _totals(con, lkey, rule.get('window'), today, team_id) - right = _totals(con, rkey, rhs.get('window'), today, team_id) + left = _totals(con, lkey, rule.get('window'), today, team_id, related) + right = _totals(con, rkey, rhs.get('window'), today, team_id, related) finally: con.close() # With no pool we can only speak about customers at least one side has seen — the zero-vs- # zero group is unknowable without being told who exists. With a pool, it is every member. universe = pool if pool is not None else (set(left) | set(right)) - return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))} + return _related_answer( + {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}, related) def _zero_satisfies(op, value): @@ -602,7 +636,7 @@ def _zero_satisfies(op, value): return _CMP[op](0.0, value) -def resolve_values(key, window, today, team_id=None, allowed_pids=None): +def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=None): """`{pid: display value}` for one (measure, window) over the caller's pool — the COLUMN version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD). @@ -621,6 +655,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): Raises for anything not admitted / an unresolvable window — the caller degrades that column to blank rather than to a number nobody computed. + + Related pins use the request's relations.measure_pin_context. A refused pin returns + unscoped numbers in a MeasureAnswer with named `refusals`; hosts must publish that channel. + A pin never replaces `allowed_pids`, including when its member set is empty. """ topic_for(key) # raises for anything not admitted # The window must RESOLVE before any query runs — same refusal as `_query_parts`, checked @@ -630,10 +668,11 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): raise _sem.ModelError( f"unresolvable window {window!r} for the {key!r} column: refusing rather than " f"defaulting to all time") + related = _related_scope(related, [key]) pool = _pool_of(allowed_pids) con = _sem._store_con() try: - totals = _totals(con, key, window, today, team_id) + totals = _totals(con, key, window, today, team_id, related) finally: con.close() scale = ADMITTED.get(key, {}).get('input_scale') or 1.0 @@ -641,7 +680,9 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None): nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal universe = pool if pool is not None else set(totals) # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts. - return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe} + # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction. + return _related_answer( + {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}, related) #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to @@ -766,19 +807,23 @@ def resolve_series_slid(key, window_spec, buckets, today, team_id=None, allowed_ return {'values': values, 'agg': agg_kind} -def resolve_tree(rules, today, team_id=None, allowed_pids=None): +def resolve_tree(rules, today, team_id=None, allowed_pids=None, related=None): """{rule_id: sorted pids} for every measure condition in a tree. Keyed by the rule's client-generated `id` so the browser can map an answer back to the condition that asked it — positional matching would silently re-associate every answer the moment a user deleted a condition. """ - out = {} + out = _relations.MeasureAnswer() if related is not None else {} for r in rules or []: rid = str(r.get('id') or '') if not rid: raise _sem.ModelError('a measure condition needs a stable id to carry its answer back') - out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids)) + matched = resolve_rule(r, today, team_id, allowed_pids, related) + out[rid] = sorted(matched) + if related is not None: + out.refusals.extend(dict(marker, rule=rid) + for marker in getattr(matched, 'refusals', ())) return out diff --git a/platform/harness/routines.py b/platform/harness/routines.py index f7b0ac1b635dd2036e91077d02ea53059e9c19ce..1742f09c1722a1cb127b42f44693bcfac46b790c 100644 --- a/platform/harness/routines.py +++ b/platform/harness/routines.py @@ -48,11 +48,11 @@ def _save(d): def schedule_report(view, cadence, channel="in-app"): import harness.views as V if view not in V.views() and view not in V.dashboards(): - raise SEM.ModelError(f"unknown view/dashboard {view!r} — save_view/compose_dashboard first") + raise SEM.ModelError(f"unknown view/dashboard {view!r}; save_view/compose_dashboard first") if cadence not in CADENCES: raise SEM.ModelError(f"cadence must be one of {CADENCES}") if channel not in CHANNELS: - raise SEM.ModelError(f"channel {channel!r} not available yet — v0 delivers in-app " + raise SEM.ModelError(f"channel {channel!r} not available yet; v0 delivers in-app " "(email/Slack ship with the delivery gate); use channel='in-app'") d = load() d["reports"][view] = {"cadence": cadence, "channel": channel, @@ -107,7 +107,7 @@ def schedule_insight(name, prompt, cadence): if cadence not in CADENCES: raise SEM.ModelError(f"cadence must be one of {CADENCES}") if not (prompt or "").strip(): - raise SEM.ModelError("an insight needs a prompt — what should the Analyst look at?") + raise SEM.ModelError("an insight needs a prompt; what should the Analyst look at?") d = load() d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence, "created_at": time.strftime("%Y-%m-%d %H:%M")} @@ -244,14 +244,14 @@ def evaluate_alert(a): fired = (last_val > thr) if direc == "above" else (last_val < thr) return {**out, "last": {"label": last_label, "value": last_val}, "threshold": thr, "direction": direc, - "status": (f"FIRED — {last_label} is {direc} {thr:,.0f}" if fired else "OK"), + "status": (f"FIRED: {last_label} is {direc} {thr:,.0f}" if fired else "OK"), "fired": bool(fired)} lim = xmr(series) sig = lim["points"][-1]["signal"] if lim["points"] else "" return {**out, "last": {"label": last_label, "value": last_val}, "center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"], "points": lim["points"], - "status": (f"SIGNAL — {sig} ({last_label})" if sig else "OK"), + "status": (f"SIGNAL: {sig} ({last_label})" if sig else "OK"), "fired": bool(sig)} diff --git a/platform/harness/runtime.py b/platform/harness/runtime.py index 4af7008b4a086bb53889161f083cde8be636f85c..9e66f9a23c1793fc387e358dca9567931bb482cf 100644 --- a/platform/harness/runtime.py +++ b/platform/harness/runtime.py @@ -307,22 +307,40 @@ class TenantRuntime: return self._store().delete_path(self.store_key(path_in_repo)) # ---------------------------------------------------------------- the Odoo source (R3) - def odoo_source(self): + def odoo_source(self, connection_id=None): """THIS tenant's Odoo connector, or None — the keychain CUTOVER's resolution seam (2026-08-04; `keychain.odoo_creds` was previously called by nothing but its test). Resolution, fail-closed: - 1. a keychain `odoo` entry (any tenant) → a connector bound to THOSE creds; - 2. tenant #0 with an empty/locked keychain → its compiled env connector (R3's - "live with env fallback", which is what keeps Royal working unchanged); + 1. the keychain `odoo` connection NAMED by `connection_id` (W54-T45 / R4), or the + tenant's DESIGNATED DEFAULT connection when no id is named → a connector bound to + THOSE creds; + 2. tenant #0 with an empty/locked keychain and NO id named → its compiled env + connector (R3's "live with env fallback", which keeps Royal working unchanged); 3. anyone else → None. NEVER the environment: env is tenant #0's connection, and handing it to another tenant is the leak this method exists to prevent. + ⛔⛔ W54-T45 — THE ENV FALLBACK NARROWS HERE AND MUST NEVER WIDEN. A caller that NAMES a + connection and gets nothing back is answered None, full stop: not the default connection, + not a sibling, and above all not tenant #0's `.env`. Under R4 an agency holds one Odoo + connection per client, and under R7 that isolation is strict — a named-but-unresolvable + connection quietly falling through to the environment would serve tenant #0's own Odoo to + a question asked about somebody else's books, with nothing red anywhere. Leg (2) is + reachable only on the UNNAMED path, which is exactly the pre-wave behaviour. + Rebuilt per call on purpose — `core.odoo` caches the CLIENT per (slug, creds fingerprint), so this stays cheap while a rotated key reconnects on the next call.""" import core.keychain as keychain + named = connection_id is not None try: - creds = keychain.odoo_creds(self) + # ⚠ THE UNNAMED PATH IS SPELLED AS THE PRE-WAVE CALL, VERBATIM, and the branch is not + # decoration: `verify_api`'s R11 leg pins the literal `keychain.odoo_creds(self)` in + # this method's source as its proof that the resolver is KEYCHAIN-FIRST. Writing only + # the two-argument form would have turned that gate red for a reason that has nothing + # to do with the rule it guards, in a file this lane may not edit. The two arms are + # the same question; the left one is the one every existing caller asks. + creds = (keychain.odoo_creds(self) if not named + else keychain.odoo_creds(self, connection_id)) except Exception: # locked or unreadable keychain — fall through, fail closed creds = None if creds: @@ -332,34 +350,49 @@ class TenantRuntime: return OdooConnector(tenant_slug=self.key, creds=creds, scope={"team_ids": list(bus.values()) or None, "team_names": {v: k for k, v in bus.items()} or None}) + # ⛔ THE NAMED LEG STOPS HERE. See the docstring: an explicit connection that did not + # resolve is None, never the environment and never another connection. + if named: + return None if self.key == "royal-imports": return (getattr(self.tenant, "sources", {}) or {}).get("odoo") return None # ------------------------------------------------------- the connector PAUSE (D-10, wave 24) - def odoo_flag_key(self): + def odoo_flag_key(self, connection_id=None): """WHICH connector flag governs this tenant's Odoo — the key `connectors[].paused` is stored under, or None when nothing would serve. THE SAME RESOLUTION AS `odoo_source()` ABOVE, expressed as an identity instead of a - client: first unlocked keychain `odoo` entry, else tenant #0's compiled env source, else - nothing. It lives here so there is ONE resolver — `routes_keychain._resolved_odoo_key` - was a second copy of these three rules in the API layer, and a pause flag written against - one resolution and read against the other freezes nothing while reporting success. + client: the keychain `odoo` connection named (or the designated default when none is), + else tenant #0's compiled env source, else nothing. It lives here so there is ONE + resolver — `routes_keychain._resolved_odoo_key` was a second copy of these three rules in + the API layer, and a pause flag written against one resolution and read against the other + freezes nothing while reporting success. + + ⛔⛔ W54-T45 — AND THE INLINE WALK IS GONE, WHICH IS WHAT MADE THE SENTENCE ABOVE TRUE. + This method claimed to be the one resolver while carrying its own copy of "the first + `odoo` entry", so `keychain` and `runtime` each decided which connection serves. Both now + ask `keychain.resolve_connection_id`, which decrypts nothing — a pause flag has to resolve + on a LOCKED keychain, which is why this is an identity call and not a credential one. + ⛔ A NAMED CONNECTION THAT DOES NOT RESOLVE IS None, exactly as in `odoo_source`: pausing + by a connection id must never silently address tenant #0's environment source instead. """ import core.keychain as keychain + named = connection_id is not None try: - entries = keychain.list_entries(self) + resolved = keychain.resolve_connection_id(self, "odoo", connection_id) except Exception: - entries = [] - first_odoo = next((e["id"] for e in entries if e.get("type") == "odoo"), None) - if first_odoo and keychain.unlocked(): - return first_odoo + resolved = None + if resolved and keychain.unlocked(): + return resolved + if named: + return None if self.key == "royal-imports" and os.environ.get("ODOO_URL"): return ENV_ODOO_FLAG_KEY return None - def odoo_paused(self): + def odoo_paused(self, connection_id=None): """Is THIS tenant's RESOLVED Odoo source paused? ⚠ RESOLVED, not "any entry" — pausing an entry that is not the one serving freezes @@ -367,9 +400,13 @@ class TenantRuntime: False, because a store hiccup must not freeze a live surface. That policy is inherited from the API-layer function this one replaced, deliberately, so there is one answer to "what happens when we cannot tell" rather than two that can disagree. + + ⭐ W54-T45 — `connection_id` passes straight through to `odoo_flag_key`, so "is the + connection serving THIS client paused" is answerable per connection under R4. No id keeps + meaning the tenant's designated default, which is every existing caller. """ try: - flag_key = self.odoo_flag_key() + flag_key = self.odoo_flag_key(connection_id) if not flag_key: return False flags = self.get(CONNECTOR_FLAGS_KEY) or {} @@ -425,7 +462,7 @@ class TenantRuntime: "(refusing rather than guessing)") raise DatastoreMismatch( f"tenant {self.key!r} expects the analytical store at {want}, but this process " - f"has {have} open, {whose}. Refusing the read — a cross-tenant answer would " + f"has {have} open, {whose}. Refusing the read; a cross-tenant answer would " f"reconcile against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or " f"call harness.datastore.use_path before serving this tenant.)") diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index c72c3d203b3a92de7c5d1b6391ef32c8a52a3f70..1de3e2bcfb725c3525cef7d5a6beb8b84031063e 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -8,7 +8,7 @@ metric carries its definition, ai_context, and an independent validate contract. Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved- dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4: list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan: -.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum). +AutoWiki/research/omni-adoption.md (Part IV + the productization addendum). Design constraints honored (from /odoo-api): - read_group dot-path GROUPBY fails → scalar aggregates here use sum_field/distinct_count; diff --git a/platform/harness/tools.py b/platform/harness/tools.py index 94e74480c49c1545e8c352b0eed06908e890e569..abe20f02086f3dd6f9e242cf6928fdc9776365a4 100644 --- a/platform/harness/tools.py +++ b/platform/harness/tools.py @@ -148,7 +148,7 @@ def run_semantic_query(topic, measures, group_by=None, grain=None, date_from=Non "window": window, "note": "every number here is drillable; cite result_id when charting"} if res["row_count"] >= (limit or 1000): # surface every truncation (plan hard line) - out["warning"] = (f"TRUNCATED: the result hit limit={limit} — the full set is larger. " + out["warning"] = (f"TRUNCATED: the result hit limit={limit}; the full set is larger. " "Re-run with a higher limit (max 5000) BEFORE ranking, comparing or " "aggregating, or your answer will be computed on a partial set.") return out @@ -165,7 +165,7 @@ def transform_result(result_id, transforms): The chain is recorded on the derived result, so saved views replay query -> transforms live.""" res = _RESULTS.get(result_id) if not res: - raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + raise SEM.ModelError(f"unknown result_id {result_id!r}; run run_semantic_query first") import harness.transforms as TR rows, applied = TR.apply(res, transforms, run_query=_run_query) new = {**res, "rows": rows, "row_count": len(rows), @@ -173,7 +173,7 @@ def transform_result(result_id, transforms): rid = _remember(new) return {"result_id": rid, "rows": rows[:100], "row_count": len(rows), "columns": sorted(rows[0]) if rows else [], - "note": "derived result — chart THIS result_id to show the transform"} + "note": "derived result; chart THIS result_id to show the transform"} def _run_query(q): @@ -190,7 +190,7 @@ def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, siz column) turns line|bar|area|scatter into small multiples.""" res = _RESULTS.get(result_id) if not res: - raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + raise SEM.ModelError(f"unknown result_id {result_id!r}; run run_semantic_query first") if kind not in CHART_KINDS: raise SEM.ModelError(f"kind must be one of {CHART_KINDS}") cols = set(res["rows"][0]) if res["rows"] else set() @@ -208,7 +208,7 @@ def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, siz if facet and kind not in ("line", "bar", "area", "scatter"): raise SEM.ModelError("facet (small multiples) works with line|bar|area|scatter only") if kind == "yoy_bars" and f"{y}_ly" not in cols: - raise SEM.ModelError(f"yoy_bars needs a {y}_ly column — run transform_result " + raise SEM.ModelError(f"yoy_bars needs a {y}_ly column; run transform_result " "[{'op':'yoy'}] on the result first") spec = {"kind": kind, "x": x, "y": y, "series": series, "title": title or f"{y or x} by {x}", "result_id": result_id, @@ -219,7 +219,7 @@ def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, siz spec[k] = v out = {"chart": spec} if kind in ("pie", "donut") and len(res["rows"]) > 6: - out["note"] = (f"{len(res['rows'])} slices — the platform will show the top 5 plus an " + out["note"] = (f"{len(res['rows'])} slices; the platform will show the top 5 plus an " "'Other' bucket; for a cleaner story run transform_result top_n first") return out @@ -229,7 +229,7 @@ def make_table(result_id, columns=None, title=None): drill IS the table). columns (optional) picks and orders a subset.""" res = _RESULTS.get(result_id) if not res: - raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first") + raise SEM.ModelError(f"unknown result_id {result_id!r}; run run_semantic_query first") rows = res["rows"] if columns: cols = set(rows[0]) if rows else set() @@ -245,7 +245,7 @@ def make_table(result_id, columns=None, title=None): def make_kpi_card(result_id, metric, compare_result_id=None): res = _RESULTS.get(result_id) if not res or not res["rows"]: - raise SEM.ModelError("result_id missing/empty — run a scalar run_semantic_query first") + raise SEM.ModelError("result_id missing/empty; run a scalar run_semantic_query first") val = res["rows"][0].get(metric) if val is None: raise SEM.ModelError(f"{metric!r} not in result") @@ -285,7 +285,7 @@ def save_view(name, chart): if "table" in chart and not spec.get("kind"): spec["kind"] = "table" if not spec.get("query"): - raise SEM.ModelError("spec carries no query — pass the exact object returned by " + raise SEM.ModelError("spec carries no query; pass the exact object returned by " "make_chart / make_kpi_card (from a fresh run_semantic_query)") d["views"][name] = {"chart": spec, "saved_at": time.strftime("%Y-%m-%d %H:%M")} _save_views(d) @@ -297,7 +297,7 @@ def compose_dashboard(name, views): d = _load_views() missing = [v for v in views if v not in d["views"]] if missing: - raise SEM.ModelError(f"unknown views {missing} — save_view them first") + raise SEM.ModelError(f"unknown views {missing}; save_view them first") d["dashboards"][name] = {"views": views, "created_at": time.strftime("%Y-%m-%d %H:%M")} _save_views(d) return {"dashboard": name, "views": views} @@ -336,7 +336,7 @@ def report_gap(kind, missing, question, workaround=None): except Exception: pass return {"logged": True, - "note": "gap recorded for the platform backlog — now tell the user in ONE sentence " + "note": "gap recorded for the platform backlog; now tell the user in ONE sentence " "what is missing and offer the nearest ask that IS answerable"} @@ -404,7 +404,7 @@ TOOLS = { ["topic", "measures"])}, "transform_result": {"fn": lambda **kw: transform_result(kw["result_id"], kw["transforms"]), "description": "Apply governed analytics transforms to a result -> a NEW result_id (a " - "CHAINABLE list of {op, ...}). The Tableau-class analytics library — pick " + "CHAINABLE list of {op, ...}). The Tableau-class analytics library; pick " "op names (full catalog + recipes in the analytics skill). Families: " "ordering/rank (sort, head, bottom_n, rank, rank_pct, ntile, top_n, " "add_total) · part-to-whole (share_of_total, cum_share) · running/moving " @@ -421,7 +421,7 @@ TOOLS = { "(running_total/share_of_total/cum_share/moving_sum/abc_classify/" "concentration) work on ADDITIVE measures (revenue/units/margin/orders); " "for a cumulative/trailing DISTINCT count (customers) or ratio use ytd/" - "rolling (they re-query) — never running_total. NEVER compute any of these " + "rolling (they re-query); never running_total. NEVER compute any of these " "yourself; transform, then chart/table the new result_id.", "parameters": _p({"result_id": {"type": "string"}, "transforms": {"type": "array", "items": {"type": "object"}}}, @@ -446,7 +446,7 @@ TOOLS = { kw.get("title")), "description": "Turn a query result into a first-class TABLE artifact (house-formatted, " "totals row, drillable). Use when the user wants exact figures, many " - "columns, or a list — not a shape. columns (optional) picks and orders.", + "columns, or a list; not a shape. columns (optional) picks and orders.", "parameters": _p({"result_id": {"type": "string"}, "columns": {"type": "array", "items": {"type": "string"}}, "title": {"type": "string"}}, ["result_id"])}, @@ -456,12 +456,12 @@ TOOLS = { "compare_result_id": {"type": "string"}}, ["result_id", "metric"])}, "report_gap": {"fn": lambda **kw: report_gap(kw["kind"], kw["missing"], kw["question"], kw.get("workaround")), - "description": "LAST RESORT — log a capability gap. Call ONLY after list_topics/" + "description": "LAST RESORT: log a capability gap. Call ONLY after list_topics/" "describe_topic confirm that NO registered dimension, metric, transform " "or chart kind can answer the user's question (e.g. stock on hand, " "which has no topic). NOT a gap: YoY/decline/growth compares " "(transform_result yoy), rankings/top-N, shares, running totals, " - "distributions — those are ANSWERABLE via transform_result. Then tell " + "distributions: those are ANSWERABLE via transform_result. Then tell " "the user plainly what is missing and offer the nearest answerable ask. " "NEVER call this for something the tools support, and NEVER guess " "instead of calling it.", @@ -488,14 +488,14 @@ TOOLS = { "instantiate_template": {"fn": lambda **kw: instantiate_template(kw["filename"], kw.get("new_name")), "description": "Stamp a tenant-agnostic template (from list_workspace) into the " - "workspace as NEW objects — never overwrites. Confirm with the user first.", + "workspace as NEW objects; never overwrites. Confirm with the user first.", "parameters": _p({"filename": {"type": "string"}, "new_name": {"type": "string"}}, ["filename"])}, "update_view": {"fn": lambda **kw: update_view(kw["name"], kw.get("changes")), "description": "UPDATE an existing saved view: patch spec keys (title/kind/x/y/series) " "and/or 'query' subkeys (measures, group_by, grain, date_from, date_to, " "team_id, filters, sort, limit; null REMOVES a key). The patched query is " - "re-executed before saving — invalid updates are rejected. Confirm first.", + "re-executed before saving; invalid updates are rejected. Confirm first.", "parameters": _p({"name": {"type": "string"}, "changes": {"type": "object"}}, ["name", "changes"])}, "update_workbook": {"fn": lambda **kw: update_workbook(kw["name"], kw.get("views"), @@ -506,7 +506,7 @@ TOOLS = { "views": {"type": "array", "items": {"type": "string"}}, "new_name": {"type": "string"}}, ["name"])}, "delete_object": {"fn": lambda **kw: delete_object(kw["kind"], kw["name"]), - "description": "DELETE a workspace object (view|dashboard|alert|report). DESTRUCTIVE — " + "description": "DELETE a workspace object (view|dashboard|alert|report). DESTRUCTIVE: " "requires the user's explicit confirmation in this conversation first.", "parameters": _p({"kind": {"type": "string", "enum": ["view", "dashboard", "alert", "report"]}, diff --git a/platform/harness/transforms.py b/platform/harness/transforms.py index a3a133b8e3c75a2590596a38fa87acac79855bb1..4671d4198c892951a14a059dcfb949f6c33d53db 100644 --- a/platform/harness/transforms.py +++ b/platform/harness/transforms.py @@ -8,7 +8,7 @@ a saved view replays query -> transforms at render time (harness/views.run_view) This is the AIOS Analyst's TABLE-CALCULATION + ANALYTICS library — the open-source Tableau-class capability set (Tableau table calcs + Analytics pane, Power BI DAX quick measures, pandas/polars window ops, scipy/statsmodels stats) mapped onto ONE governed dispatcher. Grounding brief: -.claude/wiki/research/analytics-tools.md. Design laws (2026-07-18): +AutoWiki/research/analytics-tools.md. Design laws (2026-07-18): 1. ONE tool, many ops. Every capability is a pure function rows->rows (or rows->summary) in the OPS registry, dispatched through tools.transform_result — so the small model's prompt carries @@ -105,7 +105,7 @@ def _need_additive(rows, of, op): if known_non_additive(of): raise SEM.ModelError( f"{op} sums {of!r} across rows, but {of!r} is a distinct-count or ratio measure and is " - f"NOT additive — summing it would over-count. For a cumulative/rolling distinct count " + f"NOT additive; summing it would over-count. For a cumulative/rolling distinct count " f"use the 'ytd' or 'rolling' op (they re-run the query per window); otherwise pick an " f"additive measure (revenue, units, margin, orders, cogs).") @@ -901,7 +901,7 @@ def pivot(rows, index, column, value): cell = str(col) if cell in out_map[idx]: raise SEM.ModelError( - f"pivot collision: {index}={idx!r} has two rows for {column}={col!r} — pivot cannot " + f"pivot collision: {index}={idx!r} has two rows for {column}={col!r}; pivot cannot " f"aggregate (that would hide a sum); regroup the query so (index, column) is unique") out_map[idx][cell] = r.get(value) col_names = [str(c) for c in sorted(cols, key=lambda z: (z is None, z))] @@ -1032,7 +1032,7 @@ def _require_series(res, op): if not grain: raise SEM.ModelError(f"{op} needs a time grain (run the query with grain=month|week|day)") if res.get("group_by"): - raise SEM.ModelError(f"{op} works on a SINGLE time series — drop group_by (per-group " + raise SEM.ModelError(f"{op} works on a SINGLE time series; drop group_by (per-group " "widening windows need one query per group)") return grain diff --git a/platform/harness/views.py b/platform/harness/views.py index e3a983cea85acf83606dd62eb314df082191bb0a..03730101f6322d8d2906938dba6dbd61751753b9 100644 --- a/platform/harness/views.py +++ b/platform/harness/views.py @@ -73,7 +73,7 @@ def update_view(name, changes): spec = dict(v.get("chart") or {}) q = dict(spec.get("query") or {}) if not q: - raise SEM.ModelError(f"view {name!r} carries no query — rebuild it instead") + raise SEM.ModelError(f"view {name!r} carries no query; rebuild it instead") changes = dict(changes or {}) qpatch = changes.pop("query", None) or {} bad = [k for k in changes if k not in _SPEC_PATCH_KEYS] @@ -96,7 +96,7 @@ def update_view(name, changes): for ref in ("x", "y", "series", "y2", "size", "value", "facet"): if new_spec.get(ref) and cols and new_spec[ref] not in cols: raise SEM.ModelError(f"{ref}={new_spec[ref]!r} not in the patched result columns " - f"{sorted(cols)} — patch {ref} too") + f"{sorted(cols)}; patch {ref} too") d["views"][name] = {"chart": new_spec, "saved_at": time.strftime("%Y-%m-%d %H:%M")} _save(d) @@ -114,12 +114,12 @@ def update_dashboard(name, views_list=None, new_name=None): if views_list is not None: missing = [v for v in views_list if v not in d.get("views", {})] if missing: - raise SEM.ModelError(f"unknown views {missing} — save_view them first") + raise SEM.ModelError(f"unknown views {missing}; save_view them first") dash["views"] = list(views_list) dash["updated_at"] = time.strftime("%Y-%m-%d %H:%M") if new_name and new_name != name: if new_name in d.get("dashboards", {}) or new_name in d.get("views", {}): - raise SEM.ModelError(f"{new_name!r} already exists — pick another name") + raise SEM.ModelError(f"{new_name!r} already exists; pick another name") d["dashboards"][new_name] = d["dashboards"].pop(name) try: # follow the rename into schedules import harness.routines as R @@ -148,7 +148,7 @@ def run_view(name): spec = v.get("chart") or {} q = spec.get("query") if not q: - raise SEM.ModelError(f"view {name!r} was saved without its query (pre-2026-07-11) — " + raise SEM.ModelError(f"view {name!r} was saved without its query (pre-2026-07-11); " "ask the Analyst to rebuild and re-save it") res = _run_query(q) if spec.get("kind") == "kpi" or (spec.get("metric") and spec.get("kind") != "table"): diff --git a/platform/harness/windows.py b/platform/harness/windows.py index 580ff438265c3fed527369d77fa802b267a52271..e0e615021db71e86d57d87284ac66e72468eca05 100644 --- a/platform/harness/windows.py +++ b/platform/harness/windows.py @@ -237,7 +237,7 @@ def resolve(spec, today): kind = spec["kind"] d = _parse_date(today) if d is None: - raise ValueError("resolve() needs an explicit `today` — see this module's docstring") + raise ValueError("resolve() needs an explicit `today`; see this module's docstring") if kind == "all_time": return (None, None) @@ -348,7 +348,7 @@ def resolve_anchor(mode, value, today): return None d = _parse_date(today) if d is None: - raise ValueError("resolve_anchor() needs an explicit `today` — see this module's " + raise ValueError("resolve_anchor() needs an explicit `today`; see this module's " "docstring on why the clock is never read here") if m == "today": return _iso(d) diff --git a/platform/harness/workspace.py b/platform/harness/workspace.py index aba71cccd19acdf5f5c7691911f7730f4a6c9a99..88f4f1d53604056523cf43bd4061564186b2cf5b 100644 --- a/platform/harness/workspace.py +++ b/platform/harness/workspace.py @@ -85,7 +85,7 @@ def _templatize_view(name): spec = dict(v.get("chart") or {}) q = dict(spec.get("query") or {}) if not q: - raise SEM.ModelError(f"view {name!r} carries no query — cannot templatize") + raise SEM.ModelError(f"view {name!r} carries no query; cannot templatize") stripped = {k: q.pop(k) for k in _STRIP if k in q} stripped = {k: v for k, v in stripped.items() if v} # record only real tenant baggage q["window"] = "ytd" # instantiation default; tenant picks at stamp time @@ -109,7 +109,7 @@ def export_template(kind, name): "views": [_templatize_view(vn) for vn in d.get("views", [])]} else: raise SEM.ModelError("only views and dashboards templatize (alerts/reports are one-liners " - "— recreate from the metric key)") + "; recreate from the metric key)") TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) safe = "".join(c if c.isalnum() or c in "-_ " else "_" for c in name).strip().replace(" ", "-") path = TEMPLATES_DIR / f"{kind}-{safe}.json" diff --git a/platform/model/skills/analytics.skill.yml b/platform/model/skills/analytics.skill.yml index c481cffad883374ea062182e9cd1a1eba91903ce..a09e58dd11449ddcfc62f8e1d15126756a713c28 100644 --- a/platform/model/skills/analytics.skill.yml +++ b/platform/model/skills/analytics.skill.yml @@ -1,6 +1,6 @@ # In-product SKILL: the ANALYTICS TRANSFORM LIBRARY (v1, 2026-07-18) — the Tableau-class # table-calc + analytics capability set, mapped onto the ONE governed transform_result tool. -# Grounding: .claude/wiki/research/analytics-tools.md. Every op here EXISTS in harness/transforms.py +# Grounding: AutoWiki/research/analytics-tools.md. Every op here EXISTS in harness/transforms.py # (OPS) and is unit-proven in _demo_analytics.py — recipes and code version together. # # HOW IT WORKS: run_semantic_query -> transform_result([{op, ...}, ...]) -> make_chart|make_table. diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml index ca9c8a7c094d5c8a3c1b51e715ba78da1fa554dd..e5c6d7a4be1740361e5dcc4a507eab55b4cb53fa 100644 --- a/platform/model/topics/odoo_customers.yml +++ b/platform/model/topics/odoo_customers.yml @@ -144,6 +144,16 @@ fields: type: url kind: data means: "A Google Maps link to this customer's address." + - key: lat + label: "Latitude" + type: text + kind: data + means: "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + - key: lon + label: "Longitude" + type: text + kind: data + means: "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." - key: customer_since label: "Customer since" type: date @@ -174,11 +184,6 @@ fields: type: int kind: data means: "How many days late this customer is running against their own usual ordering rhythm." - - key: est_missed - label: "Est. missed $" - type: currency - kind: data - means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - key: ar_open label: "AR current $" type: currency diff --git a/platform/model/topics/sales_lines.yml b/platform/model/topics/sales_lines.yml index cc2bb51cce26a0a86ab55a2b325746ad89769266..bb5143460f2e3843ec8e17fb180ae9d3b8e0a70c 100644 --- a/platform/model/topics/sales_lines.yml +++ b/platform/model/topics/sales_lines.yml @@ -1,5 +1,5 @@ # Topic: sales_lines — the wholesale order-line dataset (OM-0, the first topic of the semantic -# layer; see .claude/wiki/research/omni-adoption.md). A topic = the curated, scoped dataset that +# layer; see AutoWiki/research/omni-adoption.md). A topic = the curated, scoped dataset that # metrics, saved views and the AI Analyst all resolve against — ONE definition of what the # data MEANS, consumed by every surface. key: sales_lines diff --git a/platform/modules/backorders.py b/platform/modules/backorders.py index 7199947287223563cbeecd54f96ad57a595d289e..2270809227ade65d273c4cab437fb786f7c1bd21 100644 --- a/platform/modules/backorders.py +++ b/platform/modules/backorders.py @@ -193,11 +193,11 @@ def board(team_id=None, t=None): oh = onhand.get(pid_prod, 0.0) inb = inbound.get(pid_prod) if oh >= r['open_qty']: - action = 'Ship — stock on hand' + action = 'Ship: stock on hand' elif inb and inb.get('eta'): - action = f"Expedite PO — ETA {inb['eta']}" + action = f"Expedite PO: ETA {inb['eta']}" else: - action = 'Call customer — no supply inbound' + action = 'Call customer: no supply inbound' rows.append({ 'pid': O.m2o_id(r['order_partner_id']), 'customer': O.m2o_name(r['order_partner_id']), @@ -236,7 +236,7 @@ def validate(t=None, team_id=None): open_qty (same universe, independent arithmetic path).""" b = board(team_id, t) n_count = O.get_odoo().search_count('stock.picking', list(_OPEN_PICK_DOMAIN)) - checks = [{'check': 'open outgoing pickings — pull complete', + checks = [{'check': 'open outgoing pickings: pull complete', 'a': b['n_pickings'], 'b': n_count, 'gap': b['n_pickings'] - n_count, 'ok': b['n_pickings'] == n_count}] oids = sorted({r['order'] for r in b['rows']}) @@ -253,7 +253,7 @@ def validate(t=None, team_id=None): rl = O.search_read('sale.order.line', dom, ['product_uom_qty', 'qty_delivered']) ours = sum((l.get('product_uom_qty') or 0.0) - (l.get('qty_delivered') or 0.0) for l in rl) - checks.append({'check': 'open qty — server aggregate vs row arithmetic', + checks.append({'check': 'open qty: server aggregate vs row arithmetic', 'a': round(ours, 2), 'b': round(srv_open, 2), 'gap': round(ours - srv_open, 2), 'ok': abs(ours - srv_open) < 1.0}) diff --git a/platform/modules/bookings.py b/platform/modules/bookings.py index 1cc3a839c6a9e264df56e3e768ed06694aaa350a..8c28180f6cf273e3cff7a6d6dd363d9140365859 100644 --- a/platform/modules/bookings.py +++ b/platform/modules/bookings.py @@ -131,7 +131,7 @@ def validate(t=None, team_id=None): if ex: dom.append(('partner_id', 'not in', list(ex))) srv = O.sum_field('sale.order', dom, 'amount_untaxed') - return [{'check': f'{ty} season booked $ — client cumulative vs server sum', + return [{'check': f'{ty} season booked $: client cumulative vs server sum', 'a': round(b['booked_ty'], 2), 'b': round(srv, 2), 'gap': round(b['booked_ty'] - srv, 2), 'ok': abs(b['booked_ty'] - srv) < 1.0}] diff --git a/platform/modules/cohort.py b/platform/modules/cohort.py index 0873905a88d120b9cecdbcdaf928133446191b2c..19b1c4f03a76b21690ec4d2a2e09659608f9cf59 100644 --- a/platform/modules/cohort.py +++ b/platform/modules/cohort.py @@ -181,7 +181,7 @@ def _create(key, username, name, members=(), note='', st=None): raise ValueError('a cohort needs a name') existing = _all_for(key, username, st=st) if len(existing) >= MAX_COHORTS: - raise ValueError(f'you already have {MAX_COHORTS} cohorts — delete one first') + raise ValueError(f'you already have {MAX_COHORTS} cohorts; delete one first') cid = new_id(name) record = {'name': name, 'members': _clean_members(members), 'note': str(note or '')[:2000], 'created': _now(), 'updated': _now()} diff --git a/platform/modules/complexity.py b/platform/modules/complexity.py index 92dd89ae87ad67e79cf9c88505fb8f1c07dbd908..878bcbbff7dec7f52b91e72cebebeee3345726c3 100644 --- a/platform/modules/complexity.py +++ b/platform/modules/complexity.py @@ -254,7 +254,7 @@ def validate(t=None, team_id=None, pre=None): srv_bv = O.sum_field('stock.valuation.layer', [], 'value') a_bv = sum(r['book_value'] for r in b['rows']) checks.append({'check': 'complexity: Σ SKU book value == server Σ valuation layers ' - '(negatives clamped per SKU — gap = clamp effect)', + '(negatives clamped per SKU: gap = clamp effect)', 'a': round(a_bv, 2), 'b': round(srv_bv, 2), 'gap': round(a_bv - srv_bv, 2), 'ok': a_bv >= srv_bv - 1.0}) @@ -267,4 +267,4 @@ def validate(t=None, team_id=None, pre=None): 'a': round(s, 2), 'b': round(kill_rev, 2), 'gap': round(s - kill_rev, 2), 'ok': abs(s - kill_rev) <= max(1.0, kill_rev * 0.001)}) - return checks \ No newline at end of file + return checks diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py index 3fbba8be521fa16660fcd48ae4738b8af85cf2c7..9a08488ed2587c2c10032094300c10e90b59736c 100644 --- a/platform/modules/customer_data.py +++ b/platform/modules/customer_data.py @@ -1,21 +1,16 @@ -"""Customer List — build, filter and save customer lists (replaces My Day, owner IA 2026-07-23). - -The page is a customer-list WORKBENCH in two parts: - 1. BUILDER — the whole scoped book, one row per customer with a consistent metric surface - (YTD / LY revenue, at-risk $, cadence, overdue days, estimated missed $ …): filter, sort, - SELECT customers and add them to a list. - 2. LISTS — saved lists whose membership is a visible, editable FORMULA (a rule set over the - metric columns) plus hand-picked members. 'Call list' and 'Win-back' ship as TEMPLATES — - the exact formulas the old My Day queues used — and the user can change or reset them. - -Composes already-validated customers-module functions, so the page's counts derive FROM the -lists it renders (rule 8b). ⚠ **This header used to end "no separate validate()" and W30-T33 -made that false** — see `validate()` below. The claim held only while every column composed an -already-reconciled customers-module function; the ADDRESS family does not, so it needed its own -tie. Persistence: HF store key 'customer_lists' -({username: {list_name: {rules, sort, members, note}}}); templates are virtual until edited. - -The user↔agent link (users.py 'agent' field) scopes the pool to that agent's book; users with +"""Customer data: a scoped metric pool and saved table workspaces. + + 1. POOL: one row per customer, composing sales, cadence, receivables and address data. + 2. VIEWS: ordinary saved table views hold filters, sorts, layout and selected members in + 'customer_table_workspace'. The old rule-formula list readers and shipped list templates + were retired in wave 57 under R13. The 'customer_lists' bucket is retained without reads + or writes here so existing tenant data is preserved. + +The pool composes already-validated customers-module functions. Its own validate() also +reconciles the address family independently. The daily digest still uses queues(), which calls +the customers module's contact recommendations and at-risk calculations directly. + +The user-to-agent link (users.py 'agent' field) scopes the pool to that agent's book; users with no link (owner/CFO/admin) see the whole book. """ import re @@ -31,6 +26,7 @@ import modules.map as map_mod import modules.sales as S LIMIT = 500 # honest builder display cap — always shown with the full count, never silent +# Rule-formula readers retired in wave 57 under R13; keep the bucket so no tenant data is dropped. KEY = 'customer_lists' TABLE_KEY = 'customer_table_workspace' @@ -104,7 +100,6 @@ FIELDS = { 'days_since': ('Days since last order', 'num'), 'typical_gap_days': ('Typical gap (days)', 'num'), 'overdue_days': ('Overdue vs cadence (days)', 'num'), - 'est_missed': ('Est. missed $', 'num'), } NUM_OPS = ('>=', '<=', '>', '<', '=') TEXT_OPS = ('contains', '=') @@ -289,14 +284,14 @@ def pool(agent_name=None, team_id=None): buyers and anyone with an order in the 24-month cadence window — so lapsed/win-back accounts are in the pool, not just YTD actives. Reuses the customers module's validated building blocks (_cust_rev / _cadence_bulk / _partner_attrs) so every metric matches the - Customers page definitions; est_missed is the Call-list formula (min(cycles missed, 3) × AOV).""" + Customers page definitions.""" return _pool_build(agent_name, team_id, limit=None)['rows'] def pool_first(agent_name=None, team_id=None, limit=100): """Wave-7 W1 (C1 as amended): the cold-load FAST SLICE — the pool's first `limit` rows in its own default order + the true total, built from the CHEAP families only (revenue ×3, - cadence, status, est_missed, partner attributes ≈ 12s measured). The AR / product-mix / + cadence, status, partner attributes ≈ 12s measured). The AR / product-mix / salesperson / coords columns stay BLANK until the full build swaps in: `_mix_attrs` alone is a whole-book 20s read_group whose domain must never be pid-narrowed (502 — see its docstring), so "all columns on 100 rows" measured 44.6s vs 45.1s full — no win. A partial @@ -380,7 +375,7 @@ def validate(team_id=None): # customer's address behind six '(none)' cells. still_live = odoo.search_count('res.partner', [('id', 'in', rpids)]) if rpids else 0 checks.append({ - 'check': "every RETAINED customer (I13) is one Odoo genuinely cannot see — asked of " + 'check': "every RETAINED customer (I13) is one Odoo genuinely cannot see; asked of " "Odoo, never of the ledger that did the retaining", 'ours': still_live, 'theirs': 0, 'ok': still_live == 0, 'detail': {'pool_rows': len(rows), 'odoo_visible': n_visible, 'retained': len(revived), @@ -395,7 +390,7 @@ def validate(team_id=None): theirs = odoo.search_count('res.partner', [('id', 'in', pids), (field, '!=', False)]) checks.append({ 'check': f"the grid's `{key}` column is non-blank on exactly the pool customers Odoo " - f"says carry `{field}` — an INDEPENDENT search_count over the pool's own " + f"says carry `{field}`; an INDEPENDENT search_count over the pool's own " f"pid set, never a count of the rows we just built", 'ours': ours, 'theirs': theirs, 'ok': ours == theirs, # `column` is what the GATE matches on. Substring-matching the prose is how a leg @@ -420,7 +415,7 @@ def validate(team_id=None): if isinstance(v, str) and v and not v.strip(): ws[k] = ws.get(k, 0) + 1 checks.append({ - 'check': "no pool customer carries a WHITESPACE-ONLY address value — the one input on " + 'check': "no pool customer carries a WHITESPACE-ONLY address value; the one input on " "which our blank predicate and Odoo's `!= False` provably disagree", 'ours': sum(ws.values()), 'theirs': 0, 'ok': not ws, 'detail': {'per_field': ws, 'fields_checked': text_keys}, @@ -603,7 +598,6 @@ def _pool_build(agent_name, team_id, limit, fast=False): gap = c.get('typical_gap_days') overdue = c.get('overdue_days') aov = c.get('aov', 0.0) - est = (min(overdue / gap, 3.0) * aov) if (gap and overdue and overdue > 0) else 0.0 status = ('New' if rev > 0 and ly <= 0 else 'Lost' if ly > 0 and rev <= 0 else 'Declining' if 0 < rev < ly else @@ -671,7 +665,6 @@ def _pool_build(agent_name, team_id, limit, fast=False): 'last_order': c.get('last_order', ''), 'days_since': c.get('days_since'), 'typical_gap_days': gap, 'overdue_days': overdue, - 'est_missed': est, }) for r in rows: r['odoo_status'] = 'Active' @@ -760,7 +753,6 @@ def _reconcile_ledger(rows, total, agent_name, team_id): 'revenue_ytd': 0.0, 'revenue_ly': 0.0, 'yoy_pct': None, 'at_risk': 0.0, 'ltm_rev': 0.0, 'orders_24m': 0, 'aov': 0.0, 'last_order': '', 'days_since': None, 'typical_gap_days': None, 'overdue_days': None, - 'est_missed': 0.0, }) if store.available(): try: @@ -770,86 +762,6 @@ def _reconcile_ledger(rows, total, agent_name, team_id): return rows + revived, total + len(revived) -# ------------------------------------------------------------------ the rule engine -# A list's formula = [{'field','op','value'}, ...] over FIELDS + a 'sort' key ('-x' = desc). -# The two templates ARE the old My Day queue formulas, expressed as visible rules. -# -# 'note' is a SEED, not the source of truth (owner 2026-07-25). It is what a list reads -# BEFORE anyone edits it; the moment a user saves a description the stored one wins, and -# an empty stored description stays empty — the template must never resurrect. That -# precedence lives in aios_grid.views_from_defs (a saved view replaces its template-derived -# one wholesale) and is pinned by _qa_grid_view_note.py. Edit the prose here only to change -# what a NEW user starts with. -TEMPLATES = { - 'Call list': { - 'rules': [{'field': 'orders_24m', 'op': '>=', 'value': 3}, - {'field': 'overdue_days', 'op': '>', 'value': 0}, - {'field': 'days_since', 'op': '<=', 'value': 365}], - 'sort': '-est_missed', - 'note': 'Customers overdue against their OWN reorder cadence (3+ orders, a real gap ' - 'pattern, ordered within 365d), ranked by estimated missed revenue.'}, - 'Win-back': { - 'rules': [{'field': 'revenue_ly', 'op': '>=', 'value': 2000}, - {'field': 'at_risk', 'op': '>', 'value': 0}], - 'sort': '-at_risk', - 'note': 'Bought materially last year (over $2,000), down or gone this year, ranked by ' - 'dollars at risk.'}, -} - - -def _match(row, rule): - """One rule against one row. None/missing numeric values fail every numeric comparison - (a customer with no cadence is never 'overdue'); text ops are case-insensitive.""" - field, op, val = rule.get('field'), rule.get('op'), rule.get('value') - if field not in FIELDS: - return True # unknown field → rule is inert, not a crash - v = row.get(field) - if FIELDS[field][1] == 'text': - s, q = str(v or '').lower(), str(val or '').lower() - return q in s if op == 'contains' else s == q - try: - v, val = float(v), float(val) - except (TypeError, ValueError): - return False - return {'>=': v >= val, '<=': v <= val, '>': v > val, '<': v < val, - '=': v == val}.get(op, False) - - -def apply_rules(rows, rules, sort=None): - """Filter the pool through a formula (AND of all rules) and apply the list's sort.""" - out = [r for r in rows if all(_match(r, ru) for ru in (rules or []))] - if sort: - key = sort.lstrip('-') - out.sort(key=lambda r: (r.get(key) is None, - -(r.get(key) or 0) if sort.startswith('-') else (r.get(key) or 0)) - if FIELDS.get(key, ('', 'num'))[1] == 'num' - else str(r.get(key) or '').lower(), reverse=False) - return out - - -# ------------------------------------------------------------------ persistence (HF store) -def saved_lists(username): - """{list_name: {'rules','sort','members','note'}} for one user ({} when none / store down).""" - try: - return (store.get(KEY) or {}).get(username, {}) or {} - except Exception: - return {} - - -def save_list(username, name, definition): - def _up(d): - d.setdefault(username, {})[name] = definition - return d - store.update(KEY, _up) - - -def delete_list(username, name): - def _up(d): - (d.get(username) or {}).pop(name, None) - return d - store.update(KEY, _up) - - # ------------------------------------------------------------------ Airtable-style table workspace # This is intentionally separate from `customer_lists`: a LIST owns membership/formula # semantics, while a VIEW owns presentation/query state (filters, multi-sort, grouping, diff --git a/platform/modules/customers.py b/platform/modules/customers.py index ac7489d3c918568a1f1163568dc342534c9d1efc..0872f7bff3739c02b79e8ff0decbfb2197ae33d6 100644 --- a/platform/modules/customers.py +++ b/platform/modules/customers.py @@ -346,8 +346,8 @@ def customer_trends(t=None, team_id=None, agent_pids=None): # ----------------------------------------------------------- segmentation (MECE) -TIERS = [('Whale (≥$25k)', 25000), ('Large ($10–25k)', 10000), - ('Mid ($2–10k)', 2000), ('Small (<$2k)', 0)] +TIERS = [('Whale (≥$25k)', 25000), ('Large ($10k to $25k)', 10000), + ('Mid ($2k to $10k)', 2000), ('Small (<$2k)', 0)] def _tier(rev): @@ -527,7 +527,7 @@ def _cadence_bulk(t=None, team_id=None, months=24, agent_pids=None): def contact_recommendations(t=None, team_id=None, limit=40, months=24, agent_pids=None): """Prioritised follow-up list: customers overdue against their OWN purchase cadence, ranked by - estimated missed revenue (how far past due × their average order value). Reps' call list.""" + how far past their own reorder cadence they are. Reps' call list.""" t = t or P.today() cad = _cadence_bulk(t, team_id, months, agent_pids=agent_pids) attrs = _partner_attrs(list(cad)) @@ -546,17 +546,13 @@ def contact_recommendations(t=None, team_id=None, limit=40, months=24, agent_pid if c['days_since'] > 365: continue # long-dead → not a cadence follow-up ltm_rev = rev_map.get(pid, {}).get('rev', 0.0) - cycles_missed = overdue / gap # how many reorder cycles past due - # cap at 3 cycles so a long-churned account (belongs in win-back) doesn't dominate the - # "call them now" list; this rewards valuable regulars who are RECENTLY due. - est_missed = min(cycles_missed, 3.0) * c['aov'] rows.append({ 'pid': pid, 'customer': rev_map.get(pid, {}).get('name', '?'), 'last_order': c['last_order'], 'typical_gap_days': gap, 'overdue_days': overdue, - 'ltm_rev': ltm_rev, 'orders': c['n_orders'], 'est_missed': est_missed, + 'ltm_rev': ltm_rev, 'orders': c['n_orders'], 'agent': (attrs.get(pid) or {}).get('agent', '(none)'), }) - rows.sort(key=lambda x: -x['est_missed']) + rows.sort(key=lambda x: -(x['overdue_days'] or 0)) return rows[:limit] diff --git a/platform/modules/digest.py b/platform/modules/digest.py index 23ed2b3b702ca1743a654115dab9d9a79a77ddef..cacc221bbd71bc57212f60cd33260e6848b377b0 100644 --- a/platform/modules/digest.py +++ b/platform/modules/digest.py @@ -103,14 +103,13 @@ def build(user, t=None): meta = f"due {x.get('due') or '—'}" rows.append(_line(x.get('title', 'Task'), href, meta)) if calls: - rows.append(_sec('Call today — overdue vs their own cadence')) + rows.append(_sec('Call today: overdue vs their own cadence')) for r in calls: href = links.deeplink('customer_data', 'customer', r['pid'], src='digest') - meta = (f"{int(r.get('overdue_days') or 0)}d past cycle · " - f"est. missed {_m(r.get('est_missed'))}") + meta = f"{int(r.get('overdue_days') or 0)}d past cycle" rows.append(_line(r.get('customer', '?'), href, meta)) if risk: - rows.append(_sec('Win-back — down vs last year')) + rows.append(_sec('Win-back: down vs last year')) for r in risk: href = links.deeplink('customer_data', 'customer', r['pid'], src='digest') meta = f"{_m(r.get('at_risk'))} at risk · {r.get('status', '')}" @@ -140,7 +139,7 @@ def build(user, t=None): Sent only on days with something to act on. """ - subject = f"Your book this morning — {n} to act on · {t.strftime('%b %d')}" + subject = f"Your book this morning: {n} to act on · {t.strftime('%b %d')}" return subject, html_body, n diff --git a/platform/modules/expenses.py b/platform/modules/expenses.py index e75a634577d69cba60d614d95d629ce8814d6aa1..f4c0b8aa66d16f1a6252f6ad02fde288142d6716 100644 --- a/platform/modules/expenses.py +++ b/platform/modules/expenses.py @@ -387,11 +387,11 @@ def validate(t=None, team_id=None, pre=None): dir_sum = sum(r['ltm'] for r in b['rows']) checks = [ - {'check': 'operating expense LTM — client Σ == server Σ balance', + {'check': 'operating expense LTM: client Σ == server Σ balance', 'a': round(b['opex_ltm'], 2), 'b': round(srv_opex, 2), 'gap': round(b['opex_ltm'] - srv_opex, 2), 'ok': abs(b['opex_ltm'] - srv_opex) <= max(1.0, abs(srv_opex) * 0.001)}, - {'check': 'operating revenue LTM — client Σ == server income Σ', + {'check': 'operating revenue LTM: client Σ == server income Σ', 'a': round(b['rev_ltm'], 2), 'b': round(srv_rev, 2), 'gap': round(b['rev_ltm'] - srv_rev, 2), 'ok': abs(b['rev_ltm'] - srv_rev) <= max(1.0, abs(srv_rev) * 0.001)}, diff --git a/platform/modules/health.py b/platform/modules/health.py index ae447edb486de983034dfee6732fc0f95ac940d7..7c55f570e6f9791620d6fcf932630251cc442924 100644 --- a/platform/modules/health.py +++ b/platform/modules/health.py @@ -67,7 +67,7 @@ def _draft_moves(df, dt_): 'amount': r.get('amount_total') or 0} for r in rows] return _check('draft', CATEGORIES[0], 'Draft / unposted entries in period', 'high' if n else 'low', 'period', n, None, None, - 'Post or delete these before closing — draft invoices/bills/journals are not in the books yet.', + 'Post or delete these before closing; draft invoices/bills/journals are not in the books yet.', sample, {'amount': 'money'}) @@ -113,7 +113,7 @@ def _unbilled_revenue(df, dt_): sample = sorted(by_order.values(), key=lambda x: -x['unbilled'])[:25] return _check('unbilled', CATEGORIES[1], 'Delivered but not invoiced (revenue to go bill)', 'high' if total > 25000 else 'medium', 'period', len(by_order), total, 'dollar', - 'Shipped quantity exceeds invoiced quantity on these orders — bill the gap now ' + 'Shipped quantity exceeds invoiced quantity on these orders; bill the gap now ' '(or accrue it at cut-off). Line-level: partial invoices are netted correctly.', sample, {'unbilled': 'money'}) @@ -228,7 +228,7 @@ def _duplicate_bills(df, dt_): amt = sum((len(v) - 1) * (v[0]['amount_total'] or 0) for v in dups) return _check('dup_bills', CATEGORIES[3], 'Possible duplicate vendor bills (same vendor/ref/amount)', 'high' if dups else 'low', 'period', len(dups), amt, 'dollar', - 'Review before paying — duplicate bills cause double payment.', sample, {'amount': 'money'}) + 'Review before paying; duplicate bills cause double payment.', sample, {'amount': 'money'}) # ---- Inventory & COGS ------------------------------------------------------- @@ -254,7 +254,7 @@ def _negative_stock(df, dt_): sample = [{'product': O.m2o_name(r['product_id']), 'on_hand': r['quantity']} for r in rows] return _check('neg_stock', CATEGORIES[4], 'Negative on-hand stock (impossible quantities)', 'high' if n > 20 else 'medium', 'current', n, None, None, - 'Fix receipts/adjustments — negative on-hand distorts inventory valuation and COGS.', + 'Fix receipts/adjustments; negative on-hand distorts inventory valuation and COGS.', sample, {'on_hand': 'num'}) @@ -277,10 +277,10 @@ def _product_master(df, dt_): dups = [{'sku': k, 'records': len(v)} for k, v in dup.items() if len(v) > 1] c_unc = _check('uncosted', CATEGORIES[4], 'In-stock SKUs with zero cost (valuation gap)', 'medium', 'current', len(uncosted), None, None, - 'Set standard cost — these read as $0 inventory and distort margin and valuation.', + 'Set standard cost; these read as $0 inventory and distort margin and valuation.', sorted(uncosted, key=lambda x: -x['on_hand'])[:25], {'on_hand': 'num'}) c_dup = _check('dup_codes', CATEGORIES[5], 'Duplicate active SKU codes', 'medium', 'current', - len(dups), None, None, 'Merge/retire duplicates — shared codes double-count.', + len(dups), None, None, 'Merge/retire duplicates; shared codes double-count.', sorted(dups, key=lambda x: -x['records'])[:25], {'records': 'int'}) c_cat = _check('uncat', CATEGORIES[5], 'Sellable products with no real category', 'low', 'current', len(uncat), None, None, 'Assign a category so by-category reporting works.', uncat[:25], {}) diff --git a/platform/modules/inventory.py b/platform/modules/inventory.py index 16232898b6faf8d1aa808ab67305fd41cd4cffa6..85c43c19dc70feaa355cec9d0de550c681b532f0 100644 --- a/platform/modules/inventory.py +++ b/platform/modules/inventory.py @@ -157,9 +157,9 @@ def abc(t=None): # ----------------------------------------------------------- deeper analytics _CARRY_PCT = 25.0 # default annual carrying cost as a % of inventory value (capital + storage + obsolescence) -_COVERAGE_BUCKETS = [('Critical (<15d)', 0, 15), ('Lean (15–45d)', 15, 45), - ('Healthy (45–120d)', 45, 120), ('Ample (120–270d)', 120, 270), - ('Overstock (270–540d)', 270, 540), ('Excess (>540d)', 540, 1e15)] +_COVERAGE_BUCKETS = [('Critical (<15d)', 0, 15), ('Lean (15 to 45d)', 15, 45), + ('Healthy (45 to 120d)', 45, 120), ('Ample (120 to 270d)', 120, 270), + ('Overstock (270 to 540d)', 270, 540), ('Excess (>540d)', 540, 1e15)] def _enrich(df, target_days): diff --git a/platform/modules/management.py b/platform/modules/management.py index 05c8dc16f74306e95704eadde193e8ec021af9be..c6e4ef57268d200b20b6e77793d2312380984df0 100644 --- a/platform/modules/management.py +++ b/platform/modules/management.py @@ -207,7 +207,7 @@ def validate(year=None, month=None): indep = abs((g[0].get('amount') or 0.0) if g else 0.0) # display positive repro = sum(clean.get(b, {}).get(code, 0.0) for b in MP.BUS) gap = round(repro - indep, 2) - checks.append({'check': f'{code} {label} — consolidated clean vs direct Odoo sum', + checks.append({'check': f'{code} {label}: consolidated clean vs direct Odoo sum', 'a': round(repro, 2), 'b': round(indep, 2), 'gap': gap, 'ok': abs(gap) < 0.01}) # consolidation consistency: BU net == FFS + RI + GD net diff --git a/platform/modules/map.py b/platform/modules/map.py index e627e16180d3bda185a6c3b0dbe38bd7c7e59049..b5079e64b81edf48401bbb85ec5ab41e968b99a0 100644 --- a/platform/modules/map.py +++ b/platform/modules/map.py @@ -7,6 +7,9 @@ Coordinates resolve in this order, per customer: free US Census batch geocoder (these customers are almost all US/NY-metro florists). Customers with no usable address (or a geocode that fails) fall to a list under the map. +This pipeline serves Odoo `res.partner` rows only, and its US Census lookup is US only. The general +on-demand geocoder for every other database is `aios-web/api/routes_geo.py`. + Odoo stays STRICTLY read-only — geocodes are cached in the platform's own store, never written back to Odoo. The map's purpose is select-and-export (e.g. to assign agents): box/lasso-select on the map builds an Excel list. Assigning agents happens in Odoo / via that export, never as an app write. diff --git a/platform/modules/o2c.py b/platform/modules/o2c.py index eb3f0cc3ab6c6d4c7925724d92ac74c139fb2430..e97510bad2a969eb268501b111058b36df2bf167 100644 --- a/platform/modules/o2c.py +++ b/platform/modules/o2c.py @@ -203,8 +203,8 @@ def validate(t=None, team_id=None): inv_dom.append(('partner_id', 'not in', list(ex))) n_inv = O.get_odoo().search_count('account.move', inv_dom) return [ - {'check': 'LTM orders — pull complete', 'a': b['n_orders'], 'b': n_orders, + {'check': 'LTM orders: pull complete', 'a': b['n_orders'], 'b': n_orders, 'gap': b['n_orders'] - n_orders, 'ok': b['n_orders'] == n_orders}, - {'check': 'LTM posted invoices — pull complete', 'a': b['n_invoices'], 'b': n_inv, + {'check': 'LTM posted invoices: pull complete', 'a': b['n_invoices'], 'b': n_inv, 'gap': b['n_invoices'] - n_inv, 'ok': b['n_invoices'] == n_inv}, ] diff --git a/platform/modules/outreach.py b/platform/modules/outreach.py index 573e01d6a06ccca3acede7d3644c1230a84f0e3f..b397bd3aeff50027dbcae298910d937d392946a4 100644 --- a/platform/modules/outreach.py +++ b/platform/modules/outreach.py @@ -84,7 +84,7 @@ DEFAULT_TEMPLATES = { 'body_html': ( '

Hi {{ customer }},

' '

It has been {{ days_since }} days since your last order with us' - '{% if last_order %} (placed {{ last_order }}){% endif %} — based on your usual ' + '{% if last_order %} (placed {{ last_order }}){% endif %}; based on your usual ' 'rhythm, you may be running low.

' '{% if top_skus %}

Your recent favorites:

    ' '{% for s in top_skus %}
  • {{ s }}
  • {% endfor %}
{% endif %}' @@ -97,7 +97,7 @@ DEFAULT_TEMPLATES = { 'subject': 'We miss you at {{ company }}, {{ customer }}', 'body_html': ( '

Hi {{ customer }},

' - '

We noticed your orders have slowed this year and wanted to check in — our new ' + '

We noticed your orders have slowed this year and wanted to check in; our new ' 'catalog has landed and we would love to get you back to the table.

' '

Reply to this email or contact {{ agent }} to hear what is new.

' '

Thank you,
{{ company }}

'), @@ -180,8 +180,8 @@ def wrap_body(inner_html, tclass): # ------------------------------------------------------------------ audiences SEGMENT_SOURCES = { - 'reorder': 'Reorder due — overdue vs their own cadence', - 'winback': 'Win-back — bought LY, down or gone this year', + 'reorder': 'Reorder due: overdue vs their own cadence', + 'winback': 'Win-back: bought LY, down or gone this year', 'new': 'New / reactivated this year', } @@ -193,7 +193,7 @@ def build_audience(source, team_id=None, agent=None, limit=500): if source == 'reorder': rows = cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids_scope) return [{'pid': r['pid'], 'customer': r['customer'], 'agent': r.get('agent', ''), - 'metric': r.get('est_missed', 0.0), 'metric_label': 'est. missed $'} + 'metric': r.get('overdue_days', 0), 'metric_label': 'days past cadence'} for r in rows] if source == 'winback': rows = cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids_scope) diff --git a/platform/modules/pricecomp.py b/platform/modules/pricecomp.py index 72b93b8a8486fd2ed7c4d7321cedd017c295610a..2a7eb9000e5a0424c20881f8dd86fb1d64037dfc 100644 --- a/platform/modules/pricecomp.py +++ b/platform/modules/pricecomp.py @@ -152,7 +152,7 @@ def validate(t=None, team_id=None): 'product.pricelist.item', [('compute_price', '=', 'fixed'), ('applied_on', 'in', ['0_product_variant', '1_product'])]) - checks = [{'check': 'fixed pricelist rules — pull complete (pre date-filter)', + checks = [{'check': 'fixed pricelist rules: pull complete (pre date-filter)', 'a': b['n_rules'], 'b': f'<= {n}', 'gap': '', 'ok': b['n_rules'] <= n}] if b['rows']: @@ -167,7 +167,7 @@ def validate(t=None, team_id=None): qty = sum(l.get('product_uom_qty') or 0 for l in lines) rev = sum(l.get('price_subtotal') or 0 for l in lines) unit = rev / qty if qty else 0.0 - checks.append({'check': f"top exception realized price — {top['sku']}", + checks.append({'check': f"top exception realized price: {top['sku']}", 'a': round(unit, 4), 'b': round(top['realized'], 4), 'gap': round(unit - top['realized'], 4), 'ok': abs(unit - top['realized']) < 0.01}) diff --git a/platform/modules/procurement.py b/platform/modules/procurement.py index 8632737d8d1cfd4bd41ced18ece8dd5a782d92db..7286d3d82033d535fc9ef0f93ca5d7987ccf3af4 100644 --- a/platform/modules/procurement.py +++ b/platform/modules/procurement.py @@ -278,7 +278,7 @@ def data_quality(): 'fix': 'Set the vendor on the SKU (Procurement page or Vendor module) so a PO can be grouped.', 'sample': [{'default_code': c, 'name': sup.get(c, {}).get('desc')} for c in no_vendor]}, {'issue': 'No lead time in the mapping', 'count': len(no_lead), - 'fix': 'Set the supplier lead time (days) — the buy trigger needs it.', + 'fix': 'Set the supplier lead time (days); the buy trigger needs it.', 'sample': [{'default_code': c, 'name': sup.get(c, {}).get('desc')} for c in no_lead]}, ] diff --git a/platform/modules/products.py b/platform/modules/products.py index e6da735f9545d53f993f0a9f32ee9034f9da2054..46ffebc1567ad83c839c4744f94da99f935e8a6c 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -367,7 +367,7 @@ def catalogue(): n = O.get_odoo().search_count('product.product', dom) if len(prods) != n: raise ValueError( - f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against " + f"products.catalogue: the product pull is TRUNCATED; read {len(prods)} rows against " f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with " f"nothing reporting it; raise the limit before shipping this.") code_cat = _code_category() @@ -1008,8 +1008,8 @@ def sku_top_buyers(code, t=None, team_id=None, top=15): return rows[:top] -_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0), - ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)] +_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10k to $25k)', 10000.0), + ('Mid ($2k to $10k)', 2000.0), ('Small (<$2k)', 0.0)] def sku_customer_analysis(code, t=None, team_id=None, top=15): diff --git a/platform/modules/reprice.py b/platform/modules/reprice.py index 55925be809309769276b68d6ef0e2f6cd011aa55..404b7304bb6c322bedf5d405fd2b2ed41ed991a4 100644 --- a/platform/modules/reprice.py +++ b/platform/modules/reprice.py @@ -1,5 +1,5 @@ """Reprice lifecycle — the price-side discipline the pricing brief found missing -(.claude/wiki/research/pricing.md): rule hygiene BEFORE the reprice, realization tracking +(AutoWiki/research/pricing.md): rule hygiene BEFORE the reprice, realization tracking AFTER it, and the between-waves views (pocket waterfall, peer price corridor). Everything reads sale.order.line + product.pricelist.item + product masters, read-only. @@ -124,18 +124,18 @@ def rule_health(t=None): flags = [] if not variants or not any_active or (any_active and v_sold <= 0): if not variants or not any_active: - flags.append(('dead', 'Archive rule — product archived/missing')) + flags.append(('dead', 'Archive rule: product archived/missing')) elif v_sold <= 0: - flags.append(('dead', 'Archive rule — no sales in 12m (all channels)')) + flags.append(('dead', 'Archive rule: no sales in 12m (all channels)')) if str(r.get('write_date') or '')[:10] < stale_cut and v_sold > 0: - flags.append(('stale', f'Reprice — untouched >{STALE_MONTHS}m, still selling')) + flags.append(('stale', f'Reprice: untouched >{STALE_MONTHS}m, still selling')) if min_cost is not None and price > 0 and price < min_cost and v_sold > 0: - flags.append(('below_cost', 'Raise NOW — tier below current unit cost')) + flags.append(('below_cost', 'Raise NOW: tier below current unit cost')) below_cost_gap += (min_cost - price) * v_sold siblings = seen_keys.get(_target_key(r), []) if len(siblings) > 1 and any(s is not r and _windows_overlap(r, s) for s in siblings): - flags.append(('overlap', 'Resolve precedence — duplicate qty-break rule')) + flags.append(('overlap', 'Resolve precedence: duplicate qty-break rule')) if not flags: continue for kind, _v in flags: @@ -204,7 +204,7 @@ def realization(event_date, t=None, team_id=None, before_days=180, after_min_day t = t or P.today() ev = dt.date.fromisoformat(event_date) if (t - ev).days < after_min_days: - return {'rows': [], 'note': f'event only {(t - ev).days}d old — too early to score', + return {'rows': [], 'note': f'event only {(t - ev).days}d old; too early to score', 'event': event_date} rules = O.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'), @@ -390,7 +390,7 @@ def validate(t=None, team_id=None, pre_mp=None): [('compute_price', '=', 'fixed'), ('applied_on', 'in', ['0_product_variant', '1_product'])]) rh = rule_health(t) - checks = [{'check': 'pricelist rules — pull complete', + checks = [{'check': 'pricelist rules: pull complete', 'a': rh['n_rules'], 'b': n, 'gap': rh['n_rules'] - n, 'ok': rh['n_rules'] == n}] wf = waterfall(team_id, t, pre=pre_mp) @@ -406,4 +406,4 @@ def validate(t=None, team_id=None, pre_mp=None): 'a': round(s, 2), 'b': round(wf['below'], 2), 'gap': round(s - wf['below'], 2), 'ok': abs(s - wf['below']) < 1.0}) - return checks \ No newline at end of file + return checks diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index e8ac50b5559337145d61dcd81ed1dff3ee6e8638..f40b5ce5c62828b64305a531a666ceb6bb665fcd 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -31,10 +31,11 @@ The only SQL this module writes itself is the aggregate wrapper and the spec's o from __future__ import annotations +import datetime import re -#: C2's aggregate vocabulary. `metric_key` is the other arm of that union and is resolved by the -#: route against the same catalogue `routes_query.py::_compatible_metric_dtos` serves. +#: C2's aggregate vocabulary. `metric_key` is the other arm of that union, and W53-T13 answers it +#: by NAME rather than leaving it dead: see `metric_key_refusal` and its refusal's own note. AGGREGATE_FNS = ("sum", "count", "avg", "min", "max") #: ⭐ R9's PERIOD (wave 51, W51-T21). The three units a bound cell can be pinned to, and the month @@ -69,6 +70,28 @@ REFUSALS = { "aggregate_unknown": "{subject} is not an aggregate this cell can compute.", "field_unknown": "{subject} is not a column on this database.", "period_invalid": "{subject} is not a period this cell can use.", + # ── W53-T13's two refusals, and the reason each of them is NOT `field_unknown` ──────────── + # + # ⛔⛔ `field_unknown`'s sentence is "{subject} is not a column on this database", and for a + # column that IS on this database and simply cannot be TOTALLED here, that sentence is false. + # A false refusal sends the reader to the column picker to look for a column that is already + # in front of them. So the two situations are two codes: one says the column is not here, the + # other says it is here and names what stops it being folded into one number. + # + # ⚠ `{detail}` IS SUPPLIED BY `aggregate_refusal`, NEVER BY THE CALL SITE. The reason is a + # property of the COLUMN (a browser-computed formula, a rollup that folds to text, a text + # column somebody asked to sum), and computing it in one place is what stops three doors + # wording the same fact three ways ([[one-question-two-normalizers]]). + "field_not_aggregable": "{subject} cannot be totalled on this database, because {detail}", + # ⭐ C2's OTHER aggregate arm. `{metric_key}` names a number measured over a whole SET, and a + # set measure is not additive: summing a per-record average or a percentage over the rows this + # cell scopes produces a confident wrong number. The Metric COLUMN path answers the same + # question honestly, per database and per period, which is exactly ruling R7, so the sentence + # sends the reader there rather than stopping at "no". + "metric_key_unsupported": "{subject} is measured over a whole set of records rather than " + "stored on each one, so this cell cannot total it directly. Add a " + "Metric column for it on this database, then point this cell at " + "that column.", # ── C5's two sharing refusals (wave 51, W51-T25) ────────────────────────────────────────── # # ⭐⭐ THEY LIVE HERE, WITH C2's, BECAUSE THIS DICT IS THE ONE PLACE A SHEET REFUSAL IS @@ -87,13 +110,22 @@ REFUSALS = { class SheetRefusal(Exception): - """A named refusal. `code` is C2 vocabulary; `message` is the sentence a person reads.""" + """A named refusal. `code` is C2 vocabulary; `message` is the sentence a person reads. - def __init__(self, code, subject=""): + ⚠ `detail` IS OPTIONAL AND EVERY EXISTING CALL SITE IS UNAFFECTED (W53-T13). `str.format` + ignores a keyword no template names, so a template written before this argument existed is + formatted exactly as it was; only `field_not_aggregable` reads `{detail}` today, and that + sentence is unreadable without it, which is why the reason travels WITH the code rather than + being pasted at each door. + """ + + def __init__(self, code, subject="", detail=""): template = REFUSALS.get(code) or "{subject} could not be answered." self.code = code self.subject = str(subject or "") - self.message = template.format(subject=self.subject or "this database") + self.detail = str(detail or "") + self.message = template.format(subject=self.subject or "this database", + detail=self.detail) super().__init__(self.message) @@ -108,6 +140,17 @@ def clean_period(raw): ⚠ `locked` STORES ONLY TRUE. R9's lock means "a copy of this cell keeps this period"; an unlocked period is what every cell already is, so storing `false` would make a cell that was locked and unlocked compare unequal to the cell it started as. + + ⛔⛔ D-660 (W53-T16) - A PERIOD THAT NAMES NO WINDOW IS NOT A PERIOD, AND STORING ONE IS THE + WHOLE DEFECT. `{"unit": "month"}` with no `start`, or `{"locked": true}` on its own, used to be + stored as a period. `period_window` reads either as None, so the CELL still answered all time - + but the binding now carried a `period` key, and the editor reading it back sees a declared unit, + fills the missing start with TODAY'S MONTH and shows it as a chosen period. The next save then + stamps `{unit: "month", start: }` over a cell nobody meant to narrow: an all-time + total silently becomes this month's, which is exactly what D-660 names. So a period must name a + window - a `start`, the legacy `window`, or R14's `ref` - or it is dropped and the cell stays + all time. ⚠ THE ORDER MATTERS: an unknown unit or a malformed start is still REFUSED BY NAME + above, because a wrong period and an absent one are different answers. """ if not isinstance(raw, dict): return None @@ -140,7 +183,260 @@ def clean_period(raw): if raw.get("window") is not None and "unit" not in out and "start" not in out: out["window"] = raw["window"] - return out or None + #: ⛔ D-660: see the docstring. A unit on its own, a lock on its own, or the empty dict names no + #: window, so it is not a period and the cell keeps its all-time answer. + if not ({"start", "window", "ref"} & set(out)): + return None + return out + + +#: ⭐ R7's PERIOD ARITHMETIC, spelled once. How many months each unit spans, which is the whole +#: difference between the three: a period is always named by the month it OPENS on (`clean_period` +#: enforces the `YYYY-MM` spelling for that reason), so the unit only decides where it closes. +PERIOD_MONTHS = {"month": 1, "quarter": 3, "year": 12} + + +def period_window(period): + """⭐⭐ R7 — a bound cell's PERIOD as a `harness.windows` CUSTOM window, or None. + + OWNER, R7: *"must also be bounded by date time lookup"*. A Metric column carries its OWN + declared window (`{kind: "ltm"}` on both of the two live ones), and that window is what + `core/measure_resolve.py::column_values` resolves it at. So a cell that says "August 2026" over + a Metric column would otherwise print a TWELVE MONTH number under a one-month label: an answer + WIDER than the binding asked for, with nothing on screen saying so, which is standing rule 1's + second sentence turned into a defect. This function is what lets the route re-resolve that + column at the window the cell actually names. + + ⛔ THE SHAPE IS `{"kind": "custom", "from": ..., "to": ...}` BECAUSE THAT IS THE ONE SHAPE + `harness/windows.py::normalize` ACCEPTS FOR AN ABSOLUTE RANGE, and `windows.resolve` turns it + into the INCLUSIVE pair every measure query is built from. Inventing a `{unit, start}` window + kind would be a second date vocabulary beside the one the whole harness already speaks + ([[date-window-vocabulary]]). + + ⛔ `to` IS THE LAST DAY OF THE PERIOD, NOT THE FIRST DAY OF THE NEXT ONE. `windows.resolve`'s + contract is a pair of INCLUSIVE ISO dates, so an exclusive end would silently pull the whole of + the following month into a monthly cell. It is computed as "the day before the month after the + period", never as a 28/30/31 table, so February and a leap year need no special case. + + ⚠ A `{window}` PERIOD (the wave-49 spelling `clean_period` still round trips) IS RETURNED AS + IT STANDS. It already IS a window spec; normalising it here would be this module's second + opinion about a vocabulary `harness/windows.py` owns, and `normalize` refuses anything it does + not recognise anyway. + + ⚠ None MEANS "THIS CELL NAMES NO PERIOD", and the caller must read it as "leave every Metric + column at its own declared window", never as "all time". A period that carries only a `ref` + (R14's cell reference) resolves to None here too: the reference is read by the A1 layer before + a query is built, and guessing a window for an unresolved reference would answer a question + nobody asked. + """ + if not isinstance(period, dict): + return None + if period.get("window") is not None: + return period["window"] + start = str(period.get("start") or "").strip() + if not _PERIOD_START.match(start): + return None + unit = str(period.get("unit") or "month").strip().lower() + months = PERIOD_MONTHS.get(unit) + if not months: + return None + year, month = int(start[:4]), int(start[5:7]) + # The exclusive month AFTER the period, as a flat month index, then one day back off its first. + after = (year * 12 + (month - 1)) + months + end_year, end_month = divmod(after, 12) + last = datetime.date(end_year, end_month + 1, 1) - datetime.timedelta(days=1) + return {"kind": "custom", "from": f"{start}-01", "to": last.isoformat()} + + +#: The column types `harness/filter_sql.py::NUMERIC_TYPES` compiles as numbers AND +#: `odoo_relational._REGISTRY_COL_TYPE` materialises as a number. The two lists have to agree or a +#: column is stored as one thing and compared as another. +MIRROR_NUMERIC_TYPES = ("currency", "int", "pct") + +#: ⭐ THE COMPUTED KINDS THE MIRROR HAS TO BE TOLD ABOUT. `_REGISTRY_COL_TYPE` maps the ordinary +#: field types and falls back to VARCHAR for everything else, so a `metric` or `rating` column +#: would be materialised as TEXT and then either refuse to SUM or compare as a string. Both hold a +#: number, so both are declared as one. +MIRROR_TYPE = {"metric": "currency", "rating": "int"} + +#: The rollup folds that produce a NUMBER. The rest of `core/user_tables.py::ROLLUP_FNS` +#: (`latest`, `and`/`or`/`xor`, `concatenate`, `arrayjoin`, `arraycompact`, `arrayunique`) folds +#: its linked rows into text or a boolean, and a SUM over those is not a smaller number, it is a +#: DuckDB binder error or a meaningless one. +NUMERIC_ROLLUP_FNS = ("sum", "average", "stdev", "min", "max", "count", "counta", "countall") + + +def _rollup_is_numeric(field): + """True when a `rollup` column's fold produces a number.""" + bag = (field or {}).get("rollup") + if not isinstance(bag, dict): + return False + # A SOURCE-backed rollup names a semantic TOPIC and a METRIC KEY (`core/user_tables.py:: + # _clean_rollup`), and a metric is a number by construction. + if isinstance(bag.get("source"), dict): + return True + return str(bag.get("fn") or "").strip().lower() in NUMERIC_ROLLUP_FNS + + +def mirror_field_type(field): + """The type ONE tenant column should be materialised and compiled as on the mirror. + + ⭐⭐ W53-T13 — THIS IS WHAT MAKES A TENANT'S OWN COLUMN A REAL SQL COLUMN RATHER THAN TEXT. + `odoo_relational.registry_grid_spec` reads a field's `type` twice: once to pick the temp + table's column type through `_REGISTRY_COL_TYPE`, and once (through + `routes_odoo_tables._filter_columns`) to tell `filter_sql` whether the column is a number. A + `metric`, `rating` or numeric `rollup` column is missing from BOTH tables, so it lands as + VARCHAR and compares as a string: `sum` over it is a binder error and `>` over it sorts + "1000" before "9". + + ⛔ IT CHANGES NOTHING A PERSON SEES. This type is the SQL shape of one throwaway temp column; + the field's own declared type still governs the grid, the field editor and every format. A + numeric rollup is declared `currency` here only because `currency` is what `filter_sql` + already means by "a number", not because the column is money. + + ⚠ `formula` IS DELIBERATELY LEFT ALONE. Its value is worked out in the BROWSER for each row + and is never stored, so declaring it a number would materialise a column of NULLs and hand + back a confident zero. `aggregate_refusal` names it instead. + """ + kind = str((field or {}).get("type") or "").strip() + if kind == "rollup": + return "currency" if _rollup_is_numeric(field) else "text" + return MIRROR_TYPE.get(kind, kind or "text") + + +def mirror_fields(fields): + """`fields`, with every column's type replaced by the one the mirror can hold. Copies.""" + return [dict(f, type=mirror_field_type(f)) for f in (fields or []) if isinstance(f, dict)] + + +def aggregate_refusal(aggregate, field): + """The named refusal for an aggregate this database cannot answer, or None. + + ⛔⛔ IT RUNS BEFORE THE SQL IS BUILT, WHICH IS THE ONLY PLACE IT CAN RUN HONESTLY. Once every + tenant column is a real column of the materialised table (W53-T13's whole point), a `sum` over + a text column reaches DuckDB and comes back as a binder error, i.e. a 500 with no sentence, and + a `sum` over a browser-computed formula column reaches DuckDB and comes back as 0 — which is + worse, because nothing anywhere looks wrong. + + ⚠ `min` AND `max` ARE NOT REFUSED ON A TEXT COLUMN. They are defined over any ordered type in + SQL and answer a real question there ("the earliest code", "the last status"); only `sum` and + `avg` need a number. Refusing them would take away an answer this database can give. + + ⚠ `field` IS THE ASSEMBLY'S OWN FIELD DICT, not the binding's text. An unknown key is None + here and stays `aggregate_sql`'s `field_unknown` to report, because "not a column on this + database" is then exactly true. + """ + if not isinstance(field, dict): + return None + fn = str((aggregate or {}).get("fn") or "").strip().lower() + if fn in ("", "count"): + return None + label = str(field.get("label") or field.get("key") or "That column") + kind = str(field.get("type") or "").strip() + if kind == "formula": + return SheetRefusal( + "field_not_aggregable", label, + "its value is worked out in the browser for each row and is never stored, so this " + "database holds nothing to add up. Total the columns the formula reads instead.") + if fn not in ("sum", "avg"): + return None + if kind == "rollup" and not _rollup_is_numeric(field): + return SheetRefusal( + "field_not_aggregable", label, + "it folds its linked records into text rather than into a number. Change the rollup " + "to a numeric fold, or total the linked column directly.") + if mirror_field_type(field) not in MIRROR_NUMERIC_TYPES: + return SheetRefusal( + "field_not_aggregable", label, + "it holds text rather than a number. Count the records instead, or pick a number " + "column.") + return None + + +def metric_key_refusal(aggregate): + """The named refusal for C2's `{metric_key}` arm, or None when the binding is not that arm. + + ⛔⛔ IT IS ANSWERED RATHER THAN LEFT TO FALL THROUGH, and that is the whole of W53-T13's (d). + `clean_binding` accepts `{metric_key}` and nothing downstream ever resolved it, so it reached + `aggregate_sql` with no `fn` and no `field` and died as `field_unknown` with an EMPTY subject: + "That column is not a column on this database." Measured on the running app for + `{"metric_key": "revenue"}`. That sentence names nothing and is not even true. + + ⭐ WHY IT REFUSES RATHER THAN RESOLVES, and this is standing rule 9 read forwards. A metric is + measured over a SET: `harness/measure_filter.py::resolve_series`' own docstring records the + additivity law it obeys, that a non-additive measure (an average, a percentage) is computed at + POOL grain and never folded from per-record answers. A bare metric key carries no aggregate + and no window, so resolving it here would mean choosing both, and choosing `sum` would print a + wrong number for every ratio metric in the catalogue with nothing to notice. The Metric COLUMN + path answers the same question with the aggregate and the window both DECLARED, per database, + which is precisely what R7 asked for, so the sentence sends the reader one step sideways + rather than stopping them. + """ + key = str((aggregate or {}).get("metric_key") or "").strip() + if not key: + return None + return SheetRefusal("metric_key_unsupported", key) + + +def period_limits(period, aggregate, field, bounded_keys=(), depends=()): + """The `limits` rows for a PERIOD that did not narrow the number (R7, standing rule 1). + + ⛔⛔ A PERIOD THAT DOES NOTHING MUST SAY SO. A cell labelled "August 2026" over a stored column + counts every record the filter selects, in every month there has ever been. The number is not + wrong for the query that ran; it is wider than the query the person wrote, and this repo's rule + is that the widening travels WITH the value rather than being left for someone to notice. + + ⛔⛔ AND THE MIRROR OF THAT RULE IS JUST AS BINDING: A PERIOD THAT DID SOMETHING MUST NOT + CLAIM IT DID NOTHING. `depends` is every column the binding reads, the FILTER's included, and + it is here because the first version of this function looked at the aggregate's own column + alone. Measured on the running app: `count` over the filter `Sales the last 12 months > 100000` + answered 1 with no period and 0 for August 2026 - the period plainly bit, through the filter - + and this function still reported that it had been ignored. A false widening report is not a + harmless extra sentence; it teaches a reader to stop believing the true ones. + + ⚠ `bounded_keys` IS WHAT THE ROUTE ACTUALLY RE-RESOLVED at this period, never what it hoped to. + A Metric column that failed to resolve at the requested window is NOT in that set, so it is + reported here as well as by its own row: a cell whose Metric fell back to blank has not been + bounded either. + """ + if not period: + return [] + fn = str((aggregate or {}).get("fn") or "").strip().lower() + key = str((aggregate or {}).get("field") or "").strip() + read = set(depends or ()) + if key: + read.add(key) + if read & set(bounded_keys or ()): + return [] + subject = (str((field or {}).get("label") or key) if key + else ("This count" if fn == "count" else "This cell")) + return [{ + "subject": subject, + "effect": "period_ignored", + "cause": "this cell names a period, and the column it totals holds one stored value per " + "record rather than a measurement over time, so the period does not narrow it " + "and this number covers every record the filter selects", + "recommendation": "point the cell at a Metric column, which is measured over the period " + "you choose, or add a date condition to the cell's own filter", + }] + + +def metric_window_limit(subject, cause): + """The `limits` row for a Metric column that could NOT be resolved at the cell's period. + + ⛔⛔ THE ALTERNATIVE IS THE DEFECT. Falling back to the column's own declared window would + answer twelve months under a one-month label and look completely ordinary. The column goes + BLANK and this row says which column, and why, and what to do about it. + """ + return { + "subject": str(subject or "This metric column"), + "effect": "metric_not_bounded", + "cause": str(cause or "this column could not be measured over the period this cell " + "names, so it was left blank rather than answered over its own " + "wider window"), + "recommendation": "widen the cell's period, or open the database and check that the " + "metric behind this column can be measured over that range", + } def clean_on_drag(raw): diff --git a/platform/modules/spend.py b/platform/modules/spend.py index 0569ceb81e5d52a6f9af393720777ca2ba098d02..82f8fc5176320d74c4e1afb57357cf9eaa6559b5 100644 --- a/platform/modules/spend.py +++ b/platform/modules/spend.py @@ -286,7 +286,7 @@ def validate(t=None, team_id=None, pre=None): ('invoice_date', '>=', lf), ('invoice_date', '<=', lt)], 'amount_total') a = tc['spend_total'] - checks = [{'check': 'vendor-bill spend — client Σ == server Σ', + checks = [{'check': 'vendor-bill spend: client Σ == server Σ', 'a': round(a, 2), 'b': round(srv, 2), 'gap': round(a - srv, 2), 'ok': abs(a - srv) <= max(1.0, srv * 0.001)}] all_lines = _bill_lines(lf, lt) @@ -298,4 +298,4 @@ def validate(t=None, team_id=None, pre=None): checks.append({'check': 'opex cube + non-expense-type lines == all bill lines', 'a': round(lhs, 2), 'b': round(full, 2), 'gap': round(lhs - full, 2), 'ok': abs(lhs - full) < 1.0}) - return checks \ No newline at end of file + return checks diff --git a/platform/odoo_client.py b/platform/odoo_client.py index b7cba17ac397b77ac60770d958f601c3d8e2cff3..1c40354fccc1d32807f15d0a4795729d9715eeaa 100644 --- a/platform/odoo_client.py +++ b/platform/odoo_client.py @@ -61,12 +61,28 @@ class OdooClient: def __init__(self, creds=None): """`creds` (R3 keychain cutover): an explicit {url, db, user, api_key} dict — a TENANT'S stored credentials — wins over the environment. None keeps the env/.env behavior every - existing caller relies on (tenant #0's connection).""" + existing caller relies on (tenant #0's connection). + + ⛔⛔ W54-T45 / R7 — IT IS ALL-OR-NOTHING, AND THE PER-FIELD VERSION WAS A REAL LEAK SHAPE. + Each field used to read `creds.get(k) or os.environ.get(...)` INDIVIDUALLY, so an explicit + connection whose stored `db` was blank or missing silently inherited tenant #0's `ODOO_DB` + while keeping the caller's `url`, `user` and `api_key`. Under R4 an agency holds one Odoo + connection per client and under R7 that isolation is strict: a half-environment client is + one client's question authenticated against another's database, and it does not raise — + it returns real, plausible, wrong rows. So a supplied `creds` is used WHOLE, and a field + it is missing is MISSING (the `RuntimeError` below names it) rather than borrowed. With no + `creds` at all the environment builds the client exactly as it always did. + """ + from_env = creds is None creds = creds or {} - self.url = str(creds.get('url') or os.environ.get('ODOO_URL', '')).rstrip('/') - self.db = str(creds.get('db') or os.environ.get('ODOO_DB', '')) - self.user = str(creds.get('user') or os.environ.get('ODOO_USER', '')) - self.api_key = str(creds.get('api_key') or os.environ.get('ODOO_API_KEY', '')) + + def _f(key, env_key): + return str((os.environ.get(env_key, '') if from_env else creds.get(key)) or '') + + self.url = _f('url', 'ODOO_URL').rstrip('/') + self.db = _f('db', 'ODOO_DB') + self.user = _f('user', 'ODOO_USER') + self.api_key = _f('api_key', 'ODOO_API_KEY') missing = [k for k, v in [('ODOO_URL', self.url), ('ODOO_DB', self.db), ('ODOO_USER', self.user), ('ODOO_API_KEY', self.api_key)] if not v] if missing: @@ -95,12 +111,12 @@ class OdooClient: _time.sleep(3) return self._connect(_retried=True) if not self.uid: - raise RuntimeError("Odoo authentication failed — check credentials.") + raise RuntimeError("Odoo authentication failed; check credentials.") self.models = _proxy(f'{self.url}/xmlrpc/2/object') def _guard(self, method): if self.READ_ONLY and (method in WRITE_METHODS or any(method.startswith(p) for p in WRITE_PREFIXES)): - raise ReadOnlyError(f"BLOCKED write op '{method}' — dashboard is read-only.") + raise ReadOnlyError(f"BLOCKED write op '{method}'; dashboard is read-only.") def execute_kw(self, model, method, args, kwargs=None): self._guard(method) diff --git a/requirements.txt b/requirements.txt index 5f94c7afcab8b6ff3762fc8925e148ca8ee0e9f9..48425ff9155e75f9f6b5ada5e14160cd401cb1e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,9 @@ # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the # import lives inside the handler, and the gate now asserts this line exists here. fastapi>=0.139 +starlette>=1.3.1 uvicorn[standard]>=0.30 -python-dotenv>=1.0 +python-dotenv>=1.2.2 pandas>=2.0 requests>=2.28 huggingface_hub>=0.20 @@ -44,7 +45,7 @@ openpyxl>=3.1 # python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate # was green because the dev box happens to have it for unrelated reasons — the container # installs exactly this file. The environment-parity twin of the streamlit lesson above. -python-multipart>=0.0.20 +python-multipart>=0.0.31 # ⭐ WAVE 37 (D-346, answering `ASK E-1`) — THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the # Messages API over raw `requests`, which meant re-implementing retries, streaming and error # taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..19eba657181a54f4eef9d7b52c03413cddeedbdc --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,3466 @@ +{ + "name": "web", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "web", + "version": "0.0.0", + "dependencies": { + "@glideapps/glide-data-grid": "^6.0.3", + "lodash": "^4.18.1", + "marked": "^4.3.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "vega": "^6.3.1", + "vega-embed": "^7.1.0", + "vega-lite": "^6.4.3" + }, + "devDependencies": { + "@types/node": "^24.13.2", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^6.0.3", + "oxlint": "^1.71.0", + "typescript": "~6.0.2", + "vite": "^8.1.1", + "vite-plugin-singlefile": "^2.3.3" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz", + "integrity": "sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==", + "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-export-namespace-from instead.", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.29.7.tgz", + "integrity": "sha512-j0vCldybPC5b5dwCQOJ21uKtHzt7hxLygJTg9eF1ScfaikEDNfzn94XoW5Fi+seBR0nCyL23xaBFFkq7dTM8XQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/core/node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emotion/is-prop-valid": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.4.0.tgz", + "integrity": "sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0" + } + }, + "node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, + "node_modules/@glideapps/glide-data-grid": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz", + "integrity": "sha512-YXKggiNOaEemf0jP0jORq2EQKz+zXms+6mGzZc+q0mLMjmgzzoGLOQC1uYcynXSj1R61bd27JcPFsoH+Gj37Vg==", + "license": "MIT", + "dependencies": { + "@linaria/react": "^4.5.3", + "canvas-hypertxt": "^1.0.3", + "react-number-format": "^5.0.0" + }, + "peerDependencies": { + "lodash": "^4.17.19", + "marked": "^4.0.10", + "react": "^16.12.0 || 17.x || 18.x", + "react-dom": "^16.12.0 || 17.x || 18.x", + "react-responsive-carousel": "^3.2.7" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@linaria/core": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@linaria/core/-/core-4.5.4.tgz", + "integrity": "sha512-vMs/5iU0stxjfbBCxobIgY+wSQx4G8ukNwrhjPVD+6bF9QrTwi5rl0mKaCMxaGMjnfsLRiiM3i+hnWLIEYLdSg==", + "license": "MIT", + "dependencies": { + "@linaria/logger": "^4.5.0", + "@linaria/tags": "^4.5.4", + "@linaria/utils": "^4.5.3" + }, + "engines": { + "node": "^12.16.0 || >=13.7.0" + } + }, + "node_modules/@linaria/logger": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@linaria/logger/-/logger-4.5.0.tgz", + "integrity": "sha512-XdQLk242Cpcsc9a3Cz1ktOE5ysTo2TpxdeFQEPwMm8Z/+F/S6ZxBDdHYJL09srXWz3hkJr3oS2FPuMZNH1HIxw==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "picocolors": "^1.0.0" + }, + "engines": { + "node": "^12.16.0 || >=13.7.0" + } + }, + "node_modules/@linaria/react": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@linaria/react/-/react-4.5.4.tgz", + "integrity": "sha512-/dhCVCsfdGPfQCPV0q5yy+DDlFXepvfXrw/os2fC+Xo1v9J/9gyiaBBWHzcumauvNNFj8aN6vRkj89fMujPHew==", + "license": "MIT", + "dependencies": { + "@emotion/is-prop-valid": "^1.2.0", + "@linaria/core": "^4.5.4", + "@linaria/tags": "^4.5.4", + "@linaria/utils": "^4.5.3", + "minimatch": "^9.0.3", + "react-html-attributes": "^1.4.6", + "ts-invariant": "^0.10.3" + }, + "engines": { + "node": "^12.16.0 || >=13.7.0" + }, + "peerDependencies": { + "react": ">=16" + } + }, + "node_modules/@linaria/tags": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/@linaria/tags/-/tags-4.5.4.tgz", + "integrity": "sha512-HPxLB6HlJWLi6o8+8lTLegOmDnbMbuzEE+zzunaPZEGSoIIYx8HAv5VbY/sG/zNyxDElk6laiAwEVWN8h5/zxg==", + "license": "MIT", + "dependencies": { + "@babel/generator": "^7.22.9", + "@linaria/logger": "^4.5.0", + "@linaria/utils": "^4.5.3" + }, + "engines": { + "node": "^12.16.0 || >=13.7.0" + } + }, + "node_modules/@linaria/utils": { + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/@linaria/utils/-/utils-4.5.3.tgz", + "integrity": "sha512-tSpxA3Zn0DKJ2n/YBnYAgiDY+MNvkmzAHrD8R9PKrpGaZ+wz1jQEmE1vGn1cqh8dJyWK0NzPAA8sf1cqa+RmAg==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.22.9", + "@babel/generator": "^7.22.9", + "@babel/plugin-proposal-export-namespace-from": "^7.18.9", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-transform-modules-commonjs": "^7.22.5", + "@babel/template": "^7.22.5", + "@babel/traverse": "^7.22.8", + "@babel/types": "^7.22.5", + "@linaria/logger": "^4.5.0", + "babel-merge": "^3.0.0", + "find-up": "^5.0.0", + "minimatch": "^9.0.3" + }, + "engines": { + "node": "^12.16.0 || >=13.7.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.75.0.tgz", + "integrity": "sha512-lutovtFzJqlRaqpZrCqSSGaHZzl9nIxxpjLzhSRLunN6dCLylj0uzlCyQGaQDIys7rrv8kVXiFO+R4Zpn0bX7g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.75.0.tgz", + "integrity": "sha512-hXI0hDgHkw4w5nfru72aG7y+2iQJmC4waH/KV6H/hbgA6yAP5jYNx0P9yug15Hs0tWl/+mda3Jjn/2gmDT48tw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.75.0.tgz", + "integrity": "sha512-D91BWbK/dMYfCcrghspPIuKs2D9LF4Z/OabVSQjw1AO6PWxArD7teDA48bm0ySFqWDaPVqmQRl5GMWNglTXyrQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.75.0.tgz", + "integrity": "sha512-02mpwzf12BonZ6PT0TuQoomvEh2kVl2WGBIKWezCyToIS+rYkQZ6GXnARBAl9A4Ovm2V+Xe7M4KretyqmmcnJQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.75.0.tgz", + "integrity": "sha512-qZJgLnDaBsiL5YESx2t/TZ8eXkL9fEkKoXEdzegROhlz9A0lgyGnZ0dAzJrh7LJAHQl2K9RdRueN2s/9N7+odg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.75.0.tgz", + "integrity": "sha512-7XlaWA5BJD3XpCfrEqjEe6Zseeb14S7QGa304XfwKignRaKQ+eIj775BQ7nIslggWickl4IsPUFqJ+/gAyNHVg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.75.0.tgz", + "integrity": "sha512-av6Tpv8yrcMMMOadOqENBhlsLRcGFXXwoQ0hzHhsmS9FJ4Wioy8we427GbcMe2XTxmL2e60T67H1Dyr3up+tAA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.75.0.tgz", + "integrity": "sha512-WcUhd8fHT5plrA14lANevl+hOl815mVI5t2hU21oFWrZKFXIVV/Sr4rWQV0NzSvzBupbMLNc5ErEA6Ehxh5jMg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.75.0.tgz", + "integrity": "sha512-UWzp5wRHFe/ESO3+eEaxXsTkYTGLYjnTsi/I5neEacXSItQ6WNleapfOAeA4x2b8nyhJ4uQxqvtv9pHv8kWJtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.75.0.tgz", + "integrity": "sha512-XEVRwGMLKCUKrvhLAz4F6AIh8MJrQVdSZtAmPpRZt9tGPsUnamPOcl3dS/ZQzJnar/Ymgc//+xho0L60Emzuxg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.75.0.tgz", + "integrity": "sha512-mAG4DUXqfLC8cTjMD2kt3jDmVzFREYtDyeLNdLdsCcBc4Zbl2EMuiFektGBilQwkNjYnMvCqJs55U+Hyb+b+jw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.75.0.tgz", + "integrity": "sha512-95hrAvriAlI+pekSomTFIn0+bawMDlDwTNVmdjsFusTHyL2JWh7TWvRNG/Lkim72uN8OiCcO9wcaC6omLP5E3w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.75.0.tgz", + "integrity": "sha512-4b6f2+FrtruAESrCqIKcrarzfrSx+wk2QNcp+RT91/Prc+pMQMAfyZ1rG1c3tFQNl8Bc616tx40uNXyxNBRPbQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.75.0.tgz", + "integrity": "sha512-nshAhrUvXFUWOvqQ2soIw7HFNWvpvEV4o0cYSqPtzLiPF5gKyYTDOOTJ6Rn8g8K/iGvPIrbDA4v8+5MvnjJrrg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.75.0.tgz", + "integrity": "sha512-e4jNxLKnxLC6sYBQRxrI2pgIIxnmMtF8U/VwNYcjTT/CLS+spH624cYVnj07bTKwaEWT37/e025isOs6j/0xqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.75.0.tgz", + "integrity": "sha512-hZ2lH+1qLf/DiEP9UWuQTK2JWj/BgvMB4jhIV4SmNU1wfEiYYX4TynQyAZXx0j9X4qRYizAL042SKaV+8ynh4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.75.0.tgz", + "integrity": "sha512-Ilj6PNzGDS3bCU0MSJH7Msh0NhH+T/mRp2shwg+q+GHeVlPwP5LEboW96aW+3kVKFk6zYZy1Xi5pZkqZh6X8KQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.75.0.tgz", + "integrity": "sha512-QVit2nOEOiPhkmsrksPSkoGCdnZRNkspt8fwoYyP09te1VEbnSj4LAxua4rc8FKTmWkySVe05j8iz9GXYfF1AQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.75.0.tgz", + "integrity": "sha512-DSxnNkBUAYARPwJtR12Ig3deWr8w0H997xP6jy33i+e0SyYJw8FKuz4+cZtpmPEhQmvlPJE3X/2vNxDmLkd/rA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "license": "MIT" + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.4.tgz", + "integrity": "sha512-XcCQz0TBpBgljhj0gMuuDj49i6Ytqh5q1osT/Gp5uAVJUCTWxyskk/l1jwYYiu2xcNHHipdMz40EGfM1VdamVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/babel-merge": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/babel-merge/-/babel-merge-3.0.0.tgz", + "integrity": "sha512-eBOBtHnzt9xvnjpYNI5HmaPp/b2vMveE5XggzqHnQeHJ8mFIBrBv6WZEVIj5jJ2uwTItkqKo9gWzEEcBxEq0yw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "dependencies": { + "deepmerge": "^2.2.1", + "object.omit": "^3.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.1.tgz", + "integrity": "sha512-HYXq73DDpCtNzOmrFsm9eSwCvWCql0RzqjpDzXN9EadiLJ4DNat0nsZ/Bzmy+Ud12mb4/zKDY0cQ805ZzN+i0A==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.7", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz", + "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "baseline-browser-mapping": "^2.10.44", + "caniuse-lite": "^1.0.30001806", + "electron-to-chromium": "^1.5.393", + "node-releases": "^2.0.51", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001806", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", + "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/canvas-hypertxt": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/canvas-hypertxt/-/canvas-hypertxt-1.0.3.tgz", + "integrity": "sha512-+VsMpRr64jYgKq2IeFUNel3vCZH/IzS+iXSHxmUV3IUH5dXlC9xHz4AwtPZisDxZ5MWcuK0V+TXgPKFPiZnxzg==", + "license": "MIT" + }, + "node_modules/classnames": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", + "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==", + "license": "MIT" + }, + "node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo-projection": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/d3-geo-projection/-/d3-geo-projection-4.0.0.tgz", + "integrity": "sha512-p0bK60CEzph1iqmnxut7d/1kyTmm3UWtPlwdkM31AU+LW+BXazd5zJdoCn7VFxNCHXRngPHRnsNn5uGjLRGndg==", + "license": "ISC", + "dependencies": { + "commander": "7", + "d3-array": "1 - 3", + "d3-geo": "1.12.0 - 3" + }, + "bin": { + "geo2svg": "bin/geo2svg.js", + "geograticule": "bin/geograticule.js", + "geoproject": "bin/geoproject.js", + "geoquantize": "bin/geoquantize.js", + "geostitch": "bin/geostitch.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.2.1.tgz", + "integrity": "sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.395", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.395.tgz", + "integrity": "sha512-7zt9Aw+SrmxLWLN0zhaTWZQiCdryLVrYTq5R7iZakLvi2UQPYMMsROYV/2qVCzMeCiSXHwKOU+sZ4zOVVlrtKA==", + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-json-patch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz", + "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/html-element-attributes": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/html-element-attributes/-/html-element-attributes-1.3.1.tgz", + "integrity": "sha512-UrRKgp5sQmRnDy4TEwAUsu14XBUlzKB8U3hjIYDjcZ3Hbp86Jtftzxfgrv6E/ii/h78tsaZwAnAE8HwnHr0dPA==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-stringify-pretty-compact": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-4.0.0.tgz", + "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT", + "peer": true + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "license": "MIT", + "peer": true, + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/micromatch/node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.51", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", + "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.omit": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-3.0.0.tgz", + "integrity": "sha512-EO+BCv6LJfu+gBIF3ggLicFebFLN5zqzz/WWJlMFfkMyGth+oBkhxzDl0wx2W4GkLzuQs/FsSkXZb2IMWQqmBQ==", + "license": "MIT", + "dependencies": { + "is-extendable": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/oxlint": { + "version": "1.75.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.75.0.tgz", + "integrity": "sha512-m9WzjRcRYA/uqIZDa9tclrieoPJ/ln1QYTKdFx6NUOs8uY5DiHlIwRQoCrHT6OM6O3ww3l2skY5gO7G7ZphE7g==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.75.0", + "@oxlint/binding-android-arm64": "1.75.0", + "@oxlint/binding-darwin-arm64": "1.75.0", + "@oxlint/binding-darwin-x64": "1.75.0", + "@oxlint/binding-freebsd-x64": "1.75.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.75.0", + "@oxlint/binding-linux-arm-musleabihf": "1.75.0", + "@oxlint/binding-linux-arm64-gnu": "1.75.0", + "@oxlint/binding-linux-arm64-musl": "1.75.0", + "@oxlint/binding-linux-ppc64-gnu": "1.75.0", + "@oxlint/binding-linux-riscv64-gnu": "1.75.0", + "@oxlint/binding-linux-riscv64-musl": "1.75.0", + "@oxlint/binding-linux-s390x-gnu": "1.75.0", + "@oxlint/binding-linux-x64-gnu": "1.75.0", + "@oxlint/binding-linux-x64-musl": "1.75.0", + "@oxlint/binding-openharmony-arm64": "1.75.0", + "@oxlint/binding-win32-arm64-msvc": "1.75.0", + "@oxlint/binding-win32-ia32-msvc": "1.75.0", + "@oxlint/binding-win32-x64-msvc": "1.75.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=7.0.2001", + "vite-plus": "*" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + }, + "vite-plus": { + "optional": true + } + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "peer": true, + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-easy-swipe": { + "version": "0.0.21", + "resolved": "https://registry.npmjs.org/react-easy-swipe/-/react-easy-swipe-0.0.21.tgz", + "integrity": "sha512-OeR2jAxdoqUMHIn/nS9fgreI5hSpgGoL5ezdal4+oO7YSSgJR8ga+PkYGJrSrJ9MKlPcQjMQXnketrD7WNmNsg==", + "license": "MIT", + "dependencies": { + "prop-types": "^15.5.8" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/react-html-attributes": { + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/react-html-attributes/-/react-html-attributes-1.4.6.tgz", + "integrity": "sha512-uS3MmThNKFH2EZUQQw4k5pIcU7XIr208UE5dktrj/GOH1CMagqxDl4DCLpt3o2l9x+IB5nVYBeN3Cr4IutBXAg==", + "license": "MIT", + "dependencies": { + "html-element-attributes": "^1.0.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/react-number-format": { + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/react-number-format/-/react-number-format-5.4.5.tgz", + "integrity": "sha512-y8O2yHHj3w0aE9XO8d2BCcUOOdQTRSVq+WIuMlLVucAm5XNjJAy+BoOJiuQMldVYVOKTMyvVNfnbl2Oqp+YxGw==", + "license": "MIT", + "peerDependencies": { + "react": "^0.14 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^0.14 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/react-responsive-carousel": { + "version": "3.2.23", + "resolved": "https://registry.npmjs.org/react-responsive-carousel/-/react-responsive-carousel-3.2.23.tgz", + "integrity": "sha512-pqJLsBaKHWJhw/ItODgbVoziR2z4lpcJg+YwmRlSk4rKH32VE633mAtZZ9kDXjy4wFO+pgUZmDKPsPe1fPmHCg==", + "license": "MIT", + "peer": true, + "dependencies": { + "classnames": "^2.2.5", + "prop-types": "^15.5.8", + "react-easy-swipe": "^0.0.21" + } + }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, + "node_modules/rolldown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.139.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/topojson-client": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", + "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", + "license": "ISC", + "dependencies": { + "commander": "2" + }, + "bin": { + "topo2geo": "bin/topo2geo", + "topomerge": "bin/topomerge", + "topoquantize": "bin/topoquantize" + } + }, + "node_modules/topojson-client/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/ts-invariant": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/ts-invariant/-/ts-invariant-0.10.3.tgz", + "integrity": "sha512-uivwYcQaxAucv1CzRp2n/QdYPo4ILf9VXgH19zEIjFx2EJufV16P0JtJVpYHy89DItG6Kwj2oIUjrcK5au+4tQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vega": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/vega/-/vega-6.3.1.tgz", + "integrity": "sha512-mX5tvY3ISCSiPPmunuZyQfccq0XlUvJd2t5oyc6SNS0N0TwnPNoklx0mVod5n+qbzuUoiuxl7Ve/SvoRqH4RzA==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "vega-crossfilter": "~5.1.2", + "vega-dataflow": "~6.1.2", + "vega-encode": "~5.2.1", + "vega-event-selector": "~4.0.0", + "vega-expression": "~6.2.1", + "vega-force": "~5.1.2", + "vega-format": "~2.1.2", + "vega-functions": "~6.1.3", + "vega-geo": "~5.1.2", + "vega-hierarchy": "~5.1.2", + "vega-label": "~2.1.2", + "vega-loader": "~5.1.2", + "vega-parser": "~7.1.2", + "vega-projection": "~2.1.2", + "vega-regression": "~2.1.2", + "vega-runtime": "~7.1.2", + "vega-scale": "~8.1.2", + "vega-scenegraph": "~5.2.1", + "vega-statistics": "~2.0.0", + "vega-time": "~3.2.1", + "vega-transforms": "~5.2.1", + "vega-typings": "~2.2.0", + "vega-util": "~2.1.0", + "vega-view": "~6.1.2", + "vega-view-transforms": "~5.2.1", + "vega-voronoi": "~5.1.2", + "vega-wordcloud": "~5.1.2" + }, + "funding": { + "url": "https://app.hubspot.com/payments/GyPC972GD9Rt" + } + }, + "node_modules/vega-canvas": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/vega-canvas/-/vega-canvas-2.0.0.tgz", + "integrity": "sha512-9x+4TTw/USYST5nx4yN272sy9WcqSRjAR0tkQYZJ4cQIeon7uVsnohvoPQK1JZu7K1QXGUqzj08z0u/UegBVMA==", + "license": "BSD-3-Clause" + }, + "node_modules/vega-crossfilter": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-crossfilter/-/vega-crossfilter-5.1.2.tgz", + "integrity": "sha512-K14PtfBxQzb7UJq5rL1IuwDdPov6lucoHVzRaraGiBBnfc963jkWHQNfRFAuEj/nIt5BIixXBiqIrGKgfScjSA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "vega-dataflow": "^6.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-dataflow": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/vega-dataflow/-/vega-dataflow-6.1.2.tgz", + "integrity": "sha512-0/HBBlzPERh0wV5kUOh/N4aAAmb3rB18Kfpej+nWvDIjPk3dixTapNkVVkoA9CsB5dY5bY+HNqiM/Pp4XPduGQ==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-format": "^2.1.2", + "vega-loader": "^5.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-embed": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/vega-embed/-/vega-embed-7.1.0.tgz", + "integrity": "sha512-ZmEIn5XJrQt7fSh2lwtSdXG/9uf3yIqZnvXFEwBJRppiBgrEWZcZbj6VK3xn8sNTFQ+sQDXW5sl/6kmbAW3s5A==", + "license": "BSD-3-Clause", + "dependencies": { + "fast-json-patch": "^3.1.1", + "json-stringify-pretty-compact": "^4.0.0", + "semver": "^7.7.2", + "tslib": "^2.8.1", + "vega-interpreter": "^2.0.0", + "vega-schema-url-parser": "^3.0.2", + "vega-themes": "3.0.0", + "vega-tooltip": "1.0.0" + }, + "funding": { + "url": "https://app.hubspot.com/payments/GyPC972GD9Rt" + }, + "peerDependencies": { + "vega": "*", + "vega-lite": "*" + } + }, + "node_modules/vega-embed/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/vega-encode": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/vega-encode/-/vega-encode-5.2.1.tgz", + "integrity": "sha512-YuvJ66z6FQRtu0n0IYxsJp7BpliXVgaxNsur0Fqic10/a5TLMQmDYURPUcrtWViAMpqKgdNMyxnmr0ZQ4bjwCA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-interpolate": "^3.0.1", + "vega-dataflow": "^6.1.2", + "vega-scale": "^8.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-event-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/vega-event-selector/-/vega-event-selector-4.0.0.tgz", + "integrity": "sha512-CcWF4m4KL/al1Oa5qSzZ5R776q8lRxCj3IafCHs5xipoEHrkgu1BWa7F/IH5HrDNXeIDnqOpSV1pFsAWRak4gQ==", + "license": "BSD-3-Clause" + }, + "node_modules/vega-expression": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-6.2.1.tgz", + "integrity": "sha512-AK4EYh9ErCC7KU8vGigsetwG66S1wLLfPFnfjkOdydRaN4kNJp129qp4jos0wA/r7F/lIlhM3uIzw+KSA2cZEg==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/estree": "^1.0.9", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-force": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-force/-/vega-force-5.1.2.tgz", + "integrity": "sha512-693paaBvACvAtUO8vp4m/KUJ0F3tKaoXixzw+ExpP4qKkqJ/Me2MGWsVrEJh/f6KuU9bp7xv+g7OPNhqnVrpoQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-force": "^3.0.0", + "vega-dataflow": "^6.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-format": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vega-format/-/vega-format-2.1.2.tgz", + "integrity": "sha512-cJ2oqGEGYJ2NbkIIUDic03ZZX1meE+sq330gq4HFf3CvzSupTwHLWQj3/H5zug5DrqEQhLCLkiRwvmMe4pEqfg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-format": "^3.1.2", + "d3-time-format": "^4.1.0", + "vega-time": "^3.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-functions": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/vega-functions/-/vega-functions-6.1.3.tgz", + "integrity": "sha512-WOgWVxGg1T1tuICyylIppxXWXWEtl7P81Xnj0vQtoIn11VvQ0mrsBZr1aia4g8Sw0fEH1mUO7Dtj7v3o8BgBmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-color": "^3.1.0", + "d3-geo": "^3.1.1", + "vega-dataflow": "^6.1.2", + "vega-expression": "^6.2.1", + "vega-scale": "^8.1.2", + "vega-scenegraph": "^5.2.1", + "vega-selections": "^6.1.4", + "vega-statistics": "^2.0.0", + "vega-time": "^3.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-geo": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-geo/-/vega-geo-5.1.2.tgz", + "integrity": "sha512-tls2+IVuKaIOxEqdshuSk2Til8qcF7SUNaWkATKu5hwWqQEY3cp7tfdXosuTMaDTkLp+Nw84iWfe5BzfoxDYMA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-color": "^3.1.0", + "d3-geo": "^3.1.1", + "vega-canvas": "^2.0.0", + "vega-dataflow": "^6.1.2", + "vega-projection": "^2.1.2", + "vega-statistics": "^2.0.0", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-hierarchy": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-hierarchy/-/vega-hierarchy-5.1.2.tgz", + "integrity": "sha512-7a72/lA6j50HU6PQe5ChbomftHi29V1oCcibaQpDozZncHuAnsQummqfiPyTMPpNMrSRtyZy18do0IEoOvrbMg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-hierarchy": "^3.1.2", + "vega-dataflow": "^6.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-interpreter": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/vega-interpreter/-/vega-interpreter-2.3.1.tgz", + "integrity": "sha512-IYleUp+lXVIGjJUQg+5WJ/xUdteMxEGEnSyWtRhagVuUDiXD0JnERKWffIrQSCUypA86UOuyjzVDtb7oNOsaxw==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-label": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vega-label/-/vega-label-2.1.2.tgz", + "integrity": "sha512-MFsOKOCG6a5QwP5atoFFi+R58IP/Q7//65UvgrSeankX2QrDcWjBtzIqS6JlYBOAdyjoiAAghx7pnIAj5nAO6A==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-canvas": "^2.0.0", + "vega-dataflow": "^6.1.2", + "vega-scenegraph": "^5.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-lite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vega-lite/-/vega-lite-6.4.3.tgz", + "integrity": "sha512-d/7hPjfz560UERaQuTmGgIVfXAe3g2hJWeC+igDeaGohUdEoNrHLXgR/yTOBT8vV/lIuuKnw+0/xWWblkDwkMQ==", + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "json-stringify-pretty-compact": "~4.0.0", + "tslib": "~2.8.1", + "vega-event-selector": "~4.0.0", + "vega-expression": "~6.1.0", + "vega-util": "~2.1.0", + "yargs": "~18.0.0" + }, + "bin": { + "vl2pdf": "bin/vl2pdf", + "vl2png": "bin/vl2png", + "vl2svg": "bin/vl2svg", + "vl2vg": "bin/vl2vg" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://app.hubspot.com/payments/GyPC972GD9Rt" + }, + "peerDependencies": { + "vega": "^6.0.0" + } + }, + "node_modules/vega-lite/node_modules/vega-expression": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-6.1.0.tgz", + "integrity": "sha512-hHgNx/fQ1Vn1u6vHSamH7lRMsOa/yQeHGGcWVmh8fZafLdwdhCM91kZD9p7+AleNpgwiwzfGogtpATFaMmDFYg==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/estree": "^1.0.8", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-loader": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-loader/-/vega-loader-5.1.2.tgz", + "integrity": "sha512-O29bKHsSJBi391H88OZXH8vgUdopWMorC5ZJ/8XjTq8TIXVdeUrNnI8dMkUyv7yQg3ywMTji+5fHhov1TOkkyA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-dsv": "^3.0.1", + "topojson-client": "^3.1.0", + "vega-format": "^2.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-parser": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/vega-parser/-/vega-parser-7.1.2.tgz", + "integrity": "sha512-88HNkVrEwWbMmbbhPCmFHPrl/ClTcvThRv2UMZqMJJ5IqgjWLUolCA1fhAvnAbvRPe+Z1j+GbV/WWjYWAjEuyw==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-dataflow": "^6.1.2", + "vega-event-selector": "^4.0.0", + "vega-functions": "^6.1.3", + "vega-scale": "^8.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-projection": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vega-projection/-/vega-projection-2.1.2.tgz", + "integrity": "sha512-HL+FS9AgYSOa1UOIJfhOD39RMT4tDODkMcN6MCRsdlRNnukVm1+JhrixDnciBjqiZLxDhUjESJwgLQtXx8cUyw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-geo": "^3.1.1", + "d3-geo-projection": "^4.0.0", + "vega-scale": "^8.1.2" + } + }, + "node_modules/vega-regression": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vega-regression/-/vega-regression-2.1.2.tgz", + "integrity": "sha512-2dyb2sT6cxdTK5K87uNDYpW7LbTP90lplxW62imjfLAwKUq1wL74X8y03a0W5tpRfV8+f3ai+cYTv+HXB3XmDQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "vega-dataflow": "^6.1.2", + "vega-statistics": "^2.0.0", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-runtime": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/vega-runtime/-/vega-runtime-7.1.2.tgz", + "integrity": "sha512-MMfm7FeWLX2XqaY26i99GrlToexO/EWYqT7MRqEkpw+0nYgKh4zpQ47j04LB3ishnSZAj2AL0HNl9uPZLxgFhg==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-dataflow": "^6.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-scale": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/vega-scale/-/vega-scale-8.1.2.tgz", + "integrity": "sha512-ADCGgdSmhcZyHxtHbrVcDrG76er2s2eXowO3qDg0vx2eclHdXL4YNaEjTHrosNG/1TAI5LAe7aQ2BGssJFfAXA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-scale-chromatic": "^3.1.0", + "vega-time": "^3.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-scenegraph": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/vega-scenegraph/-/vega-scenegraph-5.2.1.tgz", + "integrity": "sha512-tWolI3s/cjU5g8evEjpxJqgqKQ2IJv1FoPXqvNuviLNdylP1I/h5iF5RY08Cz4jkuP9zircSskFEBsHylD1dgQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "^3.1.0", + "d3-shape": "^3.2.0", + "vega-canvas": "^2.0.0", + "vega-loader": "^5.1.2", + "vega-scale": "^8.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-schema-url-parser": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/vega-schema-url-parser/-/vega-schema-url-parser-3.0.2.tgz", + "integrity": "sha512-xAnR7KAvNPYewI3O0l5QGdT8Tv0+GCZQjqfP39cW/hbe/b3aYMAQ39vm8O2wfXUHzm04xTe7nolcsx8WQNVLRQ==", + "license": "BSD-3-Clause" + }, + "node_modules/vega-selections": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/vega-selections/-/vega-selections-6.1.4.tgz", + "integrity": "sha512-qR5feBL8xq5BjoG3rf7e2WKle/cn5Fa9pBFMBnzoOYp4nTA0GYDoKuOjZkftibsQcmdST0HzPjSrS14jH3TCQw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "3.2.4", + "vega-expression": "^6.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-statistics": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/vega-statistics/-/vega-statistics-2.0.0.tgz", + "integrity": "sha512-dGPfDXnBlgXbZF3oxtkb8JfeRXd5TYHx25Z/tIoaa9jWua4Vf/AoW2wwh8J1qmMy8J03/29aowkp1yk4DOPazQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4" + } + }, + "node_modules/vega-themes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/vega-themes/-/vega-themes-3.0.0.tgz", + "integrity": "sha512-1iFiI3BNmW9FrsLnDLx0ZKEddsCitRY3XmUAwp6qmp+p+IXyJYc9pfjlVj9E6KXBPfm4cQyU++s0smKNiWzO4g==", + "license": "BSD-3-Clause", + "funding": { + "url": "https://app.hubspot.com/payments/GyPC972GD9Rt" + }, + "peerDependencies": { + "vega": "*", + "vega-lite": "*" + } + }, + "node_modules/vega-time": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/vega-time/-/vega-time-3.2.1.tgz", + "integrity": "sha512-SXo1klLYPsmBdSj9itYSuFMmCpwnnIvQS8jWFxsCTnyWcPaBQKz0kf2M7DqTysi34KTk+6ws4kqZg11XMWnIYA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-time": "^3.1.0", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-tooltip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vega-tooltip/-/vega-tooltip-1.0.0.tgz", + "integrity": "sha512-P1R0JP29v0qnTuwzCQ0SPJlkjAzr6qeyj+H4VgUFSykHmHc1OBxda//XBaFDl/bZgIscEMvjKSjZpXd84x3aZQ==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-util": "^2.0.0" + }, + "funding": { + "url": "https://app.hubspot.com/payments/GyPC972GD9Rt" + } + }, + "node_modules/vega-transforms": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/vega-transforms/-/vega-transforms-5.2.1.tgz", + "integrity": "sha512-csKiYXJFa0YY5lubP1wc8b99SJ8nHLG/sOWvKFpW3CfDfjp8LoB0exrxgtIoClCsngAguKiuZVDwQZxBhPJZwA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "vega-dataflow": "^6.1.2", + "vega-statistics": "^2.0.0", + "vega-time": "^3.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-typings": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/vega-typings/-/vega-typings-2.2.0.tgz", + "integrity": "sha512-pX7LsqgMpzMPOyydg/drYbIjh+mmvLfKtuKr75OpCu/ZYKJ2i8UjSmNlpARYeYGLaXfuNUAM3hMN8NPZPDYiBQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/geojson": "7946.0.16", + "vega-event-selector": "^4.0.0", + "vega-expression": "^6.2.0", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-util": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-2.1.2.tgz", + "integrity": "sha512-Mdvv33BUlx4no5gz2VHdue4bJ6ejWmInuieID3JGnAXwewHKrPWE/Xr2vVWA+xS9Om1eHvj/0wVsiBGJ61v2uQ==", + "license": "BSD-3-Clause" + }, + "node_modules/vega-view": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/vega-view/-/vega-view-6.1.2.tgz", + "integrity": "sha512-BG/Kqd1csdK4uKdJ4hv72msgQX3Ry5SzCV1ngbytkn6QwuQPpFWzya/f0OSWyMQP9S79avdHXqdLCxjSeiDQRw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^3.2.4", + "d3-timer": "^3.0.1", + "vega-dataflow": "^6.1.2", + "vega-format": "^2.1.2", + "vega-functions": "^6.1.3", + "vega-runtime": "^7.1.2", + "vega-scenegraph": "^5.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-view-transforms": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/vega-view-transforms/-/vega-view-transforms-5.2.1.tgz", + "integrity": "sha512-TiguoCJ6BB7TEV+s3nNMoCoH4+q5A6XQsDMEzbVuRaSWmfIJxvWUlbZQqn4CUt65VMDr63Fy21Xpb0ntlaU5LQ==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-dataflow": "^6.1.2", + "vega-scenegraph": "^5.2.1", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-voronoi": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-voronoi/-/vega-voronoi-5.1.2.tgz", + "integrity": "sha512-8BgKb6aF/42L/PrFi3Idqag7FtbzNqDVhvPD4SfswSzGYlRDcxcn1eQQsa//q72qIZk4i5rsHeAiH2QbIjPSBg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-delaunay": "^6.0.4", + "vega-dataflow": "^6.1.2", + "vega-util": "^2.1.0" + } + }, + "node_modules/vega-wordcloud": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/vega-wordcloud/-/vega-wordcloud-5.1.2.tgz", + "integrity": "sha512-ZSw6fupf4qx4hBzL9q/oVUKPVbVePjJ/FSgfxxgRcKvE1VoYq3HWfx9qg+CQMvgDS+oGDsn2WoQJuNLYx9T3AQ==", + "license": "BSD-3-Clause", + "dependencies": { + "vega-canvas": "^2.0.0", + "vega-dataflow": "^6.1.2", + "vega-scale": "^8.1.2", + "vega-statistics": "^2.0.0", + "vega-util": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", + "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-plugin-singlefile": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/vite-plugin-singlefile/-/vite-plugin-singlefile-2.3.3.tgz", + "integrity": "sha512-XVnGH0QzbOa8fxRSsHdCarVN1BSBXNi7uLMQYlrGRN5apdHkk62XQWRJhVever0lnfuyBkwn+kvVChdm/OoOUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">18.0.0" + }, + "peerDependencies": { + "rollup": "^4.59.0", + "vite": "^5.4.21 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "license": "ISC" + }, + "node_modules/yargs": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", + "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^7.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/web/public/brand/SOURCES.md b/web/public/brand/SOURCES.md index 1b99c7befe8aa464eb42569555a375ab4132c591..b124ce254afb44fba0df95e8c1b4acf41ff1e02f 100644 --- a/web/public/brand/SOURCES.md +++ b/web/public/brand/SOURCES.md @@ -46,6 +46,7 @@ hand-drawn set was built for. | `meta` | `meta.png` | https://about.meta.com/brand/resources/meta/company-brand/ | `Meta_Company-Lockup.zip` → `1 Positive Primary/RGB/Meta_lockup_positive primary_RGB.png` (29011×12501) | **first-block crop**: the pack ships the *lockup* (glyph + wordmark); the glyph is the leftmost ink block, 6272 px wide | | `stripe` | `stripe.png` | https://stripe.com/newsroom/brand-assets | `https://images.ctfassets.net/fzn2n1nzq965/HTTOloNPhisV9P4hlMPNA/…/Stripe_icon_-_square.svg` (400×400, rasterised at 1024) | trim + fit only | | `shopify` | `shopify.png` | https://www.shopify.com/brand-assets | `shopify-shopping-bag.zip` → `02 - Glyph/png/shopify_glyph.png` (600×683) | trim + fit only | +| `amazon` | `amazon.svg` | the approved lowercase `a` and smile supplied for W58-T08 (owner instruction 19) | vector, no raster cut needed | none — shipped as the vendor SVG. **Its geometry is digest-pinned** in `src/connectors/_directory_render.cjs`, which asserts the served file and `brandMarks.tsx`'s inline fallback are the same artwork | ## ⚠ The one judgement call, stated rather than buried — `odoo` @@ -63,12 +64,32 @@ elements). It is a CUT of the vendor's artwork, at the vendor's colour, which is instruction; it is not a redraw. **If a square Odoo glyph is ever published, this is the one row in the table to revisit.** -## MISSES — none - -All nine marks in `BRAND_OF_CONNECTOR` that name a real company were obtained. Recorded explicitly -because "no misses" has to be a stated finding rather than an absence a reader has to infer — the -standing rule (wave-30 R6) is that a limit which cannot be removed is *reported* with its cause and a -recommended fix, and its second sentence is the half that gets dropped. +## MISSES + +> The nine marks above were all obtained. This section was `— none` until wave 52, and it was +> written in advance precisely so the first miss would have somewhere honest to go: the standing +> rule (wave-30 R6) is that a limit which cannot be removed is *reported* with its cause and a +> recommended fix, and its second sentence is the half that gets dropped. +> +> ⛔ **A KEY LISTED HERE MUST NOT BE IN `OFFICIAL`** in `brandMarks.tsx`. It keeps its hand-drawn +> arm, which is what renders. Adding it to `OFFICIAL` with no file behind it is a guaranteed 404 on +> every render, and `verify_meta::section_brand_door` reads THIS list rather than carrying a copy. + +**— none.** + +⭐⭐ **`amazon` WAS THIS FILE'S ONLY MISS, AND IT WAS CLOSED THE WAY THIS FILE SAID TO CLOSE IT** +(2026-09-13, wave 58 T08 / owner instruction 19 — *"find the logo with only the letter a in it"*). +The recorded rejection stands as history and is worth keeping: Amazon's own CDN serves the +**lockup**, whose wordmark smears at 15 px and is near-black on a dark chip; colour-keying it to +isolate the smile alone measured an aspect of **4.46** against a house grid running 0.88 to 1.51; +the favicon is below the 128 px grid and the touch icons 404. **The cause was that Amazon publishes +no square glyph.** + +What changed is not a new cut of the same rejected art — it is a *different asset*: the approved +lowercase `a` and smile, which is square-ish and sits on the house grid. The three steps this +section prescribed were followed exactly: the file is in this directory, its row is in the table +above, and `"amazon"` is in `OFFICIAL`. ⚠ The hand-drawn arm stays as the asset-failure fallback, +as every other mark's does. ## How to add one diff --git a/web/public/brand/amazon.svg b/web/public/brand/amazon.svg new file mode 100644 index 0000000000000000000000000000000000000000..acd353840c42970fc59cc12426ba2f9a05c64624 --- /dev/null +++ b/web/public/brand/amazon.svg @@ -0,0 +1,8 @@ + + + + + diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index 5516d42f0966db62b821a36b72f6cd2165b61839..37d5b9ee33a9685fd715ba724ad40f2fdfd8a798 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -116,6 +116,22 @@ "default": true, "description": "A Google Maps link to this customer's address." }, + { + "key": "lat", + "label": "Latitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, + { + "key": "lon", + "label": "Longitude", + "type": "text", + "source": "odoo", + "default": false, + "description": "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address." + }, { "key": "customer_since", "label": "Customer since", @@ -164,17 +180,6 @@ "default": true, "description": "How many days late this customer is running against their own usual ordering rhythm." }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, { "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", "key": "ar_open", @@ -323,7 +328,6 @@ "city": "Charlottesville", "state": "Virginia (US)", "last_order": "2026-07-21", - "est_missed": 0, "notes": "", "country": "United States", "zip": "02720", @@ -341,8 +345,8 @@ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", "days_to_pay": 34, "_created": "2023-01-15 09:10:00", - "lat": 25.7617, - "lon": -80.1918, + "lat": 38.029306, + "lon": -78.476678, "odoo_status": "Archived", "dba": "Royal", "ar_outstanding": 0 @@ -355,7 +359,6 @@ "city": "Portland", "state": "Oregon (US)", "last_order": "2026-07-19", - "est_missed": 0, "notes": "Expanding to a second storefront.", "country": "United States", "zip": "77041", @@ -373,8 +376,8 @@ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", "days_to_pay": null, "_created": "2023-02-15 09:11:00", - "lat": 27.9506, - "lon": -82.4572, + "lat": 45.515232, + "lon": -122.678385, "odoo_status": "Active", "dba": "Fisch", "ar_outstanding": 1240.5 @@ -387,7 +390,6 @@ "city": "Kansas City", "state": "Missouri (US)", "last_order": "2026-07-17", - "est_missed": 0, "notes": "", "country": "United States", "zip": "11219", @@ -405,8 +407,8 @@ "top_sku": "SATIN RIBBON 2IN", "days_to_pay": 61, "_created": "2023-03-15 09:12:00", - "lat": 28.5384, - "lon": -81.3789, + "lat": 39.099727, + "lon": -94.578567, "odoo_status": "Active", "dba": "Both", "ar_outstanding": 5120.25 @@ -419,7 +421,6 @@ "city": "Savannah", "state": "Georgia (US)", "last_order": "2026-05-30", - "est_missed": 41200, "notes": "Switched some volume to a local grower.", "country": "United States", "zip": "07649", @@ -437,8 +438,8 @@ "top_sku": "(none)", "days_to_pay": 12, "_created": "2023-04-15 09:13:00", - "lat": 30.3322, - "lon": -81.6557, + "lat": 32.083541, + "lon": -81.099834, "odoo_status": "Active", "dba": "Royal", "ar_outstanding": 9240 @@ -451,7 +452,6 @@ "city": "Seattle", "state": "Washington (US)", "last_order": "2026-07-22", - "est_missed": 0, "notes": "Top-10 account.", "country": "United States", "zip": "33125", @@ -469,8 +469,8 @@ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", "days_to_pay": 34, "_created": "2023-05-15 09:14:00", - "lat": 26.1224, - "lon": -80.1373, + "lat": 47.606209, + "lon": -122.332071, "odoo_status": "Active", "dba": "", "ar_outstanding": 0 @@ -483,7 +483,6 @@ "city": "Asheville", "state": "North Carolina (US)", "last_order": "2026-02-11", - "est_missed": 52400, "notes": "No spring order this year.", "country": "United States", "zip": "90210", @@ -501,8 +500,8 @@ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", "days_to_pay": null, "_created": "2023-06-15 09:15:00", - "lat": 27.3364, - "lon": -82.5307, + "lat": 35.595058, + "lon": -82.551487, "odoo_status": "Active", "dba": "Fisch", "ar_outstanding": 1240.5 @@ -515,7 +514,6 @@ "city": "Austin", "state": "Texas (US)", "last_order": "2025-11-04", - "est_missed": 78300, "notes": "Went with a competitor on freight terms.", "country": "United States", "zip": "08701", @@ -533,8 +531,8 @@ "top_sku": "SATIN RIBBON 2IN", "days_to_pay": 61, "_created": "2023-07-15 09:16:00", - "lat": null, - "lon": null, + "lat": 30.267153, + "lon": -97.743061, "odoo_status": "Active", "dba": "Both", "ar_outstanding": 5120.25 @@ -547,7 +545,6 @@ "city": "Denver", "state": "Colorado (US)", "last_order": "2026-07-14", - "est_missed": 0, "notes": "First order in April.", "country": "United States", "zip": "60614", @@ -565,8 +562,8 @@ "top_sku": "(none)", "days_to_pay": 12, "_created": "2023-08-15 09:17:00", - "lat": 33.749, - "lon": -84.388, + "lat": 39.739236, + "lon": -104.990251, "odoo_status": "Active", "dba": "Royal", "ar_outstanding": 9240 diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx index 73e8cff89df7d877d81455122a7c24a8f0b1f03b..fb259b5fbf47d66bc868b3a2d64254f8dc9377a2 100644 --- a/web/src/automation/AutomationSurface.tsx +++ b/web/src/automation/AutomationSurface.tsx @@ -41,6 +41,7 @@ import { import type { VoiceAgent } from "./voiceAgentTypes"; import "../styles/wave44/voice-agent.css"; import "../styles/wave45/agent-operator.css"; +import "../styles/wave57/agents-rail.css"; /** * ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST. @@ -248,6 +249,7 @@ export default function AutomationSurface() { const [activeId, setActiveId] = useState(""); const [activeVoiceId, setActiveVoiceId] = useState(""); const [railShut, setRailShut] = useState(false); + const [railQuery, setRailQuery] = useState(""); const [busy, setBusy] = useState(""); /** * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to @@ -542,6 +544,9 @@ export default function AutomationSurface() { */ const fieldAgents = items.filter((agent) => agent.system === "field_agent"); const active = items.find((a) => a.id === activeId) || null; + const railNeedle = railQuery.trim().toLowerCase(); + const railMatches = (label: string) => + !railNeedle || label.toLowerCase().includes(railNeedle); const activeVoice = voiceAgents.find((agent) => agent.automationId === activeVoiceId) || null; /** @@ -641,38 +646,65 @@ export default function AutomationSurface() { -
+
+
+ + setRailQuery(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Escape") setRailQuery(""); + }} + /> +
+
+ {/* Keep both find guards inside the callbacks: verify_automation_ui.py's W44 + check reads the literal voiceAgents.map. Chaining a filter ahead of it + takes a green gate red even though it renders the same rows. */} {items.map((a) => { + if (!railMatches(a.name)) return null; return (
{ + if (!railMatches(agent.displayName)) return null; const isActive = agent.lifecycle === "active"; return (
= [ - { key: "overview", label: "Overview" }, - { key: "keychain", label: "Keychain" }, - { key: "databases", label: "Databases" }, - { key: "relationships", label: "Relationships" }, -]; +/** + * ⭐⭐ W52-T04 · R1 / CONTRACT C1 / DONE-WHEN — WHICH TABS RENDER, AND IN WHICH ORDER. + * + * `spawns_tables === true` (Odoo, Amazon, Meta Ads at baseline) gets every section the done-when + * names, in the done-when's own order — tables and fields, relationships, sync cadence, + * credentials — with Overview kept first as the landing tab: it is the modal's existing entry + * point and every other tab already assumes the title/status header above it has been read. + * + * `spawns_tables === false` (Stripe, Shopify, Webhooks, OAuth providers, the Scraper, and every + * catalogue row) gets Overview and Keychain only. It has no tables, no relationships and no sync + * cadence to disclose, and rendering an empty version of any of those three is exactly the D-280 + * residue this ticket's done-when names. Keychain is NOT gated on `spawns_tables` — a credential + * door applies as broadly as a connector has one at all, database-tier or not. + * + * ⚠ LABELS KEEP THEIR ESTABLISHED NAMES. The done-when describes the four sections in prose + * ("tables and fields", "relationships", "sync cadence", "credentials") but that is not a + * required button caption: `verify_ui.py`'s `connector-modal` check (W49-T12, R26) does a literal + * source-text scan for `label: "Databases"` and `label: "Keychain"`, and this repo already opens + * the SAME tab from Settings > Keychains (`initialTab="keychain"`, `SettingsModal.tsx`) and from + * the card's "Manage keys"/"Sync settings" door under the word "Keychain" — renaming either label + * here would be one tab under two names, and would red a shell-wiring gate this ticket's `files:` + * fence does not include. "Databases" and "Keychain" stay; "Sync cadence" is the new section's own + * name, since nothing existing names it anything else. + */ +export function connectorTabs( + row: ConnectorRow +): ReadonlyArray<{ key: ConnectorTab; label: string }> { + if (row.spawns_tables === true) { + return [ + { key: "overview", label: "Overview" }, + { key: "databases", label: "Databases" }, + { key: "relationships", label: "Relationships" }, + { key: "cadence", label: "Sync cadence" }, + { key: "keychain", label: "Keychain" }, + ]; + } + return [ + { key: "overview", label: "Overview" }, + { key: "keychain", label: "Keychain" }, + ]; +} export interface ConnectorOverview { key: string; @@ -74,6 +131,8 @@ export interface ConnectorKeychainEntry { masked: string; scope: string; testable: boolean; + /** Server-owned admin permission for the write-only replacement flow. */ + replaceable: boolean; } /** @@ -124,8 +183,36 @@ export interface ConnectorDatabaseExclusion { export interface ConnectorDatabase { key: string; + identity?: { provider: string; connectionId: string; databaseKey: string; model: string }; + /** + * What this database is CALLED today: the tenant's override when there is one, our shipped + * default otherwise. Resolved on the server (`core/schema_overlay.database_label`), never here. + */ label: string; + /** + * ⭐⭐ W53-T27 · R9 / C2 — THREE NAMES, THREE DIFFERENT FACTS, AND THE TAB SHOWS ALL THREE. + * + * `native` is the VENDOR's own identifier for the table (`res.partner`, or an Amazon report + * type). It is never editable by anybody: it is how the source names its own thing, and a + * workspace that could rewrite it would be describing a table that does not exist over there. + */ + native: string; + /** OUR shipped default name (C2's `label`). It survives an override rather than being replaced, + * which is what lets the tab offer a way back to it. */ + defaultLabel: string; + /** This tenant's own name, or `""` when none is set. Absent means inherit the default, and it + * is the SAME override the navigation rail's rename writes (R9), not a second one. */ + override: string; + /** May THIS reader rename it? The server's own `may_open` answer, per row, because a control + * the write door would refuse is a button that always fails. */ + canRename: boolean; + /** Does the governed database exist yet? This is separate from the reader's permission. */ + exists: boolean; enabled: boolean; + /** Exact rows in the Odoo source model, or null until that model was counted successfully. */ + sourceRows: number | null; + /** Tri-state on purpose: a model nobody counted is unknown, never empty. */ + populated: boolean | null; fields: ConnectorDatabaseField[]; excluded: ConnectorDatabaseExclusion[]; /** @@ -156,14 +243,34 @@ export interface ConnectorIntrospection { canToggleTables: boolean; catalogueComplete: boolean; capturedAt: string; + censusAt: string; + censusComplete: boolean; + censusDurationMs: number; + censusModelsCounted: number; + censusModelsWithRows: number; /** What an EMPTY `excluded` list actually means, composed where the diff is computed. */ excludedBasis: string; } +/** W55-T19 / R7 — one provider's own database-enablement door. */ +export interface ConnectorEnablement { + /** Must match the connector detail envelope that carried it. */ + provider: string; + /** Read as a string so a future provider mode fails closed instead of widening this UI. */ + mode: string; + canManage: boolean; + updatePath: string; + /** The Amazon route changes only config; stored database rows remain in place. */ + preservesData: boolean; + available: boolean; +} + export interface ConnectorDetail { overview: ConnectorOverview; keychain: ConnectorKeychainHalf; databases: ConnectorDatabase[]; + enablement: ConnectorEnablement | null; + materialization?: MaterializationCapability | null; introspection: ConnectorIntrospection; relationships: ConnectorRelationshipGraph; } @@ -192,6 +299,7 @@ function parseKeychainHalf(raw: unknown): ConnectorKeychainHalf { masked: str(entry.masked), scope: str(entry.scope).trim(), testable: entry.testable === true, + replaceable: entry.replaceable === true, }); } return { @@ -239,7 +347,12 @@ function parseDatabases(raw: unknown): ConnectorDatabase[] { for (const item of raw) { const row = obj(item); const key = str(row.key).trim(); - const label = str(row.label).trim(); + // ⭐ W53-T27 — THE DEFAULT AND THE EFFECTIVE NAME EACH FALL BACK TO THE OTHER, and neither + // falls back to the KEY. A payload from a server that predates this ticket carries `label` + // alone, and the tab must then show that name in both columns rather than an empty cell or an + // internal identifier. + const defaultLabel = str(row.defaultLabel).trim() || str(row.label).trim(); + const label = str(row.label).trim() || defaultLabel; // Same rule as `parseDirectory`: a database with no name is never rendered from its key, // which would put an internal identifier on screen. if (!key || !label) continue; @@ -266,7 +379,24 @@ function parseDatabases(raw: unknown): ConnectorDatabase[] { out.push({ key, label, + identity: str(obj(row.identity).databaseKey) === key ? { + provider: str(obj(row.identity).provider), connectionId: str(obj(row.identity).connectionId), + databaseKey: key, model: str(obj(row.identity).model), + } : undefined, + // ⛔ `native` IS NOT DEFAULTED TO ANYTHING. An absent one means the server could not name + // the source's own identifier for this table, and the tab says so; inventing the key or the + // label here would put OUR word on screen under the vendor's heading. + native: str(row.native).trim(), + defaultLabel, + override: str(row.override).trim(), + canRename: row.canRename === true, + exists: row.exists === true, enabled: row.enabled === true, + sourceRows: + typeof row.sourceRows === "number" && Number.isFinite(row.sourceRows) && row.sourceRows >= 0 + ? Math.trunc(row.sourceRows) + : null, + populated: row.populated === true ? true : row.populated === false ? false : null, fields, excluded, sharesFieldsWith: parseStrings(row.sharesFieldsWith), @@ -292,10 +422,55 @@ function parseIntrospection(raw: unknown): ConnectorIntrospection { canToggleTables: source.canToggleTables === true, catalogueComplete: source.catalogueComplete === true, capturedAt: str(source.capturedAt).trim(), + censusAt: str(source.censusAt).trim(), + censusComplete: source.censusComplete === true, + censusDurationMs: + typeof source.censusDurationMs === "number" && Number.isFinite(source.censusDurationMs) + ? Math.max(0, Math.trunc(source.censusDurationMs)) + : 0, + censusModelsCounted: + typeof source.censusModelsCounted === "number" && Number.isFinite(source.censusModelsCounted) + ? Math.max(0, Math.trunc(source.censusModelsCounted)) + : 0, + censusModelsWithRows: + typeof source.censusModelsWithRows === "number" && + Number.isFinite(source.censusModelsWithRows) + ? Math.max(0, Math.trunc(source.censusModelsWithRows)) + : 0, excludedBasis: str(source.excludedBasis).trim(), }; } +/** + * Parse a provider capability bound to the selected connector, or refuse it. + * + * A path is actionable only when it stays under this provider and carries the one database-key + * slot the server declared. Missing, cross-provider and unknown modes render no control; the API + * remains the permission boundary, but the modal never paints a button it knows cannot work. + */ +export function parseConnectorEnablement( + raw: unknown, + selectedKey: string +): ConnectorEnablement | null { + const source = obj(raw); + const provider = str(source.provider).trim(); + const mode = str(source.mode).trim(); + if (provider !== selectedKey || mode !== "database" || source.preservesData !== true) return null; + const updatePath = str(source.updatePath).trim(); + const expectedPrefix = `/connectors/${encodeURIComponent(provider)}/databases/`; + const validPath = updatePath.startsWith(expectedPrefix) && updatePath.endsWith("/enablement") + && updatePath.includes("{databaseKey}"); + const canManage = source.canManage === true && source.available === true && validPath; + return { + provider, + mode, + canManage, + updatePath: canManage ? updatePath : "", + preservesData: true, + available: source.available === true, + }; +} + /** * Read the detail payload for the card that was actually clicked. * @@ -320,6 +495,12 @@ export function parseConnectorDetail(body: unknown, selectedKey: string): Connec }, keychain: parseKeychainHalf(source.keychain), databases: parseDatabases(source.databases), + enablement: parseConnectorEnablement(source.enablement, selectedKey), + materialization: obj(source.materialization).provider === selectedKey && + str(obj(source.materialization).statusPath) === `/connectors/${selectedKey}/jobs` && + str(obj(source.materialization).retryPath) === `/connectors/${selectedKey}/sync/retry` + ? { provider: selectedKey, statusPath: `/connectors/${selectedKey}/jobs`, + retryPath: `/connectors/${selectedKey}/sync/retry` } : null, introspection: parseIntrospection(source.introspection), relationships: parseRelationshipGraph(source.relationships, selectedKey), }; @@ -444,6 +625,81 @@ export function ConnectorOverviewPanel({ ); } +interface ProviderPendingBridge { + enabled: (database: ConnectorDatabase) => boolean; + stage: (database: ConnectorDatabase, next: boolean) => Promise; + busy: boolean; +} + +const ProviderPendingContext = createContext(null); + +/** Amazon's provider-specific administrator controls, committed by the modal's shared Save. */ +export function ProviderEnablementPanel({ + detail, + onChanged, +}: { + detail: ConnectorDetail; + onChanged: () => void; +}) { + const capability = detail.enablement; + const pending = useContext(ProviderPendingContext); + const [busyKey, setBusyKey] = useState(""); + const [message, setMessage] = useState(""); + if (!capability?.canManage || !capability.updatePath) return null; + + const setEnabled = async (database: ConnectorDatabase, enabled: boolean) => { + if (!pending || pending.busy) return; + setBusyKey(database.key); + setMessage(""); + await pending.stage(database, enabled); + setBusyKey(""); + // Kept in the public prop contract for the provider panel's existing callers. The shared + // Save owns the one detail re-read now, so staging deliberately does not call it. + void onChanged; + }; + + return ( +
+
+

Enablement

+

Choose which Amazon databases are enabled. Switching one off keeps its stored rows.

+
+
    + {detail.databases.map((database) => { + const enabled = pending?.enabled(database) ?? database.enabled; + return ( +
  • + {database.label} + + {enabled ? "On" : "Off"} + + +
  • + ); + })} +
+
+ {message} +
+
+ ); +} + // ── The three tabs that are not built yet. ⛔ EACH IS ITS OWN COMPONENT SLOT so W49-T13, T14 and // T15 replace a named component rather than editing a sentence out of a branch, and the switch // below never has to change shape. Plain sentences, no dashes, no emojis. @@ -473,6 +729,8 @@ type Load = | { phase: "ready"; detail: ConnectorDetail } | { phase: "error"; message: string }; +type PendingLeave = { kind: "close" } | { kind: "tab"; tab: ConnectorTab } | null; + /** * THE MODAL. Centered, portalled to , focus-trapped, dismissed by Escape or a click * outside, and it hands focus back to the control that opened it. @@ -481,15 +739,34 @@ export default function ConnectorModal({ row, onClose, initialTab = "overview", + onOpenDatabases, }: { row: ConnectorRow; onClose: () => void; /** W49-T13 opens this modal on the Keychain tab from Settings; every other door opens Overview. */ initialTab?: ConnectorTab; + /** + * ⭐⭐ W53-T28 (instructions 22/23) — TOLD, NOT ASKED. The Sync cadence tab mounts + * `OdooSyncStatus`, whose "Manage fields in the Databases tab" link used to live on the card and + * be handed the page's own opener. Inside an already-mounted modal that indirection cannot work: + * `initialTab` is a `useState` INITIALIZER with no resync, so a page-driven jump would need a + * keyed remount, and a remount re-reads `/detail`, bounces focus off the trap, and goes silently + * dead the second time the link is used because the page's tab already reads "databases". + * + * So the modal moves its OWN tab and calls this afterwards. The page uses it to keep its record + * of the chosen tab in step with what is on screen, which is what its reset-on-close is clearing. + * ⚠ Optional: `SettingsModal` opens this on Keychain with no page-level tab state at all. + */ + onOpenDatabases?: () => void; }) { const panelRef = useRef(null); const [tab, setTab] = useState(initialTab); const [load, setLoad] = useState({ phase: "loading" }); + const databasesRef = useRef(null); + const [databasePendingCount, setDatabasePendingCount] = useState(0); + const [databaseTabBusy, setDatabaseTabBusy] = useState(false); + const [databaseSaving, setDatabaseSaving] = useState(false); + const [pendingLeave, setPendingLeave] = useState(null); /** * ⭐ W49-T13 — THE RE-READ, AND IT IS NOT OPTIONAL POLISH. The Keychain tab WRITES: it stores, * tests, deletes and disconnects. Keying the fetch on `connectorKey` alone leaves a deleted key @@ -497,7 +774,11 @@ export default function ConnectorModal({ * so the effect below keys on this counter as well and every write bumps it. */ const [reads, setReads] = useState(0); - const reread = useCallback(() => setReads((n) => n + 1), []); + const [databaseReveal, setDatabaseReveal] = useState(null); + const reread = useCallback((reveal?: DatabaseReveal) => { + if (reveal) setDatabaseReveal(reveal); + setReads((n) => n + 1); + }, []); /** * ⭐⭐ W49-T19 · R31 — THE AMAZON REVIEWS PULL, OPENED FROM THE OVERVIEW TAB. * A second overlay above this one, which is why it takes the PORTAL step and this modal keeps @@ -512,6 +793,39 @@ export default function ConnectorModal({ const [pullSurface, setPullSurface] = useState(null); const canPull = isAmazonConnector(row.key); + const unsavedMessage = databasePendingCount === 1 + ? "You have 1 unsaved change to this connection." + : `You have ${databasePendingCount} unsaved changes to this connection.`; + + const requestClose = useCallback(() => { + if (databaseSaving || databaseTabBusy) return; + if (databasePendingCount) { + setPendingLeave({ kind: "close" }); + return; + } + onClose(); + }, [databasePendingCount, databaseSaving, databaseTabBusy, onClose]); + + const requestTab = useCallback((next: ConnectorTab) => { + if (databaseSaving || databaseTabBusy) return; + if (next === tab) return; + if (databasePendingCount) { + setPendingLeave({ kind: "tab", tab: next }); + return; + } + setTab(next); + }, [databasePendingCount, databaseSaving, databaseTabBusy, tab]); + + useEffect(() => { + if (!databasePendingCount) return; + const warn = (event: BeforeUnloadEvent) => { + event.preventDefault(); + event.returnValue = unsavedMessage; + }; + window.addEventListener("beforeunload", warn); + return () => window.removeEventListener("beforeunload", warn); + }, [databasePendingCount, unsavedMessage]); + useOverlayLayer({ panelRef, /** @@ -526,11 +840,12 @@ export default function ConnectorModal({ * this layer's dismissal closes THAT and stops. */ onDismiss: () => { + if (pendingLeave) return; if (pulling) { setPulling(false); return; } - onClose(); + requestClose(); }, dismissOnOutside: true, /** @@ -540,7 +855,7 @@ export default function ConnectorModal({ */ outsideElements: pullSurface ? [pullSurface] : [], initialFocus: "[data-overlay-autofocus]", - trapFocus: true, + trapFocus: pendingLeave === null, // The layer captures whatever was focused when it mounted and focuses it again on unmount, // which is the card's own open button. That is the done-when's "focus returns to the card". restoreFocus: true, @@ -549,7 +864,7 @@ export default function ConnectorModal({ const connectorKey = row.key; useEffect(() => { let dead = false; - setLoad({ phase: "loading" }); + setLoad((current) => reads > 0 && current.phase === "ready" ? current : { phase: "loading" }); void (async () => { try { const res = await fetch( @@ -609,18 +924,94 @@ export default function ConnectorModal({ return ; } if (tab === "databases") { - return ; + return ( + <> + + databasesRef.current?.tableEnabled(database) ?? database.enabled, + stage: (database, next) => + databasesRef.current?.stageTable(database, next) ?? Promise.resolve(), + busy: databaseSaving || databaseTabBusy, + }} + > + + + + + ); } if (tab === "relationships") { return ; } + if (tab === "cadence") { + /** + * ⭐⭐ W53-T28 — THE ODOO CARD'S SYNC PANEL LANDS HERE, ABOVE THE CADENCE CONTROLS. + * + * `OdooSyncStatus` reports (schedule, health, last run, errors) and links onward to field + * enablement; `SyncCadenceTab` sets (grids, cadence, server database, disconnect and + * reconnect). Neither duplicates the other, which is why the card's whole "Sync settings" + * door collapses into these two lines instead of needing anything built. Reading above + * setting is the order the card itself used. + * + * ⛔ ODOO ONLY. `OdooSyncStatus` calls `getOdooStatus`/`getOdooSchema` unconditionally, so + * mounting it for Amazon or Meta Ads would print Odoo's health under another connector's + * name. Those two keep `SyncCadenceTab`'s honest "no cadence control here yet" sentence, + * which is the whole point of that component's degraded branch. + */ + return ( + <> + {row.key === "odoo" ? ( + { + setTab("databases"); + onOpenDatabases?.(); + }} + /> + ) : null} + + + ); + } return ( setPulling(true) : undefined} /> ); - }, [canPull, load, reread, row, tab]); + }, [canPull, databaseReveal, load, onOpenDatabases, reread, row, tab]); + + const saveDatabaseChanges = useCallback(async () => { + if (!databasePendingCount || !databasesRef.current) return; + setDatabaseSaving(true); + try { + await databasesRef.current.savePending(); + } finally { + setDatabaseSaving(false); + } + }, [databasePendingCount]); + + const discardDatabaseChanges = useCallback(() => { + databasesRef.current?.discardPending(); + setDatabasePendingCount(0); + }, []); + + const confirmLeave = useCallback(() => { + if (!pendingLeave) return; + const destination = pendingLeave; + discardDatabaseChanges(); + setPendingLeave(null); + if (destination.kind === "close") onClose(); + else setTab(destination.tab); + }, [discardDatabaseChanges, onClose, pendingLeave]); const status = load.phase === "ready" ? load.detail.overview.status : row.state; const title = load.phase === "ready" ? load.detail.overview.label || row.label : row.label; @@ -647,18 +1038,41 @@ export default function ConnectorModal({ {status ? ( {status} ) : null} + {tab === "databases" ? ( +
+ + +
+ ) : null}
- {CONNECTOR_TABS.map((entry) => ( + {connectorTabs(row).map((entry) => ( @@ -693,6 +1108,15 @@ export default function ConnectorModal({ surfaceRef={setPullSurface} /> ) : null} + {unsavedMessage}

} + confirmLabel="Discard changes" + tone="destructive" + onConfirm={confirmLeave} + onCancel={() => setPendingLeave(null)} + /> ); } diff --git a/web/src/connectors/ConnectorsPage.tsx b/web/src/connectors/ConnectorsPage.tsx index 4deccdd7b7365762095b85fdd03ca473544faf42..b499356d1d25ec6a0dd06f9435ab320df567dbc7 100644 --- a/web/src/connectors/ConnectorsPage.tsx +++ b/web/src/connectors/ConnectorsPage.tsx @@ -28,7 +28,6 @@ // --------------------------------------------------------------------------- import { useEffect, useState } from "react"; -import type { MouseEvent } from "react"; import { API_V1, CREDENTIALS } from "../apiContract"; import { brandForConnector } from "./brandMarks"; import ConnectorModal from "./ConnectorModal"; @@ -41,6 +40,7 @@ import DirectoryChips, { ALL_CATEGORIES, inCategory, parseCategories, + projectCategories, type DirectoryCategory, } from "./DirectoryChips"; // ⭐⭐ W49-T19 — the run dialog opens FROM THE CARD, with no modal in between. The catalogue has @@ -48,7 +48,13 @@ import DirectoryChips, { // have made the one openable catalogue card a 404. See `ConnectorCard` below for the suppression // that keeps it out of the modal path. import MonidRunDialog from "./MonidRunDialog"; -import OdooConfigPanel, { type OdooConfig } from "./OdooConfig"; +// ⛔⛔ `OdooConfigPanel` AND `type OdooConfig` WERE IMPORTED HERE AND ARE DELETED BY W53-T28. +// They were `OdooManage`'s imports, and `OdooManage` — the card's own "Sync settings"/"Set up +// sync" door — is gone with them (see its tombstone below `ConnectorsPage`). The panel itself is +// not retired: `SyncCadenceTab` in `connectors/OdooConfig.tsx` already fetches the same +// `/admin/connectors/odoo/config` payload and renders the same `OdooConfigPanel`, and it is +// mounted by `ConnectorModal`'s Sync cadence tab. Re-importing either name here would be the +// second door this ticket exists to close. import { AutomationError, getOdooSchema, @@ -61,6 +67,12 @@ import "../styles/wave45/agent-operator.css"; // `CONNECTOR_GRAPH_CSS` string this file used to inject through `