Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- Dockerfile +17 -5
- RELEASES.json +7 -1
- VERSION +1 -1
- api/amazon_relational.py +421 -0
- api/application_registry.py +200 -9
- api/automation_engine.py +346 -12
- api/connectors_amazon.py +307 -0
- api/connectors_bd.py +7 -7
- api/connectors_ig.py +10 -11
- api/connectors_meta.py +13 -13
- api/connectors_tt.py +1 -1
- api/deps.py +37 -26
- api/formula_runtime.py +0 -0
- api/ig_master.py +2 -2
- api/main.py +76 -4
- api/meta_relational.py +52 -32
- api/oauth_connect.py +9 -9
- api/odoo_relational.py +0 -0
- api/pages.py +2 -2
- api/rollup_sql.py +2 -2
- api/routes_admin.py +94 -33
- api/routes_alerts.py +2 -2
- api/routes_assets.py +19 -2
- api/routes_auth.py +297 -57
- api/routes_connected_tables.py +17 -1
- api/routes_connectors.py +0 -0
- api/routes_external.py +399 -0
- api/routes_forms.py +5 -5
- api/routes_grid.py +410 -27
- api/routes_keychain.py +569 -67
- api/routes_nav.py +154 -0
- api/routes_oauth.py +1 -1
- api/routes_odoo_tables.py +939 -40
- api/routes_platform_admin.py +5 -5
- api/routes_products.py +3 -3
- api/routes_publish.py +5 -5
- api/routes_records.py +3 -3
- api/routes_sheet.py +1044 -31
- api/routes_slack.py +8 -8
- api/routes_tables.py +113 -0
- api/routes_uploads.py +2 -2
- api/test_odoo_agent_snapshot.py +111 -0
- api/test_odoo_customer_population.py +204 -0
- api/test_odoo_rank_refresh.py +411 -0
- api/test_odoo_relational_verdict.py +128 -0
- platform/aios_grid.py +3 -3
- platform/aios_grid_fields.json +16 -11
- platform/core/auth_google.py +5 -5
- platform/core/connector_directory.py +77 -7
- platform/core/connector_spec.py +424 -0
Dockerfile
CHANGED
|
@@ -20,12 +20,24 @@
|
|
| 20 |
# ---- stage 1: build the React bundle ----
|
| 21 |
FROM node:22-slim AS web
|
| 22 |
WORKDIR /web
|
| 23 |
-
#
|
| 24 |
-
#
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
COPY web/ ./
|
| 28 |
-
RUN
|
| 29 |
|
| 30 |
# ---- stage 2: python API serving the bundle ----
|
| 31 |
FROM python:3.12-slim
|
|
|
|
| 20 |
# ---- stage 1: build the React bundle ----
|
| 21 |
FROM node:22-slim AS web
|
| 22 |
WORKDIR /web
|
| 23 |
+
# The lockfile is the audited supply-chain boundary. It now resolves entirely from the public
|
| 24 |
+
# registry; discarding it here made every image an unreviewed dependency upgrade.
|
| 25 |
+
#
|
| 26 |
+
# ββ `npm ci` WAS TRIED HERE AND CANNOT WORK FROM A WINDOWS-RESOLVED LOCKFILE. Measured
|
| 27 |
+
# 2026-09-13: the build died with `Missing: @emnapi/runtime@1.11.3 from lock file`. npm on Windows
|
| 28 |
+
# prunes the linux-x64 rolldown binding and its transitive `@emnapi/runtime`, and `npm ci` treats
|
| 29 |
+
# any absent entry as a fatal lockfile mismatch rather than a platform gap. Regenerating with
|
| 30 |
+
# `--os=linux --cpu=x64` only inverts the failure: the lockfile then builds on Linux and breaks
|
| 31 |
+
# `npm ci` on every developer's Windows box.
|
| 32 |
+
#
|
| 33 |
+
# `npm install` WITH the lockfile present keeps the audited versions as the boundary β it honours
|
| 34 |
+
# every pin it can resolve β while repairing a platform-only gap instead of aborting. This is NOT
|
| 35 |
+
# the pre-83bfed7 `--no-package-lock`, which discarded the audit entirely.
|
| 36 |
+
# The real fix is a lockfile generated on Linux; booked as W-170.
|
| 37 |
+
COPY web/package.json web/package-lock.json ./
|
| 38 |
+
RUN npm install --no-audit --no-fund --ignore-scripts
|
| 39 |
COPY web/ ./
|
| 40 |
+
RUN npm run build # -> /web/dist
|
| 41 |
|
| 42 |
# ---- stage 2: python API serving the bundle ----
|
| 43 |
FROM python:3.12-slim
|
RELEASES.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
{
|
| 5 |
"version": "v71",
|
| 6 |
"sha": "96143dc",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "v73 (bb94de5)",
|
| 3 |
"releases": [
|
| 4 |
+
{
|
| 5 |
+
"version": "v73",
|
| 6 |
+
"sha": "bb94de5",
|
| 7 |
+
"date": "2026-09-13",
|
| 8 |
+
"subject": "release v73"
|
| 9 |
+
},
|
| 10 |
{
|
| 11 |
"version": "v71",
|
| 12 |
"sha": "96143dc",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
v73 (bb94de5)
|
api/amazon_relational.py
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""amazon_relational.py β the Amazon mirror becomes LOCKED DATABASES under Data (wave 52, R14).
|
| 2 |
+
|
| 3 |
+
`harness/amazon_store.py` pulls SP-API report documents into the tenant's DuckDB mirror; this reads
|
| 4 |
+
that mirror and spawns the `ut_amzn_*` grids a person opens. It is `meta_relational.py`'s shape
|
| 5 |
+
with one source swapped, which is itself `odoo_relational.py`'s shape.
|
| 6 |
+
|
| 7 |
+
plan(cur) the rows that WOULD be written β no store write at all
|
| 8 |
+
apply_plan(rt, built) create-or-merge every table and its rows, in ONE sync write
|
| 9 |
+
refresh(rt, why) both, with the mirror opened for this tenant
|
| 10 |
+
|
| 11 |
+
ββ **THE COLUMNS ARE DERIVED FROM THE MIRROR, NEVER TYPED OUT HERE**, exactly as Meta's are. For
|
| 12 |
+
Amazon that is doubly load-bearing: five of the seven tables have their columns MEASURED from the
|
| 13 |
+
report document's own header row at sync time (`amazon_schema.PROVENANCE`), so a hand-written field
|
| 14 |
+
list here would not merely rot, it would contradict the only measurement that exists. Add a column
|
| 15 |
+
to the report and it appears in the grid with no edit in this file.
|
| 16 |
+
|
| 17 |
+
ββ **R14 β THIS FILE IS WHERE THE CONNECTOR ENDS.** *"this is just database from Amazon, all its
|
| 18 |
+
fields mapped into our app so it can be seen under Data module and later be pulled into Sheet. All
|
| 19 |
+
database connector has this exact flow."* No dashboard, no module, no metric page. `Data` lists
|
| 20 |
+
these tables because they are `ut_*` tables; `sheet/BindingEditor.tsx:61` binds them on the `ut_`
|
| 21 |
+
prefix. Both are prefix-driven, so **neither needs a line of work here** β step 5 of `/connector` is
|
| 22 |
+
a verification, not a build.
|
| 23 |
+
|
| 24 |
+
β **R12 β AMAZON IS A STANDALONE DATABASE SET.** Owner: *"Its all different. Assume that its a
|
| 25 |
+
whole new databases which they are."* `LINKS` below therefore joins Amazon tables only to OTHER
|
| 26 |
+
AMAZON TABLES: order lines to their order, pricing to its listing. **Nothing here joins to
|
| 27 |
+
`product_data`, to `ut_odoo_*`, or to the hand-maintained ASIN mastersheet.** β That is what
|
| 28 |
+
removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
|
| 29 |
+
converted to an Amazon unit, because the two never meet.
|
| 30 |
+
|
| 31 |
+
β MATERIALISED, WITH THE CEILING REPORTED RATHER THAN ENFORCED. Same posture as Meta's: a table
|
| 32 |
+
over `MAX_ROWS` is REFUSED with its cause and its fix named, never truncated. For Amazon that is
|
| 33 |
+
standing rule 1's second half, and it is the likeliest table to hit it β an order-item history is
|
| 34 |
+
one row per line, not per order.
|
| 35 |
+
"""
|
| 36 |
+
import hashlib
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import sys
|
| 40 |
+
from pathlib import Path
|
| 41 |
+
|
| 42 |
+
_API = Path(__file__).resolve().parent
|
| 43 |
+
if str(_API.parent.parent / "platform") not in sys.path:
|
| 44 |
+
sys.path.insert(0, str(_API.parent.parent / "platform"))
|
| 45 |
+
|
| 46 |
+
import odoo_relational as _odoo # noqa: E402
|
| 47 |
+
from core import connector_spec # noqa: E402
|
| 48 |
+
from harness import amazon_schema, amazon_store, datastore # noqa: E402
|
| 49 |
+
|
| 50 |
+
Refused = _odoo.Refused
|
| 51 |
+
|
| 52 |
+
#: β THE PREFIX IS `ut_amzn_`, AND IT MUST ALSO BE IN `routes_connectors.CONNECTED_PREFIXES` β
|
| 53 |
+
#: otherwise `is_connected` falls through and every Amazon table is silently re-capped at
|
| 54 |
+
#: `MAX_ROWS`, breaking standing rule 1 with nothing raising. `verify_amazon` asserts the pair.
|
| 55 |
+
PREFIX = "ut_amzn_"
|
| 56 |
+
|
| 57 |
+
#: `ut_*` key -> the mirror table `amazon_store` writes. Derived from ONE list so a table cannot
|
| 58 |
+
#: exist on one side and not the other.
|
| 59 |
+
SOURCE = {PREFIX + t: amazon_store.table_name(t) for t in amazon_store.TABLES}
|
| 60 |
+
|
| 61 |
+
#: ββ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
|
| 62 |
+
#: (W53-T37, contract C2). `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their module-level
|
| 63 |
+
#: names because `verify_amazon`, `verify_meta.section_connector_spec` and the connector routes all
|
| 64 |
+
#: read them BY NAME β but each is a DERIVED VIEW over ONE declaration now, so the shape can no
|
| 65 |
+
#: longer be stated twice and say two things. Every comment that used to justify a literal here
|
| 66 |
+
#: (the two spines, R12, the report-vs-API spellings, the cardinalities) travelled with the values
|
| 67 |
+
#: into `connector_spec.AMAZON_TABLES`; read it there.
|
| 68 |
+
_SPEC = {s.key: s for s in connector_spec.tables_for("amazon")}
|
| 69 |
+
|
| 70 |
+
#: β A `ut_*` KEY THAT `amazon_store` SPAWNS BUT THE SPEC DOES NOT DECLARE IS ABSENT FROM ALL FOUR
|
| 71 |
+
#: DICTS BELOW, ON PURPOSE. `verify_amazon.section_relational` asserts the four key sets equal
|
| 72 |
+
#: `SOURCE`'s, so the drift reddens a gate rather than raising at import inside a live Space, and
|
| 73 |
+
#: it names the missing table instead of a KeyError deep in `apply_plan`.
|
| 74 |
+
LABELS = {k: _SPEC[k].label for k in SOURCE if k in _SPEC}
|
| 75 |
+
|
| 76 |
+
#: The column each grid shows first. β Every one is a name Amazon's own reports use, so a table
|
| 77 |
+
#: whose header does not carry it simply has no pinned column rather than a broken one.
|
| 78 |
+
PINNED = {k: _SPEC[k].pinned for k in SOURCE if k in _SPEC}
|
| 79 |
+
|
| 80 |
+
#: β THE ROW IDENTITY PER TABLE, IN FALLING PREFERENCE β a flat-file report has no `id` column the
|
| 81 |
+
#: way Graph does. When none is present the row is keyed by a hash of its own content (`_row_id`),
|
| 82 |
+
#: which still gives a STABLE id across re-syncs and is what makes the delete-then-insert merge
|
| 83 |
+
#: idempotent instead of appending a second copy of every row each time.
|
| 84 |
+
IDENTITY = {k: tuple(_SPEC[k].identity) for k in SOURCE if k in _SPEC}
|
| 85 |
+
|
| 86 |
+
#: `{ut_ key: [(this table's column, target ut_ key, the column it matches there), ...]}`.
|
| 87 |
+
#: β THE CARDINALITY IS DROPPED HERE AND KEPT IN THE SPEC. This view stays a 3-tuple because that
|
| 88 |
+
#: is the shape `verify_amazon.section_relational` unpacks; `connector_spec.edges_for` is what
|
| 89 |
+
#: `_fields_for` actually renders from, and it carries all four.
|
| 90 |
+
#: β AMAZON TO AMAZON ONLY (R12), and a link is DECLARED in the spec but VERIFIED AT SPAWN: five of
|
| 91 |
+
#: seven tables are `measured_at_sync`, so every entry is a hypothesis about a report column name
|
| 92 |
+
#: until `_fields_for` confirms BOTH sides carry it. A wrong guess here is cheap; a wrong guess in a
|
| 93 |
+
#: JOIN would not be. A table with no links is absent rather than present-and-empty, matching the
|
| 94 |
+
#: dict this replaced.
|
| 95 |
+
LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links]
|
| 96 |
+
for k in SOURCE if k in _SPEC and _SPEC[k].links}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def source_attribution(key):
|
| 100 |
+
"""Durable origin metadata for one materialised Amazon database.
|
| 101 |
+
|
| 102 |
+
DPP 1.8 permits a database separation strategy or a mechanism that tags and identifies the
|
| 103 |
+
origin of data. The DuckDB mirror uses the reserved ``amzn_*`` namespace already; this stamp
|
| 104 |
+
is the corresponding mechanism in the shared ``user_tables`` document. It is written on
|
| 105 |
+
every refresh so a user-facing label change cannot erase the Amazon origin or the exact raw
|
| 106 |
+
table/report that supplied the rows.
|
| 107 |
+
"""
|
| 108 |
+
if key not in SOURCE or key not in _SPEC:
|
| 109 |
+
raise KeyError(f"unknown Amazon database {key!r}")
|
| 110 |
+
return {
|
| 111 |
+
"key": "amazon_sp",
|
| 112 |
+
"origin": "Amazon Services API",
|
| 113 |
+
"rawTable": SOURCE[key],
|
| 114 |
+
"native": _SPEC[key].native,
|
| 115 |
+
"readOnly": True,
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 120 |
+
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 121 |
+
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
| 122 |
+
"report_date", "start_date", "end_date", "_time")
|
| 123 |
+
|
| 124 |
+
#: β MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
|
| 125 |
+
#: and an int column would round every row of a settlement. The grid shows what the report said and
|
| 126 |
+
#: converts nothing.
|
| 127 |
+
_MONEY = {"item_price", "item_tax", "shipping_price", "shipping_tax", "gift_wrap_price",
|
| 128 |
+
"gift_wrap_tax", "item_promotion_discount", "ship_promotion_discount", "price",
|
| 129 |
+
"amount", "total_amount", "principal", "shipping", "tax", "promotion", "fee",
|
| 130 |
+
"your_price", "landed_price", "listing_price", "open_date_price",
|
| 131 |
+
"sales_by_asin_ordered_product_sales_amount",
|
| 132 |
+
"sales_by_asin_ordered_product_sales_b2_b_amount"}
|
| 133 |
+
|
| 134 |
+
#: Percentages and rates are fractional too, but they are not money. `currency` is the only
|
| 135 |
+
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
|
| 136 |
+
_RATE_SUFFIX = ("_percentage", "_percentage_b2_b", "_rate", "_ctr")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _label(col):
|
| 140 |
+
return col.replace("_", " ").strip().capitalize()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _ftype(col, duck_type):
|
| 144 |
+
"""The grid field type for one mirror column. Name first, storage second.
|
| 145 |
+
|
| 146 |
+
β NAME BEATS STORAGE FOR DATES ON PURPOSE. Every flat-file column arrives as text, so a
|
| 147 |
+
purchase date is a VARCHAR in the mirror and would render as a string in the grid unless the
|
| 148 |
+
name is allowed to override. The store's `_coltype` already refuses to guess a type for text
|
| 149 |
+
it cannot parse; this is the presentation half of the same judgement.
|
| 150 |
+
"""
|
| 151 |
+
if any(col == s or col.endswith(s) for s in _DATEISH):
|
| 152 |
+
return "date"
|
| 153 |
+
if col in _MONEY or any(col.endswith(s) for s in _RATE_SUFFIX):
|
| 154 |
+
return "currency"
|
| 155 |
+
base = str(duck_type or "").upper().split("(")[0]
|
| 156 |
+
if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
|
| 157 |
+
return "int"
|
| 158 |
+
if base in ("DOUBLE", "FLOAT"):
|
| 159 |
+
return "currency"
|
| 160 |
+
return "text"
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def mirror_columns(cur, table):
|
| 164 |
+
"""[(name, type)] for a mirror table, or [] when it has never been synced."""
|
| 165 |
+
try:
|
| 166 |
+
return [(r[1], r[2]) for r in cur.execute(f"PRAGMA table_info('{table}')").fetchall()]
|
| 167 |
+
except Exception: # noqa: BLE001
|
| 168 |
+
return []
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _fields_for(cur, key):
|
| 172 |
+
"""One preset field per mirror column β the whole schema, derived. See the header."""
|
| 173 |
+
cols = mirror_columns(cur, SOURCE[key])
|
| 174 |
+
if not cols:
|
| 175 |
+
return []
|
| 176 |
+
names = {c[0] for c in cols}
|
| 177 |
+
pinned = PINNED.get(key) if PINNED.get(key) in names else None
|
| 178 |
+
out = []
|
| 179 |
+
for name, dtype in cols:
|
| 180 |
+
out.append(_odoo._preset({
|
| 181 |
+
"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
| 182 |
+
"source": "overlay",
|
| 183 |
+
"default": name == pinned or name in _DEFAULT_COLUMNS,
|
| 184 |
+
"pinned": name == pinned,
|
| 185 |
+
}, flow="amazon_relational"))
|
| 186 |
+
# β BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
|
| 187 |
+
# known until it syncs, so a link to a table that has never run would name a column that does
|
| 188 |
+
# not exist. **Absent is a normal state here, not a failure** β which is exactly what lets the
|
| 189 |
+
# map above declare hypotheses about report column names before any report has been seen.
|
| 190 |
+
# β A TABLE MAY NOW CARRY SEVERAL LINKS. The first version allowed one, which could express the
|
| 191 |
+
# transaction spine or the product spine but never both, and Amazon's data needs both.
|
| 192 |
+
# ββ THE EDGE'S KEY AND LABEL COME FROM `connector_spec.edges_for`, NOT FROM AN f-STRING HERE
|
| 193 |
+
# (W53-T37). `f"{target}_link__{from_col}"` and `f"{label} by {_label(from_col)}"` are ONE
|
| 194 |
+
# convention shared by every connector: the source column is in the key because two links to
|
| 195 |
+
# the same target (orders by id and by sku) would otherwise collide on one key and the second
|
| 196 |
+
# would silently overwrite the first. Meta carried the older `f"{target}_link"` and therefore
|
| 197 |
+
# carried that collision, latent; a shared generator is what retires it for the next connector
|
| 198 |
+
# too, instead of each module re-typing the rule and one of them getting it wrong.
|
| 199 |
+
for edge in connector_spec.edges_for(key):
|
| 200 |
+
if edge["from_col"] not in names:
|
| 201 |
+
continue
|
| 202 |
+
if not any(c[0] == edge["to_col"]
|
| 203 |
+
for c in mirror_columns(cur, SOURCE.get(edge["target"], ""))):
|
| 204 |
+
continue
|
| 205 |
+
out.append(_odoo._preset({
|
| 206 |
+
"key": edge["key"], "label": edge["label"],
|
| 207 |
+
"type": "link", "source": "overlay", "default": True,
|
| 208 |
+
"link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"]},
|
| 209 |
+
}, flow="amazon_relational"))
|
| 210 |
+
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
|
| 211 |
+
return out
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
#: Columns worth showing before a person picks their own. Deliberately short: these tables are
|
| 215 |
+
#: wide, and a default of "everything" is the same as no default at all.
|
| 216 |
+
_DEFAULT_COLUMNS = {
|
| 217 |
+
"amazon_order_id", "purchase_date", "order_status", "sku", "seller_sku", "asin", "asin1",
|
| 218 |
+
"quantity", "quantity_purchased", "item_price", "item_status", "product_name", "item_name",
|
| 219 |
+
"settlement_id", "amount", "amount_type", "amount_description", "posted_date",
|
| 220 |
+
"parent_asin", "child_asin", "search_term", "search_frequency_rank",
|
| 221 |
+
"traffic_by_asin_buy_box_percentage", "sales_by_asin_units_ordered",
|
| 222 |
+
}
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def _row_id(row, key):
|
| 226 |
+
"""A STABLE id for one mirror row.
|
| 227 |
+
|
| 228 |
+
β STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
|
| 229 |
+
merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
|
| 230 |
+
re-insert of the grid β losing any per-row state the product hangs off that id.
|
| 231 |
+
|
| 232 |
+
Falls back to a content hash rather than a counter for exactly that reason: a settlement line
|
| 233 |
+
with no id column still hashes to the same value next week if its content has not changed.
|
| 234 |
+
"""
|
| 235 |
+
for col in IDENTITY.get(key, ()):
|
| 236 |
+
val = row.get(col)
|
| 237 |
+
if val not in (None, ""):
|
| 238 |
+
return str(val)
|
| 239 |
+
payload = json.dumps({k: row.get(k) for k in sorted(row)}, sort_keys=True,
|
| 240 |
+
ensure_ascii=True, separators=(",", ":"), default=str)
|
| 241 |
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _read(cur, key):
|
| 245 |
+
"""[(row dict)] straight off the mirror, each carrying a stable `_id`."""
|
| 246 |
+
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
|
| 247 |
+
if not cols:
|
| 248 |
+
return []
|
| 249 |
+
quoted = ", ".join(f'"{c}"' for c in cols)
|
| 250 |
+
rows = cur.execute(f"SELECT {quoted} FROM {SOURCE[key]}").fetchall()
|
| 251 |
+
out = []
|
| 252 |
+
for r in rows:
|
| 253 |
+
row = {}
|
| 254 |
+
for name, val in zip(cols, r):
|
| 255 |
+
row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val))
|
| 256 |
+
row["_id"] = _row_id(row, key)
|
| 257 |
+
out.append(row)
|
| 258 |
+
return out
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def available(tenant_key="royal-imports"):
|
| 262 |
+
"""Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
|
| 263 |
+
try:
|
| 264 |
+
counts = amazon_store.status(tenant_key)
|
| 265 |
+
except Exception: # noqa: BLE001
|
| 266 |
+
return False
|
| 267 |
+
return any(bool((v or {}).get("rows")) for v in counts.values())
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def plan(cur, rt=None):
|
| 271 |
+
"""{bucket_key: rows} plus `problems`. No store write; every cap checked BEFORE any commit."""
|
| 272 |
+
built = {"problems": []}
|
| 273 |
+
ut = _odoo._ut()
|
| 274 |
+
cap = getattr(ut, "MAX_ROWS", 60_000)
|
| 275 |
+
for key in SOURCE:
|
| 276 |
+
# β COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
|
| 277 |
+
# the cap afterwards means the one table too big to keep is the one built in memory first.
|
| 278 |
+
try:
|
| 279 |
+
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
|
| 280 |
+
except Exception: # noqa: BLE001
|
| 281 |
+
continue # absent = never synced
|
| 282 |
+
if n > cap:
|
| 283 |
+
# β REPORTED WITH ITS CAUSE AND ITS FIX, NEVER TRUNCATED. Standing rule 1's second
|
| 284 |
+
# half, and `ut_amzn_order_items` is the table most likely to reach it: an order
|
| 285 |
+
# history is one row per LINE, so a mid-size seller passes 60,000 in a year.
|
| 286 |
+
built["problems"].append(
|
| 287 |
+
f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was "
|
| 288 |
+
f"truncated and nothing was written for this table. Cause: this seller's history "
|
| 289 |
+
f"is larger than a materialised ut_* table can hold. Fix: bind it read-through "
|
| 290 |
+
f"off the mirror as `ut_odoo_gl_lines` is "
|
| 291 |
+
f"(odoo_relational.READ_THROUGH_KEYS), which `routes_connected_tables` "
|
| 292 |
+
f"already serves.")
|
| 293 |
+
continue
|
| 294 |
+
rows = _read(cur, key)
|
| 295 |
+
if rows:
|
| 296 |
+
built[key] = rows
|
| 297 |
+
return built
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def plan_fingerprint(cur, built):
|
| 301 |
+
"""Digest the exact derived Amazon content that would be published.
|
| 302 |
+
|
| 303 |
+
`amazon_store.sync()` counts fetched rows, not changed rows, so it is not equality evidence.
|
| 304 |
+
Row order out of DuckDB is arbitrary and therefore non-semantic, so rows are canonicalised
|
| 305 |
+
before hashing. Callers fail safe to a refresh if this raises.
|
| 306 |
+
"""
|
| 307 |
+
tables = []
|
| 308 |
+
for key in SOURCE:
|
| 309 |
+
rows = built.get(key) or []
|
| 310 |
+
canonical = sorted(rows, key=lambda row: json.dumps(
|
| 311 |
+
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
|
| 312 |
+
tables.append({"key": key, "fields": _fields_for(cur, key), "rows": canonical})
|
| 313 |
+
payload = json.dumps(tables, sort_keys=True, ensure_ascii=True,
|
| 314 |
+
separators=(",", ":"), default=str)
|
| 315 |
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def apply_plan(rt, built, username="amazon", today=None):
|
| 319 |
+
"""Create-or-merge every table in `built`, in ONE `flush="sync"` write.
|
| 320 |
+
|
| 321 |
+
β ONE write for all seven, not seven. `Store.update` is a full download plus a full upload of
|
| 322 |
+
the tenant document, so seven would be seven round trips and seven commits per resync. It is
|
| 323 |
+
also ATOMIC: a `Refused` at MAX_TABLES aborts before anything persists, rather than leaving the
|
| 324 |
+
half-spawn `plan()` exists to prevent.
|
| 325 |
+
"""
|
| 326 |
+
if built.get("problems"):
|
| 327 |
+
raise Refused("; ".join(built["problems"]))
|
| 328 |
+
stamp = today or _odoo._iso_today()
|
| 329 |
+
written = {}
|
| 330 |
+
cur_mirror = datastore.ro_con()
|
| 331 |
+
plans = [(key, LABELS[key], _fields_for(cur_mirror, key), built[key])
|
| 332 |
+
for key in SOURCE if key in built]
|
| 333 |
+
|
| 334 |
+
def _apply_all(doc):
|
| 335 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 336 |
+
for key, label, fields, rows in plans:
|
| 337 |
+
written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
|
| 338 |
+
username, stamp)
|
| 339 |
+
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 340 |
+
# mutable display label or from the process having imported this module.
|
| 341 |
+
doc[key]["connector"] = source_attribution(key)
|
| 342 |
+
return doc
|
| 343 |
+
|
| 344 |
+
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
| 345 |
+
return written
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
|
| 349 |
+
"""Open, plan and conditionally apply the Amazon mirror.
|
| 350 |
+
|
| 351 |
+
Returns `{}` and says why when the mirror holds no Amazon tables β a tenant that never
|
| 352 |
+
connected Amazon is a normal state, not a failure.
|
| 353 |
+
"""
|
| 354 |
+
state = {"present": False, "known": False, "fingerprint": None,
|
| 355 |
+
"changed": False, "applied": False, "written": {}}
|
| 356 |
+
tenant = tenant_key or getattr(getattr(rt, "tenant", None), "key", None) or "royal-imports"
|
| 357 |
+
path = datastore.path_for(tenant)
|
| 358 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 359 |
+
datastore.use_path(path)
|
| 360 |
+
cur = datastore.ro_con()
|
| 361 |
+
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 362 |
+
log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
|
| 363 |
+
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
|
| 364 |
+
return state
|
| 365 |
+
state["present"] = True
|
| 366 |
+
built = plan(cur, rt)
|
| 367 |
+
problems = built.get("problems") or []
|
| 368 |
+
for p in problems:
|
| 369 |
+
log(f"[amazon_relational] PROBLEM {p}")
|
| 370 |
+
applicable = {k: v for k, v in built.items() if k != "problems"}
|
| 371 |
+
if not problems:
|
| 372 |
+
try:
|
| 373 |
+
state["fingerprint"] = plan_fingerprint(cur, applicable)
|
| 374 |
+
state["known"] = True
|
| 375 |
+
except Exception as e: # noqa: BLE001
|
| 376 |
+
log(f"[amazon_relational] {why}: plan fingerprint unavailable; forcing refresh "
|
| 377 |
+
f"({type(e).__name__}: {e})")
|
| 378 |
+
if state["known"] and state["fingerprint"] == previous_fingerprint:
|
| 379 |
+
log(f"[amazon_relational] {why}: Amazon plan unchanged; skipped full user_tables write")
|
| 380 |
+
return state
|
| 381 |
+
state["changed"] = True
|
| 382 |
+
state["written"] = apply_plan(rt, applicable)
|
| 383 |
+
state["applied"] = True
|
| 384 |
+
for key, counts in state["written"].items():
|
| 385 |
+
log(f"[amazon_relational] {why}: {key:<28} {counts}")
|
| 386 |
+
return state
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def refresh(rt, why="boot", tenant_key=None, log=print):
|
| 390 |
+
"""Backward-compatible facade. Use `refresh_state` for egress-aware state."""
|
| 391 |
+
return refresh_state(rt, why, tenant_key, log=log)["written"]
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def main(argv=None):
|
| 395 |
+
"""`python aios-web/api/amazon_relational.py --refresh [--tenant X]` β the manual door."""
|
| 396 |
+
import argparse
|
| 397 |
+
ap = argparse.ArgumentParser()
|
| 398 |
+
ap.add_argument("--refresh", action="store_true")
|
| 399 |
+
ap.add_argument("--plan", action="store_true")
|
| 400 |
+
ap.add_argument("--tenant", default="royal-imports")
|
| 401 |
+
a = ap.parse_args(argv)
|
| 402 |
+
os.environ.setdefault("AIOS_PREWARM", "0")
|
| 403 |
+
from harness import runtime as _rt
|
| 404 |
+
rt = _rt.get_runtime(a.tenant)
|
| 405 |
+
if a.plan:
|
| 406 |
+
path = datastore.path_for(a.tenant)
|
| 407 |
+
if Path(datastore.DB_PATH) != Path(path):
|
| 408 |
+
datastore.use_path(path)
|
| 409 |
+
built = plan(datastore.ro_con(), rt)
|
| 410 |
+
for k, v in built.items():
|
| 411 |
+
print(f" {k:<28} {len(v) if isinstance(v, list) else v}")
|
| 412 |
+
return 0
|
| 413 |
+
if a.refresh:
|
| 414 |
+
refresh(rt, why="cli", tenant_key=a.tenant)
|
| 415 |
+
return 0
|
| 416 |
+
ap.print_help()
|
| 417 |
+
return 2
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
if __name__ == "__main__":
|
| 421 |
+
sys.exit(main())
|
api/application_registry.py
CHANGED
|
@@ -7,14 +7,59 @@ import uuid
|
|
| 7 |
|
| 8 |
|
| 9 |
APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
|
|
|
|
| 10 |
_KEY = "applications_v1"
|
| 11 |
_MAX = 500
|
| 12 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
class ApplicationError(ValueError):
|
| 15 |
pass
|
| 16 |
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
def blank_doc():
|
| 19 |
return {"schema": 1, "applications": {}, "defaults": {}}
|
| 20 |
|
|
@@ -125,6 +170,129 @@ def _allowed(compatible, allowed_databases):
|
|
| 125 |
else set(compatible or ()) & set(allowed_databases))
|
| 126 |
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 129 |
is_default=False):
|
| 130 |
row=copy.deepcopy(app)
|
|
@@ -160,23 +328,34 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 160 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 161 |
|
| 162 |
|
| 163 |
-
def create(runtime, tenant, user, payload, *, semantic_records=None,
|
|
|
|
| 164 |
payload=payload if isinstance(payload,dict) else {}
|
| 165 |
-
|
| 166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 168 |
if not name: raise ApplicationError("application name is required")
|
| 169 |
app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat()
|
| 170 |
-
compatible =
|
|
|
|
| 171 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 172 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
app={"id":app_id,"tenant":str(tenant),"type":kind,"name":name,
|
| 174 |
"compatibleDatabases":compatible,
|
| 175 |
"semanticRoleBindings":_governed_bindings(
|
| 176 |
payload.get("semanticRoleBindings"), compatible, semantic_records or {}),
|
| 177 |
-
"config":_typed_config(kind,
|
| 178 |
"share":{"mode":"private","users":[]},"starredBy":[],"createdBy":user,
|
| 179 |
"createdAt":now,"updatedAt":now,"revision":1}
|
|
|
|
| 180 |
def apply(raw):
|
| 181 |
doc=_clean_doc(raw)
|
| 182 |
if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached")
|
|
@@ -188,11 +367,19 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, allowed_dat
|
|
| 188 |
def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
|
| 189 |
allowed_databases=None, target_database=None):
|
| 190 |
patch=patch if isinstance(patch,dict) else {}; landed={}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
def apply(raw):
|
| 192 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 193 |
if not app: raise ApplicationError("application does not exist")
|
| 194 |
if not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 195 |
raise ApplicationError("application does not exist")
|
|
|
|
|
|
|
| 196 |
star_only = set(patch) <= {"starred"}
|
| 197 |
if star_only:
|
| 198 |
if not may_read(app,user,is_admin): raise PermissionError("application is not available")
|
|
@@ -230,15 +417,19 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
|
|
| 230 |
|
| 231 |
|
| 232 |
def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records=None,
|
| 233 |
-
allowed_databases=None):
|
| 234 |
app=load(runtime)["applications"].get(app_id)
|
| 235 |
if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
|
| 236 |
if not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 237 |
raise ApplicationError("application does not exist")
|
| 238 |
-
|
| 239 |
"compatibleDatabases":app.get("compatibleDatabases"),
|
| 240 |
-
"semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
|
| 241 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
|
| 243 |
|
| 244 |
def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None):
|
|
|
|
| 7 |
|
| 8 |
|
| 9 |
APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
|
| 10 |
+
APPLICATION_TEMPLATE_KINDS = ("route", "catalog")
|
| 11 |
_KEY = "applications_v1"
|
| 12 |
_MAX = 500
|
| 13 |
|
| 14 |
+
# Routing and Catalog are the first authored App templates. These are wire contracts, not
|
| 15 |
+
# executable module declarations: the client renders a trusted platform surface from the stored
|
| 16 |
+
# definition. ``required`` is consumed by the creation flow and enforced again below, so a
|
| 17 |
+
# future client cannot weaken a template by skipping a wizard step.
|
| 18 |
+
_TEMPLATE_CONTRACTS = {
|
| 19 |
+
"route": {
|
| 20 |
+
"kind": "route",
|
| 21 |
+
"label": "Routing",
|
| 22 |
+
"applicationType": "route",
|
| 23 |
+
"displayMode": "map",
|
| 24 |
+
"fieldRoles": [
|
| 25 |
+
{"key": "title", "label": "Record title", "required": True, "types": []},
|
| 26 |
+
{"key": "latitude", "label": "Latitude", "required": False,
|
| 27 |
+
"types": ["text", "int", "currency", "pct", "rating", "number", "float", "decimal"]},
|
| 28 |
+
{"key": "longitude", "label": "Longitude", "required": False,
|
| 29 |
+
"types": ["text", "int", "currency", "pct", "rating", "number", "float", "decimal"]},
|
| 30 |
+
{"key": "address", "label": "Address", "required": False,
|
| 31 |
+
"types": ["text"]},
|
| 32 |
+
],
|
| 33 |
+
"defaultPlacement": {"section": "app", "order": 0},
|
| 34 |
+
},
|
| 35 |
+
"catalog": {
|
| 36 |
+
"kind": "catalog",
|
| 37 |
+
"label": "Catalog",
|
| 38 |
+
# Catalog is a template discriminator over an existing trusted renderer contract. The
|
| 39 |
+
# governed Application type vocabulary stays four-way; the client launches the Catalog
|
| 40 |
+
# surface from ``templateKind`` rather than teaching every shared type mirror a fifth word.
|
| 41 |
+
"applicationType": "map",
|
| 42 |
+
"displayMode": "map",
|
| 43 |
+
"fieldRoles": [
|
| 44 |
+
{"key": "key", "label": "Record key", "required": True, "types": []},
|
| 45 |
+
{"key": "title", "label": "Record title", "required": True, "types": []},
|
| 46 |
+
{"key": "image", "label": "Image", "required": False,
|
| 47 |
+
"types": ["image", "attachment", "url", "text"]},
|
| 48 |
+
],
|
| 49 |
+
"defaultPlacement": {"section": "app", "order": 0},
|
| 50 |
+
},
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
|
| 54 |
class ApplicationError(ValueError):
|
| 55 |
pass
|
| 56 |
|
| 57 |
|
| 58 |
+
def template_contracts():
|
| 59 |
+
"""The trusted renderer contracts offered by the Application creation flow."""
|
| 60 |
+
return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_TEMPLATE_KINDS]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
def blank_doc():
|
| 64 |
return {"schema": 1, "applications": {}, "defaults": {}}
|
| 65 |
|
|
|
|
| 170 |
else set(compatible or ()) & set(allowed_databases))
|
| 171 |
|
| 172 |
|
| 173 |
+
def _contract_fields(database, semantic_records=None, field_contracts=None):
|
| 174 |
+
"""Return ``{field key: type}`` from a caller's already permission-scoped contract.
|
| 175 |
+
|
| 176 |
+
The nav creation door supplies its served schema fields. The older Application door supplies
|
| 177 |
+
governed semantic records. Supporting both here keeps one validator while never making this
|
| 178 |
+
registry reach around either caller's database/field wall.
|
| 179 |
+
"""
|
| 180 |
+
fields = {}
|
| 181 |
+
raw = ((field_contracts or {}).get(database)
|
| 182 |
+
if isinstance(field_contracts, dict) else None)
|
| 183 |
+
if isinstance(raw, dict):
|
| 184 |
+
raw = list(raw.values())
|
| 185 |
+
for row in raw if isinstance(raw, (list, tuple)) else ():
|
| 186 |
+
if not isinstance(row, dict):
|
| 187 |
+
continue
|
| 188 |
+
key = str(row.get("key") or row.get("field") or "").strip()
|
| 189 |
+
if key:
|
| 190 |
+
fields[key] = str(row.get("type") or row.get("dataType") or "").strip().lower()
|
| 191 |
+
records = ((semantic_records or {}).get(database)
|
| 192 |
+
if isinstance(semantic_records, dict) else None)
|
| 193 |
+
for row in (records or {}).values() if isinstance(records, dict) else ():
|
| 194 |
+
if not isinstance(row, dict):
|
| 195 |
+
continue
|
| 196 |
+
key = str(row.get("field") or row.get("key") or "").strip()
|
| 197 |
+
if key and key not in fields:
|
| 198 |
+
fields[key] = str(row.get("dataType") or row.get("type") or "").strip().lower()
|
| 199 |
+
return fields
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _template_definition(kind, payload, *, semantic_records=None, field_contracts=None,
|
| 203 |
+
allowed_databases=None):
|
| 204 |
+
contract = _TEMPLATE_CONTRACTS[kind]
|
| 205 |
+
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 206 |
+
if template_kind != kind:
|
| 207 |
+
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
|
| 208 |
+
source = str(payload.get("sourceDatabase") or "").strip()[:120]
|
| 209 |
+
if not source:
|
| 210 |
+
raise ApplicationError("sourceDatabase is required")
|
| 211 |
+
if source not in set(allowed_databases or ()):
|
| 212 |
+
raise PermissionError("an Application cannot bind a database you cannot open")
|
| 213 |
+
if "compatibleDatabases" in payload:
|
| 214 |
+
compatible = _compat(payload.get("compatibleDatabases"))
|
| 215 |
+
if compatible != [source]:
|
| 216 |
+
raise ApplicationError("a template Application has exactly one sourceDatabase")
|
| 217 |
+
|
| 218 |
+
raw_map = payload.get("fieldMap")
|
| 219 |
+
if not isinstance(raw_map, dict):
|
| 220 |
+
raise ApplicationError("fieldMap must be an object")
|
| 221 |
+
roles = {row["key"]: row for row in contract["fieldRoles"]}
|
| 222 |
+
extras = sorted(set(raw_map) - set(roles))
|
| 223 |
+
if extras:
|
| 224 |
+
raise ApplicationError("fieldMap contains roles outside the template contract: "
|
| 225 |
+
+ ", ".join(extras))
|
| 226 |
+
missing = sorted(key for key, role in roles.items()
|
| 227 |
+
if role["required"] and not str(raw_map.get(key) or "").strip())
|
| 228 |
+
if missing:
|
| 229 |
+
raise ApplicationError("fieldMap is missing required roles: " + ", ".join(missing))
|
| 230 |
+
available = _contract_fields(source, semantic_records, field_contracts)
|
| 231 |
+
if not available:
|
| 232 |
+
raise ApplicationError("the source database has no readable field contract")
|
| 233 |
+
field_map = {}
|
| 234 |
+
for role, raw_field in raw_map.items():
|
| 235 |
+
field = str(raw_field or "").strip()
|
| 236 |
+
if not field:
|
| 237 |
+
continue
|
| 238 |
+
if len(field) > 160 or field not in available:
|
| 239 |
+
raise ApplicationError(f"fieldMap role {role!r} does not name a source field")
|
| 240 |
+
allowed_types = set(roles[role].get("types") or ())
|
| 241 |
+
if allowed_types and available[field] not in allowed_types:
|
| 242 |
+
raise ApplicationError(
|
| 243 |
+
f"fieldMap role {role!r} cannot use field type {available[field]!r}")
|
| 244 |
+
field_map[role] = field
|
| 245 |
+
latitude, longitude = field_map.get("latitude"), field_map.get("longitude")
|
| 246 |
+
if bool(latitude) != bool(longitude):
|
| 247 |
+
raise ApplicationError("Routing needs both coordinate fields or neither")
|
| 248 |
+
if latitude and latitude == longitude:
|
| 249 |
+
raise ApplicationError("Routing latitude and longitude must use different fields")
|
| 250 |
+
# A new Route must name at least one placement source. Stored definitions are not revalidated
|
| 251 |
+
# on read, so this create-time refusal does not retire older Applications with an empty map.
|
| 252 |
+
if kind == "route" and not (latitude and longitude) and not field_map.get("address"):
|
| 253 |
+
raise ApplicationError("Routing needs both coordinate fields or an address field")
|
| 254 |
+
|
| 255 |
+
raw_placement = payload.get("navPlacement")
|
| 256 |
+
if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
|
| 257 |
+
raise ApplicationError("navPlacement must contain section and order")
|
| 258 |
+
section = str(raw_placement.get("section") or "").strip().lower()
|
| 259 |
+
order = raw_placement.get("order")
|
| 260 |
+
if section != "app":
|
| 261 |
+
raise ApplicationError("template Applications must be placed in App")
|
| 262 |
+
if isinstance(order, bool) or not isinstance(order, int) or not 0 <= order <= 999:
|
| 263 |
+
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
|
| 264 |
+
return {"templateKind": kind, "sourceDatabase": source, "fieldMap": field_map,
|
| 265 |
+
"navPlacement": {"section": "app", "order": order}}
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def validate_application(payload, tenant, *, semantic_records=None, field_contracts=None,
|
| 269 |
+
allowed_databases=None):
|
| 270 |
+
"""Validate the server-authored identity/tenant envelope and any template definition."""
|
| 271 |
+
payload = payload if isinstance(payload, dict) else {}
|
| 272 |
+
if payload.get("id") not in (None, ""):
|
| 273 |
+
raise ApplicationError("application id is server assigned")
|
| 274 |
+
if "tenant" in payload and str(payload.get("tenant") or "") != str(tenant):
|
| 275 |
+
raise PermissionError("an Application cannot be created for another tenant")
|
| 276 |
+
template_kind = str(payload.get("templateKind") or "").strip().lower()
|
| 277 |
+
if template_kind and template_kind not in APPLICATION_TEMPLATE_KINDS:
|
| 278 |
+
raise ApplicationError("application template kind is not recognized")
|
| 279 |
+
expected_type = (_TEMPLATE_CONTRACTS[template_kind]["applicationType"]
|
| 280 |
+
if template_kind else "")
|
| 281 |
+
kind = str(payload.get("type") or expected_type).strip().lower()
|
| 282 |
+
if kind not in APPLICATION_TYPES:
|
| 283 |
+
raise ApplicationError("application type is not recognized")
|
| 284 |
+
if template_kind:
|
| 285 |
+
if kind != expected_type:
|
| 286 |
+
raise ApplicationError(
|
| 287 |
+
f"{template_kind} template Applications require type {expected_type!r}")
|
| 288 |
+
return _template_definition(
|
| 289 |
+
template_kind, payload, semantic_records=semantic_records,
|
| 290 |
+
field_contracts=field_contracts, allowed_databases=allowed_databases)
|
| 291 |
+
if any(key in payload for key in ("sourceDatabase", "fieldMap", "navPlacement")):
|
| 292 |
+
raise ApplicationError("only a template Application may carry template fields")
|
| 293 |
+
return {}
|
| 294 |
+
|
| 295 |
+
|
| 296 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 297 |
is_default=False):
|
| 298 |
row=copy.deepcopy(app)
|
|
|
|
| 328 |
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
|
| 329 |
|
| 330 |
|
| 331 |
+
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
|
| 332 |
+
allowed_databases=None):
|
| 333 |
payload=payload if isinstance(payload,dict) else {}
|
| 334 |
+
template_kind=str(payload.get("templateKind") or "").strip().lower()
|
| 335 |
+
expected_type = (_TEMPLATE_CONTRACTS.get(template_kind) or {}).get("applicationType")
|
| 336 |
+
kind=str(payload.get("type") or expected_type or "").strip().lower()
|
| 337 |
+
definition = validate_application(
|
| 338 |
+
payload, tenant, semantic_records=semantic_records, field_contracts=field_contracts,
|
| 339 |
+
allowed_databases=allowed_databases)
|
| 340 |
name=" ".join(str(payload.get("name") or "").split())[:120]
|
| 341 |
if not name: raise ApplicationError("application name is required")
|
| 342 |
app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat()
|
| 343 |
+
compatible = ([definition["sourceDatabase"]] if definition
|
| 344 |
+
else _compat(payload.get("compatibleDatabases")))
|
| 345 |
if _allowed(compatible, allowed_databases) != set(compatible):
|
| 346 |
raise PermissionError("an Application cannot bind a database you cannot open")
|
| 347 |
+
raw_config = payload.get("config")
|
| 348 |
+
if definition and raw_config is None:
|
| 349 |
+
raw_config = {"schemaVersion": 1,
|
| 350 |
+
"display": {"mode": _TEMPLATE_CONTRACTS[template_kind]["displayMode"]}}
|
| 351 |
app={"id":app_id,"tenant":str(tenant),"type":kind,"name":name,
|
| 352 |
"compatibleDatabases":compatible,
|
| 353 |
"semanticRoleBindings":_governed_bindings(
|
| 354 |
payload.get("semanticRoleBindings"), compatible, semantic_records or {}),
|
| 355 |
+
"config":_typed_config(kind, raw_config),
|
| 356 |
"share":{"mode":"private","users":[]},"starredBy":[],"createdBy":user,
|
| 357 |
"createdAt":now,"updatedAt":now,"revision":1}
|
| 358 |
+
app.update(definition)
|
| 359 |
def apply(raw):
|
| 360 |
doc=_clean_doc(raw)
|
| 361 |
if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached")
|
|
|
|
| 367 |
def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
|
| 368 |
allowed_databases=None, target_database=None):
|
| 369 |
patch=patch if isinstance(patch,dict) else {}; landed={}
|
| 370 |
+
immutable = sorted(set(patch) & {
|
| 371 |
+
"id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"
|
| 372 |
+
})
|
| 373 |
+
if immutable:
|
| 374 |
+
raise ApplicationError("Application identity and template definition are immutable: "
|
| 375 |
+
+ ", ".join(immutable))
|
| 376 |
def apply(raw):
|
| 377 |
doc=_clean_doc(raw); app=doc["applications"].get(app_id)
|
| 378 |
if not app: raise ApplicationError("application does not exist")
|
| 379 |
if not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 380 |
raise ApplicationError("application does not exist")
|
| 381 |
+
if app.get("templateKind") in APPLICATION_TEMPLATE_KINDS and "compatibleDatabases" in patch:
|
| 382 |
+
raise ApplicationError("a template Application's sourceDatabase is immutable")
|
| 383 |
star_only = set(patch) <= {"starred"}
|
| 384 |
if star_only:
|
| 385 |
if not may_read(app,user,is_admin): raise PermissionError("application is not available")
|
|
|
|
| 417 |
|
| 418 |
|
| 419 |
def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records=None,
|
| 420 |
+
field_contracts=None, allowed_databases=None):
|
| 421 |
app=load(runtime)["applications"].get(app_id)
|
| 422 |
if not may_read(app,user,is_admin): raise ApplicationError("application does not exist")
|
| 423 |
if not _allowed(app.get("compatibleDatabases"), allowed_databases):
|
| 424 |
raise ApplicationError("application does not exist")
|
| 425 |
+
payload={"type":app["type"],"name":f"{app['name']} copy",
|
| 426 |
"compatibleDatabases":app.get("compatibleDatabases"),
|
| 427 |
+
"semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}
|
| 428 |
+
for key in ("templateKind", "sourceDatabase", "fieldMap", "navPlacement"):
|
| 429 |
+
if key in app: payload[key]=copy.deepcopy(app[key])
|
| 430 |
+
return create(runtime,tenant,user,payload,
|
| 431 |
+
semantic_records=semantic_records, field_contracts=field_contracts,
|
| 432 |
+
allowed_databases=allowed_databases)
|
| 433 |
|
| 434 |
|
| 435 |
def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None):
|
api/automation_engine.py
CHANGED
|
@@ -4097,7 +4097,7 @@ POST_SNAPSHOT_FIELDS = [
|
|
| 4097 |
# ββ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO
|
| 4098 |
# orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`,
|
| 4099 |
# and `ut_ig_posts` already performs the first of them. See the verdict in
|
| 4100 |
-
# `
|
| 4101 |
# moving the window here.
|
| 4102 |
field_def("posted_at", "Posted at", "date"),
|
| 4103 |
field_def("type", "Type", "select", options=["image", "video", "carousel"]),
|
|
@@ -13770,7 +13770,7 @@ def _rollup_fold(fn, values, ftype="text"):
|
|
| 13770 |
LINKABLE_MODULES = ("customer_data", "product_data")
|
| 13771 |
|
| 13772 |
|
| 13773 |
-
def _module_limit(subject, cause, recommendation, effect="unresolvable"):
|
| 13774 |
"""A refusal in `core.user_tables::limit_report`'s vocabulary β `{subject, effect, cause,
|
| 13775 |
recommendation}`, the same four keys `perm_scope.Unresolvable.as_limit()` puts on the wire.
|
| 13776 |
|
|
@@ -13778,9 +13778,20 @@ def _module_limit(subject, cause, recommendation, effect="unresolvable"):
|
|
| 13778 |
this engine cannot reach used to resolve to `{}` and blank the column, which reads exactly
|
| 13779 |
like "there are no linked records". A reason a caller can render is the difference between a
|
| 13780 |
measurement and a lie.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13781 |
"""
|
| 13782 |
-
|
| 13783 |
-
|
|
|
|
|
|
|
|
|
|
| 13784 |
|
| 13785 |
|
| 13786 |
def _module_link_fields(key):
|
|
@@ -13901,6 +13912,112 @@ def _module_table(rt, key, memo, want_rows=False):
|
|
| 13901 |
return table, None
|
| 13902 |
|
| 13903 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13904 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 13905 |
"""A database by key, from EITHER namespace β `(table_or_None, limit_or_None)`.
|
| 13906 |
|
|
@@ -13909,9 +14026,31 @@ def _resolve_table(rt, store, key, memo, want_rows=False):
|
|
| 13909 |
the TARGET table were two separate `store.get(...)` calls, C5 named only the target, and a
|
| 13910 |
target-only fix leaves a link DECLARED ON `customer_data` returning 0 rows touched β which
|
| 13911 |
reads exactly like "nothing needed doing".
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13912 |
"""
|
| 13913 |
-
|
|
|
|
| 13914 |
if hit is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13915 |
return hit, None
|
| 13916 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 13917 |
|
|
@@ -14083,9 +14222,18 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14083 |
if not links and not rollups:
|
| 14084 |
return {}, limits
|
| 14085 |
rows = (t or {}).get("rows") or {}
|
| 14086 |
-
if not rows and t is not None and not t.get("_rows_built", True)
|
|
|
|
| 14087 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14088 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14089 |
t, limit = _resolve_table(rt, store, table_key, memo, want_rows=True)
|
| 14090 |
if limit is not None:
|
| 14091 |
limits.append(limit)
|
|
@@ -14108,12 +14256,22 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14108 |
want_rows=True)
|
| 14109 |
if linked is None:
|
| 14110 |
refused.add(lk_key)
|
| 14111 |
-
|
| 14112 |
-
|
| 14113 |
-
|
| 14114 |
-
|
| 14115 |
-
|
| 14116 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14117 |
continue
|
| 14118 |
linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
|
| 14119 |
for lf in (linked.get("fields") or [])}
|
|
@@ -14130,6 +14288,108 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14130 |
inverse_index.setdefault(target_id, []).append(
|
| 14131 |
(str(source_id), source_row or {}))
|
| 14132 |
resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14133 |
elif bag.get("on"):
|
| 14134 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 14135 |
from_key = _link_from_key(fields, bag)
|
|
@@ -14293,6 +14553,80 @@ def relation_cells(rt, table_key, tables=None):
|
|
| 14293 |
return changes, limits
|
| 14294 |
|
| 14295 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14296 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 14297 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 14298 |
|
|
|
|
| 4097 |
# ββ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO
|
| 4098 |
# orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`,
|
| 4099 |
# and `ut_ig_posts` already performs the first of them. See the verdict in
|
| 4100 |
+
# `AutoWiki/research/entity-vs-series.md` β these columns are for AGE and KIND, not for
|
| 4101 |
# moving the window here.
|
| 4102 |
field_def("posted_at", "Posted at", "date"),
|
| 4103 |
field_def("type", "Type", "select", options=["image", "video", "carousel"]),
|
|
|
|
| 13770 |
LINKABLE_MODULES = ("customer_data", "product_data")
|
| 13771 |
|
| 13772 |
|
| 13773 |
+
def _module_limit(subject, cause, recommendation, effect="unresolvable", limit=None):
|
| 13774 |
"""A refusal in `core.user_tables::limit_report`'s vocabulary β `{subject, effect, cause,
|
| 13775 |
recommendation}`, the same four keys `perm_scope.Unresolvable.as_limit()` puts on the wire.
|
| 13776 |
|
|
|
|
| 13778 |
this engine cannot reach used to resolve to `{}` and blank the column, which reads exactly
|
| 13779 |
like "there are no linked records". A reason a caller can render is the difference between a
|
| 13780 |
measurement and a lie.
|
| 13781 |
+
|
| 13782 |
+
β W53-T40 β `limit` IS OPTIONAL AND IS OMITTED UNLESS A NUMBER IS THE POINT, which keeps
|
| 13783 |
+
every refusal already shipped byte-identical. `core.user_tables.limit_report` carries the key
|
| 13784 |
+
on exactly the reports where a CEILING is the subject (`{'subject': 'rows', 'limit': 0,
|
| 13785 |
+
'effect': 'read_through', ...}`), and the two W53-T40 reports below are that shape: a
|
| 13786 |
+
read-through grain whose ceiling here is 0, and a second-hop fan-out whose ceiling is
|
| 13787 |
+
`LINK_THROUGH_MAX_IDS`. Naming the number in prose alone would leave a caller parsing English
|
| 13788 |
+
to find out how much it did not get.
|
| 13789 |
"""
|
| 13790 |
+
out = {"subject": str(subject), "effect": effect,
|
| 13791 |
+
"cause": str(cause), "recommendation": str(recommendation)}
|
| 13792 |
+
if limit is not None:
|
| 13793 |
+
out["limit"] = int(limit)
|
| 13794 |
+
return out
|
| 13795 |
|
| 13796 |
|
| 13797 |
def _module_link_fields(key):
|
|
|
|
| 13912 |
return table, None
|
| 13913 |
|
| 13914 |
|
| 13915 |
+
def _as_field_limit(lk_key, base, context):
|
| 13916 |
+
"""β W53-T40 β ONE REFUSAL, RE-SUBJECTED FROM THE DATABASE IT IS ABOUT TO THE COLUMN IT BLANKS.
|
| 13917 |
+
|
| 13918 |
+
`_resolve_table` refuses in terms of the DATABASE it could not read, which is the right
|
| 13919 |
+
subject for its own caller and the WRONG one for a link: what a client renders is a COLUMN,
|
| 13920 |
+
and "which field is blank, and why" is the entire question `relation_cells`' second return
|
| 13921 |
+
value exists to answer. Every other refusal that function raises is already keyed on the field
|
| 13922 |
+
key; this keeps a refusal that arrives from one layer down in the same vocabulary.
|
| 13923 |
+
|
| 13924 |
+
β THE CAUSE AND THE RECOMMENDATION ARE CARRIED VERBATIM, never restated. The layer that
|
| 13925 |
+
refused knows why it refused; paraphrasing it here would be a second account of one fact, and
|
| 13926 |
+
the two would drift the first time the lower one is corrected. `context` is PREPENDED, so the
|
| 13927 |
+
route is stated and then the reason follows in the words of whoever had it.
|
| 13928 |
+
"""
|
| 13929 |
+
base = base or {}
|
| 13930 |
+
return _module_limit(
|
| 13931 |
+
lk_key, (str(context or "").strip() + " " + str(base.get("cause") or "")).strip(),
|
| 13932 |
+
str(base.get("recommendation") or ""),
|
| 13933 |
+
effect=str(base.get("effect") or "unresolvable"), limit=base.get("limit"))
|
| 13934 |
+
|
| 13935 |
+
|
| 13936 |
+
def _mirror_table(rt, key, defn):
|
| 13937 |
+
"""ββ W53-T40 (d)+(b) β A READ-THROUGH DATABASE'S ROWS, FROM THE CONNECTOR MIRROR, or a
|
| 13938 |
+
REFUSAL that names the field, the cause and the fix. `(table_or_None, limit_or_None)`.
|
| 13939 |
+
|
| 13940 |
+
β THIS IS TWO ARMS AND THEY ARE THE TICKET'S (d) AND (b), IN THAT ORDER.
|
| 13941 |
+
|
| 13942 |
+
(d) **NO READER REGISTERED β REFUSE, LOUDLY.** A read-through grid's stored document is a
|
| 13943 |
+
real definition with `rows: {}` β never `None` β so the old `if linked is None` refusal
|
| 13944 |
+
in `relation_cells` could not see it: the link fell through to the ordinary join arm,
|
| 13945 |
+
`_linked_rows_by_join` indexed an empty dict, every cell resolved to `[]`, and the
|
| 13946 |
+
column was permanently blank with NO `limit_report`. That is the one case
|
| 13947 |
+
`relation_cells`' own docstring says cannot happen, and it happened because the guard
|
| 13948 |
+
was keyed on the wrong sentinel. Re-keying it means making THIS function answer the
|
| 13949 |
+
real question β "are this database's rows resolvable here?" β so the existing `is None`
|
| 13950 |
+
test becomes true for the structurally-rows-absent case rather than being duplicated.
|
| 13951 |
+
|
| 13952 |
+
(b) **A READER IS REGISTERED β WALK IT.** The rows already exist and are already served:
|
| 13953 |
+
`routes_odoo_tables.GRID_SOURCES` renders them through `routes_tables._read_through_rows`,
|
| 13954 |
+
published as `core.perm_scope._MIRROR_READER` by `routes_tables._register_mirror`. What
|
| 13955 |
+
was missing was a CALLER. This is the caller.
|
| 13956 |
+
|
| 13957 |
+
ββ THE SENTINEL IS `core.user_tables.materialises`, NOT A HAND-READ `readThrough` KEY, and
|
| 13958 |
+
the difference is a cold process. `materialises` is three-tier β the app-layer
|
| 13959 |
+
`register_read_through` set, then the definition's own `readThrough` STAMP written by
|
| 13960 |
+
`odoo_relational._ensure_table_inplace`, then the default β so a worker that never served a
|
| 13961 |
+
read-through grid still gets the right answer. A hand-read of the stamp alone would be one
|
| 13962 |
+
of those tiers pretending to be all three ([[one-evaluator-per-question]]).
|
| 13963 |
+
|
| 13964 |
+
β THE SHAPE IS COPIED FROM `core.perm_scope._read_user_table`, NOT INVENTED HERE. That
|
| 13965 |
+
function has carried exactly this branch since W38 β refuse when `_MIRROR_READER is None`,
|
| 13966 |
+
otherwise project the declared field keys and hand back the reader's rows β and a second
|
| 13967 |
+
spelling of it would be a second opinion about what a read-through read is.
|
| 13968 |
+
|
| 13969 |
+
β THE DOCUMENT IS COPIED, NEVER MUTATED. `store` is the live `user_tables` bucket; writing
|
| 13970 |
+
mirror rows into it would materialise the very grain R6 forbids materialising, in memory,
|
| 13971 |
+
silently, and then hand it to whatever reads the store next.
|
| 13972 |
+
|
| 13973 |
+
β `rt` RIDES INTO THE READER AS ITS THIRD ARGUMENT. `routes_tables._c1_mirror_rows` names
|
| 13974 |
+
that parameter `st` and passes it straight through to `_read_through_rows(..., rt=st)`, whose
|
| 13975 |
+
docstring says why it must be the SESSION'S runtime: one Space process serves every tenant
|
| 13976 |
+
and `datastore.ro_con()` reads a process-global path, so the tenant handle is the only thing
|
| 13977 |
+
standing between a relation pass and another tenant's mirror.
|
| 13978 |
+
"""
|
| 13979 |
+
from core import perm_scope # lazy: the cycle rule `_ut()` documents
|
| 13980 |
+
key = str(key or "")
|
| 13981 |
+
reader = getattr(perm_scope, "_MIRROR_READER", None)
|
| 13982 |
+
if reader is None:
|
| 13983 |
+
# (d). β ASCII ONLY: this rides out in `limits` and can reach a client (standing rule 2).
|
| 13984 |
+
return None, _module_limit(
|
| 13985 |
+
key,
|
| 13986 |
+
f"'{key}' is served READ-THROUGH from the connector mirror, so it stores no rows in "
|
| 13987 |
+
f"this workspace document (row_limit 0), and no mirror reader is registered in this "
|
| 13988 |
+
f"process. A link resolved against its stored rows would find none and render a "
|
| 13989 |
+
f"permanently blank column that looks configured",
|
| 13990 |
+
"call `perm_scope.register_mirror(...)` from the app layer before resolving a "
|
| 13991 |
+
"relation at this grain (importing `routes_tables` does it), or read the grain "
|
| 13992 |
+
"directly through the API, where every total is a SQL count over the whole table",
|
| 13993 |
+
effect="read_through", limit=0)
|
| 13994 |
+
fields = list((defn or {}).get("fields") or [])
|
| 13995 |
+
wanted = {str(f.get("key")) for f in fields if f.get("key")}
|
| 13996 |
+
try:
|
| 13997 |
+
raw = list(reader(key, wanted, rt) or ())
|
| 13998 |
+
except Exception as e: # noqa: BLE001
|
| 13999 |
+
# β A MIRROR THAT CANNOT ANSWER IS A REASON, NOT AN OUTAGE, and `perm_scope.Unresolvable`
|
| 14000 |
+
# already carries the four keys this vocabulary wants: reuse them rather than flattening
|
| 14001 |
+
# a stated cause into `str(e)`.
|
| 14002 |
+
cause = str(getattr(e, "cause", "") or "") or f"{type(e).__name__}: {e}"
|
| 14003 |
+
return None, _module_limit(
|
| 14004 |
+
key,
|
| 14005 |
+
f"'{key}' is served READ-THROUGH from the connector mirror and the mirror could not "
|
| 14006 |
+
f"answer for it: {cause}",
|
| 14007 |
+
str(getattr(e, "recommendation", "") or "")
|
| 14008 |
+
or ("retry once the connector mirror has finished opening on this process, or read "
|
| 14009 |
+
"the grain a window at a time through the API"),
|
| 14010 |
+
effect=str(getattr(e, "effect", "") or "read_through"), limit=0)
|
| 14011 |
+
# `pid` IS THE ROW ID on every read-through grid β `routes_tables` stamps it onto each row it
|
| 14012 |
+
# serves and `_read_through_rows` keeps it through the projection. A row without one cannot be
|
| 14013 |
+
# addressed by a link cell at all, so it is dropped rather than given an invented id.
|
| 14014 |
+
table = dict(defn or {})
|
| 14015 |
+
table["key"] = key
|
| 14016 |
+
table["rows"] = {str(r.get("pid")): dict(r) for r in raw if (r or {}).get("pid") is not None}
|
| 14017 |
+
table["_rows_built"] = True
|
| 14018 |
+
return table, None
|
| 14019 |
+
|
| 14020 |
+
|
| 14021 |
def _resolve_table(rt, store, key, memo, want_rows=False):
|
| 14022 |
"""A database by key, from EITHER namespace β `(table_or_None, limit_or_None)`.
|
| 14023 |
|
|
|
|
| 14026 |
the TARGET table were two separate `store.get(...)` calls, C5 named only the target, and a
|
| 14027 |
target-only fix leaves a link DECLARED ON `customer_data` returning 0 rows touched β which
|
| 14028 |
reads exactly like "nothing needed doing".
|
| 14029 |
+
|
| 14030 |
+
ββ W53-T40 β AND THAT IS ALSO WHY THE READ-THROUGH BRANCH LANDS HERE RATHER THAN IN EITHER
|
| 14031 |
+
CALLER. `_PRODUCT_LINK_BLOCKER` named this symbol as "the door BOTH of them go through", so a
|
| 14032 |
+
third namespace β the connector MIRROR β is one branch here instead of two branches that must
|
| 14033 |
+
be kept in step. A stored document is never `None`, so before this branch existed a
|
| 14034 |
+
read-through target came back as a real definition with `rows: {}` and every caller read that
|
| 14035 |
+
as "no linked records".
|
| 14036 |
+
|
| 14037 |
+
β ONLY WHEN `want_rows`. The field contract of a read-through grid is in the stored document
|
| 14038 |
+
like everybody else's, and `relation_cells` resolves the source table fields-only precisely so
|
| 14039 |
+
a table with no links pays nothing. Reaching the mirror for a fields-only resolve would put a
|
| 14040 |
+
256,593-row read behind a dict scan.
|
| 14041 |
+
|
| 14042 |
+
β MEMOISED ON A TUPLE KEY. `_module_table` memoises on the bare key; a mirror table for the
|
| 14043 |
+
same key is a different object with a different cost, and both can be wanted in one pass (the
|
| 14044 |
+
bridge of a two-hop is resolved once per link field).
|
| 14045 |
"""
|
| 14046 |
+
key = str(key or "")
|
| 14047 |
+
hit = (store or {}).get(key)
|
| 14048 |
if hit is not None:
|
| 14049 |
+
if want_rows and not _ut().materialises(key, defn=hit):
|
| 14050 |
+
cached = memo.get(("mirror", key))
|
| 14051 |
+
if cached is None:
|
| 14052 |
+
cached = memo[("mirror", key)] = _mirror_table(rt, key, hit)
|
| 14053 |
+
return cached
|
| 14054 |
return hit, None
|
| 14055 |
return _module_table(rt, key, memo, want_rows=want_rows)
|
| 14056 |
|
|
|
|
| 14222 |
if not links and not rollups:
|
| 14223 |
return {}, limits
|
| 14224 |
rows = (t or {}).get("rows") or {}
|
| 14225 |
+
if not rows and t is not None and (not t.get("_rows_built", True)
|
| 14226 |
+
or not _ut().materialises(table_key, defn=t)):
|
| 14227 |
# A registry source: NOW the pool is worth its cost, because this table really does
|
| 14228 |
# declare relations. One fetch, memoised, shared with every link and rollup below.
|
| 14229 |
+
# ββ W53-T40 (d) β AND A READ-THROUGH SOURCE TAKES THE SAME SECOND FETCH, for a reason
|
| 14230 |
+
# that is not symmetry. `_rows_built` is the REGISTRY sentinel; a read-through `ut_*`
|
| 14231 |
+
# document carries no such key, so `.get("_rows_built", True)` answered True, `rows` stayed
|
| 14232 |
+
# `{}`, and a link DECLARED ON a read-through grain returned `{}` cells with no limit β
|
| 14233 |
+
# the same silent blank on the source side that (d) ends on the target side. The predicate
|
| 14234 |
+
# is `materialises`, the one evaluator of "are this database's rows stored here", and the
|
| 14235 |
+
# cheap-by-construction guarantee is untouched because this sits BELOW the declaration
|
| 14236 |
+
# check: a table with no links and no rollups has already returned.
|
| 14237 |
t, limit = _resolve_table(rt, store, table_key, memo, want_rows=True)
|
| 14238 |
if limit is not None:
|
| 14239 |
limits.append(limit)
|
|
|
|
| 14256 |
want_rows=True)
|
| 14257 |
if linked is None:
|
| 14258 |
refused.add(lk_key)
|
| 14259 |
+
if str((limit or {}).get("effect")) == "read_through":
|
| 14260 |
+
# β W53-T40 (d) β THE NEW REFUSAL IS RE-SUBJECTED ONTO THE FIELD, and ONLY the new
|
| 14261 |
+
# one. `_module_table`'s pre-existing refusals are passed through in their own
|
| 14262 |
+
# words exactly as they always were; re-keying those would churn a report other
|
| 14263 |
+
# callers already read. This arm did not exist before this ticket, so nothing
|
| 14264 |
+
# downstream has an older reading of it to protect.
|
| 14265 |
+
limits.append(_as_field_limit(
|
| 14266 |
+
lk_key, limit,
|
| 14267 |
+
f"this link points at '{bag.get('table')}' and its rows could not be read."))
|
| 14268 |
+
else:
|
| 14269 |
+
limits.append(limit if limit is not None else _module_limit(
|
| 14270 |
+
lk_key,
|
| 14271 |
+
f"this link points at '{bag.get('table')}', which is not a database this "
|
| 14272 |
+
f"workspace holds",
|
| 14273 |
+
"re-point the link at an existing database, or delete the column; a link "
|
| 14274 |
+
"whose target is gone is not a relation"))
|
| 14275 |
continue
|
| 14276 |
linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
|
| 14277 |
for lf in (linked.get("fields") or [])}
|
|
|
|
| 14288 |
inverse_index.setdefault(target_id, []).append(
|
| 14289 |
(str(source_id), source_row or {}))
|
| 14290 |
resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows}
|
| 14291 |
+
elif bag.get("through"):
|
| 14292 |
+
# ββ W53-T40 (b)+(c) β THE TWO-HOP TRAVERSAL, AND IT IS TRAVERSAL, NOT AGGREGATION.
|
| 14293 |
+
# R12 keeps aggregation at depth 1; this walks `source -> bridge -> target` and hands
|
| 14294 |
+
# the TARGET ROWS back in the same `resolved[link][row_id] = [(id, row), ...]` shape
|
| 14295 |
+
# every one-hop arm above produces, so every rollup, cap and cell below is unchanged.
|
| 14296 |
+
#
|
| 14297 |
+
# β THE INDEX IS `_linked_rows_by_join` ON BOTH HOPS AND THE NORMALISER IS
|
| 14298 |
+
# `_join_norm` ON BOTH SIDES OF BOTH. `_join_norm`'s own docstring says why: a join is
|
| 14299 |
+
# a comparison, and a comparison normalised in two places is one edit from silently
|
| 14300 |
+
# reporting zero. A second spelling here would be four half-comparisons instead of two.
|
| 14301 |
+
#
|
| 14302 |
+
# β THE ROUTE THIS SHIPS FOR IS `customer_data -> ut_odoo_order_lines ->
|
| 14303 |
+
# product_data`, and it is the ONLY route Odoo declares between a partner and a
|
| 14304 |
+
# product: `sale.order.line` carries `order_partner_id` and `product_id`, both stored.
|
| 14305 |
+
# β THERE IS NO INVOICE ROUTE AND THERE MUST NOT BE ONE THROUGH THIS GRAIN. Invoice to
|
| 14306 |
+
# line to product is THREE hops and the first of them, `invoice_origin`, is FREE TEXT
|
| 14307 |
+
# filled on 34.1% of posted invoices β a depth-3 path over a weak hop returns a
|
| 14308 |
+
# confidently wrong number rather than an error.
|
| 14309 |
+
through_key = str(bag.get("through") or "")
|
| 14310 |
+
on_key = str(bag.get("on") or "")
|
| 14311 |
+
hop_in = str(bag.get("throughOn") or "")
|
| 14312 |
+
hop_out = str(bag.get("throughTo") or "")
|
| 14313 |
+
from_key = _link_from_key(fields, bag)
|
| 14314 |
+
if not (on_key and hop_in and hop_out and from_key):
|
| 14315 |
+
# β REFUSED, NOT BLANK β the same law the one-hop arm below follows. A route with
|
| 14316 |
+
# a missing leg has no second hop to make; resolving it to `[]` would render a
|
| 14317 |
+
# permanently blank column that looks configured. `_clean_link` refuses this shape
|
| 14318 |
+
# at the definition door, so reaching here means a bag that never went through it.
|
| 14319 |
+
refused.add(lk_key)
|
| 14320 |
+
limits.append(_module_limit(
|
| 14321 |
+
lk_key,
|
| 14322 |
+
f"this link travels through '{through_key}' but the route is incomplete. A "
|
| 14323 |
+
f"two-hop join names four columns: `from` on '{table_key}', `throughOn` and "
|
| 14324 |
+
f"`throughTo` on the bridge, and `on` on '{bag.get('table')}'. Present here: "
|
| 14325 |
+
f"from={from_key or '(none)'}, throughOn={hop_in or '(none)'}, "
|
| 14326 |
+
f"throughTo={hop_out or '(none)'}, on={on_key or '(none)'}",
|
| 14327 |
+
"declare every leg of the route on the link, or remove `through` and let it "
|
| 14328 |
+
"be the one-hop join it is actually configured as"))
|
| 14329 |
+
continue
|
| 14330 |
+
# THE BRIDGE, THROUGH THE SAME DOOR AS THE SOURCE AND THE TARGET. This is where the
|
| 14331 |
+
# read-through refusal lands for the shipped route: `ut_odoo_order_lines` stores no
|
| 14332 |
+
# rows in this document on purpose, so `_resolve_table` reaches the mirror for it, or
|
| 14333 |
+
# refuses with a cause and a fix when no reader is registered in this process.
|
| 14334 |
+
bridge, limit = _resolve_table(rt, store, through_key, memo, want_rows=True)
|
| 14335 |
+
if bridge is None:
|
| 14336 |
+
refused.add(lk_key)
|
| 14337 |
+
limits.append(_as_field_limit(
|
| 14338 |
+
lk_key, limit,
|
| 14339 |
+
f"this link travels {table_key} -> {through_key} -> {bag.get('table')} and "
|
| 14340 |
+
f"the bridge could not be read.") if limit is not None else _module_limit(
|
| 14341 |
+
lk_key,
|
| 14342 |
+
f"this link travels through '{through_key}', which is not a database this "
|
| 14343 |
+
f"workspace holds",
|
| 14344 |
+
"re-point the link's `through` at an existing database, or delete the "
|
| 14345 |
+
"column; a route across a database that is gone is not a relation"))
|
| 14346 |
+
continue
|
| 14347 |
+
bridge_idx = _linked_rows_by_join(bridge, hop_in)
|
| 14348 |
+
target_idx = _linked_rows_by_join(linked, on_key)
|
| 14349 |
+
cap = int(_ut().LINK_THROUGH_MAX_IDS)
|
| 14350 |
+
|
| 14351 |
+
def _walk(row, _bidx=bridge_idx, _tidx=target_idx, _cap=cap):
|
| 14352 |
+
"""One source row's DISTINCT target records, and whether the ceiling stopped it.
|
| 14353 |
+
|
| 14354 |
+
β DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
|
| 14355 |
+
twenty copies of one product in the cell would be a count of LINES wearing the
|
| 14356 |
+
label of a set of PRODUCTS.
|
| 14357 |
+
"""
|
| 14358 |
+
seen, out = set(), []
|
| 14359 |
+
for _bid, brow in _bidx.get(_join_norm((row or {}).get(from_key)), ()):
|
| 14360 |
+
for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
|
| 14361 |
+
if tid in seen:
|
| 14362 |
+
continue
|
| 14363 |
+
if len(out) >= _cap:
|
| 14364 |
+
return out, True
|
| 14365 |
+
seen.add(tid)
|
| 14366 |
+
out.append((tid, trow))
|
| 14367 |
+
return out, False
|
| 14368 |
+
|
| 14369 |
+
two_hop, capped = {}, 0
|
| 14370 |
+
for rid, row in rows.items():
|
| 14371 |
+
hits_2, hit_cap = _walk(row)
|
| 14372 |
+
two_hop[str(rid)] = hits_2
|
| 14373 |
+
capped += 1 if hit_cap else 0
|
| 14374 |
+
resolved[lk_key] = two_hop
|
| 14375 |
+
if capped:
|
| 14376 |
+
# ββ STANDING RULE 1'S SECOND SENTENCE, AND IT IS THE HALF THAT GETS DROPPED: a
|
| 14377 |
+
# limit that genuinely cannot be removed is REPORTED, with cause and a recommended
|
| 14378 |
+
# fix, never silently enforced. β AND THIS ONE IS LOUDER THAN THE `LINK_MAX_IDS`
|
| 14379 |
+
# display cap, because it is a different quantity: the display cap slices the CELL
|
| 14380 |
+
# and leaves `resolved` whole for the rollups, while this bounds `resolved`
|
| 14381 |
+
# ITSELF β so a rollup folded over a capped row describes a subset.
|
| 14382 |
+
# β ASCII ONLY: this rides out in `limits` and can reach a client.
|
| 14383 |
+
limits.append(_module_limit(
|
| 14384 |
+
lk_key,
|
| 14385 |
+
f"this link travels {table_key} -> {through_key} -> {bag.get('table')}, and "
|
| 14386 |
+
f"{capped} of {len(rows)} rows reached the {cap}-record fan-out ceiling on "
|
| 14387 |
+
f"the second hop. Those cells, and any rollup folded over them, describe the "
|
| 14388 |
+
f"first {cap} distinct records rather than the whole set",
|
| 14389 |
+
"narrow the route with a `filter` on the link, or read the bridge grain "
|
| 14390 |
+
"directly through the API, where every total is a SQL count over the whole "
|
| 14391 |
+
"table rather than over a resolved window",
|
| 14392 |
+
effect="capped", limit=cap))
|
| 14393 |
elif bag.get("on"):
|
| 14394 |
idx = _linked_rows_by_join(linked, str(bag["on"]))
|
| 14395 |
from_key = _link_from_key(fields, bag)
|
|
|
|
| 14553 |
return changes, limits
|
| 14554 |
|
| 14555 |
|
| 14556 |
+
def related_product_metric_values(rt, customer_id, products, fields, offer, *,
|
| 14557 |
+
team_id=None, agent=None, today=None):
|
| 14558 |
+
"""Read-only customer Γ real product metrics; the caller supplies walled server rows.
|
| 14559 |
+
|
| 14560 |
+
Product grid pids are CRCs and never enter the semantic product dimension. Keep the
|
| 14561 |
+
BU in flat filters too: an entity binding's scope_variant suppresses team_id upstream.
|
| 14562 |
+
No memo survives this request, and an unsupported metric/window stays blank.
|
| 14563 |
+
"""
|
| 14564 |
+
from harness import semantic as sem, windows
|
| 14565 |
+
import time
|
| 14566 |
+
|
| 14567 |
+
rt.assert_datastore_matches()
|
| 14568 |
+
today = today or time.strftime("%Y-%m-%d")
|
| 14569 |
+
by_key = {m["key"]: m for m in offer}
|
| 14570 |
+
ids = {int(row["pid"]): int(row["product_id"]) for row in products}
|
| 14571 |
+
filters = {"order_partner": [customer_id], "product": sorted(set(ids.values()))}
|
| 14572 |
+
if team_id is not None:
|
| 14573 |
+
filters["team"] = [team_id]
|
| 14574 |
+
if agent is not None:
|
| 14575 |
+
filters["agent"] = [agent]
|
| 14576 |
+
out = {pid: {f["key"]: None for f in fields} for pid in ids}
|
| 14577 |
+
if not ids:
|
| 14578 |
+
return out, []
|
| 14579 |
+
|
| 14580 |
+
# Membership comes from confirmed order lines, never the client link cell or SKU code.
|
| 14581 |
+
membership = sem.store_query("sales_lines", ["revenue"], group_by=["product"],
|
| 14582 |
+
filters=filters, team_id=team_id, limit=sem.MAX_GROUPS,
|
| 14583 |
+
scope_variant="all_channels")
|
| 14584 |
+
if membership.get("truncated"):
|
| 14585 |
+
raise ValueError("The related product set is incomplete.")
|
| 14586 |
+
linked = {int(r["product_id"]) for r in membership["rows"]}
|
| 14587 |
+
if set(ids.values()) - linked:
|
| 14588 |
+
raise ValueError("A product is not within this customer's permitted ordered products.")
|
| 14589 |
+
|
| 14590 |
+
groups, reasons = {}, []
|
| 14591 |
+
for field in fields:
|
| 14592 |
+
spec = field.get("measure") or {}
|
| 14593 |
+
measure = by_key.get(spec.get("key"))
|
| 14594 |
+
if not measure or measure.get("topic") != "sales_lines":
|
| 14595 |
+
reasons.append("Some Metrics cannot be evaluated for a customer and product together.")
|
| 14596 |
+
continue
|
| 14597 |
+
window = windows.normalize(spec.get("window"))
|
| 14598 |
+
bounds = windows.resolve(spec.get("window"), today) if window is not None else None
|
| 14599 |
+
if bounds is None:
|
| 14600 |
+
reasons.append("Some Metric date windows could not be resolved.")
|
| 14601 |
+
continue
|
| 14602 |
+
groups.setdefault((*bounds, measure.get("scope_variant")), []).append((field, measure))
|
| 14603 |
+
|
| 14604 |
+
for (start, end, variant), group in groups.items():
|
| 14605 |
+
keys = sorted({m.get("base_key") or m["key"] for _, m in group})
|
| 14606 |
+
result = sem.store_query("sales_lines", keys, group_by=["product"],
|
| 14607 |
+
date_from=start, date_to=end, filters=filters,
|
| 14608 |
+
team_id=team_id, exclude_services=False,
|
| 14609 |
+
limit=sem.MAX_GROUPS, scope_variant=variant)
|
| 14610 |
+
if result.get("truncated"):
|
| 14611 |
+
raise ValueError("The scoped Metric result is incomplete.")
|
| 14612 |
+
values = {int(r["product_id"]): r for r in result["rows"]}
|
| 14613 |
+
for pid, real_id in ids.items():
|
| 14614 |
+
row = values.get(real_id, {})
|
| 14615 |
+
for field, measure in group:
|
| 14616 |
+
guard = measure.get("guard")
|
| 14617 |
+
if guard is not None and not row.get(guard):
|
| 14618 |
+
continue
|
| 14619 |
+
value = row.get(measure.get("base_key") or measure["key"])
|
| 14620 |
+
if value is None:
|
| 14621 |
+
value = 0 if measure.get("empty") == "zero" else None
|
| 14622 |
+
if value is not None:
|
| 14623 |
+
value *= measure.get("scale", 1)
|
| 14624 |
+
if measure.get("round") is not None:
|
| 14625 |
+
value = round(value, measure["round"])
|
| 14626 |
+
out[pid][field["key"]] = value
|
| 14627 |
+
return out, list(dict.fromkeys(reasons))
|
| 14628 |
+
|
| 14629 |
+
|
| 14630 |
def compute_relation_cells(rt, table_key, tables=None):
|
| 14631 |
"""Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
|
| 14632 |
|
api/connectors_amazon.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""connectors_amazon.py β the API-layer FACE of the Amazon connector: the liveness probe.
|
| 2 |
+
|
| 3 |
+
ββ **THE TRANSPORT IS NOT HERE. IT IS `platform/harness/amazon_client.py`, AND THAT IS A LAYER
|
| 4 |
+
RULE, NOT A PREFERENCE.** `harness/amazon_store.py` needs the same client this probe needs, and
|
| 5 |
+
nothing under `platform/` may import an API-layer module -- layers point one way. The first version
|
| 6 |
+
of this connector put the transport here and had the store reach up for it; `verify_meta`'s
|
| 7 |
+
architecture leg caught it in one line. Everything the client exports is re-exported below, so a
|
| 8 |
+
caller that imports it from here still works.
|
| 9 |
+
|
| 10 |
+
β **THE PROBER IS REGISTERED, NEVER IMPORTED BY `core`.** `core` never imports up
|
| 11 |
+
(ARCHITECTURE.md), and the tempting shortcut -- a `sys.path.insert` plus an `import` from inside
|
| 12 |
+
`core/keychain.py` -- is an architecture inversion no gate was watching for until wave 31.
|
| 13 |
+
`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
|
| 14 |
+
down.
|
| 15 |
+
|
| 16 |
+
β **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import os
|
| 21 |
+
import sys
|
| 22 |
+
import time
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
_PLATFORM = Path(__file__).resolve().parents[2] / "platform"
|
| 26 |
+
if str(_PLATFORM) not in sys.path:
|
| 27 |
+
sys.path.insert(0, str(_PLATFORM))
|
| 28 |
+
|
| 29 |
+
# β RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
|
| 30 |
+
# map and the transport; this module adds the probe and the registration and nothing else.
|
| 31 |
+
from harness.amazon_client import ( # noqa: F401,E402
|
| 32 |
+
AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE,
|
| 33 |
+
CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
|
| 34 |
+
REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
|
| 35 |
+
ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
|
| 36 |
+
_competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH,
|
| 37 |
+
_COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH,
|
| 38 |
+
_request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory,
|
| 39 |
+
competitive_summaries, marketplace_participations, report_document,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
| 43 |
+
#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
|
| 44 |
+
#: comments in `keychain.py` warn about it in as many words.
|
| 45 |
+
ENTRY_TYPE = "amazon_sp"
|
| 46 |
+
|
| 47 |
+
#: W55-T19 / R7 / C10 β AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
|
| 48 |
+
#: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
|
| 49 |
+
#: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
|
| 50 |
+
#: models. The document lives on the request's tenant runtime, so two tenants can make opposite
|
| 51 |
+
#: choices without a process-global map ever seeing either answer.
|
| 52 |
+
ENABLEMENT_KEY = "amazon_connector_enablement"
|
| 53 |
+
ENABLEMENT_ROUTE = "/connectors/amazon_sp/databases/{databaseKey}/enablement"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class EnablementConfigError(ValueError):
|
| 57 |
+
"""The tenant's stored Amazon enablement document is malformed."""
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def database_keys():
|
| 61 |
+
"""The authored Amazon database keys, derived from the connector's one report roster."""
|
| 62 |
+
return tuple("ut_amzn_" + str(name) for name in REPORT_TYPES)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def read_database_enablement(rt):
|
| 66 |
+
"""Return this tenant's fail-closed Amazon database enablement state.
|
| 67 |
+
|
| 68 |
+
An absent document preserves the connector's shipped behaviour: all seven databases are on.
|
| 69 |
+
Once a document exists, every value must be a real boolean and every key must belong to the
|
| 70 |
+
authored Amazon roster. A malformed or unreadable document disables every row in the detail
|
| 71 |
+
response and marks the capability unavailable; it never guesses through corrupt tenant state.
|
| 72 |
+
"""
|
| 73 |
+
keys = database_keys()
|
| 74 |
+
try:
|
| 75 |
+
current = rt.get(ENABLEMENT_KEY) if rt is not None else None
|
| 76 |
+
except Exception: # noqa: BLE001
|
| 77 |
+
return {"available": False, "databases": {key: False for key in keys}}
|
| 78 |
+
# Runtime/store readers use ``{}`` for a genuinely absent document, so both empty forms are
|
| 79 |
+
# the migration default rather than malformed state.
|
| 80 |
+
if current is None or current == {}:
|
| 81 |
+
return {"available": True, "databases": {key: True for key in keys}}
|
| 82 |
+
if not isinstance(current, dict):
|
| 83 |
+
return {"available": False, "databases": {key: False for key in keys}}
|
| 84 |
+
stored = current.get("databases")
|
| 85 |
+
if not isinstance(stored, dict):
|
| 86 |
+
return {"available": False, "databases": {key: False for key in keys}}
|
| 87 |
+
if (set(stored) - set(keys)
|
| 88 |
+
or any(type(value) is not bool for value in stored.values())):
|
| 89 |
+
return {"available": False, "databases": {key: False for key in keys}}
|
| 90 |
+
return {"available": True,
|
| 91 |
+
# Absent keys stay on so a later report addition does not silently disappear for
|
| 92 |
+
# every tenant that saved an older version of this document.
|
| 93 |
+
"databases": {key: stored.get(key, True) for key in keys}}
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def set_database_enabled(rt, database_key, enabled):
|
| 97 |
+
"""Persist one tenant's Amazon database choice without touching its materialised rows."""
|
| 98 |
+
key = str(database_key or "").strip()
|
| 99 |
+
known = set(database_keys())
|
| 100 |
+
if key not in known:
|
| 101 |
+
raise KeyError(key)
|
| 102 |
+
if type(enabled) is not bool:
|
| 103 |
+
raise ValueError("enabled must be true or false")
|
| 104 |
+
|
| 105 |
+
def _set(current):
|
| 106 |
+
if current is None:
|
| 107 |
+
current = {}
|
| 108 |
+
if not isinstance(current, dict):
|
| 109 |
+
raise EnablementConfigError("the stored Amazon enablement document is malformed")
|
| 110 |
+
stored = current.get("databases", {})
|
| 111 |
+
if not isinstance(stored, dict):
|
| 112 |
+
raise EnablementConfigError("the stored Amazon database choices are malformed")
|
| 113 |
+
if (set(stored) - known
|
| 114 |
+
or any(type(value) is not bool for value in stored.values())):
|
| 115 |
+
raise EnablementConfigError("the stored Amazon database choices are malformed")
|
| 116 |
+
# A fresh document contains the whole current roster. That makes the stored answer
|
| 117 |
+
# inspectable while retaining the migration rule above for future database additions.
|
| 118 |
+
choices = {name: stored.get(name, True) for name in database_keys()}
|
| 119 |
+
choices[key] = enabled
|
| 120 |
+
return {**current, "databases": choices}
|
| 121 |
+
|
| 122 |
+
rt.update(ENABLEMENT_KEY, _set, flush="sync")
|
| 123 |
+
return read_database_enablement(rt)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def enablement_capability(rt, is_admin):
|
| 127 |
+
"""The provider-aware capability served by Amazon's connector detail response."""
|
| 128 |
+
state = read_database_enablement(rt)
|
| 129 |
+
can_manage = bool(is_admin) and state["available"]
|
| 130 |
+
return {"provider": ENTRY_TYPE,
|
| 131 |
+
"mode": "database",
|
| 132 |
+
"canManage": can_manage,
|
| 133 |
+
# A member receives no write URL. The route still enforces administrator status; this
|
| 134 |
+
# is the matching affordance wall, not the permission boundary.
|
| 135 |
+
"updatePath": ENABLEMENT_ROUTE if can_manage else "",
|
| 136 |
+
# Disabling changes only this config document. No mirror or ``ut_amzn_*`` row is
|
| 137 |
+
# removed, so re-enabling resumes against the same materialised database.
|
| 138 |
+
"preservesData": True,
|
| 139 |
+
"available": state["available"]}
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def probe_credential(fields, timeout=8):
|
| 143 |
+
"""`{ok, message}` for a stored `amazon_sp` credential: the keychain's liveness probe.
|
| 144 |
+
|
| 145 |
+
β IT LIVES HERE, IN THE API LAYER, AND IS REGISTERED INTO `core.keychain` RATHER THAN IMPORTED
|
| 146 |
+
BY IT. `core` never imports up (ARCHITECTURE.md); `keychain.register_prober` is the slot, and
|
| 147 |
+
`connectors_meta` established the idiom after a first draft did `sys.path.insert` plus an
|
| 148 |
+
`import` from inside `core`.
|
| 149 |
+
|
| 150 |
+
β ZERO marketplace participations is reported as NOT ok, deliberately, for the same reason Meta
|
| 151 |
+
reports zero ad accounts that way: a credential that authenticates but reaches nothing is an
|
| 152 |
+
authorization answer, and `ok: true` would hide the one thing the admin has to fix.
|
| 153 |
+
"""
|
| 154 |
+
f = fields or {}
|
| 155 |
+
missing = [k for k in ("client_id", "client_secret", "refresh_token") if not f.get(k)]
|
| 156 |
+
if missing:
|
| 157 |
+
return {"ok": False,
|
| 158 |
+
"message": "store " + ", ".join(f"`{m}`" for m in missing)
|
| 159 |
+
+ " alongside `seller_id` and `marketplace_id`"}
|
| 160 |
+
try:
|
| 161 |
+
parts = marketplace_participations(f, timeout=timeout)
|
| 162 |
+
except AmazonError as e:
|
| 163 |
+
return {"ok": False, "message": str(e)[:200]}
|
| 164 |
+
live = [p for p in parts
|
| 165 |
+
if ((p or {}).get("participation") or {}).get("isParticipating") is not False]
|
| 166 |
+
n = len(live)
|
| 167 |
+
if not n:
|
| 168 |
+
return {"ok": False,
|
| 169 |
+
"message": "the credential authenticates but participates in NO marketplace: "
|
| 170 |
+
"that is an authorization answer, not an empty account"}
|
| 171 |
+
names = ", ".join(sorted({str(((p or {}).get("marketplace") or {}).get("countryCode") or "?")
|
| 172 |
+
for p in live}))
|
| 173 |
+
return {"ok": True, "message": f"Amazon answered. {n} marketplace(s) reachable: {names}"}
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def register():
|
| 177 |
+
"""Fill the keychain's probe slot for this connector.
|
| 178 |
+
|
| 179 |
+
Called at import by the API layer's connector registry (`routes_connectors._register_probers`),
|
| 180 |
+
so any process that can serve the connector directory has also registered the prober. Never
|
| 181 |
+
raises: a probe slot that could not be filled costs a liveness CHECK, never the ability to
|
| 182 |
+
store a credential.
|
| 183 |
+
"""
|
| 184 |
+
try:
|
| 185 |
+
import core.keychain as kc
|
| 186 |
+
return kc.register_prober(ENTRY_TYPE, probe_credential)
|
| 187 |
+
except Exception: # noqa: BLE001
|
| 188 |
+
return False
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _env_credential():
|
| 192 |
+
"""A credential assembled from the environment, for `--check` only.
|
| 193 |
+
|
| 194 |
+
β NOT a runtime path. The product reads Amazon credentials from the per-tenant keychain and
|
| 195 |
+
from nowhere else; this exists so an operator can prove a freshly minted refresh token works
|
| 196 |
+
before pasting it into the app.
|
| 197 |
+
"""
|
| 198 |
+
# β `OPTIONAL_FIELDS` joins the read so `AMAZON_SP_SANDBOX=true` reaches `is_sandbox` and the
|
| 199 |
+
# probe talks to the sandbox host. Absent still means PRODUCTION, which is the safe default.
|
| 200 |
+
f = {k: (os.environ.get("AMAZON_SP_" + k.upper()) or "").strip()
|
| 201 |
+
for k in tuple(CREDENTIAL_FIELDS) + OPTIONAL_FIELDS}
|
| 202 |
+
return {k: v for k, v in f.items() if v}
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def probe_report(fields, report_type=None, days=30, log=print):
|
| 206 |
+
"""Drive the WHOLE report chain once and print the MEASURED header. Writes to no store.
|
| 207 |
+
|
| 208 |
+
`createReport -> poll -> getReportDocument -> download -> decompress -> parse`, then print the
|
| 209 |
+
columns the document actually carried. **This is the only way to learn a flat-file report's
|
| 210 |
+
schema short of a production sync**, and it is exactly what the connector cannot know from
|
| 211 |
+
Amazon's published artifacts: five of our seven tables have none.
|
| 212 |
+
|
| 213 |
+
β NOTHING IS PERSISTED β no mirror table, no `ut_amzn_*`, no `_sync_state` row. That is what
|
| 214 |
+
makes it safe to point at the SANDBOX. `amazon_store.sync` refuses a sandbox credential so mock
|
| 215 |
+
rows can never reach a grid; this path sidesteps that question entirely by writing nothing.
|
| 216 |
+
|
| 217 |
+
β β SANDBOX REALITY, MEASURED 2026-09-04, AND IT IS NOT WHAT "the sandbox mirrors production"
|
| 218 |
+
SUGGESTS: the static sandbox's `getReportDocument` **always returns
|
| 219 |
+
`GET_MERCHANT_LISTINGS_ALL_DATA`, whatever report type you asked for** (amzn/selling-partner-
|
| 220 |
+
api-models#231). So this probe proves the CHAIN for any report type, and yields a genuine header
|
| 221 |
+
for exactly ONE of our seven tables β `listings`. The other six still need production.
|
| 222 |
+
|
| 223 |
+
β And the parse goes through `amazon_store.parse_flat_file`, the SAME function `sync` uses.
|
| 224 |
+
A second reader here would prove something production does not do.
|
| 225 |
+
"""
|
| 226 |
+
from datetime import datetime, timedelta, timezone
|
| 227 |
+
from harness.amazon_store import parse_flat_file
|
| 228 |
+
|
| 229 |
+
rt = report_type or REPORT_TYPES["listings"]["report"]
|
| 230 |
+
sbx = is_sandbox(fields)
|
| 231 |
+
end = datetime.now(timezone.utc)
|
| 232 |
+
start = end - timedelta(days=days)
|
| 233 |
+
log(f"host : {'SANDBOX' if sbx else 'PRODUCTION'} {base_for(fields)}")
|
| 234 |
+
log(f"report : {rt}")
|
| 235 |
+
if sbx and rt != "GET_MERCHANT_LISTINGS_ALL_DATA":
|
| 236 |
+
log("WARNING: the static sandbox returns GET_MERCHANT_LISTINGS_ALL_DATA whatever you ask "
|
| 237 |
+
"for, so the document below is NOT this report type. The chain is still proven.")
|
| 238 |
+
t0 = time.time()
|
| 239 |
+
text, meta = report_document(
|
| 240 |
+
fields, rt,
|
| 241 |
+
start=start.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
| 242 |
+
end=end.strftime("%Y-%m-%dT%H:%M:%SZ"), log=log)
|
| 243 |
+
log(f"document : {len(text):,} chars, compression={meta.get('compression')} "
|
| 244 |
+
f"({time.time() - t0:.1f}s)")
|
| 245 |
+
|
| 246 |
+
cols, rows = parse_flat_file(text)
|
| 247 |
+
log("")
|
| 248 |
+
log(f"MEASURED HEADER: {len(cols)} columns, {len(rows)} data rows")
|
| 249 |
+
for i, c in enumerate(cols):
|
| 250 |
+
log(f" {i + 1:>3}. {c}")
|
| 251 |
+
if rows:
|
| 252 |
+
# β TRUNCATED, and only the first row. A report document is the SELLER'S data; a probe
|
| 253 |
+
# prints enough to prove the shape and no more.
|
| 254 |
+
log("")
|
| 255 |
+
log("first row, values truncated to 40 chars:")
|
| 256 |
+
for c in cols[:12]:
|
| 257 |
+
log(f" {c:<34} {str(rows[0].get(c))[:40]}")
|
| 258 |
+
return cols, rows
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def main(argv):
|
| 262 |
+
"""`--check` proves a credential; `--probe-report` proves the whole report chain."""
|
| 263 |
+
if "--check" not in argv and "--probe-report" not in argv:
|
| 264 |
+
print("usage: connectors_amazon.py --check prove the credential")
|
| 265 |
+
print(" connectors_amazon.py --probe-report [--report TYPE]")
|
| 266 |
+
print(" drive the full report chain and print")
|
| 267 |
+
print(" the MEASURED header. Writes nothing.")
|
| 268 |
+
print(" reads AMAZON_SP_CLIENT_ID / _CLIENT_SECRET / _REFRESH_TOKEN / _SELLER_ID /")
|
| 269 |
+
print(" _MARKETPLACE_ID, plus AMAZON_SP_SANDBOX=true for the sandbox host")
|
| 270 |
+
return 2
|
| 271 |
+
f = _env_credential()
|
| 272 |
+
have = [k for k in CREDENTIAL_FIELDS if f.get(k)]
|
| 273 |
+
print("credential fields present: " + (", ".join(have) if have else "NONE")
|
| 274 |
+
+ " (values are never printed)")
|
| 275 |
+
if not {"client_id", "client_secret", "refresh_token"} <= set(have):
|
| 276 |
+
print("REFUSED: client_id, client_secret and refresh_token are all required")
|
| 277 |
+
return 2
|
| 278 |
+
t0 = time.time()
|
| 279 |
+
res = probe_credential(f, timeout=_TIMEOUT)
|
| 280 |
+
print(f"{'OK' if res.get('ok') else 'REFUSED'}: {res.get('message')} "
|
| 281 |
+
f"({time.time() - t0:.1f}s)")
|
| 282 |
+
if not res.get("ok"):
|
| 283 |
+
return 1
|
| 284 |
+
print("\nreport types this connector will pull (owner ruling R4):")
|
| 285 |
+
for table, spec in REPORT_TYPES.items():
|
| 286 |
+
rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)"
|
| 287 |
+
print(f" {table:<16} {rep}")
|
| 288 |
+
|
| 289 |
+
if "--probe-report" in argv:
|
| 290 |
+
which = None
|
| 291 |
+
for i, a in enumerate(argv):
|
| 292 |
+
if a == "--report" and i + 1 < len(argv):
|
| 293 |
+
which = argv[i + 1]
|
| 294 |
+
print("")
|
| 295 |
+
try:
|
| 296 |
+
probe_report(f, report_type=which)
|
| 297 |
+
except Exception as e: # noqa: BLE001
|
| 298 |
+
# β REPORTED, NOT RAISED. The probe is a diagnostic: a refusal is a RESULT the operator
|
| 299 |
+
# needs to read (a role they lack, a throttle, an unsupported sandbox report type), not
|
| 300 |
+
# a traceback that buries it.
|
| 301 |
+
print(f"REPORT PROBE REFUSED: {type(e).__name__}: {e}")
|
| 302 |
+
return 1
|
| 303 |
+
return 0
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
if __name__ == "__main__":
|
| 307 |
+
sys.exit(main(sys.argv[1:]))
|
api/connectors_bd.py
CHANGED
|
@@ -182,7 +182,7 @@ def bd_call(path, params=None, body=None):
|
|
| 182 |
if not key:
|
| 183 |
# β THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends
|
| 184 |
# somebody to read Instagram's status page instead of setting an env var.
|
| 185 |
-
return None, "AIOS_BRIGHTDATA_KEY is not configured
|
| 186 |
qs = "&".join(f"{k}={requests.utils.quote(str(v))}"
|
| 187 |
for k, v in (params or {}).items() if v not in (None, ""))
|
| 188 |
url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "")
|
|
@@ -220,7 +220,7 @@ def bd_call(path, params=None, body=None):
|
|
| 220 |
# paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow.
|
| 221 |
if len(raw) >= BD_MAX_KB * 1024:
|
| 222 |
return None, (f"the profile source's answer exceeded this client's {BD_MAX_KB // 1024} MB read "
|
| 223 |
-
f"ceiling and was TRUNCATED
|
| 224 |
f"from a partial document")
|
| 225 |
text = raw.decode("utf-8", "replace").strip()
|
| 226 |
if not text:
|
|
@@ -336,7 +336,7 @@ def bd_snapshot_progress(sid):
|
|
| 336 |
if isinstance(codes, dict) and codes else "")
|
| 337 |
if errors or named or status in ("failed", "error"):
|
| 338 |
return "failed", 0, ("the source finished this request and collected nothing"
|
| 339 |
-
+ (f"
|
| 340 |
+ "; the target is unreachable, private, or no longer exists")
|
| 341 |
return "done", 0, "the source finished this request and found no records for it"
|
| 342 |
|
|
@@ -388,7 +388,7 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
|
|
| 388 |
deferred.append({"snapshotId": sid, "datasetId": str(dataset_id),
|
| 389 |
"urls": requested_urls})
|
| 390 |
return [], (f"the profile source deferred this batch to {sid} and it was not ready within "
|
| 391 |
-
f"{int(budget)}s
|
| 392 |
f"or collect {sid} from the control panel")
|
| 393 |
|
| 394 |
|
|
@@ -575,7 +575,7 @@ def bd_filter_start(predicates, operator="and", records_limit=5, dataset_id="",
|
|
| 575 |
if not str(dataset_id or "").strip():
|
| 576 |
# Fail-closed, and it names what is missing. A transport that guesses a corpus is a
|
| 577 |
# transport that bills the wrong one.
|
| 578 |
-
return "", "the search was not started
|
| 579 |
nodes = bd_group(expand_predicates(predicates), operator)
|
| 580 |
filt = {"operator": operator, "filters": nodes}
|
| 581 |
handles = [str(h).strip().lstrip("@").lower() for h in (exclude_handles or [])]
|
|
@@ -627,7 +627,7 @@ def bd_filter_status(snapshot_id):
|
|
| 627 |
return "empty", 0, ""
|
| 628 |
if status == "failed" and err:
|
| 629 |
# NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name.
|
| 630 |
-
extra = ("
|
| 631 |
"conditions or lower the limit" if "FUNDS" in err.upper() else "")
|
| 632 |
return status, 0, f"the search failed: {err}{extra}"
|
| 633 |
return status, int(d.get("dataset_size") or 0), ""
|
|
@@ -646,7 +646,7 @@ def bd_filter_rows(snapshot_id):
|
|
| 646 |
return [], note
|
| 647 |
rows = _bd_rows(payload)
|
| 648 |
if not rows:
|
| 649 |
-
return [], "the search is ready but its rows are not downloadable yet
|
| 650 |
"delivery lags its own `ready` status by minutes"
|
| 651 |
return rows, ""
|
| 652 |
|
|
|
|
| 182 |
if not key:
|
| 183 |
# β THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends
|
| 184 |
# somebody to read Instagram's status page instead of setting an env var.
|
| 185 |
+
return None, "AIOS_BRIGHTDATA_KEY is not configured; the paid rung is closed"
|
| 186 |
qs = "&".join(f"{k}={requests.utils.quote(str(v))}"
|
| 187 |
for k, v in (params or {}).items() if v not in (None, ""))
|
| 188 |
url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "")
|
|
|
|
| 220 |
# paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow.
|
| 221 |
if len(raw) >= BD_MAX_KB * 1024:
|
| 222 |
return None, (f"the profile source's answer exceeded this client's {BD_MAX_KB // 1024} MB read "
|
| 223 |
+
f"ceiling and was TRUNCATED; ask for fewer records; nothing was parsed "
|
| 224 |
f"from a partial document")
|
| 225 |
text = raw.decode("utf-8", "replace").strip()
|
| 226 |
if not text:
|
|
|
|
| 336 |
if isinstance(codes, dict) and codes else "")
|
| 337 |
if errors or named or status in ("failed", "error"):
|
| 338 |
return "failed", 0, ("the source finished this request and collected nothing"
|
| 339 |
+
+ (f"; it reported {named}" if named else "")
|
| 340 |
+ "; the target is unreachable, private, or no longer exists")
|
| 341 |
return "done", 0, "the source finished this request and found no records for it"
|
| 342 |
|
|
|
|
| 388 |
deferred.append({"snapshotId": sid, "datasetId": str(dataset_id),
|
| 389 |
"urls": requested_urls})
|
| 390 |
return [], (f"the profile source deferred this batch to {sid} and it was not ready within "
|
| 391 |
+
f"{int(budget)}s; the records are collected, not lost; ask for a smaller batch "
|
| 392 |
f"or collect {sid} from the control panel")
|
| 393 |
|
| 394 |
|
|
|
|
| 575 |
if not str(dataset_id or "").strip():
|
| 576 |
# Fail-closed, and it names what is missing. A transport that guesses a corpus is a
|
| 577 |
# transport that bills the wrong one.
|
| 578 |
+
return "", "the search was not started; no corpus was named for this platform"
|
| 579 |
nodes = bd_group(expand_predicates(predicates), operator)
|
| 580 |
filt = {"operator": operator, "filters": nodes}
|
| 581 |
handles = [str(h).strip().lstrip("@").lower() for h in (exclude_handles or [])]
|
|
|
|
| 627 |
return "empty", 0, ""
|
| 628 |
if status == "failed" and err:
|
| 629 |
# NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name.
|
| 630 |
+
extra = ("; the matched set is too large for this account's balance; narrow the "
|
| 631 |
"conditions or lower the limit" if "FUNDS" in err.upper() else "")
|
| 632 |
return status, 0, f"the search failed: {err}{extra}"
|
| 633 |
return status, int(d.get("dataset_size") or 0), ""
|
|
|
|
| 646 |
return [], note
|
| 647 |
rows = _bd_rows(payload)
|
| 648 |
if not rows:
|
| 649 |
+
return [], "the search is ready but its rows are not downloadable yet; the vendor's " \
|
| 650 |
"delivery lags its own `ready` status by minutes"
|
| 651 |
return rows, ""
|
| 652 |
|
api/connectors_ig.py
CHANGED
|
@@ -658,7 +658,7 @@ def bd_profile_posts(handle, count, deferred=None, wait=None, post_type="", raw_
|
|
| 658 |
deferred.append({"snapshotId": sid, "datasetId": str(BD_DS_POSTS),
|
| 659 |
"urls": [f"https://www.instagram.com/{handle}/"], "kind": "posts"})
|
| 660 |
return [], (f"the post source deferred this request to {sid} and it was not ready within "
|
| 661 |
-
f"{int(budget)}s
|
| 662 |
|
| 663 |
|
| 664 |
def group_types(groups):
|
|
@@ -815,7 +815,7 @@ APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240)
|
|
| 815 |
#: a rung that returns this into `res["gone"]`, which is what lets a run stop re-buying a dead
|
| 816 |
#: handle every morning ([[flag-shipped-without-its-writer]] β the flag needs a writer AND a
|
| 817 |
#: reader, so both are in this file's call graph).
|
| 818 |
-
ACCOUNT_GONE_NOTE = ("Instagram has no account with that handle
|
| 819 |
"not exist (it may have been deleted, renamed, or suspended)")
|
| 820 |
|
| 821 |
|
|
@@ -841,7 +841,7 @@ def apify_posts(urls):
|
|
| 841 |
return [], ""
|
| 842 |
key = apify_key()
|
| 843 |
if not key:
|
| 844 |
-
return [], "AIOS_APIFY_KEY is not configured
|
| 845 |
url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
|
| 846 |
f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
|
| 847 |
body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)),
|
|
@@ -895,7 +895,7 @@ def apify_reels(handle, limit):
|
|
| 895 |
return [], "no handle to read reels for"
|
| 896 |
key = apify_key()
|
| 897 |
if not key:
|
| 898 |
-
return [], "AIOS_APIFY_KEY is not configured
|
| 899 |
try:
|
| 900 |
n = max(1, min(int(limit), MAX_POST_WINDOW))
|
| 901 |
except (TypeError, ValueError):
|
|
@@ -949,7 +949,7 @@ def apify_profile(handle):
|
|
| 949 |
return None, "no handle"
|
| 950 |
key = apify_key()
|
| 951 |
if not key:
|
| 952 |
-
return None, "AIOS_APIFY_KEY is not configured
|
| 953 |
url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
|
| 954 |
f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
|
| 955 |
body = {"directUrls": [f"https://www.instagram.com/{h}/"],
|
|
@@ -1065,7 +1065,7 @@ def bd_corpus_profile(handle, log=print):
|
|
| 1065 |
time.sleep(BD_CORPUS_POLL)
|
| 1066 |
break
|
| 1067 |
return None, (f"the profile corpus did not deliver {h!r} within "
|
| 1068 |
-
f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building
|
| 1069 |
|
| 1070 |
|
| 1071 |
def _tag_metric_deferrals(items, start, kind, influencer_key):
|
|
@@ -1266,7 +1266,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
|
| 1266 |
sid = str(cached.get(DEFERRED_MARK))
|
| 1267 |
return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
|
| 1268 |
"via": "brightdata", "deferredProfile": [sid],
|
| 1269 |
-
"note": (f"the profile source deferred this batch to {sid}
|
| 1270 |
f"collected, not lost; the next run picks it up")}
|
| 1271 |
if isinstance(cached, dict) and cached:
|
| 1272 |
rows, note = [cached], ""
|
|
@@ -1292,7 +1292,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
|
| 1292 |
# against the schema. Losing it behind the corpus fallback would make a mapping
|
| 1293 |
# regression look like a vendor outage.
|
| 1294 |
scrape_note = note or ("the scrape answered, but no follower/following counts were "
|
| 1295 |
-
"readable in it (the field names may have moved
|
| 1296 |
"instagram-capture.md)")
|
| 1297 |
cnode, cnote = bd_corpus_profile(handle, log=log)
|
| 1298 |
if cnode:
|
|
@@ -1303,7 +1303,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
|
| 1303 |
# a read taken now. Said out loud on the attempt, because "when was this true" is the
|
| 1304 |
# one question a stored measurement must always be able to answer.
|
| 1305 |
note = (f"the live scrape did not answer ({_s(scrape_note, 120)}), so this came from "
|
| 1306 |
-
f"the vendor's pre-collected profile corpus
|
| 1307 |
f"capture, not a read taken just now")
|
| 1308 |
else:
|
| 1309 |
# β THE DEFERRAL IS REGISTERED ONLY WHEN THE PAID RUNG GIVES UP. If the corpus
|
|
@@ -1323,7 +1323,7 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
|
|
| 1323 |
# is discarded, so a row where it is the only "readable" field is a row we cannot read.
|
| 1324 |
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 1325 |
"note": "the profile source answered, but no follower/following counts were readable in "
|
| 1326 |
-
"it (the field names may have moved
|
| 1327 |
posts = []
|
| 1328 |
for p in (node.get("posts") or [])[:max_posts]:
|
| 1329 |
got = _bd_post_identity(p)
|
|
@@ -1652,4 +1652,3 @@ def pull_profile(url, max_posts=None, log=print,
|
|
| 1652 |
"no provider could read this profile "
|
| 1653 |
f"({'; '.join(attempts) or 'blocked'})")}
|
| 1654 |
|
| 1655 |
-
|
|
|
|
| 658 |
deferred.append({"snapshotId": sid, "datasetId": str(BD_DS_POSTS),
|
| 659 |
"urls": [f"https://www.instagram.com/{handle}/"], "kind": "posts"})
|
| 660 |
return [], (f"the post source deferred this request to {sid} and it was not ready within "
|
| 661 |
+
f"{int(budget)}s; the records are collected, not lost")
|
| 662 |
|
| 663 |
|
| 664 |
def group_types(groups):
|
|
|
|
| 815 |
#: a rung that returns this into `res["gone"]`, which is what lets a run stop re-buying a dead
|
| 816 |
#: handle every morning ([[flag-shipped-without-its-writer]] β the flag needs a writer AND a
|
| 817 |
#: reader, so both are in this file's call graph).
|
| 818 |
+
ACCOUNT_GONE_NOTE = ("Instagram has no account with that handle; the source reports it does "
|
| 819 |
"not exist (it may have been deleted, renamed, or suspended)")
|
| 820 |
|
| 821 |
|
|
|
|
| 841 |
return [], ""
|
| 842 |
key = apify_key()
|
| 843 |
if not key:
|
| 844 |
+
return [], "AIOS_APIFY_KEY is not configured; the Apify rung is closed"
|
| 845 |
url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
|
| 846 |
f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
|
| 847 |
body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)),
|
|
|
|
| 895 |
return [], "no handle to read reels for"
|
| 896 |
key = apify_key()
|
| 897 |
if not key:
|
| 898 |
+
return [], "AIOS_APIFY_KEY is not configured; the Apify rung is closed"
|
| 899 |
try:
|
| 900 |
n = max(1, min(int(limit), MAX_POST_WINDOW))
|
| 901 |
except (TypeError, ValueError):
|
|
|
|
| 949 |
return None, "no handle"
|
| 950 |
key = apify_key()
|
| 951 |
if not key:
|
| 952 |
+
return None, "AIOS_APIFY_KEY is not configured; the Apify rung is closed"
|
| 953 |
url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items"
|
| 954 |
f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}")
|
| 955 |
body = {"directUrls": [f"https://www.instagram.com/{h}/"],
|
|
|
|
| 1065 |
time.sleep(BD_CORPUS_POLL)
|
| 1066 |
break
|
| 1067 |
return None, (f"the profile corpus did not deliver {h!r} within "
|
| 1068 |
+
f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building; it is not lost)")
|
| 1069 |
|
| 1070 |
|
| 1071 |
def _tag_metric_deferrals(items, start, kind, influencer_key):
|
|
|
|
| 1266 |
sid = str(cached.get(DEFERRED_MARK))
|
| 1267 |
return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
|
| 1268 |
"via": "brightdata", "deferredProfile": [sid],
|
| 1269 |
+
"note": (f"the profile source deferred this batch to {sid}; the record is "
|
| 1270 |
f"collected, not lost; the next run picks it up")}
|
| 1271 |
if isinstance(cached, dict) and cached:
|
| 1272 |
rows, note = [cached], ""
|
|
|
|
| 1292 |
# against the schema. Losing it behind the corpus fallback would make a mapping
|
| 1293 |
# regression look like a vendor outage.
|
| 1294 |
scrape_note = note or ("the scrape answered, but no follower/following counts were "
|
| 1295 |
+
"readable in it (the field names may have moved; see "
|
| 1296 |
"instagram-capture.md)")
|
| 1297 |
cnode, cnote = bd_corpus_profile(handle, log=log)
|
| 1298 |
if cnode:
|
|
|
|
| 1303 |
# a read taken now. Said out loud on the attempt, because "when was this true" is the
|
| 1304 |
# one question a stored measurement must always be able to answer.
|
| 1305 |
note = (f"the live scrape did not answer ({_s(scrape_note, 120)}), so this came from "
|
| 1306 |
+
f"the vendor's pre-collected profile corpus; the values are its last "
|
| 1307 |
f"capture, not a read taken just now")
|
| 1308 |
else:
|
| 1309 |
# β THE DEFERRAL IS REGISTERED ONLY WHEN THE PAID RUNG GIVES UP. If the corpus
|
|
|
|
| 1323 |
# is discarded, so a row where it is the only "readable" field is a row we cannot read.
|
| 1324 |
return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
|
| 1325 |
"note": "the profile source answered, but no follower/following counts were readable in "
|
| 1326 |
+
"it (the field names may have moved; see instagram-capture.md)"}
|
| 1327 |
posts = []
|
| 1328 |
for p in (node.get("posts") or [])[:max_posts]:
|
| 1329 |
got = _bd_post_identity(p)
|
|
|
|
| 1652 |
"no provider could read this profile "
|
| 1653 |
f"({'; '.join(attempts) or 'blocked'})")}
|
| 1654 |
|
|
|
api/connectors_meta.py
CHANGED
|
@@ -325,7 +325,7 @@ def probe():
|
|
| 325 |
report["errors"].append(f"reaching /me/adaccounts: {e}")
|
| 326 |
return report
|
| 327 |
if not report["accounts"]:
|
| 328 |
-
report["errors"].append("the token authenticates but reaches ZERO ad accounts
|
| 329 |
"permissions/scope answer, not an empty business")
|
| 330 |
return report
|
| 331 |
act = report["accounts"][0]
|
|
@@ -373,7 +373,7 @@ def probe():
|
|
| 373 |
def as_markdown(report):
|
| 374 |
"""The wave asset. β Field NAMES and TYPES only β no row values, no account identifiers beyond
|
| 375 |
the ad account name the owner already knows, and no query strings."""
|
| 376 |
-
L = ["# W31-T40
|
| 377 |
f"Graph version: `{report['graphVersion']}` Β· token present: "
|
| 378 |
f"**{'yes' if report['token'] else 'NO'}** Β· probe calls: **{len(CALLS)}**", ""]
|
| 379 |
if report["errors"]:
|
|
@@ -392,25 +392,25 @@ def as_markdown(report):
|
|
| 392 |
f"{a.get('account_status','')} | {a.get('currency','')} | "
|
| 393 |
f"{a.get('timezone_name','')} |")
|
| 394 |
L.append("")
|
| 395 |
-
L += ["## Method
|
| 396 |
"β **Graph v21.0 publishes no introspection table.** `GET /act_β¦?metadata=1` returns "
|
| 397 |
-
"keys `['account_id','id']` and `GET /<campaign_id>?metadata=1` returns `['id']`
|
| 398 |
"`metadata.fields` is EMPTY. The first run of this probe reported *0 fields* for all "
|
| 399 |
"five levels and looked like a broken probe; it was a withdrawn API.", "",
|
| 400 |
"β So every level below is measured by **ACCEPTANCE**: the names in "
|
| 401 |
"`connectors_meta.ENTITY_CANDIDATES` are offered, and what Graph takes is the catalog. "
|
| 402 |
"A `(#100)` refusal NAMES the offending field, so a batch converges in one retry; a "
|
| 403 |
"`(#200)` permission refusal names nothing, so the batch is **bisected** rather than "
|
| 404 |
-
"dropped
|
| 405 |
"ββ **ACCEPTED IS NOT RETURNED.** A 20-field batch on a live campaign answers HTTP 200 "
|
| 406 |
"with 14 keys, because Graph omits nulls. Every list below is what was **asked and "
|
| 407 |
-
"accepted**
|
| 408 |
"returned keys drops roughly a third of the schema, which is R2's *do not drop any "
|
| 409 |
"column* broken with nothing going red.", ""]
|
| 410 |
for key, label, _edge in LEVELS:
|
| 411 |
lv = report["levels"].get(key) or {}
|
| 412 |
fields = lv.get("accepted") or []
|
| 413 |
-
L += [f"## {label}
|
| 414 |
if lv.get("note"):
|
| 415 |
L += [f"β {lv['note']}", ""]
|
| 416 |
if fields:
|
|
@@ -420,12 +420,12 @@ def as_markdown(report):
|
|
| 420 |
"guessed at):", ""]
|
| 421 |
L += [f"- `{k}`" for k in sorted(lv["rejected"])] + [""]
|
| 422 |
if lv.get("gated"):
|
| 423 |
-
L += ["β **Permission-gated
|
| 424 |
"sentence):", ""]
|
| 425 |
-
L += [f"- `{k}`
|
| 426 |
ins = report.get("insights") or {}
|
| 427 |
if ins:
|
| 428 |
-
L += [f"## Insights at `time_increment=1`
|
| 429 |
f"{ins.get('candidatesAsked', 0)} asked", "",
|
| 430 |
"β Insights publishes no `metadata=1` field table, so this catalog is measured by "
|
| 431 |
"ACCEPTANCE: every name below was offered to the API and taken. A name absent from "
|
|
@@ -433,7 +433,7 @@ def as_markdown(report):
|
|
| 433 |
"```", ", ".join(ins.get("accepted") or []), "```", ""]
|
| 434 |
if ins.get("rejected"):
|
| 435 |
L += ["### Refused by name", ""]
|
| 436 |
-
L += [f"- `{k}`
|
| 437 |
return "\n".join(L)
|
| 438 |
|
| 439 |
|
|
@@ -461,7 +461,7 @@ def probe_credential(fields, timeout=8):
|
|
| 461 |
return {"ok": False, "message": str(e)[:200]}
|
| 462 |
n = len(got.get("data") or [])
|
| 463 |
return {"ok": n > 0,
|
| 464 |
-
"message": (f"Meta answered
|
| 465 |
"the token authenticates but reaches NO ad account: that is a "
|
| 466 |
"scope/permission answer, not an empty business")}
|
| 467 |
|
|
@@ -494,7 +494,7 @@ def main(argv):
|
|
| 494 |
print(f"REFUSED: {e}")
|
| 495 |
return 1
|
| 496 |
report = probe()
|
| 497 |
-
out = (Path(__file__).resolve().parents[2] / "
|
| 498 |
/ "wave31" / "proto" / "meta-ads-schema.md")
|
| 499 |
out.parent.mkdir(parents=True, exist_ok=True)
|
| 500 |
out.write_text(as_markdown(report), encoding="utf-8")
|
|
|
|
| 325 |
report["errors"].append(f"reaching /me/adaccounts: {e}")
|
| 326 |
return report
|
| 327 |
if not report["accounts"]:
|
| 328 |
+
report["errors"].append("the token authenticates but reaches ZERO ad accounts; a "
|
| 329 |
"permissions/scope answer, not an empty business")
|
| 330 |
return report
|
| 331 |
act = report["accounts"][0]
|
|
|
|
| 373 |
def as_markdown(report):
|
| 374 |
"""The wave asset. β Field NAMES and TYPES only β no row values, no account identifiers beyond
|
| 375 |
the ad account name the owner already knows, and no query strings."""
|
| 376 |
+
L = ["# W31-T40: the Meta Ads schema, MEASURED", "",
|
| 377 |
f"Graph version: `{report['graphVersion']}` Β· token present: "
|
| 378 |
f"**{'yes' if report['token'] else 'NO'}** Β· probe calls: **{len(CALLS)}**", ""]
|
| 379 |
if report["errors"]:
|
|
|
|
| 392 |
f"{a.get('account_status','')} | {a.get('currency','')} | "
|
| 393 |
f"{a.get('timezone_name','')} |")
|
| 394 |
L.append("")
|
| 395 |
+
L += ["## Method: and why it is not `?metadata=1`", "",
|
| 396 |
"β **Graph v21.0 publishes no introspection table.** `GET /act_β¦?metadata=1` returns "
|
| 397 |
+
"keys `['account_id','id']` and `GET /<campaign_id>?metadata=1` returns `['id']`; "
|
| 398 |
"`metadata.fields` is EMPTY. The first run of this probe reported *0 fields* for all "
|
| 399 |
"five levels and looked like a broken probe; it was a withdrawn API.", "",
|
| 400 |
"β So every level below is measured by **ACCEPTANCE**: the names in "
|
| 401 |
"`connectors_meta.ENTITY_CANDIDATES` are offered, and what Graph takes is the catalog. "
|
| 402 |
"A `(#100)` refusal NAMES the offending field, so a batch converges in one retry; a "
|
| 403 |
"`(#200)` permission refusal names nothing, so the batch is **bisected** rather than "
|
| 404 |
+
"dropped; otherwise the Ad Account level reports a confident, wrong `0 fields`.", "",
|
| 405 |
"ββ **ACCEPTED IS NOT RETURNED.** A 20-field batch on a live campaign answers HTTP 200 "
|
| 406 |
"with 14 keys, because Graph omits nulls. Every list below is what was **asked and "
|
| 407 |
+
"accepted**; the response's own keys are never inspected. Building columns from "
|
| 408 |
"returned keys drops roughly a third of the schema, which is R2's *do not drop any "
|
| 409 |
"column* broken with nothing going red.", ""]
|
| 410 |
for key, label, _edge in LEVELS:
|
| 411 |
lv = report["levels"].get(key) or {}
|
| 412 |
fields = lv.get("accepted") or []
|
| 413 |
+
L += [f"## {label}: **{len(fields)} accepted** of {lv.get('asked', 0)} asked", ""]
|
| 414 |
if lv.get("note"):
|
| 415 |
L += [f"β {lv['note']}", ""]
|
| 416 |
if fields:
|
|
|
|
| 420 |
"guessed at):", ""]
|
| 421 |
L += [f"- `{k}`" for k in sorted(lv["rejected"])] + [""]
|
| 422 |
if lv.get("gated"):
|
| 423 |
+
L += ["β **Permission-gated: a REPORTED limit, not a silent omission** (R6's second "
|
| 424 |
"sentence):", ""]
|
| 425 |
+
L += [f"- `{k}`: {v}" for k, v in sorted(lv["gated"].items())] + [""]
|
| 426 |
ins = report.get("insights") or {}
|
| 427 |
if ins:
|
| 428 |
+
L += [f"## Insights at `time_increment=1`: {len(ins.get('accepted') or [])} accepted of "
|
| 429 |
f"{ins.get('candidatesAsked', 0)} asked", "",
|
| 430 |
"β Insights publishes no `metadata=1` field table, so this catalog is measured by "
|
| 431 |
"ACCEPTANCE: every name below was offered to the API and taken. A name absent from "
|
|
|
|
| 433 |
"```", ", ".join(ins.get("accepted") or []), "```", ""]
|
| 434 |
if ins.get("rejected"):
|
| 435 |
L += ["### Refused by name", ""]
|
| 436 |
+
L += [f"- `{k}`: {v}" for k, v in sorted((ins.get("rejected") or {}).items())] + [""]
|
| 437 |
return "\n".join(L)
|
| 438 |
|
| 439 |
|
|
|
|
| 461 |
return {"ok": False, "message": str(e)[:200]}
|
| 462 |
n = len(got.get("data") or [])
|
| 463 |
return {"ok": n > 0,
|
| 464 |
+
"message": (f"Meta answered; {n} ad account(s) reachable" if n else
|
| 465 |
"the token authenticates but reaches NO ad account: that is a "
|
| 466 |
"scope/permission answer, not an empty business")}
|
| 467 |
|
|
|
|
| 494 |
print(f"REFUSED: {e}")
|
| 495 |
return 1
|
| 496 |
report = probe()
|
| 497 |
+
out = (Path(__file__).resolve().parents[2] / "AutoWiki" / "research" / "waves"
|
| 498 |
/ "wave31" / "proto" / "meta-ads-schema.md")
|
| 499 |
out.parent.mkdir(parents=True, exist_ok=True)
|
| 500 |
out.write_text(as_markdown(report), encoding="utf-8")
|
api/connectors_tt.py
CHANGED
|
@@ -7,7 +7,7 @@ exists at all rather than another thousand lines inside the engine.
|
|
| 7 |
β **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β 40 profile / 43
|
| 8 |
post / 17 comment fields, each with the vendor's own type, description and `pii` flag β was read
|
| 9 |
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
-
`
|
| 11 |
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
|
|
|
| 7 |
β **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β 40 profile / 43
|
| 8 |
post / 17 comment fields, each with the vendor's own type, description and `pii` flag β was read
|
| 9 |
live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
|
| 10 |
+
`AutoWiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
|
| 11 |
close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
|
| 12 |
name below reads through a candidate list it is because the vendor has two names for one fact
|
| 13 |
(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
|
api/deps.py
CHANGED
|
@@ -103,10 +103,40 @@ class Session:
|
|
| 103 |
# the Customer/Product pool builders, connector routes, provider routes, Odoo, or an environment
|
| 104 |
# lookup from this boundary. A cache miss is a named refusal, never permission to refresh from the
|
| 105 |
# source of truth. That is what "data the app already stores or serves" means in executable form.
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
"customer_data": {"label": "Odoo customers", "cache_prefix": "pool"},
|
| 108 |
"product_data": {"label": "Odoo products", "cache_prefix": "product_pool"},
|
| 109 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
|
| 112 |
def _assistant_refuse(status, code, message):
|
|
@@ -746,24 +776,16 @@ def _user_for(claims):
|
|
| 746 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 747 |
stateless cookie revocable.
|
| 748 |
|
| 749 |
-
β THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities
|
| 750 |
issues.** Any divergence produces the worst failure shape there is β a login that returns 200
|
| 751 |
-
with a cookie and then 401s every request after it.
|
| 752 |
-
emergency-master branch below ran only when NO record existed, while `verify` grants the master
|
| 753 |
-
identity even when one does), so it is now written as a deliberate mirror of `verify`'s
|
| 754 |
-
structure: try the record, else the master.
|
| 755 |
|
| 756 |
-
TWO CASES
|
| 757 |
* THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
|
| 758 |
the account is active, and its CURRENT epoch equals the cookie's β otherwise the session
|
| 759 |
is revoked (a password change or a deactivation bumped it).
|
| 760 |
-
* THE
|
| 761 |
-
|
| 762 |
-
store outage, a self-deactivation or a bumped epoch locking the owner out of the product.
|
| 763 |
-
β AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That
|
| 764 |
-
is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's
|
| 765 |
-
epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it.
|
| 766 |
-
Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`.
|
| 767 |
* anything else β None. An unknown username is NOT admitted just because its signature was
|
| 768 |
valid: a signature proves the cookie is ours, not that the account still exists.
|
| 769 |
"""
|
|
@@ -801,17 +823,6 @@ def _user_for(claims):
|
|
| 801 |
if claim_tenant != pub.get("tenant"):
|
| 802 |
return None
|
| 803 |
return pub
|
| 804 |
-
if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
|
| 805 |
-
and claim_tenant == "royal-imports"):
|
| 806 |
-
# The emergency master is TENANT #0's break-glass, not the platform's: a synthetic
|
| 807 |
-
# all-access identity minted into another tenant's session would be the exact
|
| 808 |
-
# cross-tenant widening the rule above closes.
|
| 809 |
-
# β WAVE 40 (R2): the same mark `users.verify`'s master branch carries. The two dicts are
|
| 810 |
-
# documented mirrors of each other, so the key rides on both or they are not mirrors.
|
| 811 |
-
return {"username": "admin", "name": "Administrator", "role": "admin",
|
| 812 |
-
"bus": "all", "modules": "all", "tenant": "royal-imports",
|
| 813 |
-
"emergency_master": True,
|
| 814 |
-
"epoch": int(claims.get("e") or 0)}
|
| 815 |
return None
|
| 816 |
|
| 817 |
|
|
@@ -832,7 +843,7 @@ def _user_for(claims):
|
|
| 832 |
# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
|
| 833 |
# write once per window (still bounded); a restart re-arms every user's first request; and the
|
| 834 |
# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
|
| 835 |
-
# renders it as. Bounded
|
| 836 |
# with many accounts.
|
| 837 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
| 838 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
|
|
|
| 103 |
# the Customer/Product pool builders, connector routes, provider routes, Odoo, or an environment
|
| 104 |
# lookup from this boundary. A cache miss is a named refusal, never permission to refresh from the
|
| 105 |
# source of truth. That is what "data the app already stores or serves" means in executable form.
|
| 106 |
+
# These two keys name registry modules backed by runtime pools. They are DATABASES the product
|
| 107 |
+
# may open, but they are not materialised `ut_*` documents and they are not DuckDB read-through
|
| 108 |
+
# mirrors. Keeping that distinction in the request boundary prevents a route resolver from
|
| 109 |
+
# treating `product_data` as though `_sources()` could bind it to a mirror table.
|
| 110 |
+
MODULE_CACHE_DATABASES = {
|
| 111 |
"customer_data": {"label": "Odoo customers", "cache_prefix": "pool"},
|
| 112 |
"product_data": {"label": "Odoo products", "cache_prefix": "product_pool"},
|
| 113 |
}
|
| 114 |
+
# The Assistant predates the public taxonomy above. Its private name remains an alias so its
|
| 115 |
+
# snapshot-only contract and every existing caller stay byte-for-byte unchanged.
|
| 116 |
+
_ASSISTANT_BUILT_INS = MODULE_CACHE_DATABASES
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def module_cache_source(database: str):
|
| 120 |
+
"""Metadata for a runtime-cache database, or None for every materialised/mirror key."""
|
| 121 |
+
meta = MODULE_CACHE_DATABASES.get(str(database or ""))
|
| 122 |
+
return dict(meta) if meta else None
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def module_cache_rows(session: Session, database: str):
|
| 126 |
+
"""The current viewer's scoped rows for one known runtime-cache database.
|
| 127 |
+
|
| 128 |
+
This is deliberately the interactive grid boundary, not the Assistant snapshot reader:
|
| 129 |
+
`scoped_table` applies the same team/agent and row grants as the module page, while
|
| 130 |
+
`Session.require` keeps the account and tenant-catalogue checks in their one chokepoint.
|
| 131 |
+
Unknown keys fail before either check, so an absent relation target cannot be mislabeled as
|
| 132 |
+
a permission denial.
|
| 133 |
+
"""
|
| 134 |
+
key = str(database or "")
|
| 135 |
+
if module_cache_source(key) is None:
|
| 136 |
+
raise KeyError(key)
|
| 137 |
+
session.require(key)
|
| 138 |
+
import core.perm_scope as perm_scope
|
| 139 |
+
return list(perm_scope.scoped_table(session.user, key, st=session.runtime) or [])
|
| 140 |
|
| 141 |
|
| 142 |
def _assistant_refuse(status, code, message):
|
|
|
|
| 776 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 777 |
stateless cookie revocable.
|
| 778 |
|
| 779 |
+
β THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities the login route
|
| 780 |
issues.** Any divergence produces the worst failure shape there is β a login that returns 200
|
| 781 |
+
with a cookie and then 401s every request after it.
|
|
|
|
|
|
|
|
|
|
| 782 |
|
| 783 |
+
TWO CASES:
|
| 784 |
* THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
|
| 785 |
the account is active, and its CURRENT epoch equals the cookie's β otherwise the session
|
| 786 |
is revoked (a password change or a deactivation bumped it).
|
| 787 |
+
* THE QA RUNNER, only when both non-production deployment gates remain enabled and its
|
| 788 |
+
password-derived epoch still matches. Live deployments cannot enable it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
* anything else β None. An unknown username is NOT admitted just because its signature was
|
| 790 |
valid: a signature proves the cookie is ours, not that the account still exists.
|
| 791 |
"""
|
|
|
|
| 823 |
if claim_tenant != pub.get("tenant"):
|
| 824 |
return None
|
| 825 |
return pub
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 826 |
return None
|
| 827 |
|
| 828 |
|
|
|
|
| 843 |
# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
|
| 844 |
# write once per window (still bounded); a restart re-arms every user's first request; and the
|
| 845 |
# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
|
| 846 |
+
# renders it as. Bounded independently so it cannot grow into a leak on a deployment
|
| 847 |
# with many accounts.
|
| 848 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
| 849 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
api/formula_runtime.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/ig_master.py
CHANGED
|
@@ -70,7 +70,7 @@ def append_run(tenant, snaps, posts, psnaps):
|
|
| 70 |
own budget, and a per-row writer would spend it the same way). Empty inputs cost nothing.
|
| 71 |
"""
|
| 72 |
if not configured():
|
| 73 |
-
return "off", "no master store on this deployment
|
| 74 |
slug = str(tenant or "").strip() or "unknown"
|
| 75 |
stamped = {
|
| 76 |
SNAP_BUCKET: ([{**r, "tenant": slug} for r in snaps or []], "snapshot_key"),
|
|
@@ -87,7 +87,7 @@ def append_run(tenant, snaps, posts, psnaps):
|
|
| 87 |
existing = dict(h.get(bucket) or {})
|
| 88 |
merged, counts = upsert(existing, rows_in, key_field, cap=MASTER_MAX_ROWS)
|
| 89 |
if counts.get("capped"):
|
| 90 |
-
capped.append(f"{bucket} is FULL ({MASTER_MAX_ROWS:,})
|
| 91 |
f"{counts['capped']} rows lost")
|
| 92 |
h.update(bucket, lambda cur, m=merged: m, flush="sync")
|
| 93 |
except Exception as e: # noqa: BLE001
|
|
|
|
| 70 |
own budget, and a per-row writer would spend it the same way). Empty inputs cost nothing.
|
| 71 |
"""
|
| 72 |
if not configured():
|
| 73 |
+
return "off", "no master store on this deployment; tenant copy only"
|
| 74 |
slug = str(tenant or "").strip() or "unknown"
|
| 75 |
stamped = {
|
| 76 |
SNAP_BUCKET: ([{**r, "tenant": slug} for r in snaps or []], "snapshot_key"),
|
|
|
|
| 87 |
existing = dict(h.get(bucket) or {})
|
| 88 |
merged, counts = upsert(existing, rows_in, key_field, cap=MASTER_MAX_ROWS)
|
| 89 |
if counts.get("capped"):
|
| 90 |
+
capped.append(f"{bucket} is FULL ({MASTER_MAX_ROWS:,}); "
|
| 91 |
f"{counts['capped']} rows lost")
|
| 92 |
h.update(bucket, lambda cur, m=merged: m, flush="sync")
|
| 93 |
except Exception as e: # noqa: BLE001
|
api/main.py
CHANGED
|
@@ -66,6 +66,7 @@ import routes_sheet # noqa: E402 (wave 49 instr 1 / W49-T02 / C1 β the Sheet
|
|
| 66 |
import routes_customers # noqa: E402
|
| 67 |
import routes_grid # noqa: E402
|
| 68 |
import routes_keychain # noqa: E402 (wave 18 C7 β keychain + connectors admin surfaces)
|
|
|
|
| 69 |
import routes_statements # noqa: E402 (EXIT-6 β the statement sender, off Streamlit)
|
| 70 |
import routes_nav # noqa: E402
|
| 71 |
import routes_pages # noqa: E402
|
|
@@ -461,6 +462,7 @@ app.include_router(routes_agent_harness.router) # R8 / C4 β versioned agent h
|
|
| 461 |
# tiles = OpenStreetMap Β· routing = OSRM Β· geocoding = Nominatim, none of them keyed, all of them
|
| 462 |
# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden).
|
| 463 |
app.include_router(routes_geo.router) # R14 / A11 β D's router, A's line (map providers)
|
|
|
|
| 464 |
|
| 465 |
|
| 466 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
@@ -751,6 +753,37 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations
|
|
| 751 |
"changed": True, "applied": False, "written": {}}
|
| 752 |
|
| 753 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 754 |
def _sweep_automation_schemas(why):
|
| 755 |
"""ββ WAVE 32 Β· `W32-T07` β MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES.
|
| 756 |
|
|
@@ -895,7 +928,21 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 895 |
_rt = _runtime.get_runtime("royal-imports")
|
| 896 |
from core import user_tables as _ut
|
| 897 |
_ut.record_odoo_sync_status(phase="syncing", st=_rt)
|
| 898 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 899 |
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
|
| 900 |
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
|
| 901 |
errors=schema_errors, st=_rt)
|
|
@@ -939,6 +986,10 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 939 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 940 |
if isinstance(meta_state_out, dict):
|
| 941 |
meta_state_out.update(meta_state)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 942 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 943 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 944 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
|
@@ -1223,6 +1274,11 @@ def _store_resync_loop():
|
|
| 1223 |
last_relational_success_at = None
|
| 1224 |
last_meta_fingerprint = None
|
| 1225 |
last_meta_applied_at = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1226 |
while True:
|
| 1227 |
# ββ WAVE 32 Β· OWNER ITEM 11 / R11 β THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 1228 |
#
|
|
@@ -1277,7 +1333,13 @@ def _store_resync_loop():
|
|
| 1277 |
_RESYNC_NOTICE[0] = _cad["disagreement"]
|
| 1278 |
if _cad["disagreement"]:
|
| 1279 |
print(f"[aios-api] store resync cadence: {_cad['disagreement']}")
|
| 1280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1281 |
passes += 1
|
| 1282 |
RESYNC_STATE["passes"] = passes
|
| 1283 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
|
@@ -1328,6 +1390,7 @@ def _store_resync_loop():
|
|
| 1328 |
_meta_reason = _meta_rebuild_force_reason(
|
| 1329 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
| 1330 |
_meta_previous = last_meta_fingerprint if _meta_reason is None else None
|
|
|
|
| 1331 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 1332 |
last_relational_success_at, _meta_now)
|
| 1333 |
_meta_state = {}
|
|
@@ -1335,13 +1398,22 @@ def _store_resync_loop():
|
|
| 1335 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
| 1336 |
if _rebuild_odoo_relational("resync", _meta_previous, _meta_state):
|
| 1337 |
last_relational_success_at = _t.time()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1338 |
else:
|
| 1339 |
# A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
|
| 1340 |
# whether this cadence may skip both the full-document write and relation pass.
|
| 1341 |
try:
|
| 1342 |
from harness import runtime as _runtime
|
| 1343 |
-
|
| 1344 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1345 |
except Exception as e: # noqa: BLE001
|
| 1346 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 1347 |
f"{type(e).__name__}: {e}")
|
|
|
|
| 66 |
import routes_customers # noqa: E402
|
| 67 |
import routes_grid # noqa: E402
|
| 68 |
import routes_keychain # noqa: E402 (wave 18 C7 β keychain + connectors admin surfaces)
|
| 69 |
+
import routes_external # noqa: E402 (wave 56 R3/C7 β per-user external AI credential)
|
| 70 |
import routes_statements # noqa: E402 (EXIT-6 β the statement sender, off Streamlit)
|
| 71 |
import routes_nav # noqa: E402
|
| 72 |
import routes_pages # noqa: E402
|
|
|
|
| 462 |
# tiles = OpenStreetMap Β· routing = OSRM Β· geocoding = Nominatim, none of them keyed, all of them
|
| 463 |
# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden).
|
| 464 |
app.include_router(routes_geo.router) # R14 / A11 β D's router, A's line (map providers)
|
| 465 |
+
app.include_router(routes_external.router)
|
| 466 |
|
| 467 |
|
| 468 |
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
|
|
|
| 753 |
"changed": True, "applied": False, "written": {}}
|
| 754 |
|
| 755 |
|
| 756 |
+
def _rebuild_amazon_relational(why, rt, previous_fingerprint=None):
|
| 757 |
+
"""Spawn/refresh the `ut_amzn_*` locked databases off the SAME mirror the two passes above used.
|
| 758 |
+
|
| 759 |
+
β CALLED IN THE SAME PLACE AND FOR THE SAME REASON AS `_rebuild_meta_relational`, which is
|
| 760 |
+
D-29 and not tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and the
|
| 761 |
+
caller has already established which tenant's file this process holds open. Spawning Amazon
|
| 762 |
+
here inherits that binding instead of rebinding it under live readers, which is the documented
|
| 763 |
+
way to serve one tenant's rows to another with nothing raised.
|
| 764 |
+
|
| 765 |
+
β SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Amazon has no `amzn_*`
|
| 766 |
+
tables in its mirror; `refresh_state` returns `{}` and says so once. That is the normal state
|
| 767 |
+
for every tenant but this one, and it must not print an error every 30 minutes for all of them.
|
| 768 |
+
|
| 769 |
+
β ITS OWN try/except, for the reason the passes above have theirs: an Amazon refusal must not
|
| 770 |
+
cancel an Odoo or Meta rebuild that already succeeded. Amazon is the likeliest of the three to
|
| 771 |
+
refuse, because R5 ships Brand Analytics UNPROVEN and a missing role answers 403.
|
| 772 |
+
"""
|
| 773 |
+
try:
|
| 774 |
+
import amazon_relational as _amzn
|
| 775 |
+
state = _amzn.refresh_state(rt, why, previous_fingerprint=previous_fingerprint)
|
| 776 |
+
counts = state["written"]
|
| 777 |
+
if counts:
|
| 778 |
+
print(f"[aios-api] amazon relational rebuild done ({why}): "
|
| 779 |
+
+ ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
| 780 |
+
return state
|
| 781 |
+
except Exception as e: # noqa: BLE001
|
| 782 |
+
print(f"[aios-api] amazon relational rebuild FAILED ({why}): {type(e).__name__}: {e}")
|
| 783 |
+
return {"present": False, "known": False, "fingerprint": None,
|
| 784 |
+
"changed": True, "applied": False, "written": {}}
|
| 785 |
+
|
| 786 |
+
|
| 787 |
def _sweep_automation_schemas(why):
|
| 788 |
"""ββ WAVE 32 Β· `W32-T07` β MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES.
|
| 789 |
|
|
|
|
| 928 |
_rt = _runtime.get_runtime("royal-imports")
|
| 929 |
from core import user_tables as _ut
|
| 930 |
_ut.record_odoo_sync_status(phase="syncing", st=_rt)
|
| 931 |
+
# ββ W54-T46 / R4 / R7 β EVERY Odoo CONNECTION, not just the first. `all_connections`
|
| 932 |
+
# delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
|
| 933 |
+
# connections in slot order and records a failure against the connection it belongs
|
| 934 |
+
# to rather than aborting the rest. The DEFAULT connection's counts stay at the top
|
| 935 |
+
# level of the returned dict, so the log line below is unchanged; the per-connection
|
| 936 |
+
# map rides beside them under `connections`.
|
| 937 |
+
# β A FLAG RATHER THAN CALLING THE ALL-CONNECTIONS FUNCTION BY NAME HERE, and the
|
| 938 |
+
# reason is a gate: `verify_odoo_relational` locates this call by its literal spelling
|
| 939 |
+
# and reads the code that follows it, to prove the relation cells are computed after
|
| 940 |
+
# the spawn. Spelling the boot call differently moves that window off the code it is
|
| 941 |
+
# meant to measure. β AND THE CALL IS NAMED IN WORDS ABOVE RATHER THAN WRITTEN OUT,
|
| 942 |
+
# because a gate that counts RAW occurrences of a call would count the one in this
|
| 943 |
+
# comment too. That gate happens to strip comments first; the next one may not.
|
| 944 |
+
# See `odoo_relational.refresh` and its `all_connections` arm.
|
| 945 |
+
counts = _rel.refresh(_rt, "royal-imports", all_connections=True)
|
| 946 |
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
|
| 947 |
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
|
| 948 |
errors=schema_errors, st=_rt)
|
|
|
|
| 986 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 987 |
if isinstance(meta_state_out, dict):
|
| 988 |
meta_state_out.update(meta_state)
|
| 989 |
+
# β AFTER Meta and inside the same tenant binding β see `_rebuild_amazon_relational`.
|
| 990 |
+
# It carries no fingerprint here on purpose: this branch has just done a full Odoo
|
| 991 |
+
# rebuild, so a skip would be optimising the rarest pass at the cost of a stale grid.
|
| 992 |
+
_rebuild_amazon_relational(why, _rt)
|
| 993 |
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
|
| 994 |
# continue. This return value answers the narrower resync-policy question: did the Odoo
|
| 995 |
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
|
|
|
|
| 1274 |
last_relational_success_at = None
|
| 1275 |
last_meta_fingerprint = None
|
| 1276 |
last_meta_applied_at = None
|
| 1277 |
+
# β AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
|
| 1278 |
+
# on different cadences behind one variable would let a quiet Meta pass suppress an Amazon
|
| 1279 |
+
# rebuild that was due -- one question with two answers, which is the defect class this
|
| 1280 |
+
# codebase already names. Unknown is never carried forward as equality evidence.
|
| 1281 |
+
last_amzn_fingerprint = None
|
| 1282 |
while True:
|
| 1283 |
# ββ WAVE 32 Β· OWNER ITEM 11 / R11 β THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
|
| 1284 |
#
|
|
|
|
| 1333 |
_RESYNC_NOTICE[0] = _cad["disagreement"]
|
| 1334 |
if _cad["disagreement"]:
|
| 1335 |
print(f"[aios-api] store resync cadence: {_cad['disagreement']}")
|
| 1336 |
+
# W58: the first resident pass is the boot catch-up when AIOS_BOOT_SYNC=0. Sleeping here
|
| 1337 |
+
# unconditionally left a newly installed seed untouched for the full hourly cadence;
|
| 1338 |
+
# stored fields such as customer_rank can change without partner write_date advancing,
|
| 1339 |
+
# so staging served a provably stale customer population for its first hour. Subsequent
|
| 1340 |
+
# passes still sleep at the freshly resolved cadence, and manual still skips only Odoo.
|
| 1341 |
+
if passes:
|
| 1342 |
+
_t.sleep(_every)
|
| 1343 |
passes += 1
|
| 1344 |
RESYNC_STATE["passes"] = passes
|
| 1345 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
|
|
|
|
| 1390 |
_meta_reason = _meta_rebuild_force_reason(
|
| 1391 |
_meta_pull_confirmed, last_meta_fingerprint, last_meta_applied_at, _meta_now)
|
| 1392 |
_meta_previous = last_meta_fingerprint if _meta_reason is None else None
|
| 1393 |
+
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
|
| 1394 |
_reason = _resync_rebuild_reason(sync_report, delete_report,
|
| 1395 |
last_relational_success_at, _meta_now)
|
| 1396 |
_meta_state = {}
|
|
|
|
| 1398 |
print(f"[aios-api] odoo relational rebuild required ({_reason})")
|
| 1399 |
if _rebuild_odoo_relational("resync", _meta_previous, _meta_state):
|
| 1400 |
last_relational_success_at = _t.time()
|
| 1401 |
+
else:
|
| 1402 |
+
# A prior successful projection no longer proves the changed mirror was applied.
|
| 1403 |
+
# Keep recovery due on the next cadence even when that sync has no new rows.
|
| 1404 |
+
last_relational_success_at = None
|
| 1405 |
else:
|
| 1406 |
# A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
|
| 1407 |
# whether this cadence may skip both the full-document write and relation pass.
|
| 1408 |
try:
|
| 1409 |
from harness import runtime as _runtime
|
| 1410 |
+
_rt_resync = _runtime.get_runtime("royal-imports")
|
| 1411 |
+
_meta_state = _rebuild_meta_relational("resync", _rt_resync, _meta_previous)
|
| 1412 |
+
# A zero Odoo report says nothing about Amazon either. Same reasoning, same pass,
|
| 1413 |
+
# and it inherits the tenant binding the line above just established.
|
| 1414 |
+
_amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
|
| 1415 |
+
last_amzn_fingerprint = (_amzn_state["fingerprint"]
|
| 1416 |
+
if _amzn_state.get("known") else None)
|
| 1417 |
except Exception as e: # noqa: BLE001
|
| 1418 |
print(f"[aios-api] meta relational refresh failed (resync): "
|
| 1419 |
f"{type(e).__name__}: {e}")
|
api/meta_relational.py
CHANGED
|
@@ -40,10 +40,14 @@ if str(_API.parent.parent / "platform") not in sys.path:
|
|
| 40 |
sys.path.insert(0, str(_API.parent.parent / "platform"))
|
| 41 |
|
| 42 |
import odoo_relational as _odoo # noqa: E402
|
|
|
|
| 43 |
from harness import datastore, meta_store # noqa: E402
|
| 44 |
|
| 45 |
Refused = _odoo.Refused
|
| 46 |
|
|
|
|
|
|
|
|
|
|
| 47 |
ACCOUNTS_KEY = "ut_meta_ad_accounts"
|
| 48 |
CAMPAIGNS_KEY = "ut_meta_campaigns"
|
| 49 |
ADSETS_KEY = "ut_meta_adsets"
|
|
@@ -51,36 +55,47 @@ ADS_KEY = "ut_meta_ads"
|
|
| 51 |
CREATIVES_KEY = "ut_meta_creatives"
|
| 52 |
INSIGHTS_KEY = "ut_meta_insights"
|
| 53 |
|
| 54 |
-
#:
|
| 55 |
-
SOURCE
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
#: The column each grid shows first. Everything else keeps the API's own order, which is the order
|
| 71 |
#: the schema was measured in β a reader comparing the grid to Graph sees the same sequence.
|
| 72 |
-
PINNED = {
|
| 73 |
-
ADS_KEY: "name", CREATIVES_KEY: "name", INSIGHTS_KEY: "date_start"}
|
| 74 |
|
| 75 |
-
#:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
#: person can walk account -> campaign -> ad set -> ad in the product the way they do in Ads
|
| 77 |
-
#: Manager. β Only emitted when
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
|
|
|
|
|
|
| 84 |
|
| 85 |
#: DuckDB type -> `core.user_tables.UT_FIELD_TYPES` member. β `_clean_field` returns None for an
|
| 86 |
#: unknown type, which DELETES the column on the next read rather than erroring β so this map only
|
|
@@ -131,7 +146,7 @@ def _fields_for(cur, key):
|
|
| 131 |
if not cols:
|
| 132 |
return []
|
| 133 |
pinned = PINNED.get(key)
|
| 134 |
-
|
| 135 |
out = []
|
| 136 |
for name, dtype in cols:
|
| 137 |
f = {"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
|
@@ -139,13 +154,18 @@ def _fields_for(cur, key):
|
|
| 139 |
"impressions", "clicks", "date_start"),
|
| 140 |
"pinned": name == pinned}
|
| 141 |
out.append(_odoo._preset(f, flow="meta_relational"))
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
out.append(_odoo._preset(
|
| 146 |
-
{"key":
|
| 147 |
"source": "overlay", "default": True,
|
| 148 |
-
"link": {"table": target, "on":
|
|
|
|
| 149 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="meta_relational"))
|
| 150 |
return out
|
| 151 |
|
|
@@ -277,7 +297,7 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
|
|
| 277 |
datastore.use_path(path)
|
| 278 |
cur = datastore.ro_con()
|
| 279 |
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 280 |
-
log(f"[meta_relational] {why}: no Meta tables in {tenant}'s mirror
|
| 281 |
f"(run `python platform/harness/meta_store.py --sync --tenant {tenant}` first)")
|
| 282 |
return state
|
| 283 |
state["present"] = True
|
|
|
|
| 40 |
sys.path.insert(0, str(_API.parent.parent / "platform"))
|
| 41 |
|
| 42 |
import odoo_relational as _odoo # noqa: E402
|
| 43 |
+
from core import connector_spec # noqa: E402
|
| 44 |
from harness import datastore, meta_store # noqa: E402
|
| 45 |
|
| 46 |
Refused = _odoo.Refused
|
| 47 |
|
| 48 |
+
#: The six tables this module spawns, as names rather than repeated string literals. β They are the
|
| 49 |
+
#: same six keys `connector_spec.META_TABLES` declares; `INSIGHTS_KEY` is the one still USED below,
|
| 50 |
+
#: so a typo in it grows `SOURCE` a seventh entry and `verify_meta`'s coverage leg reddens.
|
| 51 |
ACCOUNTS_KEY = "ut_meta_ad_accounts"
|
| 52 |
CAMPAIGNS_KEY = "ut_meta_campaigns"
|
| 53 |
ADSETS_KEY = "ut_meta_adsets"
|
|
|
|
| 55 |
CREATIVES_KEY = "ut_meta_creatives"
|
| 56 |
INSIGHTS_KEY = "ut_meta_insights"
|
| 57 |
|
| 58 |
+
#: ββ **THE TABLE SHAPE IS NOW READ FROM `platform/core/connector_spec.py`, NOT TYPED HERE**
|
| 59 |
+
#: (W53-T37, contract C2). `SOURCE`, `LABELS`, `PINNED`, `IDENTITY` and `LINKS` keep their
|
| 60 |
+
#: module-level names because gates and routes read them BY NAME β but each is a DERIVED VIEW over
|
| 61 |
+
#: ONE declaration now, and Amazon's copy of the same five dicts is derived from the same file. Two
|
| 62 |
+
#: connectors stating one shape in two dialects is what C2 exists to end.
|
| 63 |
+
_SPEC = {s.key: s for s in connector_spec.tables_for("meta")}
|
| 64 |
+
|
| 65 |
+
#: ut_* key -> the mirror table `meta_store` writes, in the spec's declaration order (which is the
|
| 66 |
+
#: order `meta_store` syncs them, and the order `plan` therefore walks).
|
| 67 |
+
SOURCE = {k: s.source for k, s in _SPEC.items()}
|
| 68 |
+
#: β ONE ENTRY STAYS DERIVED FROM THE LOADER RATHER THAN FROM THE SPEC, ON PURPOSE. The daily
|
| 69 |
+
#: insights grain is the only Meta mirror table whose name `meta_store` also owns
|
| 70 |
+
#: (`meta_store.INSIGHTS_TABLE`), and reading it from there keeps `verify_meta`'s Meta-source parity
|
| 71 |
+
#: leg a REAL cross-check for that table instead of comparing the spec with itself. The other five
|
| 72 |
+
#: mirror names exist only in the spec, so there is nothing to hold them against.
|
| 73 |
+
SOURCE[INSIGHTS_KEY] = meta_store.INSIGHTS_TABLE
|
| 74 |
+
|
| 75 |
+
LABELS = {k: s.label for k, s in _SPEC.items()}
|
| 76 |
|
| 77 |
#: The column each grid shows first. Everything else keeps the API's own order, which is the order
|
| 78 |
#: the schema was measured in β a reader comparing the grid to Graph sees the same sequence.
|
| 79 |
+
PINNED = {k: s.pinned for k, s in _SPEC.items()}
|
|
|
|
| 80 |
|
| 81 |
+
#: β THE ROW IDENTITY PER TABLE, IN FALLING PREFERENCE, NEW IN W53-T37 β this module had no
|
| 82 |
+
#: `IDENTITY` at all before the spec declared one. β NOTHING IN THIS MODULE READS IT YET: `_read`
|
| 83 |
+
#: keys every row on the Graph `id` inline, so this is the spec's answer made reachable by name for
|
| 84 |
+
#: the callers that will ask (the relation engine, the connector detail door), not a behaviour
|
| 85 |
+
#: change. Wiring a consumer that did not exist is a different ticket.
|
| 86 |
+
IDENTITY = {k: tuple(s.identity) for k, s in _SPEC.items()}
|
| 87 |
+
|
| 88 |
+
#: `{ut_ key: [(this table's column, target ut_ key, the column it matches there), ...]}`, so a
|
| 89 |
#: person can walk account -> campaign -> ad set -> ad in the product the way they do in Ads
|
| 90 |
+
#: Manager. β Only emitted when this table's own mirror carries the column.
|
| 91 |
+
#: ββ **A LIST PER KEY, NOT A SCALAR 3-TUPLE** (W53-T37, and it is the point of the ticket). The
|
| 92 |
+
#: old scalar could hold exactly one edge per table, and the key it generated was
|
| 93 |
+
#: `f"{target}_link"` β so a SECOND edge to the same target would have collided on both the field
|
| 94 |
+
#: key and the field label and silently overwritten the first. Meta's chain is linear today, so
|
| 95 |
+
#: nothing collided and no gate was red; it was latent, and Amazon had already fixed the same
|
| 96 |
+
#: defect. The shape and the generated key are now Amazon's, from one shared generator.
|
| 97 |
+
LINKS = {k: [(c, t, o) for c, t, o, _cardinality in s.links]
|
| 98 |
+
for k, s in _SPEC.items() if s.links}
|
| 99 |
|
| 100 |
#: DuckDB type -> `core.user_tables.UT_FIELD_TYPES` member. β `_clean_field` returns None for an
|
| 101 |
#: unknown type, which DELETES the column on the next read rather than erroring β so this map only
|
|
|
|
| 146 |
if not cols:
|
| 147 |
return []
|
| 148 |
pinned = PINNED.get(key)
|
| 149 |
+
names = {c[0] for c in cols}
|
| 150 |
out = []
|
| 151 |
for name, dtype in cols:
|
| 152 |
f = {"key": name, "label": _label(name), "type": _ftype(name, dtype),
|
|
|
|
| 154 |
"impressions", "clicks", "date_start"),
|
| 155 |
"pinned": name == pinned}
|
| 156 |
out.append(_odoo._preset(f, flow="meta_relational"))
|
| 157 |
+
# ββ EVERY EDGE OF THIS TABLE, FROM THE SHARED GENERATOR (W53-T37). `connector_spec.edges_for`
|
| 158 |
+
# mints `f"{target}_link__{from_col}"` and `f"{target label} by {source column}"` β Amazon's
|
| 159 |
+
# convention, now everyone's. β THE SOURCE COLUMN IS IN THE KEY ON PURPOSE: this loop used to
|
| 160 |
+
# read a single scalar link and key it `f"{target}_link"`, so a second edge to one target would
|
| 161 |
+
# have overwritten the first on both the key and the label, with nothing raising.
|
| 162 |
+
for edge in connector_spec.edges_for(key):
|
| 163 |
+
if edge["from_col"] in names:
|
| 164 |
out.append(_odoo._preset(
|
| 165 |
+
{"key": edge["key"], "label": edge["label"], "type": "link",
|
| 166 |
"source": "overlay", "default": True,
|
| 167 |
+
"link": {"table": edge["target"], "on": edge["to_col"],
|
| 168 |
+
"from": edge["from_col"]}}, flow="meta_relational"))
|
| 169 |
out.append(_odoo._preset(_odoo._refreshed_field(), flow="meta_relational"))
|
| 170 |
return out
|
| 171 |
|
|
|
|
| 297 |
datastore.use_path(path)
|
| 298 |
cur = datastore.ro_con()
|
| 299 |
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
|
| 300 |
+
log(f"[meta_relational] {why}: no Meta tables in {tenant}'s mirror; nothing to spawn "
|
| 301 |
f"(run `python platform/harness/meta_store.py --sync --tenant {tenant}` first)")
|
| 302 |
return state
|
| 303 |
state["present"] = True
|
api/oauth_connect.py
CHANGED
|
@@ -164,7 +164,7 @@ def start(provider, uname, redirect_uri, next_path=""):
|
|
| 164 |
if p is None:
|
| 165 |
return "", f"{provider!r} is not a connectable provider"
|
| 166 |
if not configured(provider):
|
| 167 |
-
return "", (f"{p['label']} OAuth is not configured on this deployment
|
| 168 |
f"{p['env_id']} / {p['env_secret']} first")
|
| 169 |
_prune_states()
|
| 170 |
cid, _secret = client(provider)
|
|
@@ -229,11 +229,11 @@ def callback(rt, uname, state, code):
|
|
| 229 |
entry = _STATES.pop(str(state or ""), None) # single-use, whatever happens next
|
| 230 |
home = safe_next((entry or {}).get("next"))
|
| 231 |
if not entry:
|
| 232 |
-
return "", home, "that connect attempt is unknown or expired
|
| 233 |
if entry.get("uname") != str(uname or ""):
|
| 234 |
# A state minted for one session consumed by another is exactly the CSRF the state
|
| 235 |
# exists to refuse. Fail closed; the honest user just clicks Connect again.
|
| 236 |
-
return "", home, "that connect attempt belongs to a different session
|
| 237 |
provider = entry.get("provider") or "google"
|
| 238 |
p = provider_def(provider)
|
| 239 |
if p is None or not configured(provider):
|
|
@@ -250,11 +250,11 @@ def callback(rt, uname, state, code):
|
|
| 250 |
access = str(payload.get("access_token") or "")
|
| 251 |
refresh = str(payload.get("refresh_token") or "")
|
| 252 |
if not access:
|
| 253 |
-
return "", home, "the provider answered without an access token
|
| 254 |
if not refresh:
|
| 255 |
# Google answers access-only when consent was silently reused β an access-only slot
|
| 256 |
# dies within the hour, silently, so it is refused with the way out named.
|
| 257 |
-
return "", home, ("no refresh token came back
|
| 258 |
"permissions page and connect again")
|
| 259 |
expiry = time.time() + float(payload.get("expires_in") or 3600) - 60
|
| 260 |
email = _userinfo_email(provider, access)
|
|
@@ -302,11 +302,11 @@ def creds(rt, uname, provider="google"):
|
|
| 302 |
try:
|
| 303 |
slot = kc.read_user_secret(rt, uname, slot_name(provider))
|
| 304 |
except kc.KeychainLocked as e:
|
| 305 |
-
return "", f"the keychain is locked ({e})
|
| 306 |
if not slot:
|
| 307 |
-
return "", (f"no {p['label']} connection for this user
|
| 308 |
if slot.get("reconnect"):
|
| 309 |
-
return "", f"the {p['label']} connection expired
|
| 310 |
try:
|
| 311 |
expiry = float(slot.get("expiry") or 0)
|
| 312 |
except (TypeError, ValueError):
|
|
@@ -314,7 +314,7 @@ def creds(rt, uname, provider="google"):
|
|
| 314 |
if time.time() >= expiry:
|
| 315 |
slot, err = _refresh(rt, uname, provider, slot)
|
| 316 |
if err:
|
| 317 |
-
return "", f"the {p['label']} connection could not refresh
|
| 318 |
return str(slot.get("access") or ""), ""
|
| 319 |
|
| 320 |
|
|
|
|
| 164 |
if p is None:
|
| 165 |
return "", f"{provider!r} is not a connectable provider"
|
| 166 |
if not configured(provider):
|
| 167 |
+
return "", (f"{p['label']} OAuth is not configured on this deployment; the owner adds "
|
| 168 |
f"{p['env_id']} / {p['env_secret']} first")
|
| 169 |
_prune_states()
|
| 170 |
cid, _secret = client(provider)
|
|
|
|
| 229 |
entry = _STATES.pop(str(state or ""), None) # single-use, whatever happens next
|
| 230 |
home = safe_next((entry or {}).get("next"))
|
| 231 |
if not entry:
|
| 232 |
+
return "", home, "that connect attempt is unknown or expired; start again"
|
| 233 |
if entry.get("uname") != str(uname or ""):
|
| 234 |
# A state minted for one session consumed by another is exactly the CSRF the state
|
| 235 |
# exists to refuse. Fail closed; the honest user just clicks Connect again.
|
| 236 |
+
return "", home, "that connect attempt belongs to a different session; start again"
|
| 237 |
provider = entry.get("provider") or "google"
|
| 238 |
p = provider_def(provider)
|
| 239 |
if p is None or not configured(provider):
|
|
|
|
| 250 |
access = str(payload.get("access_token") or "")
|
| 251 |
refresh = str(payload.get("refresh_token") or "")
|
| 252 |
if not access:
|
| 253 |
+
return "", home, "the provider answered without an access token; nothing was stored"
|
| 254 |
if not refresh:
|
| 255 |
# Google answers access-only when consent was silently reused β an access-only slot
|
| 256 |
# dies within the hour, silently, so it is refused with the way out named.
|
| 257 |
+
return "", home, ("no refresh token came back; disconnect the app on the provider's "
|
| 258 |
"permissions page and connect again")
|
| 259 |
expiry = time.time() + float(payload.get("expires_in") or 3600) - 60
|
| 260 |
email = _userinfo_email(provider, access)
|
|
|
|
| 302 |
try:
|
| 303 |
slot = kc.read_user_secret(rt, uname, slot_name(provider))
|
| 304 |
except kc.KeychainLocked as e:
|
| 305 |
+
return "", f"the keychain is locked ({e}); no token can be read"
|
| 306 |
if not slot:
|
| 307 |
+
return "", (f"no {p['label']} connection for this user; connect it in Settings")
|
| 308 |
if slot.get("reconnect"):
|
| 309 |
+
return "", f"the {p['label']} connection expired; reconnect it in Settings"
|
| 310 |
try:
|
| 311 |
expiry = float(slot.get("expiry") or 0)
|
| 312 |
except (TypeError, ValueError):
|
|
|
|
| 314 |
if time.time() >= expiry:
|
| 315 |
slot, err = _refresh(rt, uname, provider, slot)
|
| 316 |
if err:
|
| 317 |
+
return "", f"the {p['label']} connection could not refresh; {err}"
|
| 318 |
return str(slot.get("access") or ""), ""
|
| 319 |
|
| 320 |
|
api/odoo_relational.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/pages.py
CHANGED
|
@@ -159,7 +159,7 @@ def chart(key, spec, series, fields, rows, x_order=None, drill=None, total=None,
|
|
| 159 |
for f in fields:
|
| 160 |
if f.get("type") not in VIZ_TYPES:
|
| 161 |
raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a "
|
| 162 |
-
f"viz FieldType {VIZ_TYPES}
|
| 163 |
rows = list(rows)
|
| 164 |
out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series),
|
| 165 |
"fields": list(fields), "rows": rows,
|
|
@@ -317,7 +317,7 @@ _BUILDERS = {}
|
|
| 317 |
#: goes red β which is the only way a "deliberately empty" claim can stay true.
|
| 318 |
BUILDERS_EMPTY_BECAUSE = (
|
| 319 |
"the Sales page was deleted by owner ruling in wave 16; no dashboard page is published "
|
| 320 |
-
"today. The route, the envelope and the grant wall stay
|
| 321 |
"a builder here (ARCHITECTURE.md Β§7), not by rebuilding this file."
|
| 322 |
)
|
| 323 |
|
|
|
|
| 159 |
for f in fields:
|
| 160 |
if f.get("type") not in VIZ_TYPES:
|
| 161 |
raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a "
|
| 162 |
+
f"viz FieldType {VIZ_TYPES}; `money` is the kpi/table vocabulary")
|
| 163 |
rows = list(rows)
|
| 164 |
out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series),
|
| 165 |
"fields": list(fields), "rows": rows,
|
|
|
|
| 317 |
#: goes red β which is the only way a "deliberately empty" claim can stay true.
|
| 318 |
BUILDERS_EMPTY_BECAUSE = (
|
| 319 |
"the Sales page was deleted by owner ruling in wave 16; no dashboard page is published "
|
| 320 |
+
"today. The route, the envelope and the grant wall stay; a new page ships by registering "
|
| 321 |
"a builder here (ARCHITECTURE.md Β§7), not by rebuilding this file."
|
| 322 |
)
|
| 323 |
|
api/rollup_sql.py
CHANGED
|
@@ -81,7 +81,7 @@ def group_values(bag, today=None):
|
|
| 81 |
rng = W.resolve({"kind": window}, today or _today())
|
| 82 |
if rng is None:
|
| 83 |
raise RollupSourceError(
|
| 84 |
-
f"window {window!r} could not be resolved
|
| 85 |
f"rollup to all time")
|
| 86 |
date_from, date_to = rng
|
| 87 |
|
|
@@ -96,7 +96,7 @@ def group_values(bag, today=None):
|
|
| 96 |
# β THE REFUSAL THAT MATTERS. See the module header: a truncated GROUP set is a wrong
|
| 97 |
# answer per parent, not a shortened one, and it would look completely normal on screen.
|
| 98 |
raise RollupSourceError(
|
| 99 |
-
f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups
|
| 100 |
f"write cells from a truncated result")
|
| 101 |
|
| 102 |
# β THE KEY COLUMN IS NOT ALWAYS `<dim>_id`, and reading it unconditionally was a SILENT
|
|
|
|
| 81 |
rng = W.resolve({"kind": window}, today or _today())
|
| 82 |
if rng is None:
|
| 83 |
raise RollupSourceError(
|
| 84 |
+
f"window {window!r} could not be resolved; refusing rather than widening this "
|
| 85 |
f"rollup to all time")
|
| 86 |
date_from, date_to = rng
|
| 87 |
|
|
|
|
| 96 |
# β THE REFUSAL THAT MATTERS. See the module header: a truncated GROUP set is a wrong
|
| 97 |
# answer per parent, not a shortened one, and it would look completely normal on screen.
|
| 98 |
raise RollupSourceError(
|
| 99 |
+
f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups; refusing to "
|
| 100 |
f"write cells from a truncated result")
|
| 101 |
|
| 102 |
# β THE KEY COLUMN IS NOT ALWAYS `<dim>_id`, and reading it unconditionally was a SILENT
|
api/routes_admin.py
CHANGED
|
@@ -75,10 +75,7 @@ def system_data_inspector(session: Session = Depends(require_session)):
|
|
| 75 |
_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$")
|
| 76 |
#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`.
|
| 77 |
_UNAME_MAX = 254
|
| 78 |
-
#:
|
| 79 |
-
#: enforces nothing; this does, and the two are allowed to differ because only one of them is
|
| 80 |
-
#: reachable from the internet.
|
| 81 |
-
_MIN_PW = 8
|
| 82 |
_ROLES = ("user", "admin")
|
| 83 |
|
| 84 |
|
|
@@ -98,7 +95,7 @@ def _require_store():
|
|
| 98 |
ok = False
|
| 99 |
if not ok:
|
| 100 |
raise err(503, "store_unavailable",
|
| 101 |
-
"the tenant store is unavailable
|
| 102 |
|
| 103 |
|
| 104 |
def _registry():
|
|
@@ -131,6 +128,12 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
|
|
| 131 |
# The session-revocation handle. An admin needs it: it is the only visible evidence
|
| 132 |
# that "sign them out everywhere" actually happened.
|
| 133 |
"epoch": int(rec.get("epoch") or 0),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
# S3 ask 4 β one sentence for the roster's Access column, so the list does not need
|
| 135 |
# a /perms round trip per row to fill one cell. Computed from the same record the
|
| 136 |
# editor will open, so the two cannot disagree.
|
|
@@ -213,10 +216,16 @@ def _clean_username(v):
|
|
| 213 |
return uname
|
| 214 |
|
| 215 |
|
| 216 |
-
def _clean_password(v
|
|
|
|
| 217 |
pw = str(v or "")
|
| 218 |
-
|
| 219 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
return pw
|
| 221 |
|
| 222 |
|
|
@@ -249,7 +258,7 @@ def _clean_bus(v):
|
|
| 249 |
raise err(400, "bad_bus", "a business-unit id must be a number")
|
| 250 |
if b not in users.BU_LABELS:
|
| 251 |
raise err(400, "bad_bus",
|
| 252 |
-
f"{b} is not a known business unit {sorted(users.BU_LABELS)}
|
| 253 |
"unrecognised id would widen access to every BU")
|
| 254 |
out.append(b)
|
| 255 |
return sorted(set(out))
|
|
@@ -268,7 +277,7 @@ def _clean_modules(v):
|
|
| 268 |
raise err(400, "bad_modules", "modules must be 'all' or a list of module keys")
|
| 269 |
if not isinstance(v, (list, tuple)) or not v:
|
| 270 |
raise err(400, "bad_modules",
|
| 271 |
-
"modules must be 'all' or a NON-EMPTY list of module keys
|
| 272 |
"reads as UNRESTRICTED, which is the opposite of locking an account down")
|
| 273 |
known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS)
|
| 274 |
out, bad = [], []
|
|
@@ -326,17 +335,17 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi
|
|
| 326 |
body = body or {}
|
| 327 |
_require_store()
|
| 328 |
uname = _clean_username(body.get("username"))
|
| 329 |
-
|
|
|
|
|
|
|
| 330 |
role = _clean_role(body.get("role") or "user")
|
| 331 |
bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all")
|
| 332 |
modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all")
|
| 333 |
-
name = str(body.get("name") or "").strip() or uname
|
| 334 |
agent = str(body.get("agent") or "").strip() or None
|
| 335 |
-
email = str(body.get("email") or "").strip() or None
|
| 336 |
|
| 337 |
_reg_before = _registry()
|
| 338 |
if uname in _reg_before:
|
| 339 |
-
raise err(409, "user_exists", f"{uname} already exists
|
| 340 |
# ββ WAVE 37 Β· T03 (R8) β THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME.
|
| 341 |
# `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only
|
| 342 |
# scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was
|
|
@@ -365,13 +374,16 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi
|
|
| 365 |
f"address for this account.")
|
| 366 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's β
|
| 367 |
# the tenant is stamped from the session, not taken from the body.
|
| 368 |
-
|
| 369 |
-
|
|
|
|
|
|
|
|
|
|
| 370 |
reg = _registry()
|
| 371 |
if uname not in reg:
|
| 372 |
# The store accepted the write and does not have it. Reporting 201 here would be the
|
| 373 |
# "200 over a write that evaporated" failure the whole store-outage rule exists to prevent.
|
| 374 |
-
raise err(503, "store_unavailable", "the account was not saved
|
| 375 |
return {"user": _view(uname, reg[uname])}
|
| 376 |
|
| 377 |
|
|
@@ -404,12 +416,11 @@ def update_user(username: str, body: dict = Body(default=None),
|
|
| 404 |
is_self = uname == session.uname
|
| 405 |
role = _clean_role(body["role"]) if "role" in body else None
|
| 406 |
active = bool(body["active"]) if "active" in body else None
|
| 407 |
-
# THE ONE-CLICK LOCKOUT GUARD.
|
| 408 |
-
#
|
| 409 |
-
# their own access with a toggle and having to go find the master password.
|
| 410 |
if is_self and role is not None and role != "admin":
|
| 411 |
raise err(400, "self_demote",
|
| 412 |
-
"you cannot remove your own administrator role
|
| 413 |
if is_self and active is False:
|
| 414 |
raise err(400, "self_deactivate", "you cannot deactivate your own account")
|
| 415 |
kw = {}
|
|
@@ -456,15 +467,65 @@ def set_password(username: str, body: dict = Body(default=None),
|
|
| 456 |
# Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one β
|
| 457 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 458 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 459 |
-
pw = _clean_password((body or {}).get("password")
|
|
|
|
|
|
|
| 460 |
before = int(reg[uname].get("epoch") or 0)
|
| 461 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
after = int((_registry().get(uname) or {}).get("epoch") or 0)
|
| 463 |
if after <= before:
|
| 464 |
# The whole value of this route is the revocation. If the epoch did not move, the sessions
|
| 465 |
# were not revoked, and answering 204 would say they were.
|
| 466 |
raise err(503, "store_unavailable",
|
| 467 |
-
"the password change did not persist
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 468 |
return Response(status_code=204)
|
| 469 |
|
| 470 |
|
|
@@ -818,7 +879,7 @@ def _module_fields(key, session=None):
|
|
| 818 |
if session is None:
|
| 819 |
raise err(400, "ungoverned_module",
|
| 820 |
f"{key!r} is a tenant database, so its columns can only be read for a "
|
| 821 |
-
f"known tenant
|
| 822 |
import core.user_tables as user_tables
|
| 823 |
defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime))
|
| 824 |
if not defn:
|
|
@@ -1702,7 +1763,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
|
|
| 1702 |
unknown = sorted(set(v) - governed)
|
| 1703 |
if unknown:
|
| 1704 |
raise err(400, "bad_perms",
|
| 1705 |
-
f"unknown module keys: {unknown}
|
| 1706 |
f"{sorted(governed)}")
|
| 1707 |
out = {}
|
| 1708 |
for key, raw in v.items():
|
|
@@ -1817,7 +1878,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
|
|
| 1817 |
if tree not in (None, {}, []):
|
| 1818 |
if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list):
|
| 1819 |
raise err(400, "bad_filter",
|
| 1820 |
-
f"{key}: filter must be {{conj?, nodes:[β¦]}}
|
| 1821 |
f"the root conjunction, and an 'or' wall read as 'and' restricts "
|
| 1822 |
f"nothing it was meant to")
|
| 1823 |
conj = tree.get("conj", "and")
|
|
@@ -1885,7 +1946,7 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
|
|
| 1885 |
if _leaf_count(cleaned) != _leaf_count(nodes):
|
| 1886 |
raise err(400, "bad_filter",
|
| 1887 |
f"{key}: the filter contains conditions this module cannot evaluate "
|
| 1888 |
-
f"(an unknown field, an unknown operator, or a cohort leaf
|
| 1889 |
f"are per-user and cannot be a permanent rule). Refused rather than "
|
| 1890 |
f"saved with the bad conditions silently removed, which would store "
|
| 1891 |
f"a weaker wall than the one on screen.")
|
|
@@ -1938,8 +1999,8 @@ def _refuse_unshapeable_bu(perms_out):
|
|
| 1938 |
f"{key}: this filter restricts the business unit in a way the data layer "
|
| 1939 |
f"cannot push into the query, so the rows would be correct while the "
|
| 1940 |
f"revenue figures on them stayed consolidated across both units. Express "
|
| 1941 |
-
f"the business-unit condition as a TOP-LEVEL condition
|
| 1942 |
-
f"or an 'any of' over the brands
|
| 1943 |
|
| 1944 |
|
| 1945 |
def _guard_bu_widening(rec, cleaned, confirmed):
|
|
@@ -1983,7 +2044,7 @@ def _guard_bu_widening(rec, cleaned, confirmed):
|
|
| 1983 |
raise err(400, "bu_widening",
|
| 1984 |
f"{key}: this would REMOVE the business-unit restriction and give the account "
|
| 1985 |
f"both units. If that is intended, resend with confirm_widen: true. If it is "
|
| 1986 |
-
f"not, reload the account's permissions first
|
| 1987 |
f"the last change drops the conditions it did not know about.")
|
| 1988 |
# A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees
|
| 1989 |
# the connection to derive_pool_scope, which is what the removed condition was feeding.
|
|
@@ -2346,7 +2407,7 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2346 |
# writing a wall for themselves that would bite the moment their role changed.
|
| 2347 |
if uname == session.uname:
|
| 2348 |
raise err(400, "self_perms",
|
| 2349 |
-
"you cannot set permissions on your own account
|
| 2350 |
|
| 2351 |
_mods = _perm_modules(session, surfaces=True)
|
| 2352 |
# ββ W36-T22 / C2 β ONE key set. It was two ("listed" and "enforced") with a refusal in the
|
|
@@ -2371,7 +2432,7 @@ def put_perms(username: str, body: dict = Body(default=None),
|
|
| 2371 |
# The store took the write and did not record it. A 200 here would tell an administrator
|
| 2372 |
# the wall is up when it is not β the one direction this must never fail in.
|
| 2373 |
raise err(503, "store_unavailable",
|
| 2374 |
-
"the permissions were not saved
|
| 2375 |
return {"username": uname, "perms": rec.get("perms") or {},
|
| 2376 |
"perms_v": int(rec.get("perms_v") or 0),
|
| 2377 |
# W46-T35 β `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
|
|
|
|
| 75 |
_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$")
|
| 76 |
#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`.
|
| 77 |
_UNAME_MAX = 254
|
| 78 |
+
#: Password policy lives in core.users so this route and self-service rotation cannot disagree.
|
|
|
|
|
|
|
|
|
|
| 79 |
_ROLES = ("user", "admin")
|
| 80 |
|
| 81 |
|
|
|
|
| 95 |
ok = False
|
| 96 |
if not ok:
|
| 97 |
raise err(503, "store_unavailable",
|
| 98 |
+
"the tenant store is unavailable; accounts cannot be read or changed")
|
| 99 |
|
| 100 |
|
| 101 |
def _registry():
|
|
|
|
| 128 |
# The session-revocation handle. An admin needs it: it is the only visible evidence
|
| 129 |
# that "sign them out everywhere" actually happened.
|
| 130 |
"epoch": int(rec.get("epoch") or 0),
|
| 131 |
+
# Administrators need the rotation state, never the credential history behind it.
|
| 132 |
+
"passwordChangedAt": users.password_changed_at(rec),
|
| 133 |
+
"passwordExpired": users.password_expired(rec),
|
| 134 |
+
# The reset controls need enrolment state, never the secret, hashes or code count.
|
| 135 |
+
"mfaEnrolled": users.mfa_enrolled(rec),
|
| 136 |
+
"mfaEnrolledAt": rec.get("mfa_enrolled_at") or None,
|
| 137 |
# S3 ask 4 β one sentence for the roster's Access column, so the list does not need
|
| 138 |
# a /perms round trip per row to fill one cell. Computed from the same record the
|
| 139 |
# editor will open, so the two cannot disagree.
|
|
|
|
| 216 |
return uname
|
| 217 |
|
| 218 |
|
| 219 |
+
def _clean_password(v, username=None, name=None, email=None, record=None,
|
| 220 |
+
allow_early_rotation=False):
|
| 221 |
pw = str(v or "")
|
| 222 |
+
failures = users.password_failures(pw, username=username, name=name, email=email,
|
| 223 |
+
record=record,
|
| 224 |
+
allow_early_rotation=allow_early_rotation)
|
| 225 |
+
if failures:
|
| 226 |
+
# Name every broken rule so the caller can correct the value without guessing the policy.
|
| 227 |
+
raise err(400, "weak_password",
|
| 228 |
+
"That password does not meet the policy: " + str(users.PasswordPolicyError(failures)))
|
| 229 |
return pw
|
| 230 |
|
| 231 |
|
|
|
|
| 258 |
raise err(400, "bad_bus", "a business-unit id must be a number")
|
| 259 |
if b not in users.BU_LABELS:
|
| 260 |
raise err(400, "bad_bus",
|
| 261 |
+
f"{b} is not a known business unit {sorted(users.BU_LABELS)}; an "
|
| 262 |
"unrecognised id would widen access to every BU")
|
| 263 |
out.append(b)
|
| 264 |
return sorted(set(out))
|
|
|
|
| 277 |
raise err(400, "bad_modules", "modules must be 'all' or a list of module keys")
|
| 278 |
if not isinstance(v, (list, tuple)) or not v:
|
| 279 |
raise err(400, "bad_modules",
|
| 280 |
+
"modules must be 'all' or a NON-EMPTY list of module keys; an empty list "
|
| 281 |
"reads as UNRESTRICTED, which is the opposite of locking an account down")
|
| 282 |
known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS)
|
| 283 |
out, bad = [], []
|
|
|
|
| 335 |
body = body or {}
|
| 336 |
_require_store()
|
| 337 |
uname = _clean_username(body.get("username"))
|
| 338 |
+
name = str(body.get("name") or "").strip() or uname
|
| 339 |
+
email = str(body.get("email") or "").strip() or None
|
| 340 |
+
pw = _clean_password(body.get("password"), username=uname, name=name, email=email)
|
| 341 |
role = _clean_role(body.get("role") or "user")
|
| 342 |
bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all")
|
| 343 |
modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all")
|
|
|
|
| 344 |
agent = str(body.get("agent") or "").strip() or None
|
|
|
|
| 345 |
|
| 346 |
_reg_before = _registry()
|
| 347 |
if uname in _reg_before:
|
| 348 |
+
raise err(409, "user_exists", f"{uname} already exists; PATCH it to change it")
|
| 349 |
# ββ WAVE 37 Β· T03 (R8) β THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME.
|
| 350 |
# `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only
|
| 351 |
# scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was
|
|
|
|
| 374 |
f"address for this account.")
|
| 375 |
# Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's β
|
| 376 |
# the tenant is stamped from the session, not taken from the body.
|
| 377 |
+
try:
|
| 378 |
+
users.create_user(uname, pw, name, role=role, bus=bus, modules=modules,
|
| 379 |
+
agent=agent, email=email, tenant=_tenant_of(session.user))
|
| 380 |
+
except users.PasswordPolicyError as exc:
|
| 381 |
+
raise err(400, "weak_password", "That password does not meet the policy: " + str(exc))
|
| 382 |
reg = _registry()
|
| 383 |
if uname not in reg:
|
| 384 |
# The store accepted the write and does not have it. Reporting 201 here would be the
|
| 385 |
# "200 over a write that evaporated" failure the whole store-outage rule exists to prevent.
|
| 386 |
+
raise err(503, "store_unavailable", "the account was not saved; try again")
|
| 387 |
return {"user": _view(uname, reg[uname])}
|
| 388 |
|
| 389 |
|
|
|
|
| 416 |
is_self = uname == session.uname
|
| 417 |
role = _clean_role(body["role"]) if "role" in body else None
|
| 418 |
active = bool(body["active"]) if "active" in body else None
|
| 419 |
+
# THE ONE-CLICK LOCKOUT GUARD. There is deliberately no APP_PASSWORD bypass, so preventing a
|
| 420 |
+
# stored administrator from removing their own access is part of keeping recovery possible.
|
|
|
|
| 421 |
if is_self and role is not None and role != "admin":
|
| 422 |
raise err(400, "self_demote",
|
| 423 |
+
"you cannot remove your own administrator role; ask another admin")
|
| 424 |
if is_self and active is False:
|
| 425 |
raise err(400, "self_deactivate", "you cannot deactivate your own account")
|
| 426 |
kw = {}
|
|
|
|
| 467 |
# Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one β
|
| 468 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 469 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 470 |
+
pw = _clean_password((body or {}).get("password"), username=uname,
|
| 471 |
+
name=reg[uname].get("name"), email=reg[uname].get("email"),
|
| 472 |
+
record=reg[uname], allow_early_rotation=True)
|
| 473 |
before = int(reg[uname].get("epoch") or 0)
|
| 474 |
+
try:
|
| 475 |
+
# This route just checked strength AND reuse against its record. Spend the history's
|
| 476 |
+
# PBKDF2 passes once; the core still merges the rotation into the fresh update document.
|
| 477 |
+
users.set_password(uname, pw, enforce=False)
|
| 478 |
+
except users.PasswordPolicyError as exc:
|
| 479 |
+
raise err(400, "weak_password", "That password does not meet the policy: " + str(exc))
|
| 480 |
after = int((_registry().get(uname) or {}).get("epoch") or 0)
|
| 481 |
if after <= before:
|
| 482 |
# The whole value of this route is the revocation. If the epoch did not move, the sessions
|
| 483 |
# were not revoked, and answering 204 would say they were.
|
| 484 |
raise err(503, "store_unavailable",
|
| 485 |
+
"the password change did not persist; outstanding sessions were NOT revoked")
|
| 486 |
+
return Response(status_code=204)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
@router.post("/admin/users/{username}/mfa/reset", status_code=204)
|
| 490 |
+
def reset_mfa(username: str, session: Session = Depends(admin_gate)):
|
| 491 |
+
"""Clear a lost factor and revoke every session; the person enrols their own phone again.
|
| 492 |
+
|
| 493 |
+
The administrator never sees or sets a user's secret. Removing it opens the enrolment door
|
| 494 |
+
at next sign-in; the atomic epoch bump closes every session that predates this reset.
|
| 495 |
+
"""
|
| 496 |
+
reg = _registry()
|
| 497 |
+
uname = str(username or "").strip().lower()
|
| 498 |
+
if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user):
|
| 499 |
+
# A cross-tenant username answers exactly like a missing one: naming another company's
|
| 500 |
+
# account, even only to refuse the reset, would itself disclose that the account exists.
|
| 501 |
+
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 502 |
+
before = int(reg[uname].get("epoch") or 0)
|
| 503 |
+
users.clear_mfa(uname)
|
| 504 |
+
after = int((_registry().get(uname) or {}).get("epoch") or 0)
|
| 505 |
+
if after <= before:
|
| 506 |
+
# Revocation is the whole value of this door. A 204 without a moved epoch would claim
|
| 507 |
+
# the lost phone's sessions died when the store did not persist that change.
|
| 508 |
+
raise err(503, "store_unavailable", "the reset did not persist - outstanding sessions were NOT "
|
| 509 |
+
"revoked")
|
| 510 |
+
return Response(status_code=204)
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
@router.post("/admin/users/{username}/password/expire", status_code=204)
|
| 514 |
+
def expire_password(username: str, session: Session = Depends(admin_gate)):
|
| 515 |
+
"""Force the account through password rotation without choosing its replacement secret."""
|
| 516 |
+
reg = _registry()
|
| 517 |
+
uname = str(username or "").strip().lower()
|
| 518 |
+
if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user):
|
| 519 |
+
# Same missing-account answer across tenants: the reset door cannot enumerate others.
|
| 520 |
+
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 521 |
+
before = int(reg[uname].get("epoch") or 0)
|
| 522 |
+
users.set_must_change_password(uname, True)
|
| 523 |
+
after = int((_registry().get(uname) or {}).get("epoch") or 0)
|
| 524 |
+
if after <= before:
|
| 525 |
+
# A flag without revocation would leave the old sessions usable. Prove the write before
|
| 526 |
+
# telling an administrator the reset is in effect, just like the password setter above.
|
| 527 |
+
raise err(503, "store_unavailable",
|
| 528 |
+
"the forced password rotation did not persist. Outstanding sessions were NOT revoked")
|
| 529 |
return Response(status_code=204)
|
| 530 |
|
| 531 |
|
|
|
|
| 879 |
if session is None:
|
| 880 |
raise err(400, "ungoverned_module",
|
| 881 |
f"{key!r} is a tenant database, so its columns can only be read for a "
|
| 882 |
+
f"known tenant; this caller passed no session")
|
| 883 |
import core.user_tables as user_tables
|
| 884 |
defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime))
|
| 885 |
if not defn:
|
|
|
|
| 1763 |
unknown = sorted(set(v) - governed)
|
| 1764 |
if unknown:
|
| 1765 |
raise err(400, "bad_perms",
|
| 1766 |
+
f"unknown module keys: {unknown}; this tenant's permission editor governs "
|
| 1767 |
f"{sorted(governed)}")
|
| 1768 |
out = {}
|
| 1769 |
for key, raw in v.items():
|
|
|
|
| 1878 |
if tree not in (None, {}, []):
|
| 1879 |
if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list):
|
| 1880 |
raise err(400, "bad_filter",
|
| 1881 |
+
f"{key}: filter must be {{conj?, nodes:[β¦]}}; a bare list would lose "
|
| 1882 |
f"the root conjunction, and an 'or' wall read as 'and' restricts "
|
| 1883 |
f"nothing it was meant to")
|
| 1884 |
conj = tree.get("conj", "and")
|
|
|
|
| 1946 |
if _leaf_count(cleaned) != _leaf_count(nodes):
|
| 1947 |
raise err(400, "bad_filter",
|
| 1948 |
f"{key}: the filter contains conditions this module cannot evaluate "
|
| 1949 |
+
f"(an unknown field, an unknown operator, or a cohort leaf; cohorts "
|
| 1950 |
f"are per-user and cannot be a permanent rule). Refused rather than "
|
| 1951 |
f"saved with the bad conditions silently removed, which would store "
|
| 1952 |
f"a weaker wall than the one on screen.")
|
|
|
|
| 1999 |
f"{key}: this filter restricts the business unit in a way the data layer "
|
| 2000 |
f"cannot push into the query, so the rows would be correct while the "
|
| 2001 |
f"revenue figures on them stayed consolidated across both units. Express "
|
| 2002 |
+
f"the business-unit condition as a TOP-LEVEL condition; 'DBA is Fisch', "
|
| 2003 |
+
f"or an 'any of' over the brands; and keep any other rules alongside it.")
|
| 2004 |
|
| 2005 |
|
| 2006 |
def _guard_bu_widening(rec, cleaned, confirmed):
|
|
|
|
| 2044 |
raise err(400, "bu_widening",
|
| 2045 |
f"{key}: this would REMOVE the business-unit restriction and give the account "
|
| 2046 |
f"both units. If that is intended, resend with confirm_widen: true. If it is "
|
| 2047 |
+
f"not, reload the account's permissions first; saving a filter loaded before "
|
| 2048 |
f"the last change drops the conditions it did not know about.")
|
| 2049 |
# A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees
|
| 2050 |
# the connection to derive_pool_scope, which is what the removed condition was feeding.
|
|
|
|
| 2407 |
# writing a wall for themselves that would bite the moment their role changed.
|
| 2408 |
if uname == session.uname:
|
| 2409 |
raise err(400, "self_perms",
|
| 2410 |
+
"you cannot set permissions on your own account; ask another admin")
|
| 2411 |
|
| 2412 |
_mods = _perm_modules(session, surfaces=True)
|
| 2413 |
# ββ W36-T22 / C2 β ONE key set. It was two ("listed" and "enforced") with a refusal in the
|
|
|
|
| 2432 |
# The store took the write and did not record it. A 200 here would tell an administrator
|
| 2433 |
# the wall is up when it is not β the one direction this must never fail in.
|
| 2434 |
raise err(503, "store_unavailable",
|
| 2435 |
+
"the permissions were not saved; the account is UNCHANGED")
|
| 2436 |
return {"username": uname, "perms": rec.get("perms") or {},
|
| 2437 |
"perms_v": int(rec.get("perms_v") or 0),
|
| 2438 |
# W46-T35 β `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
|
api/routes_alerts.py
CHANGED
|
@@ -511,7 +511,7 @@ def _require_filtered_view(session: Session, topic: str, view_id: str):
|
|
| 511 |
# Genuinely unexpected. Still a 503, and now it carries the exception TYPE β without it,
|
| 512 |
# the one path that reaches this branch is also the one path with nothing to debug from.
|
| 513 |
raise err(503, "unavailable",
|
| 514 |
-
f"the table could not be read ({type(e).__name__})
|
| 515 |
view = _view_by_id(g, view_id)
|
| 516 |
if not isinstance(view, dict):
|
| 517 |
raise err(404, "no_view", "that view does not exist on this table")
|
|
@@ -572,7 +572,7 @@ def _require_filtered_view(session: Session, topic: str, view_id: str):
|
|
| 572 |
|
| 573 |
if not _any_active(nodes):
|
| 574 |
raise err(400, "no_filter",
|
| 575 |
-
"this view has no active filter, so no record can ever ENTER it
|
| 576 |
"condition to the view first, then create the alert")
|
| 577 |
|
| 578 |
|
|
|
|
| 511 |
# Genuinely unexpected. Still a 503, and now it carries the exception TYPE β without it,
|
| 512 |
# the one path that reaches this branch is also the one path with nothing to debug from.
|
| 513 |
raise err(503, "unavailable",
|
| 514 |
+
f"the table could not be read ({type(e).__name__}); try again in a moment")
|
| 515 |
view = _view_by_id(g, view_id)
|
| 516 |
if not isinstance(view, dict):
|
| 517 |
raise err(404, "no_view", "that view does not exist on this table")
|
|
|
|
| 572 |
|
| 573 |
if not _any_active(nodes):
|
| 574 |
raise err(400, "no_filter",
|
| 575 |
+
"this view has no active filter, so no record can ever ENTER it; add a "
|
| 576 |
"condition to the view first, then create the alert")
|
| 577 |
|
| 578 |
|
api/routes_assets.py
CHANGED
|
@@ -152,14 +152,31 @@ def duplicate_application(application_id: str,
|
|
| 152 |
|
| 153 |
|
| 154 |
@router.delete("/applications/{application_id}")
|
| 155 |
-
def delete_application(application_id: str,
|
| 156 |
session: Session = Depends(require_session)):
|
| 157 |
try:
|
| 158 |
current = _application_databases(session.runtime, application_id)
|
| 159 |
allowed, _ = application_access_context(session, current)
|
| 160 |
if not allowed:
|
| 161 |
raise applications.ApplicationError("application does not exist")
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
allowed_databases=allowed)
|
| 164 |
except (applications.ApplicationError, PermissionError) as exc:
|
| 165 |
_application_error(exc)
|
|
|
|
| 152 |
|
| 153 |
|
| 154 |
@router.delete("/applications/{application_id}")
|
| 155 |
+
def delete_application(application_id: str, body: dict = Body(default=None),
|
| 156 |
session: Session = Depends(require_session)):
|
| 157 |
try:
|
| 158 |
current = _application_databases(session.runtime, application_id)
|
| 159 |
allowed, _ = application_access_context(session, current)
|
| 160 |
if not allowed:
|
| 161 |
raise applications.ApplicationError("application does not exist")
|
| 162 |
+
confirm_name = (body or {}).get("confirmName")
|
| 163 |
+
|
| 164 |
+
class NameConfirmedDelete:
|
| 165 |
+
"""Keep confirmation inside the registry's existing atomic update and permission gate."""
|
| 166 |
+
|
| 167 |
+
def update(self, key, delete):
|
| 168 |
+
def confirmed(raw):
|
| 169 |
+
# The registry validates access and copies the document before removing the App.
|
| 170 |
+
# A refusal below never reaches persistence, even if its name changed mid-request.
|
| 171 |
+
deleted = delete(raw)
|
| 172 |
+
app = raw["applications"][application_id]
|
| 173 |
+
if not isinstance(confirm_name, str) or confirm_name != app.get("name"):
|
| 174 |
+
raise applications.ApplicationError("enter the exact App name to confirm deletion")
|
| 175 |
+
return deleted
|
| 176 |
+
|
| 177 |
+
return session.runtime.update(key, confirmed)
|
| 178 |
+
|
| 179 |
+
return applications.delete(NameConfirmedDelete(), application_id, session.uname, session.admin,
|
| 180 |
allowed_databases=allowed)
|
| 181 |
except (applications.ApplicationError, PermissionError) as exc:
|
| 182 |
_application_error(exc)
|
api/routes_auth.py
CHANGED
|
@@ -7,11 +7,7 @@ replaces, and it is deliberately the same account store so the migration is a sw
|
|
| 7 |
CREDENTIAL check, not of the user model.
|
| 8 |
"""
|
| 9 |
import base64 as _b64
|
| 10 |
-
import hmac
|
| 11 |
-
import os
|
| 12 |
import re as _re
|
| 13 |
-
import time
|
| 14 |
-
from collections import OrderedDict
|
| 15 |
|
| 16 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 17 |
|
|
@@ -20,49 +16,46 @@ from deps import Session, err, require_session, users, perms
|
|
| 20 |
|
| 21 |
router = APIRouter(prefix="/api/v1/auth")
|
| 22 |
|
| 23 |
-
#: THE SERVER OWNS BACKOFF
|
| 24 |
-
#:
|
| 25 |
-
#:
|
| 26 |
-
#:
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
_FAILS = OrderedDict()
|
| 34 |
-
_MAX_TRACKED = 2048
|
| 35 |
-
_LOCKOUT_AFTER = 8
|
| 36 |
-
_LOCKOUT_SECONDS = 60
|
| 37 |
|
| 38 |
|
| 39 |
def _throttled(key, now=None):
|
| 40 |
-
|
| 41 |
-
hit = _FAILS.get(key)
|
| 42 |
-
if not hit:
|
| 43 |
-
return 0
|
| 44 |
-
count, last = hit
|
| 45 |
-
if count < _LOCKOUT_AFTER:
|
| 46 |
return 0
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _note_failure(key, now=None):
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
_FAILS.move_to_end(key)
|
| 60 |
-
while len(_FAILS) > _MAX_TRACKED:
|
| 61 |
-
_FAILS.popitem(last=False)
|
| 62 |
|
| 63 |
|
| 64 |
def _clear_failures(key):
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
|
| 67 |
|
| 68 |
def _public_user(user):
|
|
@@ -143,20 +136,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 143 |
body = body or {}
|
| 144 |
uname = str(body.get("username") or "").strip().lower()
|
| 145 |
pw = str(body.get("password") or "")
|
| 146 |
-
|
| 147 |
-
# Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves,
|
| 148 |
-
# and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs.
|
| 149 |
-
throttle_key = ("*", uname)
|
| 150 |
-
wait = _throttled(throttle_key)
|
| 151 |
-
if wait:
|
| 152 |
-
raise err(429, "too_many_attempts",
|
| 153 |
-
f"too many failed attempts β try again in {wait} seconds")
|
| 154 |
|
| 155 |
# W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
|
| 156 |
# tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
|
| 157 |
# a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
|
| 158 |
# APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
|
| 159 |
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
if not user:
|
| 161 |
user = users.verify(uname, pw) if (uname and pw) else None
|
| 162 |
if not user:
|
|
@@ -179,7 +172,46 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 179 |
_note_failure(throttle_key)
|
| 180 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 181 |
|
| 182 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
|
| 184 |
# β WAVE 40, R2 β SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 185 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
|
@@ -191,21 +223,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 191 |
# the bump; minting from it would issue every cookie in the product at the OLD epoch, which
|
| 192 |
# `_user_for` refuses on the very next request β "200, then 401 forever".
|
| 193 |
#
|
| 194 |
-
#
|
| 195 |
-
#
|
| 196 |
-
#
|
| 197 |
-
# NOTHING on tenant #0's `users` key, which a bump's read-modify-write would violate.
|
| 198 |
-
# * `emergency_master` β `deps._user_for` admits that identity whatever the record's epoch
|
| 199 |
-
# says, so a bump revokes nothing there and only risks a store write on the break-glass
|
| 200 |
-
# path that exists precisely for a store outage.
|
| 201 |
#
|
| 202 |
# A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
|
| 203 |
# deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
|
| 204 |
# store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
|
| 205 |
# a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
|
| 206 |
# lost, and the log says so rather than leaving it silent.
|
|
|
|
|
|
|
|
|
|
| 207 |
session_epoch = int(user.get("epoch") or 0)
|
| 208 |
-
if not
|
| 209 |
try:
|
| 210 |
bumped = users.bump_epoch(user["username"])
|
| 211 |
if bumped is not None:
|
|
@@ -218,9 +249,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 218 |
aios_session.set_cookie(response, request, value)
|
| 219 |
# Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β the
|
| 220 |
# person may have typed an email address and `users.verify` resolved it to the registry key;
|
| 221 |
-
# stamping what was typed would write onto a key that does not exist.
|
| 222 |
-
#
|
| 223 |
-
# so the one thing this cannot do is turn a good credential into a failed sign-in.
|
| 224 |
# The synthetic QA identity has no normal account record to stamp. More importantly, a
|
| 225 |
# registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
|
| 226 |
if not user.get("qa_runner"):
|
|
@@ -234,6 +264,216 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 234 |
return {"user": _public_user(user)}
|
| 235 |
|
| 236 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
@router.post("/logout", status_code=204)
|
| 238 |
def logout(request: Request):
|
| 239 |
"""204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
|
|
|
|
| 7 |
CREDENTIAL check, not of the user model.
|
| 8 |
"""
|
| 9 |
import base64 as _b64
|
|
|
|
|
|
|
| 10 |
import re as _re
|
|
|
|
|
|
|
| 11 |
|
| 12 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 13 |
|
|
|
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1/auth")
|
| 18 |
|
| 19 |
+
#: THE SERVER OWNS BACKOFF: the login page never paces itself. The budget lives in the shared
|
| 20 |
+
#: tenant-0 auth store, not this process, so worker rotation or a deploy cannot buy ten more tries.
|
| 21 |
+
#: Username and email aliases resolve to one canonical account key; only a SHA-256 digest of that
|
| 22 |
+
#: normalised key is persisted. Any store failure fails the authentication door closed.
|
| 23 |
+
def _throttle_key(identifier):
|
| 24 |
+
try:
|
| 25 |
+
return users.canonical_login_identifier(identifier)
|
| 26 |
+
except Exception:
|
| 27 |
+
raise err(503, "auth_state_unavailable",
|
| 28 |
+
"sign in protection is temporarily unavailable. Try again later")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
def _throttled(key, now=None):
|
| 32 |
+
if key is None: # staging-only synthetic QA identity
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
return 0
|
| 34 |
+
try:
|
| 35 |
+
return users.login_lockout_remaining(key, now=now)
|
| 36 |
+
except Exception:
|
| 37 |
+
raise err(503, "auth_state_unavailable",
|
| 38 |
+
"sign in protection is temporarily unavailable. Try again later")
|
| 39 |
|
| 40 |
|
| 41 |
def _note_failure(key, now=None):
|
| 42 |
+
if key is None:
|
| 43 |
+
return 0
|
| 44 |
+
try:
|
| 45 |
+
return users.note_login_failure(key, now=now)
|
| 46 |
+
except Exception:
|
| 47 |
+
raise err(503, "auth_state_unavailable",
|
| 48 |
+
"sign in protection is temporarily unavailable. Try again later")
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _clear_failures(key):
|
| 52 |
+
if key is None:
|
| 53 |
+
return
|
| 54 |
+
try:
|
| 55 |
+
users.clear_login_failures(key)
|
| 56 |
+
except Exception:
|
| 57 |
+
raise err(503, "auth_state_unavailable",
|
| 58 |
+
"sign in protection is temporarily unavailable. Try again later")
|
| 59 |
|
| 60 |
|
| 61 |
def _public_user(user):
|
|
|
|
| 136 |
body = body or {}
|
| 137 |
uname = str(body.get("username") or "").strip().lower()
|
| 138 |
pw = str(body.get("password") or "")
|
| 139 |
+
code = str(body.get("mfaCode") or "").strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
# W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
|
| 142 |
# tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
|
| 143 |
# a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
|
| 144 |
# APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
|
| 145 |
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
| 146 |
+
# A correctly proved QA runner is a staging fixture and remains zero-touch against tenant #0.
|
| 147 |
+
# Every other attempt resolves username/email aliases to one durable account budget.
|
| 148 |
+
throttle_key = None if user else _throttle_key(uname)
|
| 149 |
+
wait = _throttled(throttle_key)
|
| 150 |
+
if wait:
|
| 151 |
+
raise err(429, "too_many_attempts",
|
| 152 |
+
f"too many failed attempts. Try again in {wait} seconds")
|
| 153 |
if not user:
|
| 154 |
user = users.verify(uname, pw) if (uname and pw) else None
|
| 155 |
if not user:
|
|
|
|
| 172 |
_note_failure(throttle_key)
|
| 173 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 174 |
|
| 175 |
+
# W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or
|
| 176 |
+
# minting so an expired password gets the change door and no session, without counting a
|
| 177 |
+
# successful authentication against the guessing budget. The staging QA fixture has no
|
| 178 |
+
# stored password clock; every production identity does.
|
| 179 |
+
if not user.get("qa_runner"):
|
| 180 |
+
try:
|
| 181 |
+
rec = (users.registry() or {}).get(user["username"])
|
| 182 |
+
except Exception:
|
| 183 |
+
rec = None
|
| 184 |
+
if users.password_expired(rec):
|
| 185 |
+
raise err(403, "password_expired", "Your password has expired. Set a new one to continue.")
|
| 186 |
+
if rec and not users.password_changed_at(rec):
|
| 187 |
+
# Legacy records start their clock at first sign-in after this ships. The background
|
| 188 |
+
# merge is fail-silent like touch_login, so backfilling can never deny a good login.
|
| 189 |
+
try:
|
| 190 |
+
users.stamp_password_age(user["username"])
|
| 191 |
+
except Exception:
|
| 192 |
+
pass
|
| 193 |
+
|
| 194 |
+
# W57-T36 / C12: every STORED account needs its second factor before any cookie is minted.
|
| 195 |
+
# qa_runner is the sole exception and is double-gated out of live deployments.
|
| 196 |
+
if not user.get("qa_runner"):
|
| 197 |
+
try:
|
| 198 |
+
rec = (users.registry() or {}).get(user["username"])
|
| 199 |
+
except Exception:
|
| 200 |
+
rec = None
|
| 201 |
+
if not users.mfa_enrolled(rec):
|
| 202 |
+
# Correct credentials needing setup are not a guess. Counting them would lock the
|
| 203 |
+
# person out of the unauthenticated enrolment door that repairs their account.
|
| 204 |
+
raise err(403, "mfa_enrolment_required", "This account needs two step sign in. Set up your "
|
| 205 |
+
"authenticator app to continue.")
|
| 206 |
+
if not code:
|
| 207 |
+
# A missing factor is a prompt, not a failed attempt at guessing one.
|
| 208 |
+
raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
|
| 209 |
+
# TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
|
| 210 |
+
# ordinary code already proves the factor, and never reset the guess budget on password.
|
| 211 |
+
if not (users.totp_valid(rec.get("mfa_secret"), code)
|
| 212 |
+
or users.consume_recovery_code(user["username"], code)):
|
| 213 |
+
_note_failure(throttle_key)
|
| 214 |
+
raise err(401, "mfa_invalid", _MFA_INVALID)
|
| 215 |
|
| 216 |
# β WAVE 40, R2 β SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
|
| 217 |
# account already holds, so a forgotten sign-in on someone else's computer cannot lock the
|
|
|
|
| 223 |
# the bump; minting from it would issue every cookie in the product at the OLD epoch, which
|
| 224 |
# `_user_for` refuses on the very next request β "200, then 401 forever".
|
| 225 |
#
|
| 226 |
+
# qa_runner is the sole exception: it has no registry record to bump (its epoch is derived
|
| 227 |
+
# from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes NOTHING on
|
| 228 |
+
# tenant #0's `users` key. Live deployments cannot enable this double-gated fixture.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
#
|
| 230 |
# A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
|
| 231 |
# deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
|
| 232 |
# store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
|
| 233 |
# a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
|
| 234 |
# lost, and the log says so rather than leaving it silent.
|
| 235 |
+
# A complete password+factor proof clears the durable budget BEFORE a cookie is minted. If
|
| 236 |
+
# that security write is unavailable, authentication fails closed.
|
| 237 |
+
_clear_failures(throttle_key)
|
| 238 |
session_epoch = int(user.get("epoch") or 0)
|
| 239 |
+
if not user.get("qa_runner"):
|
| 240 |
try:
|
| 241 |
bumped = users.bump_epoch(user["username"])
|
| 242 |
if bumped is not None:
|
|
|
|
| 249 |
aios_session.set_cookie(response, request, value)
|
| 250 |
# Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β the
|
| 251 |
# person may have typed an email address and `users.verify` resolved it to the registry key;
|
| 252 |
+
# stamping what was typed would write onto a key that does not exist. `touch_login` is
|
| 253 |
+
# fail-silent, so it cannot turn a good credential into a failed sign-in.
|
|
|
|
| 254 |
# The synthetic QA identity has no normal account record to stamp. More importantly, a
|
| 255 |
# registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
|
| 256 |
if not user.get("qa_runner"):
|
|
|
|
| 264 |
return {"user": _public_user(user)}
|
| 265 |
|
| 266 |
|
| 267 |
+
# ------------------------------------------------------------------ second factor (W57-T36 / C12)
|
| 268 |
+
_MFA_INVALID = ("That code is not right. Check your authenticator app and try the next one.")
|
| 269 |
+
_MFA_ALREADY_ENROLLED = ("Two step sign in is already on for this account. Ask an administrator "
|
| 270 |
+
"to reset it if you lost your phone.")
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _mfa_credentials(uname, password):
|
| 274 |
+
"""The MFA doors share login's credential check and guessing budget, including aliases."""
|
| 275 |
+
# The synthetic runner has no stored factor and is refused without touching tenant #0.
|
| 276 |
+
user = users.qa_identity(uname, password) if (uname and password) else None
|
| 277 |
+
if user:
|
| 278 |
+
raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.")
|
| 279 |
+
throttle_key = _throttle_key(uname)
|
| 280 |
+
wait = _throttled(throttle_key)
|
| 281 |
+
if wait:
|
| 282 |
+
raise err(429, "too_many_attempts",
|
| 283 |
+
f"too many failed attempts. Try again in {wait} seconds")
|
| 284 |
+
user = users.verify(uname, password) if (uname and password) else None
|
| 285 |
+
if not user:
|
| 286 |
+
try:
|
| 287 |
+
users._hash(password or "x", "00" * 16)
|
| 288 |
+
except Exception:
|
| 289 |
+
pass
|
| 290 |
+
_note_failure(throttle_key)
|
| 291 |
+
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 292 |
+
return user, throttle_key
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _mfa_record(uname):
|
| 296 |
+
"""A stored identity was already proved. Failure to re-read it cannot assert absence."""
|
| 297 |
+
try:
|
| 298 |
+
rec = (users.registry() or {}).get(uname)
|
| 299 |
+
except Exception:
|
| 300 |
+
rec = None
|
| 301 |
+
if not rec:
|
| 302 |
+
raise err(503, "store_unavailable", "the account could not be read. Try again")
|
| 303 |
+
return rec
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def _mfa_session_tenant(user, rec, throttle_key):
|
| 307 |
+
"""Both MFA cookie writers retain login's tenant and T35 password-expiry walls."""
|
| 308 |
+
tenant = str(user.get("tenant") or "royal-imports").strip().lower()
|
| 309 |
+
from harness import runtime
|
| 310 |
+
try:
|
| 311 |
+
runtime.get_runtime(tenant)
|
| 312 |
+
except KeyError:
|
| 313 |
+
_note_failure(throttle_key)
|
| 314 |
+
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 315 |
+
# Confirming a phone must not become a second login door that accepts an expired password.
|
| 316 |
+
# Check before writing the factor, so the person can rotate and resume the same setup.
|
| 317 |
+
if users.password_expired(rec):
|
| 318 |
+
raise err(403, "password_expired", "Your password has expired. Set a new one to continue.")
|
| 319 |
+
return tenant
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _mfa_set(user, secret, codes, *, require_unenrolled=False):
|
| 323 |
+
"""Translate a raced credential change into a retry, never a claimed successful setup."""
|
| 324 |
+
try:
|
| 325 |
+
# Use the RESOLVED username: the posted identifier can be an email alias with no record.
|
| 326 |
+
return users.set_mfa(user["username"], secret, codes,
|
| 327 |
+
require_unenrolled=require_unenrolled,
|
| 328 |
+
expected_epoch=int(user.get("epoch") or 0))
|
| 329 |
+
except users.MFAStateError as exc:
|
| 330 |
+
if str(exc) == "already_enrolled":
|
| 331 |
+
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
| 332 |
+
raise err(409, "mfa_changed", "This account changed during setup. Sign in again to continue.")
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _mfa_session(request, response, user, rec, tenant, landed, throttle_key):
|
| 336 |
+
"""Mint only from the MFA write's landed epoch, never the stale login projection."""
|
| 337 |
+
if landed is None:
|
| 338 |
+
# A missing record/no-op write cannot support a cookie or a promise that codes were saved.
|
| 339 |
+
raise err(503, "store_unavailable", "the second factor was not saved. Try again")
|
| 340 |
+
# Clear only after the whole factor proof and its write succeed, but before minting a cookie.
|
| 341 |
+
_clear_failures(throttle_key)
|
| 342 |
+
value, _claims = aios_session.mint(tenant, user["username"], int(landed))
|
| 343 |
+
aios_session.set_cookie(response, request, value)
|
| 344 |
+
users.touch_login(user["username"])
|
| 345 |
+
import deps as _deps
|
| 346 |
+
_deps.note_active(user["username"])
|
| 347 |
+
if not users.password_changed_at(rec):
|
| 348 |
+
# This is a successful first sign-in too: preserve T35's legacy clock backfill.
|
| 349 |
+
users.stamp_password_age(user["username"])
|
| 350 |
+
# Responses carrying a new secret or recovery codes must not survive in an HTTP cache.
|
| 351 |
+
response.headers["Cache-Control"] = "no-store"
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
@router.post("/mfa/enroll")
|
| 355 |
+
def mfa_enroll(response: Response, body: dict = Body(default=None)):
|
| 356 |
+
"""Offer a candidate secret without changing the account or issuing recovery codes.
|
| 357 |
+
|
| 358 |
+
Stateless by design: the client posts this candidate back with its password and phone code.
|
| 359 |
+
A pending-secret dict would belong to one worker and fail when confirmation reaches another.
|
| 360 |
+
"""
|
| 361 |
+
body = body or {}
|
| 362 |
+
uname = str(body.get("username") or "").strip().lower()
|
| 363 |
+
user, _ = _mfa_credentials(uname, str(body.get("password") or ""))
|
| 364 |
+
rec = _mfa_record(user["username"])
|
| 365 |
+
if users.mfa_enrolled(rec):
|
| 366 |
+
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
| 367 |
+
secret = users.new_mfa_secret()
|
| 368 |
+
response.headers["Cache-Control"] = "no-store"
|
| 369 |
+
return {"secret": secret, "otpauthUri": users.otpauth_uri(secret, user["username"]),
|
| 370 |
+
"issuer": users.MFA_ISSUER, "digits": users.MFA_DIGITS, "period": users.MFA_PERIOD}
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
@router.post("/mfa/enroll/confirm")
|
| 374 |
+
def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(default=None)):
|
| 375 |
+
"""Prove installation, enrol once and sign in under the new epoch.
|
| 376 |
+
|
| 377 |
+
This response is the only time those codes exist in clear anywhere. The record holds SHA-256
|
| 378 |
+
hashes. No other route returns them, no log line prints them, and there is no way to read them
|
| 379 |
+
back. Explicit regeneration issues a DIFFERENT set, never a second reading of this one.
|
| 380 |
+
"""
|
| 381 |
+
body = body or {}
|
| 382 |
+
uname = str(body.get("username") or "").strip().lower()
|
| 383 |
+
user, throttle_key = _mfa_credentials(uname, str(body.get("password") or ""))
|
| 384 |
+
rec = _mfa_record(user["username"])
|
| 385 |
+
if users.mfa_enrolled(rec):
|
| 386 |
+
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
| 387 |
+
secret = str(body.get("secret") or "").strip().upper()
|
| 388 |
+
if not users.totp_valid(secret, str(body.get("code") or "")):
|
| 389 |
+
_note_failure(throttle_key)
|
| 390 |
+
raise err(401, "mfa_invalid", _MFA_INVALID)
|
| 391 |
+
tenant = _mfa_session_tenant(user, rec, throttle_key)
|
| 392 |
+
codes = users.new_recovery_codes()
|
| 393 |
+
# The store guard repeats the check atomically: two workers cannot issue two first sets.
|
| 394 |
+
landed = _mfa_set(user, secret, codes, require_unenrolled=True)
|
| 395 |
+
_mfa_session(request, response, user, rec, tenant, landed, throttle_key)
|
| 396 |
+
return {"user": _public_user(user), "recoveryCodes": codes}
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
@router.get("/mfa")
|
| 400 |
+
def mfa_status(session: Session = Depends(require_session)):
|
| 401 |
+
"""The calling account's state only: never a secret, code or hash."""
|
| 402 |
+
# This identity has no stored factor; do not read tenant #0 for the QA runner.
|
| 403 |
+
if session.user.get("qa_runner"):
|
| 404 |
+
rec = {}
|
| 405 |
+
else:
|
| 406 |
+
rec = _mfa_record(session.uname)
|
| 407 |
+
return {"enrolled": users.mfa_enrolled(rec), "enrolledAt": rec.get("mfa_enrolled_at") or None,
|
| 408 |
+
"recoveryCodesRemaining": users.recovery_codes_remaining(rec)}
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
@router.post("/mfa/recovery-codes")
|
| 412 |
+
def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(default=None),
|
| 413 |
+
session: Session = Depends(require_session)):
|
| 414 |
+
"""Issue a new set once after password re-authentication, keeping the installed phone.
|
| 415 |
+
|
| 416 |
+
Regeneration is a credential change: set_mfa bumps the epoch and signs OTHER sessions out.
|
| 417 |
+
Re-mint the CALLING session from that landed epoch or the user's own action signs them out.
|
| 418 |
+
The clear codes appear only in this response; the store and all later reads hold only hashes.
|
| 419 |
+
"""
|
| 420 |
+
uname = session.uname
|
| 421 |
+
user, throttle_key = _mfa_credentials(uname, str((body or {}).get("password") or ""))
|
| 422 |
+
# The session supplies the identity. Body username/tenant fields can never choose a target.
|
| 423 |
+
rec = _mfa_record(uname)
|
| 424 |
+
if not users.mfa_enrolled(rec):
|
| 425 |
+
raise err(409, "mfa_not_enrolled", "Set up your authenticator app before making recovery codes.")
|
| 426 |
+
tenant = _mfa_session_tenant(user, rec, throttle_key)
|
| 427 |
+
codes = users.new_recovery_codes()
|
| 428 |
+
landed = _mfa_set(user, rec["mfa_secret"], codes)
|
| 429 |
+
_mfa_session(request, response, user, rec, tenant, landed, throttle_key)
|
| 430 |
+
return {"recoveryCodes": codes}
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
@router.get("/password-policy")
|
| 434 |
+
def password_policy():
|
| 435 |
+
"""An account-independent checklist the sign-in page can show before authentication."""
|
| 436 |
+
return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH,
|
| 437 |
+
"minAgeDays": users.PASSWORD_MIN_AGE_DAYS,
|
| 438 |
+
"maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY}
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
@router.post("/password", status_code=204)
|
| 442 |
+
def change_password(body: dict = Body(default=None)):
|
| 443 |
+
"""Rotate with the current credential, including when expiry refuses a login session."""
|
| 444 |
+
body = body or {}
|
| 445 |
+
uname = str(body.get("username") or "").strip().lower()
|
| 446 |
+
pw = str(body.get("currentPassword") or "")
|
| 447 |
+
new_pw = str(body.get("newPassword") or "")
|
| 448 |
+
# The staging runner has no stored password and is refused without touching tenant #0.
|
| 449 |
+
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
| 450 |
+
if user:
|
| 451 |
+
raise err(400, "no_stored_password", "This account has no stored password to change here.")
|
| 452 |
+
# Share login's canonical durable bucket: a second unauthenticated door and an email alias
|
| 453 |
+
# must not buy another password-guessing budget.
|
| 454 |
+
throttle_key = _throttle_key(uname)
|
| 455 |
+
wait = _throttled(throttle_key)
|
| 456 |
+
if wait:
|
| 457 |
+
raise err(429, "too_many_attempts",
|
| 458 |
+
f"too many failed attempts. Try again in {wait} seconds")
|
| 459 |
+
user = users.verify(uname, pw) if (uname and pw) else None
|
| 460 |
+
if not user:
|
| 461 |
+
try:
|
| 462 |
+
users._hash(pw or "x", "00" * 16)
|
| 463 |
+
except Exception:
|
| 464 |
+
pass
|
| 465 |
+
_note_failure(throttle_key)
|
| 466 |
+
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 467 |
+
try:
|
| 468 |
+
# verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed
|
| 469 |
+
# alias, or this door could return 204 without changing any password or revoking sessions.
|
| 470 |
+
users.set_password(user["username"], new_pw)
|
| 471 |
+
except users.PasswordPolicyError as exc:
|
| 472 |
+
raise err(400, "weak_password", "That password does not meet the policy: " + str(exc))
|
| 473 |
+
_clear_failures(throttle_key)
|
| 474 |
+
return Response(status_code=204)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
@router.post("/logout", status_code=204)
|
| 478 |
def logout(request: Request):
|
| 479 |
"""204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
|
api/routes_connected_tables.py
CHANGED
|
@@ -32,11 +32,27 @@ behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be
|
|
| 32 |
"""
|
| 33 |
from fastapi import APIRouter, Depends, Query
|
| 34 |
|
| 35 |
-
from deps import Session, require_session
|
| 36 |
|
| 37 |
router = APIRouter(prefix="/api/v1")
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
@router.get("/connected-tables/{table_key}/rows")
|
| 41 |
def connected_table_rows(table_key: str,
|
| 42 |
offset: int = Query(default=0, ge=0),
|
|
|
|
| 32 |
"""
|
| 33 |
from fastapi import APIRouter, Depends, Query
|
| 34 |
|
| 35 |
+
from deps import Session, err, require_session
|
| 36 |
|
| 37 |
router = APIRouter(prefix="/api/v1")
|
| 38 |
|
| 39 |
|
| 40 |
+
@router.get("/connected-tables/{table_key}/identity")
|
| 41 |
+
def connected_table_identity(table_key: str, session: Session = Depends(require_session)):
|
| 42 |
+
"""Provider/database identity is separate from the row identity in the paged rows contract."""
|
| 43 |
+
from core import user_tables
|
| 44 |
+
from routes_tables import _defn_or_refuse
|
| 45 |
+
|
| 46 |
+
# Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
|
| 47 |
+
definition = _defn_or_refuse(session, table_key, defs_only=True)
|
| 48 |
+
identity = user_tables.connector_database_identity(
|
| 49 |
+
table_key, rt=session.runtime, definition=definition)
|
| 50 |
+
if identity is None:
|
| 51 |
+
raise err(404, "unknown_connector_database", "that database has no connector identity")
|
| 52 |
+
return {**identity, "readOnly": bool(
|
| 53 |
+
definition.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE)}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
@router.get("/connected-tables/{table_key}/rows")
|
| 57 |
def connected_table_rows(table_key: str,
|
| 58 |
offset: int = Query(default=0, ge=0),
|
api/routes_connectors.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
api/routes_external.py
ADDED
|
@@ -0,0 +1,399 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-user credential management and the bearer-authenticated external read door."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from urllib.parse import quote
|
| 5 |
+
|
| 6 |
+
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
|
| 7 |
+
|
| 8 |
+
import deps
|
| 9 |
+
from deps import Session, err, require_session
|
| 10 |
+
from core import external_access
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
router = APIRouter(prefix="/api/v1/external", tags=["external"])
|
| 14 |
+
|
| 15 |
+
_CREDENTIAL_PREFIX = "lpb1."
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _auth_refusal():
|
| 19 |
+
raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def require_external_session(
|
| 23 |
+
authorization: str | None = Header(default=None, alias="Authorization"),
|
| 24 |
+
) -> Session:
|
| 25 |
+
"""Resolve a bearer credential without consulting or extending the browser session cookie."""
|
| 26 |
+
if not isinstance(authorization, str):
|
| 27 |
+
_auth_refusal()
|
| 28 |
+
scheme, separator, token = authorization.strip().partition(" ")
|
| 29 |
+
if not separator or scheme.lower() != "bearer" or not token.strip():
|
| 30 |
+
_auth_refusal()
|
| 31 |
+
try:
|
| 32 |
+
verified = external_access.verify(token.strip())
|
| 33 |
+
except external_access.ExternalAuthError:
|
| 34 |
+
_auth_refusal()
|
| 35 |
+
return Session(
|
| 36 |
+
tenant=verified.tenant,
|
| 37 |
+
user=verified.user,
|
| 38 |
+
claims={
|
| 39 |
+
"t": verified.tenant,
|
| 40 |
+
"u": verified.user["username"],
|
| 41 |
+
"external": True,
|
| 42 |
+
"external_key_id": verified.key_id,
|
| 43 |
+
},
|
| 44 |
+
runtime=verified.runtime,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def require_external_analyst(
|
| 49 |
+
session: Session = Depends(require_external_session),
|
| 50 |
+
) -> Session:
|
| 51 |
+
"""Apply the same Analyst grant wall used by the in-app Assistant and Query."""
|
| 52 |
+
session.require("analyst")
|
| 53 |
+
return session
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _can_use_analyst(session: Session) -> bool:
|
| 57 |
+
try:
|
| 58 |
+
session.require("analyst")
|
| 59 |
+
except HTTPException as exc:
|
| 60 |
+
if exc.status_code == 403:
|
| 61 |
+
return False
|
| 62 |
+
raise
|
| 63 |
+
return True
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _management_refusal(exc: external_access.ExternalAccessError):
|
| 67 |
+
if exc.code == "external_label_invalid":
|
| 68 |
+
raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
|
| 69 |
+
if exc.code == "external_key_not_found":
|
| 70 |
+
raise err(404, exc.code, "That credential was not found.")
|
| 71 |
+
if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
|
| 72 |
+
raise err(409, exc.code, "This account cannot create an external credential.")
|
| 73 |
+
raise err(503, "external_access_unavailable", "External access is unavailable right now.")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _resolve_external_scope(
|
| 77 |
+
session: Session,
|
| 78 |
+
databases=None,
|
| 79 |
+
*,
|
| 80 |
+
include_schema: bool = False,
|
| 81 |
+
strict_schema: bool = False,
|
| 82 |
+
):
|
| 83 |
+
"""Resolve the status and field contract used by every external scope description.
|
| 84 |
+
|
| 85 |
+
`strict_schema` preserves the schema route's named refusals for unknown, forbidden and
|
| 86 |
+
temporarily unavailable sources. The portable document uses the non-strict form so one
|
| 87 |
+
unavailable source can still be reported with the reason supplied by source status.
|
| 88 |
+
"""
|
| 89 |
+
status = deps.assistant_source_status(session, databases=databases)
|
| 90 |
+
definitions = None
|
| 91 |
+
|
| 92 |
+
def fallback_label(database: str) -> str:
|
| 93 |
+
nonlocal definitions
|
| 94 |
+
built_in = deps.module_cache_source(database)
|
| 95 |
+
if built_in:
|
| 96 |
+
return str(built_in.get("label") or database)
|
| 97 |
+
if definitions is None:
|
| 98 |
+
try:
|
| 99 |
+
candidate = session.runtime.get_projection("user_tables", drop=("rows",))
|
| 100 |
+
except Exception:
|
| 101 |
+
candidate = {}
|
| 102 |
+
definitions = candidate if isinstance(candidate, dict) else {}
|
| 103 |
+
definition = definitions.get(database)
|
| 104 |
+
if isinstance(definition, dict):
|
| 105 |
+
return str(definition.get("label") or database)
|
| 106 |
+
return database
|
| 107 |
+
|
| 108 |
+
resolved = []
|
| 109 |
+
for database, raw in sorted(status.items()):
|
| 110 |
+
row = {
|
| 111 |
+
"database": database,
|
| 112 |
+
"permitted": bool(raw.get("permitted")),
|
| 113 |
+
"answerable": bool(raw.get("answerable")),
|
| 114 |
+
"visible": bool(raw.get("visible")),
|
| 115 |
+
"reason": raw.get("reason") or "",
|
| 116 |
+
}
|
| 117 |
+
if include_schema:
|
| 118 |
+
if row["answerable"] or strict_schema:
|
| 119 |
+
scoped = deps.assistant_read_scope(
|
| 120 |
+
session, database, fields=None, filters=None,
|
| 121 |
+
)
|
| 122 |
+
row["schema"] = {
|
| 123 |
+
key: value for key, value in scoped.items()
|
| 124 |
+
if key not in {"records", "filters"}
|
| 125 |
+
}
|
| 126 |
+
row["label"] = str(scoped.get("label") or database)
|
| 127 |
+
row["fields"] = list(scoped.get("fields") or [])
|
| 128 |
+
else:
|
| 129 |
+
row["label"] = fallback_label(database)
|
| 130 |
+
row["fields"] = []
|
| 131 |
+
resolved.append(row)
|
| 132 |
+
return resolved
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _markdown_inline(value) -> str:
|
| 136 |
+
text = str(value or "").replace(chr(0x2014), " - ").replace(chr(0x2013), "-")
|
| 137 |
+
return " ".join(text.replace("`", "'").split())
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _portable_skill_document(session: Session, base_url: str) -> str:
|
| 141 |
+
scoped = [
|
| 142 |
+
row for row in _resolve_external_scope(session, include_schema=True)
|
| 143 |
+
if row["permitted"]
|
| 144 |
+
]
|
| 145 |
+
example = next((row for row in scoped if row["answerable"]), None)
|
| 146 |
+
if example is None and scoped:
|
| 147 |
+
example = scoped[0]
|
| 148 |
+
|
| 149 |
+
surface_url = f"{base_url.rstrip('/')}/api/v1/external"
|
| 150 |
+
lines = [
|
| 151 |
+
"---",
|
| 152 |
+
"name: loopable-external-data",
|
| 153 |
+
"description: Read this user's governed Loopable data through the external API.",
|
| 154 |
+
"---",
|
| 155 |
+
"",
|
| 156 |
+
"# Loopable external data",
|
| 157 |
+
"",
|
| 158 |
+
("This surface is a read-only view of the Loopable data this user can already see. "
|
| 159 |
+
"It applies the same database, field, and row permissions as the Loopable app."),
|
| 160 |
+
"",
|
| 161 |
+
"## Authentication",
|
| 162 |
+
"",
|
| 163 |
+
"```http",
|
| 164 |
+
"Authorization: Bearer <your Loopable credential>",
|
| 165 |
+
"```",
|
| 166 |
+
"",
|
| 167 |
+
("Mint the credential in Loopable Settings. It is shown once, and revoking it stops "
|
| 168 |
+
"the next request."),
|
| 169 |
+
"",
|
| 170 |
+
f"Base URL: `{surface_url}`",
|
| 171 |
+
"",
|
| 172 |
+
"## First call: list databases",
|
| 173 |
+
"",
|
| 174 |
+
"Endpoint: `GET /api/v1/external/databases`",
|
| 175 |
+
"",
|
| 176 |
+
"```sh",
|
| 177 |
+
"curl --request GET \\",
|
| 178 |
+
" --header 'Authorization: Bearer <your Loopable credential>' \\",
|
| 179 |
+
f" '{surface_url}/databases'",
|
| 180 |
+
"```",
|
| 181 |
+
"",
|
| 182 |
+
"Call this first. It returns only databases this user may read or already sees in the app.",
|
| 183 |
+
"",
|
| 184 |
+
"## Read a database schema",
|
| 185 |
+
"",
|
| 186 |
+
"Endpoint: `GET /api/v1/external/databases/{database}/schema`",
|
| 187 |
+
"",
|
| 188 |
+
]
|
| 189 |
+
|
| 190 |
+
if example is not None:
|
| 191 |
+
database_path = quote(str(example["database"]), safe="")
|
| 192 |
+
fields = example.get("fields") or []
|
| 193 |
+
field_key = next(
|
| 194 |
+
(str(field.get("key")) for field in fields
|
| 195 |
+
if isinstance(field, dict) and field.get("key")),
|
| 196 |
+
None,
|
| 197 |
+
)
|
| 198 |
+
query_body = {
|
| 199 |
+
"fields": [field_key] if field_key else [],
|
| 200 |
+
"filters": None,
|
| 201 |
+
}
|
| 202 |
+
lines.extend([
|
| 203 |
+
"```sh",
|
| 204 |
+
"curl --request GET \\",
|
| 205 |
+
" --header 'Authorization: Bearer <your Loopable credential>' \\",
|
| 206 |
+
f" '{surface_url}/databases/{database_path}/schema'",
|
| 207 |
+
"```",
|
| 208 |
+
"",
|
| 209 |
+
"## Query a database",
|
| 210 |
+
"",
|
| 211 |
+
"Endpoint: `POST /api/v1/external/databases/{database}/query`",
|
| 212 |
+
"",
|
| 213 |
+
"```sh",
|
| 214 |
+
"curl --request POST \\",
|
| 215 |
+
" --header 'Authorization: Bearer <your Loopable credential>' \\",
|
| 216 |
+
" --header 'Content-Type: application/json' \\",
|
| 217 |
+
f" --data '{json.dumps(query_body, separators=(',', ':'))}' \\",
|
| 218 |
+
f" '{surface_url}/databases/{database_path}/query'",
|
| 219 |
+
"```",
|
| 220 |
+
"",
|
| 221 |
+
])
|
| 222 |
+
if not example["answerable"]:
|
| 223 |
+
lines.extend([
|
| 224 |
+
("This exact schema and query path names a permitted source, but it is not "
|
| 225 |
+
"answerable right now. The reason appears in the scope below."),
|
| 226 |
+
"",
|
| 227 |
+
])
|
| 228 |
+
else:
|
| 229 |
+
lines.extend([
|
| 230 |
+
"No schema call is available because this user's current scope has no databases.",
|
| 231 |
+
"",
|
| 232 |
+
"## Query a database",
|
| 233 |
+
"",
|
| 234 |
+
"Endpoint: `POST /api/v1/external/databases/{database}/query`",
|
| 235 |
+
"",
|
| 236 |
+
"No query call is available because this user's current scope has no databases.",
|
| 237 |
+
"",
|
| 238 |
+
])
|
| 239 |
+
|
| 240 |
+
lines.extend(["## This user's current read scope", ""])
|
| 241 |
+
if not scoped:
|
| 242 |
+
lines.extend(["No databases are currently permitted.", ""])
|
| 243 |
+
for row in scoped:
|
| 244 |
+
database = _markdown_inline(row["database"])
|
| 245 |
+
label = _markdown_inline(row.get("label") or row["database"])
|
| 246 |
+
lines.append(f"### `{database}`: {label}")
|
| 247 |
+
lines.append("")
|
| 248 |
+
if row["answerable"]:
|
| 249 |
+
fields = [
|
| 250 |
+
field for field in (row.get("fields") or [])
|
| 251 |
+
if isinstance(field, dict) and field.get("key")
|
| 252 |
+
]
|
| 253 |
+
lines.append("Fields this user may read:")
|
| 254 |
+
lines.append("")
|
| 255 |
+
if fields:
|
| 256 |
+
for field in fields:
|
| 257 |
+
key = _markdown_inline(field.get("key"))
|
| 258 |
+
field_label = _markdown_inline(field.get("label") or field.get("key"))
|
| 259 |
+
lines.append(f"- `{key}`: {field_label}")
|
| 260 |
+
else:
|
| 261 |
+
lines.append("- No readable fields are currently exposed.")
|
| 262 |
+
else:
|
| 263 |
+
reason = _markdown_inline(row.get("reason") or "This source is not answerable right now.")
|
| 264 |
+
lines.extend([
|
| 265 |
+
f"Not answerable right now: {reason}",
|
| 266 |
+
"",
|
| 267 |
+
"The readable field contract is unavailable until this source is answerable.",
|
| 268 |
+
])
|
| 269 |
+
lines.append("")
|
| 270 |
+
|
| 271 |
+
lines.extend([
|
| 272 |
+
"## Limits",
|
| 273 |
+
"",
|
| 274 |
+
("This surface performs no writes. It returns no data the user cannot already see in "
|
| 275 |
+
"Loopable, and it provides no export beyond what the user could already export from "
|
| 276 |
+
"the app."),
|
| 277 |
+
"",
|
| 278 |
+
])
|
| 279 |
+
document = "\n".join(lines)
|
| 280 |
+
if _CREDENTIAL_PREFIX in document:
|
| 281 |
+
raise RuntimeError("portable skill document contained a credential value")
|
| 282 |
+
return document
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
@router.post("/keys", status_code=201)
|
| 286 |
+
def create_key(body=Body(default=None), session: Session = Depends(require_session)):
|
| 287 |
+
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
|
| 288 |
+
session.require("analyst")
|
| 289 |
+
label = body.get("label") if isinstance(body, dict) else None
|
| 290 |
+
try:
|
| 291 |
+
key, token = external_access.mint(session.runtime, session.uname, session.tenant, label)
|
| 292 |
+
except external_access.ExternalAccessError as exc:
|
| 293 |
+
_management_refusal(exc)
|
| 294 |
+
return {"key": key, "secret": token}
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
@router.get("/keys")
|
| 298 |
+
def list_keys(session: Session = Depends(require_session)):
|
| 299 |
+
"""List only the signed-in human's metadata. Stored hashes and secrets never leave core."""
|
| 300 |
+
try:
|
| 301 |
+
keys = external_access.list_for_user(session.runtime, session.uname, session.tenant)
|
| 302 |
+
except external_access.ExternalAccessError as exc:
|
| 303 |
+
_management_refusal(exc)
|
| 304 |
+
return {"keys": keys, "can_mint": _can_use_analyst(session)}
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
@router.post("/keys/{key_id}/revoke")
|
| 308 |
+
def revoke_key(key_id: str, session: Session = Depends(require_session)):
|
| 309 |
+
"""Tombstone one key owned by the signed-in human. Bearer auth is not accepted here."""
|
| 310 |
+
try:
|
| 311 |
+
key = external_access.revoke(session.runtime, session.uname, session.tenant, key_id)
|
| 312 |
+
except external_access.ExternalAccessError as exc:
|
| 313 |
+
_management_refusal(exc)
|
| 314 |
+
return {"key": key}
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
@router.get("/skill.md")
|
| 318 |
+
def portable_skill(request: Request, session: Session = Depends(require_session)):
|
| 319 |
+
"""Download the signed-in human's current governed external read instructions."""
|
| 320 |
+
session.require("analyst")
|
| 321 |
+
document = _portable_skill_document(session, str(request.base_url))
|
| 322 |
+
return Response(
|
| 323 |
+
content=document,
|
| 324 |
+
media_type="text/markdown",
|
| 325 |
+
headers={"Content-Disposition": 'attachment; filename="skill.md"'},
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
@router.get("/whoami")
|
| 330 |
+
def whoami(session: Session = Depends(require_external_analyst)):
|
| 331 |
+
"""The one observable bearer route in this ticket; later tickets add governed data reads."""
|
| 332 |
+
return {
|
| 333 |
+
"username": session.uname,
|
| 334 |
+
"name": session.user.get("name") or session.uname,
|
| 335 |
+
"tenant": session.tenant,
|
| 336 |
+
"key_id": session.claims["external_key_id"],
|
| 337 |
+
"read_only": True,
|
| 338 |
+
}
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
@router.get("/databases")
|
| 342 |
+
def list_databases(session: Session = Depends(require_external_analyst)):
|
| 343 |
+
"""List only sources this user may read or already sees elsewhere in the product."""
|
| 344 |
+
databases = []
|
| 345 |
+
for row in _resolve_external_scope(session):
|
| 346 |
+
if not (row.get("permitted") or row.get("visible")):
|
| 347 |
+
continue
|
| 348 |
+
public = {
|
| 349 |
+
"database": row["database"],
|
| 350 |
+
"permitted": bool(row.get("permitted")),
|
| 351 |
+
"answerable": bool(row.get("answerable")),
|
| 352 |
+
}
|
| 353 |
+
if not public["answerable"]:
|
| 354 |
+
public["reason"] = row.get("reason") or ""
|
| 355 |
+
databases.append(public)
|
| 356 |
+
return {"databases": databases, "read_only": True}
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
@router.get("/databases/{database}/schema")
|
| 360 |
+
def database_schema(database: str, session: Session = Depends(require_external_analyst)):
|
| 361 |
+
"""Return the field contract after resolving the ordinary strict Assistant read wall."""
|
| 362 |
+
resolved = _resolve_external_scope(
|
| 363 |
+
session, databases=[database], include_schema=True, strict_schema=True,
|
| 364 |
+
)
|
| 365 |
+
return resolved[0]["schema"]
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
@router.post("/databases/{database}/query")
|
| 369 |
+
def query_database(
|
| 370 |
+
database: str,
|
| 371 |
+
body: dict | None = Body(default=None),
|
| 372 |
+
session: Session = Depends(require_external_analyst),
|
| 373 |
+
):
|
| 374 |
+
"""Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
|
| 375 |
+
payload = body or {}
|
| 376 |
+
return deps.assistant_read_scope(
|
| 377 |
+
session,
|
| 378 |
+
database,
|
| 379 |
+
fields=payload.get("fields"),
|
| 380 |
+
filters=payload.get("filters"),
|
| 381 |
+
)
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
# C7 / R3: one import-time chokepoint makes the external bearer surface structurally read only.
|
| 385 |
+
# POST remains available for bounded query bodies and for the cookie-only credential lifecycle
|
| 386 |
+
# above. No bearer dependency appears on those lifecycle routes.
|
| 387 |
+
_READ_ONLY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS"})
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def _assert_read_only_router() -> None:
|
| 391 |
+
for route in router.routes:
|
| 392 |
+
methods = set(getattr(route, "methods", set()) or set())
|
| 393 |
+
refused = methods - _READ_ONLY_METHODS
|
| 394 |
+
if refused:
|
| 395 |
+
joined = ", ".join(sorted(refused))
|
| 396 |
+
raise RuntimeError(f"external route {route.path} advertises refused method {joined}")
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
_assert_read_only_router()
|
api/routes_forms.py
CHANGED
|
@@ -331,7 +331,7 @@ def _clean_values(fields: list, raw: dict):
|
|
| 331 |
text = "" if v is None else str(v)
|
| 332 |
text = text.strip() if isinstance(v, str) else text
|
| 333 |
if len(text) > MAX_VALUE_LEN:
|
| 334 |
-
return None, f"β{label}β is too long
|
| 335 |
if text == "":
|
| 336 |
if f.get("required"):
|
| 337 |
return None, f"β{label}β is required."
|
|
@@ -508,7 +508,7 @@ def get_form(token: str, request: Request):
|
|
| 508 |
"""Render-time payload for a public form. Rate-limited like the POST: an unauthenticated GET
|
| 509 |
that walks every tenant's workspace is the cheapest way to make this process do work."""
|
| 510 |
if not _rate_ok(_client_ip(request), time.time()):
|
| 511 |
-
raise err(429, "too_many_requests", "too many requests
|
| 512 |
found = _resolve(token)
|
| 513 |
if not found:
|
| 514 |
raise _refuse()
|
|
@@ -556,7 +556,7 @@ async def submit_form(token: str, request: Request):
|
|
| 556 |
"""
|
| 557 |
now = time.time()
|
| 558 |
if not _rate_ok(_client_ip(request), now):
|
| 559 |
-
raise err(429, "too_many_requests", "too many requests
|
| 560 |
body = await _bounded_body(request)
|
| 561 |
found = _resolve(token)
|
| 562 |
if not found:
|
|
@@ -600,7 +600,7 @@ async def submit_form(token: str, request: Request):
|
|
| 600 |
# window's job, and a refusal costs the attacker the same either way.
|
| 601 |
if not _daily_ok(token, time.time()):
|
| 602 |
raise err(429, "form_daily_cap",
|
| 603 |
-
"this form has reached today's submission limit
|
| 604 |
|
| 605 |
import core.user_tables as user_tables
|
| 606 |
|
|
@@ -616,7 +616,7 @@ async def submit_form(token: str, request: Request):
|
|
| 616 |
actor = f"form:{submitter or token[:8]}"[:120]
|
| 617 |
row_id = user_tables.add_row(table_key, clean, username=actor, st=rt)
|
| 618 |
if not row_id:
|
| 619 |
-
raise err(409, "not_accepted", "that submission could not be saved
|
| 620 |
|
| 621 |
# W23-W7 β A's FROZEN signature, called verbatim (CP1-a). Failure here must not turn a
|
| 622 |
# SAVED row into an error for the submitter: the answer is theirs and it is already stored.
|
|
|
|
| 331 |
text = "" if v is None else str(v)
|
| 332 |
text = text.strip() if isinstance(v, str) else text
|
| 333 |
if len(text) > MAX_VALUE_LEN:
|
| 334 |
+
return None, f"β{label}β is too long; {MAX_VALUE_LEN} characters at most."
|
| 335 |
if text == "":
|
| 336 |
if f.get("required"):
|
| 337 |
return None, f"β{label}β is required."
|
|
|
|
| 508 |
"""Render-time payload for a public form. Rate-limited like the POST: an unauthenticated GET
|
| 509 |
that walks every tenant's workspace is the cheapest way to make this process do work."""
|
| 510 |
if not _rate_ok(_client_ip(request), time.time()):
|
| 511 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 512 |
found = _resolve(token)
|
| 513 |
if not found:
|
| 514 |
raise _refuse()
|
|
|
|
| 556 |
"""
|
| 557 |
now = time.time()
|
| 558 |
if not _rate_ok(_client_ip(request), now):
|
| 559 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 560 |
body = await _bounded_body(request)
|
| 561 |
found = _resolve(token)
|
| 562 |
if not found:
|
|
|
|
| 600 |
# window's job, and a refusal costs the attacker the same either way.
|
| 601 |
if not _daily_ok(token, time.time()):
|
| 602 |
raise err(429, "form_daily_cap",
|
| 603 |
+
"this form has reached today's submission limit; try again tomorrow")
|
| 604 |
|
| 605 |
import core.user_tables as user_tables
|
| 606 |
|
|
|
|
| 616 |
actor = f"form:{submitter or token[:8]}"[:120]
|
| 617 |
row_id = user_tables.add_row(table_key, clean, username=actor, st=rt)
|
| 618 |
if not row_id:
|
| 619 |
+
raise err(409, "not_accepted", "that submission could not be saved; try again later")
|
| 620 |
|
| 621 |
# W23-W7 β A's FROZEN signature, called verbatim (CP1-a). Failure here must not turn a
|
| 622 |
# SAVED row into an error for the submitter: the answer is theirs and it is already stored.
|
api/routes_grid.py
CHANGED
|
@@ -2644,7 +2644,76 @@ def grid_hop_topics(session: Session = Depends(require_session)):
|
|
| 2644 |
_PRESET_LINK_MODULES = ("customer_data", "product_data")
|
| 2645 |
|
| 2646 |
|
| 2647 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2648 |
"""May this session open `key` at all? The SAME two gates `_pivot_side` applies, minus the rows.
|
| 2649 |
|
| 2650 |
β THE ROWS ARE THE POINT OF THE DIFFERENCE. `_pivot_side` calls `perm_scope.scoped_table`,
|
|
@@ -2655,23 +2724,30 @@ def _may_open(session: Session, key: str):
|
|
| 2655 |
β FAIL CLOSED. Any refusal, any unknown table, any error is False. A target this account may
|
| 2656 |
not open must not appear in the strip, because the tab would 403 on the first click and read as
|
| 2657 |
a broken product rather than as a permission.
|
| 2658 |
-
"""
|
| 2659 |
-
import core.perm_scope as perm_scope
|
| 2660 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2661 |
try:
|
| 2662 |
session.require(key)
|
| 2663 |
except HTTPException:
|
| 2664 |
return False
|
| 2665 |
except Exception: # noqa: BLE001
|
| 2666 |
return False
|
| 2667 |
-
|
| 2668 |
-
perm_scope.scoped_fields(session.user, key, st=session.runtime)
|
| 2669 |
-
except Exception: # noqa: BLE001
|
| 2670 |
-
return False
|
| 2671 |
-
return True
|
| 2672 |
|
| 2673 |
|
| 2674 |
-
def _source_fields(session: Session, table: str):
|
| 2675 |
"""The link-bearing field list for `table`, under this session's field wall.
|
| 2676 |
|
| 2677 |
ββ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
|
|
@@ -2682,13 +2758,12 @@ def _source_fields(session: Session, table: str):
|
|
| 2682 |
and look entirely correct doing it.
|
| 2683 |
β APPENDED, NOT MERGED BY KEY: a tenant cannot author a field on a registry module, so the two
|
| 2684 |
lists cannot collide, and a union keyed on `key` would hide that if they ever did.
|
| 2685 |
-
"""
|
| 2686 |
-
import core.perm_scope as perm_scope
|
| 2687 |
|
| 2688 |
-
|
| 2689 |
-
|
| 2690 |
-
|
| 2691 |
-
|
|
|
|
| 2692 |
if table in _PRESET_LINK_MODULES:
|
| 2693 |
try:
|
| 2694 |
import odoo_relational
|
|
@@ -2721,21 +2796,284 @@ def related_entries(session: Session, table: str):
|
|
| 2721 |
return [e for e in entries if _may_open(session, e["target"])]
|
| 2722 |
|
| 2723 |
|
| 2724 |
-
|
| 2725 |
-
|
| 2726 |
-
"""C6 β every database related to `table`, one entry per edge, under the caller's scope.
|
| 2727 |
|
| 2728 |
-
`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2729 |
|
| 2730 |
β ONE ENTRY PER EDGE, NOT PER TARGET. R2: *"two links between the same pair give two tabs
|
| 2731 |
subtitled by the field"* β so `key` is the EDGE's identity and `via.key` repeats it, because
|
| 2732 |
-
`via` is the whole of what
|
| 2733 |
-
|
| 2734 |
|
| 2735 |
-
β `served` IS PART OF THE ANSWER, AND R6'S SECOND SENTENCE IS WHY. A declared edge whose
|
| 2736 |
-
|
| 2737 |
-
|
| 2738 |
-
|
| 2739 |
|
| 2740 |
β THE CALLER'S OWN ACCESS IS CHECKED FIRST. Listing what `table` is related to tells you what
|
| 2741 |
`table` is, so the listing is behind the same gate the grid is.
|
|
@@ -2743,8 +3081,53 @@ def grid_related(table: str, session: Session = Depends(require_session)):
|
|
| 2743 |
table = str(table or "").strip()
|
| 2744 |
if not table:
|
| 2745 |
raise err(400, "bad_table", "name the database to list relations for")
|
| 2746 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2747 |
raise err(403, "not_permitted",
|
| 2748 |
f"your account may not open '{table}', so it cannot be told what that database "
|
| 2749 |
f"is related to. Ask an administrator for access to it")
|
| 2750 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2644 |
_PRESET_LINK_MODULES = ("customer_data", "product_data")
|
| 2645 |
|
| 2646 |
|
| 2647 |
+
def _register_retired_tables():
|
| 2648 |
+
"""Push `odoo_relational.RETIRED_KEYS` down into `core.relations`, at import and per request.
|
| 2649 |
+
|
| 2650 |
+
ββ W54-T37 / C5 / INSTRUCTION 22 - THIS IS WHY THE OWNER SAW A PERMISSION WALL ABOUT A
|
| 2651 |
+
DATABASE THAT DOES NOT EXIST. `ut_odoo_customers` and `ut_odoo_products` are RETIRED: popped
|
| 2652 |
+
from every tenant document on every sync, replaced by the `customer_data` / `product_data`
|
| 2653 |
+
registry modules, and still DESCRIBED by the connector spec, which is legitimate (a spec
|
| 2654 |
+
describes a shape). The relation enumeration therefore minted edges at them, `_may_open` below
|
| 2655 |
+
failed closed on a key it could not resolve, and `relations.forbidden_table_refusal` told the
|
| 2656 |
+
reader to ask an administrator for access to a table nobody can grant.
|
| 2657 |
+
|
| 2658 |
+
β ONE DECLARATION, PUSHED, NEVER COPIED. `odoo_relational.RETIRED_KEYS` stays the only place
|
| 2659 |
+
the two keys are named; `core` cannot read it (`core` never imports up), so the app layer hands
|
| 2660 |
+
it down exactly as `core.user_tables.register_connected_prefix` is handed its prefixes. Copying
|
| 2661 |
+
the keys into `core` would make a second declaration of one fact.
|
| 2662 |
+
|
| 2663 |
+
β GUARDED, AND CALLED TWICE FOR THE SAME REASON `_ensure_odoo_spec` exists: a process that
|
| 2664 |
+
cannot import `odoo_relational` or `core` must still START, and one that imports them later
|
| 2665 |
+
must still get the right answer. Registration is idempotent (it is a set union) and returns the
|
| 2666 |
+
whole registered set, so a caller can assert what it just joined.
|
| 2667 |
+
"""
|
| 2668 |
+
try:
|
| 2669 |
+
import odoo_relational
|
| 2670 |
+
import core.relations as relations
|
| 2671 |
+
return relations.register_retired_tables(*odoo_relational.RETIRED_KEYS)
|
| 2672 |
+
except Exception: # noqa: BLE001
|
| 2673 |
+
return frozenset()
|
| 2674 |
+
|
| 2675 |
+
|
| 2676 |
+
#: The import-time push. β Its VALUE is not read anywhere on the request path: it is here so the
|
| 2677 |
+
#: registration has happened before the first listing, and so an operator can see what landed.
|
| 2678 |
+
RETIRED_TABLES_AT_IMPORT = _register_retired_tables()
|
| 2679 |
+
|
| 2680 |
+
|
| 2681 |
+
def _walled_fields(session: Session, key: str, st=None, memo: dict = None):
|
| 2682 |
+
"""`perm_scope.scoped_fields(key)`, or **None** when the field wall REFUSED β memoised.
|
| 2683 |
+
|
| 2684 |
+
ββ W54-T39 β THIS FUNCTION EXISTS BECAUSE THE SAME WALL WAS BEING ASKED TWICE PER TABLE AND
|
| 2685 |
+
THE ANSWER WAS THROWN AWAY BOTH TIMES. `_may_open` below wanted only the verdict and
|
| 2686 |
+
`_source_fields` below wanted only the list; each called `scoped_fields` itself, so a listing
|
| 2687 |
+
that touches six tables paid for TWELVE wall resolutions. MEASURED against tenant #0 on
|
| 2688 |
+
2026-09-06: one `scoped_fields` is ~135 ms on a `ut_*` grid and ~550 ms on `product_data`, so
|
| 2689 |
+
the duplicate pass alone was more than half of the 3.5 s the record rail took to open.
|
| 2690 |
+
|
| 2691 |
+
β THE VERDICT AND THE LIST ARE ONE ANSWER, AND THEY MUST STAY ONE CALL. Resolving them
|
| 2692 |
+
separately is not merely slow, it is two readings of one wall that can disagree if a grant
|
| 2693 |
+
changes between them β the [[one-question-two-normalizers]] shape, on a permission.
|
| 2694 |
+
β `None` RATHER THAN `[]` IS THE WHOLE POINT OF THE RETURN TYPE. A table the wall refuses and
|
| 2695 |
+
a table with no columns are different facts, and `_may_open` is the caller that has to tell
|
| 2696 |
+
them apart; collapsing both to an empty list would hide every refused target as an empty one.
|
| 2697 |
+
|
| 2698 |
+
β FAIL CLOSED, exactly as before: any exception at all is a refusal, never a widened answer.
|
| 2699 |
+
β `memo` IS ONE REQUEST'S, handed down by `_relation_context`. Passing None simply means no
|
| 2700 |
+
memo, which is what every caller outside a listing pass wants.
|
| 2701 |
+
"""
|
| 2702 |
+
if memo is not None and key in memo:
|
| 2703 |
+
return memo[key]
|
| 2704 |
+
import core.perm_scope as perm_scope
|
| 2705 |
+
|
| 2706 |
+
try:
|
| 2707 |
+
answer = list(perm_scope.scoped_fields(
|
| 2708 |
+
session.user, key, st=session.runtime if st is None else st) or ())
|
| 2709 |
+
except Exception: # noqa: BLE001
|
| 2710 |
+
answer = None
|
| 2711 |
+
if memo is not None:
|
| 2712 |
+
memo[key] = answer
|
| 2713 |
+
return answer
|
| 2714 |
+
|
| 2715 |
+
|
| 2716 |
+
def _may_open(session: Session, key: str, st=None, memo: dict = None):
|
| 2717 |
"""May this session open `key` at all? The SAME two gates `_pivot_side` applies, minus the rows.
|
| 2718 |
|
| 2719 |
β THE ROWS ARE THE POINT OF THE DIFFERENCE. `_pivot_side` calls `perm_scope.scoped_table`,
|
|
|
|
| 2724 |
β FAIL CLOSED. Any refusal, any unknown table, any error is False. A target this account may
|
| 2725 |
not open must not appear in the strip, because the tab would 403 on the first click and read as
|
| 2726 |
a broken product rather than as a permission.
|
|
|
|
|
|
|
| 2727 |
|
| 2728 |
+
ββ AND IT STILL CANNOT TELL "ABSENT" FROM "REFUSED", WHICH IS CORRECT AND IS **NOT** WHAT
|
| 2729 |
+
INSTRUCTION 22 ASKED TO CHANGE. Conflating "not granted" with "could not be read" is deliberate
|
| 2730 |
+
and the refusal sentence says both out loud. What was missing is a third case that never
|
| 2731 |
+
belonged here at all: a table this deployment does not HAVE. That is answered upstream, by
|
| 2732 |
+
`_register_retired_tables` above keeping such a key from ever being enumerated as an edge
|
| 2733 |
+
target, and by `relations.diagnose_path` naming it when a request asks for one anyway.
|
| 2734 |
+
|
| 2735 |
+
β W54-T39 β THE SECOND GATE IS NOW ASKED THROUGH `_walled_fields`, WHICH IS THE SAME QUESTION
|
| 2736 |
+
AND THE SAME FAIL-CLOSED ANSWER, shared with `_source_fields` so one table costs one wall
|
| 2737 |
+
resolution instead of two. β THE GRANT GATE STILL RIDES `session.require`, which resolves
|
| 2738 |
+
against the session's OWN runtime rather than any handle passed in: a lent snapshot may make a
|
| 2739 |
+
read cheaper and must never be able to make a grant broader.
|
| 2740 |
+
"""
|
| 2741 |
try:
|
| 2742 |
session.require(key)
|
| 2743 |
except HTTPException:
|
| 2744 |
return False
|
| 2745 |
except Exception: # noqa: BLE001
|
| 2746 |
return False
|
| 2747 |
+
return _walled_fields(session, key, st=st, memo=memo) is not None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2748 |
|
| 2749 |
|
| 2750 |
+
def _source_fields(session: Session, table: str, st=None, memo: dict = None):
|
| 2751 |
"""The link-bearing field list for `table`, under this session's field wall.
|
| 2752 |
|
| 2753 |
ββ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
|
|
|
|
| 2758 |
and look entirely correct doing it.
|
| 2759 |
β APPENDED, NOT MERGED BY KEY: a tenant cannot author a field on a registry module, so the two
|
| 2760 |
lists cannot collide, and a union keyed on `key` would hide that if they ever did.
|
|
|
|
|
|
|
| 2761 |
|
| 2762 |
+
β W54-T39 β A REFUSED WALL STILL READS AS `[]` HERE, unchanged. `_walled_fields` answers None
|
| 2763 |
+
for that case so `_may_open` can tell it from an empty table; this door has never needed the
|
| 2764 |
+
distinction and must not start reporting one, because a refusal is `_may_open`'s word.
|
| 2765 |
+
"""
|
| 2766 |
+
fields = list(_walled_fields(session, table, st=st, memo=memo) or ())
|
| 2767 |
if table in _PRESET_LINK_MODULES:
|
| 2768 |
try:
|
| 2769 |
import odoo_relational
|
|
|
|
| 2796 |
return [e for e in entries if _may_open(session, e["target"])]
|
| 2797 |
|
| 2798 |
|
| 2799 |
+
def _ensure_odoo_spec():
|
| 2800 |
+
"""Make sure Odoo's connector spec is registered in `core.connector_spec` before it is READ.
|
|
|
|
| 2801 |
|
| 2802 |
+
ββ AN EMPTY `edges_for("odoo")` IS AN IMPORT THAT NEVER HAPPENED, NOT A CONNECTOR WITH NO
|
| 2803 |
+
EDGES, AND THIS FUNCTION EXISTS BECAUSE THAT FAILURE IS SILENT. `platform/core/connector_spec.py`
|
| 2804 |
+
cannot import `odoo_relational` (`core` never imports up), so Odoo's rows reach that registry by
|
| 2805 |
+
exactly one route: `odoo_relational` being imported IN A PROCESS WHERE `core` IS ON THE PATH.
|
| 2806 |
+
MEASURED: a bare `import odoo_relational` from `aios-web/api` leaves `SPEC_REGISTRATION` at
|
| 2807 |
+
`{'registered': False, 'error': "core.connector_spec is not importable from here: ..."}`, and
|
| 2808 |
+
`edges_for("odoo")` then answers `[]` with nothing red anywhere. The relation answer would come
|
| 2809 |
+
back short by every spec edge and look entirely correct doing it.
|
| 2810 |
+
|
| 2811 |
+
β IT IS DONE HERE, NOT IN `core/relations.py`, for the layering reason above: `core` never
|
| 2812 |
+
imports up, so the module that READS the edges cannot be the module that registers them. This
|
| 2813 |
+
file already imports `odoo_relational` inside `_source_fields`, so the dependency is not new.
|
| 2814 |
+
β IDEMPOTENT: `connector_spec.register` REPLACES a connector, and `register_connector_spec`
|
| 2815 |
+
never raises. So calling it per request is cheap and the failure is recorded rather than thrown.
|
| 2816 |
+
"""
|
| 2817 |
+
try:
|
| 2818 |
+
import odoo_relational
|
| 2819 |
+
# β BESIDE THE SPEC REGISTRATION AND FOR ITS REASON. Both are pushes from this layer into
|
| 2820 |
+
# `core`, both are idempotent, and both are wrong to skip in a process that imported this
|
| 2821 |
+
# module before `platform/` was on the path. Registering the retired keys HERE as well as
|
| 2822 |
+
# at import is what makes the enumeration below correct in either process shape.
|
| 2823 |
+
_register_retired_tables()
|
| 2824 |
+
return bool(odoo_relational.register_connector_spec())
|
| 2825 |
+
except Exception: # noqa: BLE001
|
| 2826 |
+
return False
|
| 2827 |
+
|
| 2828 |
+
|
| 2829 |
+
def _unwalled_fields(defs: dict, key: str):
|
| 2830 |
+
"""`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only.
|
| 2831 |
+
|
| 2832 |
+
ββ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of
|
| 2833 |
+
`relations.diagnose_path` / `wall_refusals`, which chooses a refusal CODE and names a table or
|
| 2834 |
+
a hop. Nothing built from it reaches `related`, and `verify_odoo_relational` proves that by
|
| 2835 |
+
comparing the served list with and without the whole diff rather than by trusting this note.
|
| 2836 |
+
|
| 2837 |
+
β `all_defs` IS ALREADY UNSCOPED (it is the tenant's definitions, projected free of `rows`),
|
| 2838 |
+
so the raw `fields` on a definition IS the pre-wall list; `_source_fields`' narrowing happens
|
| 2839 |
+
later, in `perm_scope.scoped_fields`. A registry module has no `all_defs` row at all, which is
|
| 2840 |
+
why the pre-set link fields are appended here exactly as `_source_fields` appends them.
|
| 2841 |
+
"""
|
| 2842 |
+
# β `isinstance` RATHER THAN `or {}`, the guard `relations.reverse_links` already carries on the
|
| 2843 |
+
# same input: a definition that is not a dict makes `.get` an AttributeError, and this function
|
| 2844 |
+
# runs inside the LISTING, so the crash would land on a door that is otherwise answering fine.
|
| 2845 |
+
defn = defs.get(key)
|
| 2846 |
+
fields = list((defn.get("fields") or ()) if isinstance(defn, dict) else ())
|
| 2847 |
+
if key in _PRESET_LINK_MODULES:
|
| 2848 |
+
try:
|
| 2849 |
+
import odoo_relational
|
| 2850 |
+
fields = fields + list(odoo_relational.module_link_fields(key) or ())
|
| 2851 |
+
except Exception: # noqa: BLE001
|
| 2852 |
+
pass
|
| 2853 |
+
return fields
|
| 2854 |
+
|
| 2855 |
+
|
| 2856 |
+
def _relation_context(session: Session, table: str, memo: dict = None):
|
| 2857 |
+
"""`defs` plus the three callbacks `relations` needs, memoised for the life of ONE request.
|
| 2858 |
+
|
| 2859 |
+
β BOTH WALLS ARE MEMOISED. `_may_open` runs `session.require` plus `perm_scope.scoped_fields`
|
| 2860 |
+
and `_source_fields` runs the second of those again; a depth-2 enumeration asks about the same
|
| 2861 |
+
dozen tables repeatedly, so without the memo one listing pays for the same permission answer
|
| 2862 |
+
many times over. W53-T18 added a SECOND enumeration over the same tables, which is exactly why
|
| 2863 |
+
the memo now lives out here instead of inside the builder: the diff pass costs no extra
|
| 2864 |
+
permission call at all.
|
| 2865 |
+
|
| 2866 |
+
ββ `open_fields_for` IS A **UNION**, NOT A REPLACEMENT, AND THAT IS LOAD-BEARING. The diff in
|
| 2867 |
+
`relations.diagnose_path` calls a path that is present unfiltered and absent filtered a WALL
|
| 2868 |
+
removal; if the unfiltered field list could ever be NARROWER than the walled one, a path the
|
| 2869 |
+
wall really did remove would be absent from both and get reported as an edge that never
|
| 2870 |
+
existed. Taking the union makes the unfiltered enumeration a superset by construction, so the
|
| 2871 |
+
diff is monotone and the only thing it can be wrong about is saying "unknown" too rarely.
|
| 2872 |
+
|
| 2873 |
+
ββ W54-T39 β **THE DEFINITIONS ARE READ ONCE AND THEN LENT, WHICH IS WHERE THE RECORD RAIL'S
|
| 2874 |
+
THREE AND A HALF SECONDS WERE.** `perm_scope.scoped_fields` reaches `user_tables.all_defs`
|
| 2875 |
+
for every table it is asked about, and `all_defs` is a store projection: one Neon round trip,
|
| 2876 |
+
MEASURED at ~135 ms each on 2026-09-06. A depth-2 listing off `customer_data` asked about six
|
| 2877 |
+
tables and paid that round trip TWELVE times for the identical document this function had
|
| 2878 |
+
already read on its first line. `user_tables.lend` exists for exactly this β `nav_entries` has
|
| 2879 |
+
paired `all_defs` with it since W32-T02 β so the handle below serves that one snapshot to
|
| 2880 |
+
every wall resolution in the pass and each `ut_*` table's `scoped_fields` drops from ~135 ms
|
| 2881 |
+
to 0.
|
| 2882 |
+
β IT IS A READ HANDLE AND IT NEVER REACHES A WRITE. `lend_defs`' own note is the rule: every
|
| 2883 |
+
value it serves is a `_Projected`, so a caller that wanted rows gets a loud `KeyError` rather
|
| 2884 |
+
than a silently empty grid β and `scoped_fields` is a `want_rows=False` door by construction.
|
| 2885 |
+
Nothing here is handed to a mutation; `grid_related` is a GET.
|
| 2886 |
+
β IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves
|
| 2887 |
+
only `user_tables` and (lazily, once) `object_shares`, every other store key falls through to
|
| 2888 |
+
the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all
|
| 2889 |
+
ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list
|
| 2890 |
+
EQUAL, key for key and value for value, to the one it returned under the bare runtime.
|
| 2891 |
+
|
| 2892 |
+
β AND THE SNAPSHOT IS THE POINT, not a side effect. One listing answers about one set of
|
| 2893 |
+
definitions; re-reading them mid-pass could enumerate an edge from a table that vanished two
|
| 2894 |
+
hops ago and report the inconsistency as a refusal.
|
| 2895 |
+
"""
|
| 2896 |
+
import core.user_tables as user_tables
|
| 2897 |
+
|
| 2898 |
+
try:
|
| 2899 |
+
defs = dict(user_tables.all_defs(st=session.runtime) or {})
|
| 2900 |
+
except Exception: # noqa: BLE001
|
| 2901 |
+
defs = {}
|
| 2902 |
+
try:
|
| 2903 |
+
lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs})
|
| 2904 |
+
except Exception: # noqa: BLE001
|
| 2905 |
+
# A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback
|
| 2906 |
+
# is: the pass simply goes back to paying a round trip per wall resolution.
|
| 2907 |
+
lent = session.runtime
|
| 2908 |
+
|
| 2909 |
+
# β ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo`
|
| 2910 |
+
# below still memoise their own composition (the grant gate, the pre-set link append); this is
|
| 2911 |
+
# the `scoped_fields` answer underneath both of them, which is the expensive part.
|
| 2912 |
+
# β THE CALLER MAY HAND ITS OWN IN, and `grid_related` does. That route has to answer 403 for
|
| 2913 |
+
# the SOURCE table before it lists anything, so it resolves that one wall a moment before this
|
| 2914 |
+
# function is built; without the shared bag the source table would be resolved twice, which on
|
| 2915 |
+
# `product_data` is a second ~550 ms for an answer already in hand.
|
| 2916 |
+
wall_memo = memo if memo is not None else {}
|
| 2917 |
+
open_memo, field_memo, wide_memo = {}, {}, {}
|
| 2918 |
+
|
| 2919 |
+
def may_open(key):
|
| 2920 |
+
if key not in open_memo:
|
| 2921 |
+
open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo)
|
| 2922 |
+
return open_memo[key]
|
| 2923 |
+
|
| 2924 |
+
def fields_for(key):
|
| 2925 |
+
if key not in field_memo:
|
| 2926 |
+
field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo)
|
| 2927 |
+
return field_memo[key]
|
| 2928 |
+
|
| 2929 |
+
def open_fields_for(key):
|
| 2930 |
+
if key not in wide_memo:
|
| 2931 |
+
walled = list(fields_for(key) or ())
|
| 2932 |
+
seen = {str(f.get("key")) for f in walled if isinstance(f, dict)}
|
| 2933 |
+
extra = [f for f in _unwalled_fields(defs, key)
|
| 2934 |
+
if isinstance(f, dict) and str(f.get("key")) not in seen]
|
| 2935 |
+
wide_memo[key] = walled + extra
|
| 2936 |
+
return wide_memo[key]
|
| 2937 |
+
|
| 2938 |
+
return {"defs": defs, "may_open": may_open, "fields_for": fields_for,
|
| 2939 |
+
"open_fields_for": open_fields_for}
|
| 2940 |
+
|
| 2941 |
+
|
| 2942 |
+
def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None):
|
| 2943 |
+
"""C3's `/related` list for `table` at `depth`, narrowed to what this session may open.
|
| 2944 |
+
|
| 2945 |
+
β THE WALL IS APPLIED TO EVERY TABLE ON A PATH, INTERMEDIATE INCLUDED, and it is applied by
|
| 2946 |
+
`relations.related_paths` calling back into `_may_open` rather than by filtering the finished
|
| 2947 |
+
list: a two-hop path through a table this reader cannot open must never be enumerated, because
|
| 2948 |
+
the enumeration itself tells them what that table is related to.
|
| 2949 |
+
|
| 2950 |
+
β THIS IS THE **SERVED** ANSWER AND W53-T18 DID NOT TOUCH IT. The diff pass that names the
|
| 2951 |
+
refusals is `related_refusals` below; it reads this list and never feeds it.
|
| 2952 |
+
"""
|
| 2953 |
+
import core.relations as relations
|
| 2954 |
+
|
| 2955 |
+
_ensure_odoo_spec()
|
| 2956 |
+
table = str(table or "")
|
| 2957 |
+
ctx = ctx or _relation_context(session, table)
|
| 2958 |
+
return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"],
|
| 2959 |
+
depth=depth, may_open=ctx["may_open"],
|
| 2960 |
+
fields_for=ctx["fields_for"])
|
| 2961 |
+
|
| 2962 |
+
|
| 2963 |
+
def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None):
|
| 2964 |
+
"""W53-T18 / R13 / C3 β WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`.
|
| 2965 |
+
|
| 2966 |
+
`[{code, message, path, table?, hop?}]`, where `code` is one of `relations.REFUSAL_CODES`.
|
| 2967 |
+
|
| 2968 |
+
ββ *"AN EMPTY LIST IS NEVER AN ACCEPTABLE ANSWER TO A PERMISSION-FILTERED TRAVERSAL."* An
|
| 2969 |
+
HTTP error covers the request that PINNED one path; the LISTING answers 200 with a shorter
|
| 2970 |
+
array, and a shorter array is exactly the silent empty PostgREST hands out under RLS and Odoo
|
| 2971 |
+
hands out for a forbidden One2many. So a path the wall removed is REPORTED here rather than
|
| 2972 |
+
omitted, and a table with nothing on the other side says `no_related_rows` out loud.
|
| 2973 |
+
β AND A 403 FOR THE WHOLE STRIP WOULD BE EQUALLY WRONG: the other tabs are legitimately
|
| 2974 |
+
visible, and refusing them all in order to explain one is a wall with a bug in it.
|
| 2975 |
+
|
| 2976 |
+
β IT CANNOT WIDEN `related`. The unfiltered enumeration lives entirely inside
|
| 2977 |
+
`relations.wall_refusals`, which is handed the SERVED list and only ever reports paths that are
|
| 2978 |
+
NOT in it. Nothing here is added to, or removed from, what the caller is served.
|
| 2979 |
+
"""
|
| 2980 |
+
import core.relations as relations
|
| 2981 |
+
|
| 2982 |
+
table = str(table or "")
|
| 2983 |
+
ctx = ctx or _relation_context(session, table)
|
| 2984 |
+
return relations.wall_refusals(
|
| 2985 |
+
table, fields=ctx["fields_for"](table), defs=ctx["defs"], depth=depth,
|
| 2986 |
+
may_open=ctx["may_open"], fields_for=ctx["fields_for"], entries=entries,
|
| 2987 |
+
open_fields=ctx["open_fields_for"](table), open_fields_for=ctx["open_fields_for"])
|
| 2988 |
+
|
| 2989 |
+
|
| 2990 |
+
#: β THE HTTP STATUS EACH REFUSAL CODE EARNS ON A **PIN**. The two permission answers are 403, the
|
| 2991 |
+
#: same status the top-of-route `_may_open` gate on `table` itself already uses; a path that never
|
| 2992 |
+
#: existed stays the 400 this route already answered with, unchanged; and "every hop was open and
|
| 2993 |
+
#: there is nothing there" is a 404, because it is neither a bad request nor a refusal.
|
| 2994 |
+
#: β `no_related_rows` cannot be reached from THIS route's pin branch (a path that resolves is
|
| 2995 |
+
#: served, so the diff never reports it) and is mapped anyway, because `diagnose_path` is the one
|
| 2996 |
+
#: diagnosis and the row door will raise the same bag from another ticket's fence.
|
| 2997 |
+
#: ββ W54-T37 / C5 - `retired_target` IS A **404**, AND IT IS DELIBERATELY NOT A 403. The whole
|
| 2998 |
+
#: complaint in instruction 22 is a permission status on a table that does not exist, so any 4xx
|
| 2999 |
+
#: in the permission family would repeat the lie in a header. β 410 Gone is the more precise word
|
| 3000 |
+
#: for a database that was retired rather than never present, and it is NOT used: the client half
|
| 3001 |
+
#: of this contract is another lane's, every existing handler here branches on the CODE rather than
|
| 3002 |
+
#: the status, and introducing the one status nothing in this app has ever returned would risk a
|
| 3003 |
+
#: fetch wrapper treating it as a transport failure. The code carries the meaning; the status only
|
| 3004 |
+
#: has to be honest.
|
| 3005 |
+
_REFUSAL_STATUS = {"forbidden_table": 403, "hop_filtered": 403, "retired_target": 404,
|
| 3006 |
+
"no_related_rows": 404, "unknown_via": 400}
|
| 3007 |
+
|
| 3008 |
+
|
| 3009 |
+
def _refusal_error(diag: dict):
|
| 3010 |
+
"""`err(...)` for a `diagnose_path` bag, carrying its named table, hop and candidates.
|
| 3011 |
+
|
| 3012 |
+
β THE X2 SHAPE IS UNCHANGED: `{"error": {"code", "message"}}`, with the extra keys added as
|
| 3013 |
+
SIBLINGS inside `error`. C3 asks for *"the ENUMERATED candidates with their join columns and
|
| 3014 |
+
the exact retry strings, not prose"*, and a client cannot draw a retry button out of an English
|
| 3015 |
+
clause, so the structure rides beside the sentence rather than inside it.
|
| 3016 |
+
|
| 3017 |
+
ββ W54-T40 β `detail` IS COPIED TOO, AND IT IS THE ROW W54-T38 BOOKED. C5 gives every refusal
|
| 3018 |
+
a SHORT `message` and a long `detail`, and the client renders the pair as a `<details>` with the
|
| 3019 |
+
sentence in the summary; this function copied four keys and dropped the diagnosis, so a refusal
|
| 3020 |
+
arriving through THIS door rendered as the sentence alone while the identical refusal arriving
|
| 3021 |
+
through the listing kept its disclosure. One key, and the two doors stop disagreeing about how
|
| 3022 |
+
much of one contract they carry.
|
| 3023 |
+
β A BAG WITH NO DIAGNOSIS STILL RENDERS AS THE SENTENCE ALONE: the loop copies a key only when
|
| 3024 |
+
the bag carries one, so nothing gains a caret with nothing behind it.
|
| 3025 |
+
"""
|
| 3026 |
+
exc = err(_REFUSAL_STATUS.get(str(diag.get("code")), 400), str(diag.get("code")),
|
| 3027 |
+
str(diag.get("message") or ""))
|
| 3028 |
+
for key in ("table", "hop", "path", "candidates", "detail"):
|
| 3029 |
+
if diag.get(key) not in (None, ""):
|
| 3030 |
+
exc.detail["error"][key] = diag[key]
|
| 3031 |
+
return exc
|
| 3032 |
+
|
| 3033 |
+
|
| 3034 |
+
@router.get("/grid/{table}/related")
|
| 3035 |
+
def grid_related(table: str,
|
| 3036 |
+
depth: int = Query(default=2),
|
| 3037 |
+
via: list[str] = Query(default=None),
|
| 3038 |
+
session: Session = Depends(require_session)):
|
| 3039 |
+
"""C6 + C3 β every database reachable from `table` in at most `depth` NAMED hops.
|
| 3040 |
+
|
| 3041 |
+
`{ok, table, depth, related: [{key, label, via, target, served, where?, cardinality, path,
|
| 3042 |
+
hops}], refusals: [{code, message, path, table?, hop?}]}`.
|
| 3043 |
+
|
| 3044 |
+
ββ R12: *"the server enumerates what is reachable within two hops and offers it; the client
|
| 3045 |
+
pins one by naming each edge. Never a path search."* So `depth` is 1 or 2 and nothing else, the
|
| 3046 |
+
answer is an ENUMERATION rather than a route this server chose, and `path` is what the client
|
| 3047 |
+
sends back to pin one. β `depth`'s default of 2 mirrors `relations.MAX_DEPTH`;
|
| 3048 |
+
`verify_odoo_relational` asserts the two agree rather than letting two literals drift.
|
| 3049 |
+
|
| 3050 |
+
β A BAD `depth` IS A NAMED 400, NEVER A 500 AND NEVER A SILENT CLAMP. Clamping 3 to 2 would
|
| 3051 |
+
answer a different question than the one asked and call it success, which is the whole failure
|
| 3052 |
+
mode R12's refusal exists to prevent.
|
| 3053 |
+
|
| 3054 |
+
β `via` IS AN ARRAY OF EDGE KEYS, one per hop, resolved by `relations.pick_path` against THIS
|
| 3055 |
+
enumeration, so an edge this account cannot be offered is an edge it cannot pin. A SINGLE
|
| 3056 |
+
`?via=<key>` arrives as a one-element list and is therefore a one-hop path, which is C3's
|
| 3057 |
+
back-compat leg with no second matching rule behind it.
|
| 3058 |
+
|
| 3059 |
+
ββ W53-T18 / R13 β `refusals` IS A SIBLING OF `related`, AND `related` DID NOT CHANGE. R13:
|
| 3060 |
+
*"a permission-filtered traversal must be distinguishable on the wire from an empty one"*. A
|
| 3061 |
+
shorter `related` array is exactly the silent empty PostgREST hands out under RLS, so every
|
| 3062 |
+
path the wall removed is NAMED here instead: `forbidden_table` naming the closed table (which
|
| 3063 |
+
may be the INTERMEDIATE of a two-hop path), `hop_filtered` naming the hop the field wall took,
|
| 3064 |
+
and `no_related_rows` said out loud when the answer is genuinely nothing. β THE LIST IS NOT
|
| 3065 |
+
THE ANSWER TO A PIN: a `via` that cannot resolve fails the REQUEST with the same codes through
|
| 3066 |
+
`_refusal_error`, and a pin that resolves reports `refusals: []`.
|
| 3067 |
|
| 3068 |
β ONE ENTRY PER EDGE, NOT PER TARGET. R2: *"two links between the same pair give two tabs
|
| 3069 |
subtitled by the field"* β so `key` is the EDGE's identity and `via.key` repeats it, because
|
| 3070 |
+
`via` is the whole of what a resolver is handed back and it has to identify an edge on its own.
|
| 3071 |
+
Two bridges off `customer_data` really do share `{direction: "bridge", on: "partner_id"}`.
|
| 3072 |
|
| 3073 |
+
β `served` IS PART OF THE ANSWER, AND R6'S SECOND SENTENCE IS WHY. A declared edge whose target
|
| 3074 |
+
is served from its stored copy cannot be pinned yet (that door takes no predicate), and the
|
| 3075 |
+
caller is told which ones those are rather than discovering it as an empty tab. Open invoices is
|
| 3076 |
+
the one such edge today, and a two-hop entry is `served` only when BOTH its hops are.
|
| 3077 |
|
| 3078 |
β THE CALLER'S OWN ACCESS IS CHECKED FIRST. Listing what `table` is related to tells you what
|
| 3079 |
`table` is, so the listing is behind the same gate the grid is.
|
|
|
|
| 3081 |
table = str(table or "").strip()
|
| 3082 |
if not table:
|
| 3083 |
raise err(400, "bad_table", "name the database to list relations for")
|
| 3084 |
+
# β W54-T39 β THE GATE IS UNMOVED AND UNCHANGED; only its ANSWER is kept. `wall_memo` rides
|
| 3085 |
+
# into `_relation_context` below so this table's field wall, which the 403 has just resolved
|
| 3086 |
+
# at full price, is not resolved a second time three lines later. The order is untouched: the
|
| 3087 |
+
# caller's own access is still the first thing this route decides.
|
| 3088 |
+
wall_memo = {}
|
| 3089 |
+
if not _may_open(session, table, memo=wall_memo):
|
| 3090 |
raise err(403, "not_permitted",
|
| 3091 |
f"your account may not open '{table}', so it cannot be told what that database "
|
| 3092 |
f"is related to. Ask an administrator for access to it")
|
| 3093 |
+
import core.relations as relations
|
| 3094 |
+
|
| 3095 |
+
try:
|
| 3096 |
+
depth = int(depth)
|
| 3097 |
+
except (TypeError, ValueError):
|
| 3098 |
+
depth = -1
|
| 3099 |
+
if depth not in (1, relations.MAX_DEPTH):
|
| 3100 |
+
raise err(400, "bad_depth",
|
| 3101 |
+
f"a relation listing is offered at depth 1 (edges of '{table}' itself) or depth "
|
| 3102 |
+
f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and "
|
| 3103 |
+
f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}")
|
| 3104 |
+
|
| 3105 |
+
ctx = _relation_context(session, table, memo=wall_memo)
|
| 3106 |
+
entries = related_path_entries(session, table, depth, ctx)
|
| 3107 |
+
if via:
|
| 3108 |
+
wanted = [str(v) for v in via if str(v or "").strip()]
|
| 3109 |
+
if len(wanted) > relations.MAX_DEPTH:
|
| 3110 |
+
raise err(400, "over_depth", relations.over_depth_refusal(table, wanted))
|
| 3111 |
+
hit = relations.pick_path(entries, wanted)
|
| 3112 |
+
if hit is None:
|
| 3113 |
+
# ββ W53-T18 / R13 / C3 β THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED
|
| 3114 |
+
# ANSWERS, CHOSEN BY A DIFF. `relations.diagnose_path` enumerates the reachable set
|
| 3115 |
+
# TWICE, once with this session's wall and once without it, and compares: a path the
|
| 3116 |
+
# wall removed answers `forbidden_table` NAMING the closed table or `hop_filtered`
|
| 3117 |
+
# NAMING the hop, while a path absent from both really is an edge that does not exist
|
| 3118 |
+
# and keeps the `unknown_via` code this route already used, now carrying the
|
| 3119 |
+
# enumerated candidates with their join columns and the exact retry strings.
|
| 3120 |
+
# β THE UNFILTERED PASS CHOOSES THE CODE AND NAMES THE TABLE. IT SERVES NOTHING:
|
| 3121 |
+
# `entries` above is the walled enumeration and is what a resolvable `via` returns.
|
| 3122 |
+
raise _refusal_error(relations.diagnose_path(
|
| 3123 |
+
table, wanted, fields=ctx["fields_for"](table), defs=ctx["defs"],
|
| 3124 |
+
may_open=ctx["may_open"], fields_for=ctx["fields_for"], depth=depth,
|
| 3125 |
+
entries=entries, open_fields=ctx["open_fields_for"](table),
|
| 3126 |
+
open_fields_for=ctx["open_fields_for"]))
|
| 3127 |
+
# β A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other
|
| 3128 |
+
# tabs the wall removed are the LISTING's answer, not this one, and repeating them here
|
| 3129 |
+
# would attach an unasked-for permission report to every successful read.
|
| 3130 |
+
return {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []}
|
| 3131 |
+
return {"ok": True, "table": table, "depth": depth, "related": entries,
|
| 3132 |
+
"refusals": related_refusals(session, table, depth, entries, ctx)}
|
| 3133 |
+
|
api/routes_keychain.py
CHANGED
|
@@ -10,6 +10,7 @@ in the payload (`pausedNote`): pausing marks intent and warns; the source cutove
|
|
| 10 |
the keychain cutover wave R3 staged.
|
| 11 |
"""
|
| 12 |
import os
|
|
|
|
| 13 |
|
| 14 |
from fastapi import Body, Depends
|
| 15 |
from fastapi import APIRouter
|
|
@@ -99,8 +100,10 @@ SCOPE_KEY = "keychain_scopes"
|
|
| 99 |
|
| 100 |
#: ββ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL.
|
| 101 |
#:
|
| 102 |
-
#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *
|
| 103 |
-
#:
|
|
|
|
|
|
|
| 104 |
#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they
|
| 105 |
#: would silently become the credential the entire workspace's databases are built from, which is
|
| 106 |
#: a credential elevation wearing a scope picker.
|
|
@@ -108,7 +111,40 @@ SCOPE_KEY = "keychain_scopes"
|
|
| 108 |
#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is
|
| 109 |
#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door
|
| 110 |
#: from the only side B owns, and the resolver-side guard is booked for A.
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
|
| 113 |
|
| 114 |
def clean_scope(raw, default=DEFAULT_SCOPE):
|
|
@@ -169,13 +205,30 @@ def visible_entries(rt, uname, is_admin=False):
|
|
| 169 |
owner = str(e.get("createdBy") or "")
|
| 170 |
if not may_see(scope, owner, uname):
|
| 171 |
continue
|
| 172 |
-
row = {**e, "scope": scope, "owner": owner}
|
| 173 |
if not (is_admin or str(owner) == str(uname)):
|
| 174 |
row["preview"] = ""
|
| 175 |
out.append(row)
|
| 176 |
return out
|
| 177 |
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
def _write_scope(rt, entry_id, scope, owner):
|
| 180 |
"""Persist one entry's scope. A `business` entry CLEARS its row rather than writing the
|
| 181 |
default, so the store holds one spelling of the common case."""
|
|
@@ -203,11 +256,140 @@ def _kc():
|
|
| 203 |
return keychain
|
| 204 |
|
| 205 |
|
| 206 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
"""`(source, flag_key)` β which source would serve this tenant's Odoo queries, in both the
|
| 208 |
shapes this module needs: the display string (`env` / `keychain:<id>`) and the key the pause
|
| 209 |
flag is stored under.
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
β D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`,
|
| 212 |
beside the `odoo_source()` it must agree with. This function had its own copy of the same
|
| 213 |
three rules β first unlocked keychain odoo entry, else env for tenant #0, else nothing β and
|
|
@@ -215,7 +397,7 @@ def _resolved_odoo_key(rt):
|
|
| 215 |
against another, freezing nothing while the UI reports success. The two SHAPES stay here
|
| 216 |
because they are this module's presentation concern; the DECISION does not.
|
| 217 |
"""
|
| 218 |
-
flag_key = rt.odoo_flag_key()
|
| 219 |
if not flag_key:
|
| 220 |
return None, None
|
| 221 |
return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key
|
|
@@ -514,6 +696,134 @@ def _with_connector(rows):
|
|
| 514 |
return out
|
| 515 |
|
| 516 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 517 |
@router.get("/admin/keychain")
|
| 518 |
def list_keychain(session: Session = Depends(require_session)):
|
| 519 |
"""β W32-T11 / R4 β SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation.
|
|
@@ -527,7 +837,16 @@ def list_keychain(session: Session = Depends(require_session)):
|
|
| 527 |
#: the vocabulary and the permission, so the client renders a picker it can honour
|
| 528 |
#: rather than offering an option the server will refuse (R4: business is admin-only).
|
| 529 |
"scopes": list(SCOPES), "canBusiness": bool(session.admin),
|
| 530 |
-
"tenantWideTypes": list(TENANT_WIDE_TYPES)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 531 |
|
| 532 |
|
| 533 |
@router.post("/admin/keychain", status_code=201)
|
|
@@ -540,29 +859,55 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 540 |
if scope is None:
|
| 541 |
raise err(400, "bad_scope",
|
| 542 |
f"scope must be one of {', '.join(SCOPES)}")
|
| 543 |
-
if scope == "business" and not session.admin:
|
| 544 |
-
raise err(403, "not_admin",
|
| 545 |
-
"a business-wide connection applies to everyone in this workspace, so only an "
|
| 546 |
-
"administrator can create one. You can add it as a personal connection instead.")
|
| 547 |
etype = str(body.get("type") or "").strip().lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 548 |
if scope == "personal" and etype in TENANT_WIDE_TYPES:
|
| 549 |
# β REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this
|
| 550 |
# credential is what the WHOLE workspace's databases are built from, so "personal" would
|
| 551 |
# be a label on a tenant-wide key rather than a boundary around it.
|
|
|
|
|
|
|
|
|
|
| 552 |
raise err(400, "scope_not_available",
|
| 553 |
f"a {etype} connection is what this whole workspace's databases are read "
|
| 554 |
-
f"through, so it is always business-wide
|
| 555 |
f"An administrator can add it for everyone.")
|
| 556 |
try:
|
| 557 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 558 |
body.get("fields"), session.uname)
|
| 559 |
except kc.KeychainLocked as e:
|
| 560 |
raise err(503, "keychain_locked",
|
| 561 |
-
f"the keychain is locked
|
| 562 |
except ValueError as e:
|
| 563 |
raise err(400, "bad_entry", str(e))
|
| 564 |
except Exception:
|
| 565 |
-
raise err(503, "store_unavailable", "the entry was not saved
|
| 566 |
# ββ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE β i.e. the failure mode
|
| 567 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 568 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
|
@@ -584,49 +929,179 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
|
|
| 584 |
except Exception: # noqa: BLE001
|
| 585 |
pass
|
| 586 |
raise err(503, "store_unavailable",
|
| 587 |
-
"the key was not saved
|
| 588 |
"nothing was kept. Try again.")
|
| 589 |
-
return {"entry": {**row, "scope": scope, "owner": session.uname}}
|
| 590 |
|
| 591 |
|
| 592 |
@router.put("/admin/keychain/{entry_id}")
|
| 593 |
def update_key(entry_id: str, body: dict = Body(default=None),
|
| 594 |
session: Session = Depends(require_session)):
|
| 595 |
-
"""Contract C1's scope door
|
| 596 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 597 |
row = _own_row(session, entry_id)
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
"
|
| 604 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 605 |
if not _may_touch(session, row):
|
| 606 |
raise err(403, "not_yours", "this connection is not yours to change")
|
| 607 |
-
if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES:
|
| 608 |
raise err(400, "scope_not_available",
|
| 609 |
f"a {row.get('type')} connection is what this whole workspace's databases are "
|
| 610 |
f"read through, so it is always business-wide.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 611 |
owner = row.get("owner") or session.uname
|
|
|
|
|
|
|
| 612 |
try:
|
| 613 |
_write_scope(session.runtime, entry_id, scope, owner)
|
| 614 |
except Exception:
|
| 615 |
-
raise err(503, "store_unavailable", "the change was not saved
|
| 616 |
-
return {"entry": {**row, "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 617 |
|
| 618 |
|
| 619 |
@router.delete("/admin/keychain/{entry_id}")
|
| 620 |
-
def delete_key(entry_id: str, session: Session = Depends(require_session)):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 621 |
row = _own_row(session, entry_id)
|
| 622 |
if not _may_touch(session, row):
|
| 623 |
raise err(403, "not_yours", "this connection is not yours to delete")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 624 |
try:
|
| 625 |
-
|
| 626 |
-
_write_scope(
|
| 627 |
except Exception:
|
| 628 |
-
raise err(503, "store_unavailable", "
|
| 629 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
|
| 631 |
|
| 632 |
@router.post("/admin/keychain/{entry_id}/test")
|
|
@@ -666,7 +1141,7 @@ def _unsynced_customer_records(session):
|
|
| 666 |
"shown": min(len(orphans), 50)}
|
| 667 |
except Exception as e:
|
| 668 |
return {"known": False, "count": None, "rows": [],
|
| 669 |
-
"note": f"pool unavailable
|
| 670 |
|
| 671 |
|
| 672 |
@router.get("/admin/connectors")
|
|
@@ -678,21 +1153,27 @@ def connectors(session: Session = Depends(require_session)):
|
|
| 678 |
# every colleague's personal connection on the screen next door to the one that hides them.
|
| 679 |
entries = visible_entries(session.runtime, session.uname, bool(session.admin))
|
| 680 |
# R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries β
|
| 681 |
-
# mirrors TenantRuntime.odoo_source() exactly:
|
| 682 |
-
# for tenant #0 only, else nothing (fail closed β never another tenant's
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 683 |
# β W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source
|
| 684 |
# serves this workspace" is one fact for everybody, and deriving it from a per-USER list would
|
| 685 |
# make the answer depend on who opened the pane. Personal entries are excluded for the same
|
| 686 |
# reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source.
|
| 687 |
-
|
| 688 |
-
first_odoo = next((e["id"] for e in kc.list_entries(session.runtime)
|
| 689 |
-
if e["type"] == "odoo"
|
| 690 |
-
and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None)
|
| 691 |
# β W32-T10 β the env leg goes through `env_odoo_available` now, so this route and the
|
| 692 |
# connectors DIRECTORY answer "does the environment serve this tenant?" with one function
|
| 693 |
# instead of two spellings that agreed until a second tenant signed in.
|
| 694 |
-
if
|
| 695 |
-
resolved = f"keychain:{
|
| 696 |
elif env_odoo_available(session.runtime):
|
| 697 |
resolved = "env"
|
| 698 |
else:
|
|
@@ -741,13 +1222,13 @@ def connectors(session: Session = Depends(require_session)):
|
|
| 741 |
# so in as many words). So a paused connector plus a restarted server is exactly the
|
| 742 |
# blank screen this sentence promised could not happen. A warning that over-promises
|
| 743 |
# is worse than none: it is the sentence somebody quotes when the screen disagrees.
|
| 744 |
-
"pausedNote": ("Pausing a connector never deletes data
|
| 745 |
"views stay, and nothing reaches the source while it is paused. "
|
| 746 |
"Anything this server has already read keeps showing: the customer "
|
| 747 |
"workspace serves its pause-time snapshot, the product list serves "
|
| 748 |
"the last copy read since startup, and measure columns keep answering "
|
| 749 |
"from the mirror as it stood when you paused. What has NOT been read "
|
| 750 |
-
"cannot be shown
|
| 751 |
"a restart, reports that the source is paused instead of showing "
|
| 752 |
"figures. Resume to start reading live again.")}
|
| 753 |
# ββ W32-T11 β ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to
|
|
@@ -779,14 +1260,31 @@ def _rel():
|
|
| 779 |
return rel
|
| 780 |
|
| 781 |
|
| 782 |
-
def _odoo_entry(session):
|
| 783 |
"""The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing
|
| 784 |
-
is). Business-scoped by construction β `TENANT_WIDE_TYPES` refuses a personal one.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 785 |
kc = _kc()
|
|
|
|
|
|
|
|
|
|
| 786 |
scopes = _scope_rows(session.runtime)
|
| 787 |
for e in kc.list_entries(session.runtime):
|
| 788 |
-
if e["
|
| 789 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 790 |
return None
|
| 791 |
|
| 792 |
|
|
@@ -872,7 +1370,7 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 872 |
f"this connector has no grid called {', '.join(unknown)}")
|
| 873 |
clean_grids = {str(k): bool(v) for k, v in grids.items()}
|
| 874 |
if clean_grids and not any(clean_grids.get(k, True) for k in known):
|
| 875 |
-
notes.append("every grid is switched off
|
| 876 |
"sync, and the databases you already have are left untouched")
|
| 877 |
|
| 878 |
every = body.get("syncEvery")
|
|
@@ -886,7 +1384,7 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 886 |
# without a word is how a limit becomes invisible.
|
| 887 |
clean_every = rel.DEFAULT_SYNC
|
| 888 |
notes.append(f"{every!r} is not an interval this connector offers, and anything under "
|
| 889 |
-
f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all
|
| 890 |
f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead")
|
| 891 |
|
| 892 |
server_db = body.get("serverDb")
|
|
@@ -904,7 +1402,7 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 904 |
try:
|
| 905 |
session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
| 906 |
except Exception:
|
| 907 |
-
raise err(503, "store_unavailable", "the change was not saved
|
| 908 |
|
| 909 |
if server_db:
|
| 910 |
notes.append(_rewrite_server_db(session, server_db))
|
|
@@ -924,11 +1422,11 @@ def odoo_config_put(body: dict = Body(default=None),
|
|
| 924 |
if clean_every is not None:
|
| 925 |
if not rel.is_royal(session.tenant):
|
| 926 |
notes.append("this interval is saved, but the sync loop currently reads its schedule "
|
| 927 |
-
"from one workspace for the whole deployment
|
| 928 |
"often YOUR data refreshes until per-workspace scheduling ships")
|
| 929 |
else:
|
| 930 |
notes.append("this interval is saved and it is the one the deployment's sync loop "
|
| 931 |
-
"uses
|
| 932 |
"deployment, not only this one")
|
| 933 |
if clean_every == "manual":
|
| 934 |
notes.append("β 'manual' does not yet stop the background sync: the loop has no "
|
|
@@ -946,10 +1444,14 @@ def _rewrite_server_db(session, server_db):
|
|
| 946 |
So this is add-then-delete through the module's own doors, with the side rows (pause flag,
|
| 947 |
scope) carried across because they are keyed by ENTRY ID.
|
| 948 |
β THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the
|
| 949 |
-
delete this tenant has TWO odoo entries
|
| 950 |
-
|
| 951 |
-
|
| 952 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 953 |
"""
|
| 954 |
kc = _kc()
|
| 955 |
entry = _odoo_entry(session)
|
|
@@ -959,7 +1461,7 @@ def _rewrite_server_db(session, server_db):
|
|
| 959 |
try:
|
| 960 |
fields = kc.read_fields(session.runtime, entry["id"]) or {}
|
| 961 |
except kc.KeychainLocked as e:
|
| 962 |
-
raise err(503, "keychain_locked", f"the keychain is locked
|
| 963 |
if not fields:
|
| 964 |
raise err(400, "bad_entry", "this credential could not be read back to be changed")
|
| 965 |
if str(fields.get("db") or "") == server_db:
|
|
@@ -969,7 +1471,7 @@ def _rewrite_server_db(session, server_db):
|
|
| 969 |
new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname)
|
| 970 |
except Exception:
|
| 971 |
raise err(503, "store_unavailable",
|
| 972 |
-
"the server database was not changed
|
| 973 |
# carry the side rows across, then retire the old entry
|
| 974 |
try:
|
| 975 |
flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {}
|
|
@@ -982,7 +1484,7 @@ def _rewrite_server_db(session, server_db):
|
|
| 982 |
_write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "")
|
| 983 |
except Exception: # noqa: BLE001
|
| 984 |
return (f"the connection now points at {server_db}, but the previous credential could "
|
| 985 |
-
f"not be removed
|
| 986 |
return f"the connection now points at the {server_db} database"
|
| 987 |
|
| 988 |
|
|
@@ -1047,7 +1549,7 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1047 |
try:
|
| 1048 |
session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
| 1049 |
except Exception:
|
| 1050 |
-
raise err(503, "store_unavailable", "nothing was disconnected
|
| 1051 |
|
| 1052 |
paused_mirror = False
|
| 1053 |
if flag_key:
|
|
@@ -1069,7 +1571,7 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1069 |
removed = entry["id"]
|
| 1070 |
except Exception:
|
| 1071 |
raise err(503, "store_unavailable",
|
| 1072 |
-
"the databases are frozen but the stored credential was not removed
|
| 1073 |
"delete it under Keychains")
|
| 1074 |
|
| 1075 |
# β THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the
|
|
@@ -1081,11 +1583,11 @@ def odoo_disconnect(session: Session = Depends(require_session)):
|
|
| 1081 |
"still readable, and nothing is being refreshed.")
|
| 1082 |
if not paused_mirror and flag_key:
|
| 1083 |
note += (" β The live mirror could not be paused, so the two read-through databases "
|
| 1084 |
-
"(order lines and GL lines) may keep advancing
|
| 1085 |
"Keychains to stop them.")
|
| 1086 |
note += (" Reconnecting adds the key back and resumes into the same databases." if entry
|
| 1087 |
else " This workspace's Odoo credential comes from the deployment environment, so "
|
| 1088 |
-
"there was no stored key to remove
|
| 1089 |
"resumes them into the same tables.")
|
| 1090 |
return {"frozen": True, "removedEntry": removed,
|
| 1091 |
# β ON THE WIRE, so the client and a gate can both see which half happened. A boolean
|
|
@@ -1107,7 +1609,7 @@ def odoo_reconnect(session: Session = Depends(require_session)):
|
|
| 1107 |
try:
|
| 1108 |
_rel_reconnect(session.runtime)
|
| 1109 |
except Exception:
|
| 1110 |
-
raise err(503, "store_unavailable", "the change was not saved
|
| 1111 |
# ββ W33-T65 β AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only
|
| 1112 |
# `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the
|
| 1113 |
# two read-through grids would sit at the disconnect date while the panel said "connected
|
|
@@ -1130,7 +1632,7 @@ def odoo_reconnect(session: Session = Depends(require_session)):
|
|
| 1130 |
return {"frozen": False, "connected": connected, "resumedMirror": resumed,
|
| 1131 |
"note": ("Odoo is connected again and the databases you already had will refresh in "
|
| 1132 |
"place." if connected else
|
| 1133 |
-
"The freeze is lifted, but there is no Odoo credential yet
|
| 1134 |
"Keychains and the databases resume into the same tables.")}
|
| 1135 |
|
| 1136 |
|
|
@@ -1167,5 +1669,5 @@ def pause_connector(key: str, body: dict = Body(default=None),
|
|
| 1167 |
try:
|
| 1168 |
session.runtime.update(_CONNECTOR_FLAGS_KEY, _up)
|
| 1169 |
except Exception:
|
| 1170 |
-
raise err(503, "store_unavailable", "the change was not saved
|
| 1171 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
|
|
|
| 10 |
the keychain cutover wave R3 staged.
|
| 11 |
"""
|
| 12 |
import os
|
| 13 |
+
import time
|
| 14 |
|
| 15 |
from fastapi import Body, Depends
|
| 16 |
from fastapi import APIRouter
|
|
|
|
| 100 |
|
| 101 |
#: ββ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL.
|
| 102 |
#:
|
| 103 |
+
#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *a connection of that type* for
|
| 104 |
+
#: the TENANT β the one a caller NAMES, or the tenant's designated default when none is named
|
| 105 |
+
#: (W54-T45 / R4; it was "the first entry of that type" before) β that is what spawns
|
| 106 |
+
#: `ut_odoo_*` / `ut_meta_*` and what every measure column is
|
| 107 |
#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they
|
| 108 |
#: would silently become the credential the entire workspace's databases are built from, which is
|
| 109 |
#: a credential elevation wearing a scope picker.
|
|
|
|
| 111 |
#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is
|
| 112 |
#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door
|
| 113 |
#: from the only side B owns, and the resolver-side guard is booked for A.
|
| 114 |
+
#: ββ W52-T02 β `amazon_sp` JOINS, CLOSING A PARITY HOLE THIS FILE HAD WITH `core/keychain.py`
|
| 115 |
+
#: (whose own `TENANT_WIDE_TYPES` already carried three entries, `amazon_sp` included, "JOINED IN
|
| 116 |
+
#: WAVE 52" by its own header comment). Until this line, a non-admin could POST a PERSONAL
|
| 117 |
+
#: `amazon_sp` entry through THIS door β `add_key`'s refusal only fires for a type in this tuple β
|
| 118 |
+
#: while `core/keychain`'s shared resolver (`_first_creds` then, `_creds_for` since W54-T45)
|
| 119 |
+
#: already skipped a personal `amazon_sp` entry when resolving
|
| 120 |
+
#: the tenant's Amazon credential. So the write succeeded, the UI told the member they had
|
| 121 |
+
#: connected their own Amazon, and the resolver silently served nothing for it: not the elevation
|
| 122 |
+
#: (the resolver-side guard held), but a door that stores a credential the product then ignores,
|
| 123 |
+
#: which is its own kind of wrong. `routes_connectors.py::_tokens` imports this tuple rather than
|
| 124 |
+
#: keeping its own, so its `manage` field for the Amazon card corrects itself in the same change.
|
| 125 |
+
TENANT_WIDE_TYPES = ("odoo", "meta_ads", "amazon_sp")
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
#: ββ W53-T29 / OWNER RULING R6 β WHICH SCOPES ONE SOURCE MAY EVER CARRY, SAID AS DATA.
|
| 129 |
+
#:
|
| 130 |
+
#: R6: *"Each catalogue source declares which scopes it allows (Odoo company-only; Gmail both)."*
|
| 131 |
+
#: β IT IS DERIVED FROM `TENANT_WIDE_TYPES`, NEVER A SECOND LIST BESIDE IT. That tuple is already
|
| 132 |
+
#: the authored answer to "may this source be personal" β `add_key` and `update_key` REFUSE on it
|
| 133 |
+
#: three lines apart β so a hand-written `{"odoo": ["business"], ...}` map here would be the
|
| 134 |
+
#: [[one-question-two-normalizers]] shape the day one of the two changes and the other does not.
|
| 135 |
+
#: The function exists so the CLIENT can be told the fact rather than re-deriving it from a
|
| 136 |
+
#: `tenantWideTypes` list it would have to reason about itself.
|
| 137 |
+
def allowed_scopes(entry_type):
|
| 138 |
+
"""The scopes `entry_type` may legally be stored under, as a subset of `SCOPES`.
|
| 139 |
+
|
| 140 |
+
β ELIGIBILITY, NOT PERMISSION. This is the SOURCE's own rule and it does not know who is
|
| 141 |
+
asking: whether THIS reader may create a business-wide connection is `session.admin`, decided
|
| 142 |
+
at the door in `add_key`. A card needs both answers and they are different questions, which is
|
| 143 |
+
why `routes_connectors._detail_connect` narrows this one by the admin flag rather than
|
| 144 |
+
replacing it.
|
| 145 |
+
"""
|
| 146 |
+
word = str(entry_type or "").strip().lower()
|
| 147 |
+
return ("business",) if word in TENANT_WIDE_TYPES else SCOPES
|
| 148 |
|
| 149 |
|
| 150 |
def clean_scope(raw, default=DEFAULT_SCOPE):
|
|
|
|
| 205 |
owner = str(e.get("createdBy") or "")
|
| 206 |
if not may_see(scope, owner, uname):
|
| 207 |
continue
|
| 208 |
+
row = {**_identified(e), "scope": scope, "owner": owner}
|
| 209 |
if not (is_admin or str(owner) == str(uname)):
|
| 210 |
row["preview"] = ""
|
| 211 |
out.append(row)
|
| 212 |
return out
|
| 213 |
|
| 214 |
|
| 215 |
+
#: ββ W54-T44 / R4 / CONTRACT C4 β THE IDENTITY, IN THE WIRE'S OWN SPELLING.
|
| 216 |
+
#:
|
| 217 |
+
#: `core/keychain.py` stamps `connection_id` and `slot` on every entry so that two connections of
|
| 218 |
+
#: one kind are separately addressable (R4's agency tenant: many Odoos, one workspace). This layer
|
| 219 |
+
#: renames the first of them to `connectionId` because that is what every other camelCase field on
|
| 220 |
+
#: these payloads does (`createdBy`, `canBusiness`, `tenantWideTypes`), and a payload carrying both
|
| 221 |
+
#: spellings would be [[a-constant-two-features-share]] on the wire, where a client picking the
|
| 222 |
+
#: wrong one still looks right until the day only one is filled.
|
| 223 |
+
#: β `slot` is one word, so it crosses unchanged.
|
| 224 |
+
def _identified(row):
|
| 225 |
+
"""One keychain row, with its identity in wire spelling. Gains keys, never loses one."""
|
| 226 |
+
out = dict(row)
|
| 227 |
+
out["connectionId"] = str(out.pop("connection_id", "") or out.get("id") or "")
|
| 228 |
+
out["slot"] = int(out.get("slot") or 1)
|
| 229 |
+
return out
|
| 230 |
+
|
| 231 |
+
|
| 232 |
def _write_scope(rt, entry_id, scope, owner):
|
| 233 |
"""Persist one entry's scope. A `business` entry CLEARS its row rather than writing the
|
| 234 |
default, so the store holds one spelling of the common case."""
|
|
|
|
| 256 |
return keychain
|
| 257 |
|
| 258 |
|
| 259 |
+
def _ps():
|
| 260 |
+
"""`core.perm_scope`, lazily. Same idiom as `_kc()` and `_rel()`, and for the same reason: the
|
| 261 |
+
permission layer pulls the grid stack in behind it, and this router is imported at boot."""
|
| 262 |
+
import core.perm_scope as perm_scope
|
| 263 |
+
return perm_scope
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
#: β THE ONE KIND THAT STANDS BEHIND NO CONNECTOR CARD. `_connector_by_entry_type` names every
|
| 267 |
+
#: other member of `keychain.ENTRY_TYPES`; `generic` is a credential a person stored by hand, so it
|
| 268 |
+
#: has no card and no label to borrow. Spelled here rather than dropped from the vocabulary,
|
| 269 |
+
#: because a grant list narrower than the door it governs is a gap nobody can see: `add_key` accepts
|
| 270 |
+
#: every `ENTRY_TYPES` member, so every one of them must be grantable.
|
| 271 |
+
CONNECTOR_KIND_LABELS = {"generic": "Other credential"}
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def connector_grant_kinds():
|
| 275 |
+
"""The grant vocabulary: `[{key, label, tenantWide}]`, one row per connectable kind.
|
| 276 |
+
|
| 277 |
+
ββ W54-T47 / R4 β THE THIRD AXIS'S OWN WORDS. Owner: *"access becomes Database / Module /
|
| 278 |
+
Connector."* The first two are registry keys the permission editor already renders; this one is
|
| 279 |
+
a CREDENTIAL KIND, so its vocabulary is `keychain.ENTRY_TYPES` and not the registry at all.
|
| 280 |
+
|
| 281 |
+
β DERIVED FROM `ENTRY_TYPES`, NEVER TYPED OUT. That set is what `add_entry` validates against,
|
| 282 |
+
so a hand-written list here would be [[one-question-two-normalizers]] the day a connector is
|
| 283 |
+
added: a kind the door accepts and the editor cannot grant is a permission nobody can give.
|
| 284 |
+
β `tenantWide` RIDES ALONG so the editor can say what a grant on that kind really means. A
|
| 285 |
+
tenant-wide source is never personal, so granting it hands the member the workspace's own
|
| 286 |
+
credential slot; that is exactly what R4 asks for and exactly what an admin should read before
|
| 287 |
+
ticking it.
|
| 288 |
+
"""
|
| 289 |
+
owners = _connector_by_entry_type()
|
| 290 |
+
out = []
|
| 291 |
+
for etype in sorted(_kc().ENTRY_TYPES):
|
| 292 |
+
label = str((owners.get(etype) or {}).get("label") or "").strip()
|
| 293 |
+
out.append({"key": etype,
|
| 294 |
+
"label": label or CONNECTOR_KIND_LABELS.get(etype, etype),
|
| 295 |
+
"tenantWide": etype in TENANT_WIDE_TYPES})
|
| 296 |
+
return out
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _grant_subject(session, username):
|
| 300 |
+
"""The registry key for `username`, or a 404 that cannot distinguish another tenant's account
|
| 301 |
+
from a missing one.
|
| 302 |
+
|
| 303 |
+
β THE 404 RULE AND THE TENANT DEFAULT ARE BORROWED, NOT RE-SPELLED. `routes_admin.get_perms`
|
| 304 |
+
already answers *"a cross-tenant username answers exactly like a missing one, 404, never 403,
|
| 305 |
+
because 'that account exists in another company' is itself a leak"*, and `_tenant_of` is
|
| 306 |
+
already the ONE spelling of the pre-wave default. A second copy here is how a tenant wall grows
|
| 307 |
+
a gap. Imported lazily, exactly as `_connector_by_entry_type` imports `routes_connectors`.
|
| 308 |
+
"""
|
| 309 |
+
import routes_admin as ra
|
| 310 |
+
uname = str(username or "").strip().lower()
|
| 311 |
+
reg = ra._registry()
|
| 312 |
+
if uname not in reg or ra._tenant_of(reg[uname]) != ra._tenant_of(session.user):
|
| 313 |
+
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 314 |
+
return uname
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def _grant_admin(session):
|
| 318 |
+
"""Deciding who may connect a source is an administrator's act, refused at the server.
|
| 319 |
+
|
| 320 |
+
β THE CLIENT HIDES; THIS FORBIDS. `PermsEditor.tsx` renders the section only for an admin as a
|
| 321 |
+
courtesy, and its own header says every control there is one.
|
| 322 |
+
"""
|
| 323 |
+
if not session.admin:
|
| 324 |
+
raise err(403, "not_admin",
|
| 325 |
+
"only an administrator can decide who may add connections to this workspace")
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
@router.get("/admin/users/{username}/connector-grants")
|
| 329 |
+
def get_connector_grants(username: str, session: Session = Depends(require_session)):
|
| 330 |
+
"""One account's Connector grants, plus the vocabulary the editor renders them from."""
|
| 331 |
+
_grant_admin(session)
|
| 332 |
+
uname = _grant_subject(session, username)
|
| 333 |
+
return {"username": uname,
|
| 334 |
+
"kinds": connector_grant_kinds(),
|
| 335 |
+
"granted": sorted(_ps().connector_grants(uname, st=session.runtime))}
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
@router.put("/admin/users/{username}/connector-grants")
|
| 339 |
+
def put_connector_grants(username: str, body: dict = Body(default=None),
|
| 340 |
+
session: Session = Depends(require_session)):
|
| 341 |
+
"""Replace one account's Connector grants wholesale (W54-T47 / R4 / contract C4).
|
| 342 |
+
|
| 343 |
+
WHOLESALE, for the reason `put_perms` is: *"'remove this restriction' has to be expressible,
|
| 344 |
+
and a merge cannot express a deletion without a second vocabulary for it."* An empty list is
|
| 345 |
+
therefore a legal body and it REVOKES.
|
| 346 |
+
|
| 347 |
+
β AN EMPTIED GRANT CLEARS ITS ROW rather than storing `[]`. An absent row and "no grant" mean
|
| 348 |
+
the same thing (`perm_scope.CONNECTOR_GRANT_KEY`'s own note), so there is one spelling of the
|
| 349 |
+
common case and revoking returns the store to the shape it had before anybody granted anything.
|
| 350 |
+
|
| 351 |
+
β AN UNKNOWN KIND IS NAMED, NEVER DROPPED (W30/R6's second sentence). Silently discarding it
|
| 352 |
+
would tell an administrator they granted something the door will never honour.
|
| 353 |
+
"""
|
| 354 |
+
_grant_admin(session)
|
| 355 |
+
uname = _grant_subject(session, username)
|
| 356 |
+
body = body if isinstance(body, dict) else {}
|
| 357 |
+
raw = body.get("granted")
|
| 358 |
+
if not isinstance(raw, list):
|
| 359 |
+
raise err(400, "bad_grants",
|
| 360 |
+
"say which connectors this account may add, as a list of connector kinds")
|
| 361 |
+
known = {k["key"] for k in connector_grant_kinds()}
|
| 362 |
+
want = sorted({str(k).strip().lower() for k in raw if str(k or "").strip()})
|
| 363 |
+
unknown = [k for k in want if k not in known]
|
| 364 |
+
if unknown:
|
| 365 |
+
raise err(400, "unknown_connector",
|
| 366 |
+
f"this workspace has no connector called {', '.join(unknown)}")
|
| 367 |
+
|
| 368 |
+
def _up(cur):
|
| 369 |
+
cur = cur if isinstance(cur, dict) else {}
|
| 370 |
+
if want:
|
| 371 |
+
cur[uname] = want
|
| 372 |
+
else:
|
| 373 |
+
cur.pop(uname, None)
|
| 374 |
+
return cur
|
| 375 |
+
|
| 376 |
+
try:
|
| 377 |
+
session.runtime.update(_ps().CONNECTOR_GRANT_KEY, _up, flush="sync")
|
| 378 |
+
except Exception:
|
| 379 |
+
raise err(503, "store_unavailable", "the change was not saved, please try again")
|
| 380 |
+
return {"username": uname, "kinds": connector_grant_kinds(), "granted": want}
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _resolved_odoo_key(rt, connection_id=None):
|
| 384 |
"""`(source, flag_key)` β which source would serve this tenant's Odoo queries, in both the
|
| 385 |
shapes this module needs: the display string (`env` / `keychain:<id>`) and the key the pause
|
| 386 |
flag is stored under.
|
| 387 |
|
| 388 |
+
β W54-T45 / R4 β `connection_id` NAMES which connection is being asked about; no id keeps
|
| 389 |
+
meaning the tenant's designated default, so no route's behaviour moves. The mirroring is the
|
| 390 |
+
point: this still delegates to `TenantRuntime.odoo_flag_key`, which now refuses to fall back
|
| 391 |
+
to the environment for a named-but-unresolvable connection, and so does this by construction.
|
| 392 |
+
|
| 393 |
β D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`,
|
| 394 |
beside the `odoo_source()` it must agree with. This function had its own copy of the same
|
| 395 |
three rules β first unlocked keychain odoo entry, else env for tenant #0, else nothing β and
|
|
|
|
| 397 |
against another, freezing nothing while the UI reports success. The two SHAPES stay here
|
| 398 |
because they are this module's presentation concern; the DECISION does not.
|
| 399 |
"""
|
| 400 |
+
flag_key = rt.odoo_flag_key(connection_id)
|
| 401 |
if not flag_key:
|
| 402 |
return None, None
|
| 403 |
return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key
|
|
|
|
| 696 |
return out
|
| 697 |
|
| 698 |
|
| 699 |
+
_REGISTERED = {"done": False}
|
| 700 |
+
# A stale Remove must not inherit consent forever; ten minutes leaves time to read the warning
|
| 701 |
+
# and retry the existing client while binding that consent to the same user and database set.
|
| 702 |
+
REVOKE_CONFIRM_SECONDS = 600
|
| 703 |
+
_REVOCATIONS_KEY = "keychain_revocations"
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def _odoo_table_owner():
|
| 707 |
+
"""Odoo's per-connection table resolver, or None when it cannot be imported.
|
| 708 |
+
|
| 709 |
+
β ITS ABSENCE MUST NOT TAKE THE REGISTRY DOWN WITH IT, and that is why this is a function
|
| 710 |
+
rather than one more name in the block below's `import`. `odoo_relational` is the heavy end of
|
| 711 |
+
this codebase (it pulls the mirror and the connector spec); a failure to import it inside one
|
| 712 |
+
`try` around all three families would leave AMAZON unregistered, and an unregistered Amazon
|
| 713 |
+
family is a revocation that purges nothing while reporting a clean `registry: True`. That is
|
| 714 |
+
the silent-no-op shape instruction 30 exists to remove.
|
| 715 |
+
β Falling back to None is a REAL degradation and a safe one: without the resolver every
|
| 716 |
+
`ut_odoo_*` table resolves to the tenant's designated default connection instead of its own,
|
| 717 |
+
which can only ever widen the KEEP list a disclosure prints. Odoo purges nothing (R10), so a
|
| 718 |
+
coarse answer here cannot delete a row.
|
| 719 |
+
"""
|
| 720 |
+
try:
|
| 721 |
+
import odoo_relational
|
| 722 |
+
return odoo_relational.connection_for_table_key
|
| 723 |
+
except Exception: # noqa: BLE001
|
| 724 |
+
return None
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
def _register_credential_tables():
|
| 728 |
+
"""Declare revocation policy lazily; a failed import remains retryable and reportable.
|
| 729 |
+
|
| 730 |
+
routes_connectors imports this module at boot, and odoo_relational is heavy. An import-time
|
| 731 |
+
registration would put a cycle on every route before anyone even opens Keychain.
|
| 732 |
+
"""
|
| 733 |
+
if _REGISTERED["done"]:
|
| 734 |
+
return True
|
| 735 |
+
try:
|
| 736 |
+
from core import user_tables as ut
|
| 737 |
+
import routes_connectors as rc
|
| 738 |
+
# Instruction 30 / C13 names the Amazon Data Protection Policy, not every connector.
|
| 739 |
+
ut.register_credential_tables(
|
| 740 |
+
"amazon_sp", rc.CONNECTED_PREFIXES["amazon_sp"], on_revoke=ut.REVOKE_PURGE,
|
| 741 |
+
why="The Amazon Data Protection Policy requires deleting the seller data "
|
| 742 |
+
"authorised by a revoked credential.")
|
| 743 |
+
# W32 R10's second sentence and D-259 require these grids to survive a reconnect.
|
| 744 |
+
ut.register_credential_tables(
|
| 745 |
+
"odoo", rc.CONNECTED_PREFIXES["odoo"], on_revoke=ut.REVOKE_KEEP,
|
| 746 |
+
owner=_odoo_table_owner(),
|
| 747 |
+
why="Wave 32 R10 says reconnecting resumes into the same tables. Use "
|
| 748 |
+
"POST /admin/connectors/odoo/disconnect to freeze the data "
|
| 749 |
+
"instead of deleting it.")
|
| 750 |
+
# No Meta ruling exists this wave. KEEP records the reversible direction explicitly.
|
| 751 |
+
ut.register_credential_tables(
|
| 752 |
+
"meta_ads", rc.CONNECTED_PREFIXES["meta_ads"], on_revoke=ut.REVOKE_KEEP,
|
| 753 |
+
why="No ruling has decided Meta Ads data deletion on revocation, so the reversible "
|
| 754 |
+
"direction is taken and its databases are kept.")
|
| 755 |
+
except Exception as exc:
|
| 756 |
+
_REGISTERED["error"] = type(exc).__name__
|
| 757 |
+
return False
|
| 758 |
+
_REGISTERED["done"] = True
|
| 759 |
+
_REGISTERED.pop("error", None)
|
| 760 |
+
return True
|
| 761 |
+
|
| 762 |
+
|
| 763 |
+
def _credential_revocation(row, rt):
|
| 764 |
+
"""A failed census is UNKNOWN, never evidence of an empty seller footprint."""
|
| 765 |
+
registered = _register_credential_tables()
|
| 766 |
+
etype, cid = str(row.get("type") or ""), str(row["id"])
|
| 767 |
+
if registered:
|
| 768 |
+
try:
|
| 769 |
+
from core import user_tables as ut
|
| 770 |
+
return registered, ut.credential_footprint(etype, cid, rt=rt, st=rt)
|
| 771 |
+
except Exception as exc:
|
| 772 |
+
why = f"The credential footprint could not be read ({type(exc).__name__})."
|
| 773 |
+
else:
|
| 774 |
+
failure = _REGISTERED.get("error", "unavailable")
|
| 775 |
+
why = f"The credential table registry could not be built ({failure})."
|
| 776 |
+
return registered, {
|
| 777 |
+
"type": etype, "connectionId": cid, "action": "none", "purge": [], "keep": [],
|
| 778 |
+
"rows": 0, "rowsKnown": False,
|
| 779 |
+
"why": why + " Data cleanup is not known. "
|
| 780 |
+
"Restore the store or registry and retry cleanup.",
|
| 781 |
+
}
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
def _revocation_residue(entry_type):
|
| 785 |
+
"""Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
|
| 786 |
+
|
| 787 |
+
Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
|
| 788 |
+
deletion stays operator-only because it is permanent, exact-tenant work outside the product
|
| 789 |
+
store transaction.
|
| 790 |
+
"""
|
| 791 |
+
if entry_type != "amazon_sp":
|
| 792 |
+
return []
|
| 793 |
+
return [{
|
| 794 |
+
"where": "The tenant DuckDB mirror",
|
| 795 |
+
"why": "Connection removal clears product databases but not the raw Amazon mirror.",
|
| 796 |
+
"fix": "After every Amazon credential is revoked, an operator must run "
|
| 797 |
+
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 798 |
+
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
| 799 |
+
}]
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
def _write_revocation_warning(rt, entry_id, warning=None):
|
| 803 |
+
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
|
| 804 |
+
now = int(time.time())
|
| 805 |
+
|
| 806 |
+
def _up(cur):
|
| 807 |
+
kept = {}
|
| 808 |
+
for key, value in (cur or {}).items():
|
| 809 |
+
try:
|
| 810 |
+
age = now - int(value.get("at", 0))
|
| 811 |
+
except (AttributeError, TypeError, ValueError):
|
| 812 |
+
continue
|
| 813 |
+
if 0 <= age < REVOKE_CONFIRM_SECONDS:
|
| 814 |
+
kept[key] = value
|
| 815 |
+
kept.pop(str(entry_id), None)
|
| 816 |
+
if warning is not None:
|
| 817 |
+
kept[str(entry_id)] = warning
|
| 818 |
+
return kept
|
| 819 |
+
|
| 820 |
+
try:
|
| 821 |
+
rt.update(_REVOCATIONS_KEY, _up, flush="sync")
|
| 822 |
+
return True
|
| 823 |
+
except Exception:
|
| 824 |
+
return False
|
| 825 |
+
|
| 826 |
+
|
| 827 |
@router.get("/admin/keychain")
|
| 828 |
def list_keychain(session: Session = Depends(require_session)):
|
| 829 |
"""β W32-T11 / R4 β SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation.
|
|
|
|
| 837 |
#: the vocabulary and the permission, so the client renders a picker it can honour
|
| 838 |
#: rather than offering an option the server will refuse (R4: business is admin-only).
|
| 839 |
"scopes": list(SCOPES), "canBusiness": bool(session.admin),
|
| 840 |
+
"tenantWideTypes": list(TENANT_WIDE_TYPES),
|
| 841 |
+
# β W54-T47 / R4 β WHICH KINDS THIS READER MAY CREATE BUSINESS-WIDE WITHOUT BEING AN
|
| 842 |
+
# ADMIN. β ADDITIVE, AND `canBusiness` IS DELIBERATELY UNMOVED: that flag has meant
|
| 843 |
+
# "this account is an administrator" since wave 32 and three gates read it as such, so
|
| 844 |
+
# widening it would make one boolean answer two questions. A grant is per KIND and
|
| 845 |
+
# arrives as a list for the same reason the wall is per kind.
|
| 846 |
+
# β COURTESY, NEVER THE WALL. `add_key` consults `perm_scope.may_add_connection`
|
| 847 |
+
# regardless of what any client believes about this field.
|
| 848 |
+
"connectorGrants": sorted(_ps().connector_grants(session.uname,
|
| 849 |
+
st=session.runtime))}
|
| 850 |
|
| 851 |
|
| 852 |
@router.post("/admin/keychain", status_code=201)
|
|
|
|
| 859 |
if scope is None:
|
| 860 |
raise err(400, "bad_scope",
|
| 861 |
f"scope must be one of {', '.join(SCOPES)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 862 |
etype = str(body.get("type") or "").strip().lower()
|
| 863 |
+
# ββ W54-T47 / R4 / CONTRACT C4 β THE CONNECTOR GRANT IS CONSULTED HERE AND ONLY HERE, ON THE
|
| 864 |
+
# PATH THAT WAS ALREADY REFUSED.
|
| 865 |
+
#
|
| 866 |
+
# ββ WHICH IS WHAT MAKES IT ABSENCE-PRESERVING, and that property is the whole design. A
|
| 867 |
+
# member with NO grant meets byte-identical behaviour to the day before this shipped: 403
|
| 868 |
+
# `not_admin` here, 400 `scope_not_available` below, and their own personal connection of an
|
| 869 |
+
# ordinary type still succeeds. Nothing was storable before this ticket, so every account in
|
| 870 |
+
# every tenant carries no grant β reading that absence as DENY anywhere else on this route
|
| 871 |
+
# would revoke personal connections for the entire product with nobody having decided
|
| 872 |
+
# anything (`core/perm_scope.may_metrics` records the identical failure one field over).
|
| 873 |
+
#
|
| 874 |
+
# β SO THE GRANT ONLY EVER WIDENS: an explicit per-kind grant is what lets a named non-admin
|
| 875 |
+
# create a business-wide connection OF THAT KIND, which is R4's agency case said as a rule
|
| 876 |
+
# (*"many Gmail connections, or even many Odoo connections for one tenant, depending on the
|
| 877 |
+
# admin's configuration per user"*).
|
| 878 |
+
#
|
| 879 |
+
# β `etype` MOVED ABOVE THIS TEST rather than staying below it, because the grant is per KIND
|
| 880 |
+
# and a check that cannot see the kind can only ever be a second `session.admin`.
|
| 881 |
+
if scope == "business" and not session.admin and not _ps().may_add_connection(
|
| 882 |
+
session.user, etype, st=session.runtime):
|
| 883 |
+
raise err(403, "not_admin",
|
| 884 |
+
"a business-wide connection applies to everyone in this workspace, so it takes "
|
| 885 |
+
"an administrator, or an account an administrator has granted this connector "
|
| 886 |
+
"under Settings. "
|
| 887 |
+
+ ("Ask an administrator to add it for everyone."
|
| 888 |
+
if etype in TENANT_WIDE_TYPES
|
| 889 |
+
else "You can add it as a personal connection instead."))
|
| 890 |
if scope == "personal" and etype in TENANT_WIDE_TYPES:
|
| 891 |
# β REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this
|
| 892 |
# credential is what the WHOLE workspace's databases are built from, so "personal" would
|
| 893 |
# be a label on a tenant-wide key rather than a boundary around it.
|
| 894 |
+
# β AND THE GRANT DOES NOT REACH THIS REFUSAL. `allowed_scopes` is the SOURCE's own rule
|
| 895 |
+
# and knows nothing about who is asking; a grant says a person may connect this kind for
|
| 896 |
+
# the workspace, never that a workspace-wide credential can become one person's.
|
| 897 |
raise err(400, "scope_not_available",
|
| 898 |
f"a {etype} connection is what this whole workspace's databases are read "
|
| 899 |
+
f"through, so it is always business-wide and cannot be a personal connection. "
|
| 900 |
f"An administrator can add it for everyone.")
|
| 901 |
try:
|
| 902 |
row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
|
| 903 |
body.get("fields"), session.uname)
|
| 904 |
except kc.KeychainLocked as e:
|
| 905 |
raise err(503, "keychain_locked",
|
| 906 |
+
f"the keychain is locked; {e}. A secret is never stored unencrypted.")
|
| 907 |
except ValueError as e:
|
| 908 |
raise err(400, "bad_entry", str(e))
|
| 909 |
except Exception:
|
| 910 |
+
raise err(503, "store_unavailable", "the entry was not saved; try again")
|
| 911 |
# ββ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE β i.e. the failure mode
|
| 912 |
# of a side bucket is to publish a credential, not to hide one. So the second write is not
|
| 913 |
# best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was
|
|
|
|
| 929 |
except Exception: # noqa: BLE001
|
| 930 |
pass
|
| 931 |
raise err(503, "store_unavailable",
|
| 932 |
+
"the key was not saved; its sharing setting could not be stored, so "
|
| 933 |
"nothing was kept. Try again.")
|
| 934 |
+
return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}}
|
| 935 |
|
| 936 |
|
| 937 |
@router.put("/admin/keychain/{entry_id}")
|
| 938 |
def update_key(entry_id: str, body: dict = Body(default=None),
|
| 939 |
session: Session = Depends(require_session)):
|
| 940 |
+
"""Contract C1's scope door, and W54-T44 / C4's RENAME door. Both are settings ABOUT a
|
| 941 |
+
connection rather than the credential inside it: a stored secret is never re-openable, so
|
| 942 |
+
"change the key itself" still means "replace it", which is `DELETE` + `POST`.
|
| 943 |
+
|
| 944 |
+
β R4 asks for the rename by name β *"Odoo 1, Odoo 2, and obviously user can rename this"* β
|
| 945 |
+
and this is where it belongs: the walls a rename needs (`_own_row`'s 404 for an entry you may
|
| 946 |
+
not see, `_may_touch`'s ownership rule) are already standing here, and a second door would be
|
| 947 |
+
a second copy of both.
|
| 948 |
+
β `{"label": ...}` and `{"scope": ...}` are independent; either alone is a valid request and
|
| 949 |
+
both together are one change. A body carrying NEITHER is a 400 rather than a silent no-op,
|
| 950 |
+
because "nothing happened" and "your change was saved" must never look the same.
|
| 951 |
+
"""
|
| 952 |
row = _own_row(session, entry_id)
|
| 953 |
+
body = body or {}
|
| 954 |
+
wants_scope, wants_label = "scope" in body, "label" in body
|
| 955 |
+
if not wants_scope and not wants_label:
|
| 956 |
+
raise err(400, "nothing_to_change",
|
| 957 |
+
"say what to change about this connection: a new name, or who it is shared "
|
| 958 |
+
"with.")
|
| 959 |
+
scope = None
|
| 960 |
+
if wants_scope:
|
| 961 |
+
scope = clean_scope(body.get("scope"), default=None)
|
| 962 |
+
if scope is None:
|
| 963 |
+
raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}")
|
| 964 |
+
if scope == "business" and not session.admin:
|
| 965 |
+
raise err(403, "not_admin",
|
| 966 |
+
"a business-wide connection applies to everyone in this workspace, so only "
|
| 967 |
+
"an administrator can make one business-wide.")
|
| 968 |
if not _may_touch(session, row):
|
| 969 |
raise err(403, "not_yours", "this connection is not yours to change")
|
| 970 |
+
if wants_scope and scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES:
|
| 971 |
raise err(400, "scope_not_available",
|
| 972 |
f"a {row.get('type')} connection is what this whole workspace's databases are "
|
| 973 |
f"read through, so it is always business-wide.")
|
| 974 |
+
label = row.get("label") or ""
|
| 975 |
+
if wants_label:
|
| 976 |
+
# β THE NAME MOVES AND NOTHING ELSE. `rename_entry` touches one field on one row, so the
|
| 977 |
+
# connection keeps its `connectionId`, its `slot` and its encrypted credential, and every
|
| 978 |
+
# other connection of the same kind is left byte for byte as it was.
|
| 979 |
+
try:
|
| 980 |
+
renamed = _kc().rename_entry(session.runtime, entry_id, body.get("label"))
|
| 981 |
+
except ValueError as e:
|
| 982 |
+
raise err(400, "bad_label", str(e))
|
| 983 |
+
except Exception:
|
| 984 |
+
raise err(503, "store_unavailable", "the new name was not saved, please try again")
|
| 985 |
+
if renamed is None:
|
| 986 |
+
raise err(404, "no_entry", "no such key")
|
| 987 |
+
label = renamed.get("label") or label
|
| 988 |
owner = row.get("owner") or session.uname
|
| 989 |
+
if not wants_scope:
|
| 990 |
+
return {"entry": {**row, "label": label}}
|
| 991 |
try:
|
| 992 |
_write_scope(session.runtime, entry_id, scope, owner)
|
| 993 |
except Exception:
|
| 994 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 995 |
+
return {"entry": {**row, "label": label, "scope": scope,
|
| 996 |
+
"owner": owner if scope == "personal" else ""}}
|
| 997 |
+
|
| 998 |
+
|
| 999 |
+
@router.get("/admin/keychain/{entry_id}/revocation")
|
| 1000 |
+
def key_revocation(entry_id: str, session: Session = Depends(require_session)):
|
| 1001 |
+
"""Disclose what revocation removes, behind the delete's own visibility and ownership walls.
|
| 1002 |
+
|
| 1003 |
+
Like table_footprint, only someone who could delete may case the joint. An unavailable
|
| 1004 |
+
census carries rowsKnown=False, because an empty list alone would fabricate reassurance.
|
| 1005 |
+
"""
|
| 1006 |
+
row = _own_row(session, entry_id)
|
| 1007 |
+
if not _may_touch(session, row):
|
| 1008 |
+
raise err(403, "not_yours", "this connection is not yours to delete")
|
| 1009 |
+
registered, footprint = _credential_revocation(row, session.runtime)
|
| 1010 |
+
residue = (_revocation_residue(row["type"])
|
| 1011 |
+
if footprint["purge"] or not footprint["rowsKnown"] else [])
|
| 1012 |
+
return {"entry": {key: row.get(key, "") for key in ("id", "label", "type")},
|
| 1013 |
+
**footprint, "registry": registered, "residue": residue}
|
| 1014 |
|
| 1015 |
|
| 1016 |
@router.delete("/admin/keychain/{entry_id}")
|
| 1017 |
+
def delete_key(entry_id: str, session: Session = Depends(require_session), confirm: str = ""):
|
| 1018 |
+
kc, rt = _kc(), session.runtime
|
| 1019 |
+
entries = kc.list_entries(rt)
|
| 1020 |
+
# Instruction 30 needs idempotent revocation. An absent id's 200 is distinguishable from
|
| 1021 |
+
# another person's personal entry's 404, but ids have 12 random hex characters and are not
|
| 1022 |
+
# enumerable. A 404 on a lost response's retry would break this policy path's idempotence.
|
| 1023 |
+
if not any(entry["id"] == str(entry_id) for entry in entries):
|
| 1024 |
+
return {"ok": True, "alreadyGone": True,
|
| 1025 |
+
"purged": {"tables": [], "rows": 0, "rowsKnown": True,
|
| 1026 |
+
"refused": [], "failed": []}}
|
| 1027 |
row = _own_row(session, entry_id)
|
| 1028 |
if not _may_touch(session, row):
|
| 1029 |
raise err(403, "not_yours", "this connection is not yours to delete")
|
| 1030 |
+
registered, footprint = _credential_revocation(row, rt)
|
| 1031 |
+
tables = sorted(table["key"] for table in footprint["purge"])
|
| 1032 |
+
purges = footprint["action"] == "purge" and bool(tables)
|
| 1033 |
+
if purges and confirm != "purge":
|
| 1034 |
+
now = int(time.time())
|
| 1035 |
+
try:
|
| 1036 |
+
warning = (rt.get(_REVOCATIONS_KEY) or {}).get(str(entry_id)) or {}
|
| 1037 |
+
repeated = (warning.get("user") == session.uname
|
| 1038 |
+
and 0 <= now - int(warning.get("at", 0)) < REVOKE_CONFIRM_SECONDS
|
| 1039 |
+
and warning.get("tables") == tables)
|
| 1040 |
+
except Exception:
|
| 1041 |
+
repeated = False
|
| 1042 |
+
if not repeated:
|
| 1043 |
+
# No lane owns KeychainTab.tsx this wave. Its dialog still says "Data already read
|
| 1044 |
+
# stays; nothing new will be read." Purging silently behind that sentence would make
|
| 1045 |
+
# the product lie on screen. Refusing once with the count makes disclosure true for
|
| 1046 |
+
# the client that exists; a repeat by this user over this set is the second door.
|
| 1047 |
+
stored = _write_revocation_warning(rt, entry_id, {
|
| 1048 |
+
"user": session.uname, "at": now, "tables": tables,
|
| 1049 |
+
})
|
| 1050 |
+
# β THE KIND COMES FROM THE CREDENTIAL, NOT FROM A LITERAL. Amazon is the only
|
| 1051 |
+
# family that purges today, so "Amazon databases" would have read correctly and
|
| 1052 |
+
# become a lie in the one change that flips a second family to REVOKE_PURGE.
|
| 1053 |
+
kind = kc.kind_label(row.get("type") or "")
|
| 1054 |
+
noun = "database" if len(tables) == 1 else "databases"
|
| 1055 |
+
count = (f" holding {footprint['rows']:,} rows. " if footprint["rowsKnown"]
|
| 1056 |
+
else ". The row count is not known. ")
|
| 1057 |
+
next_step = ("Choose Remove again within 10 minutes to confirm, or pass confirm=purge."
|
| 1058 |
+
if stored else "The warning could not be saved. "
|
| 1059 |
+
"Pass confirm=purge to confirm explicitly.")
|
| 1060 |
+
raise err(409, "confirm_purge",
|
| 1061 |
+
f"Removing this connection also deletes {len(tables):,} {kind} {noun}"
|
| 1062 |
+
+ count + "That data cannot be recovered. " + next_step)
|
| 1063 |
+
|
| 1064 |
+
class _Ownership:
|
| 1065 |
+
# Revoking the default changes keychain's resolver immediately. Lend only metadata from
|
| 1066 |
+
# BEFORE revocation so core can recompute ownership after the credential is gone. Table
|
| 1067 |
+
# definitions and deletions still use the live tenant runtime; no caller-supplied plan
|
| 1068 |
+
# and no encrypted credential is carried into the purge.
|
| 1069 |
+
def get(self, name):
|
| 1070 |
+
if name == kc.KEY:
|
| 1071 |
+
return {"entries": {entry["id"]: entry for entry in entries}}
|
| 1072 |
+
return rt.get(name)
|
| 1073 |
+
|
| 1074 |
+
# Credential FIRST, data second. A crash leaves revoked access and residual tables that can
|
| 1075 |
+
# be swept again; the reverse leaves live credentials over deleted data we cannot read back.
|
| 1076 |
+
# The absent-id retry above is only a no-op receipt, not proof that a crashed sweep completed.
|
| 1077 |
try:
|
| 1078 |
+
kc.delete_entry(rt, entry_id)
|
| 1079 |
+
_write_scope(rt, entry_id, DEFAULT_SCOPE, "")
|
| 1080 |
except Exception:
|
| 1081 |
+
raise err(503, "store_unavailable", "The delete did not land. Try again.")
|
| 1082 |
+
if not purges:
|
| 1083 |
+
_write_revocation_warning(rt, entry_id)
|
| 1084 |
+
if footprint["rowsKnown"]:
|
| 1085 |
+
return {"ok": True}
|
| 1086 |
+
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
|
| 1087 |
+
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
|
| 1088 |
+
return {"ok": True, "registry": registered, "footprint": footprint,
|
| 1089 |
+
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
|
| 1090 |
+
"refused": [], "failed": []},
|
| 1091 |
+
"residue": _revocation_residue(row["type"])}
|
| 1092 |
+
from core import user_tables as ut
|
| 1093 |
+
try:
|
| 1094 |
+
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
|
| 1095 |
+
except Exception as exc:
|
| 1096 |
+
# Access is already revoked. Report every unresolved table without leaking a store path.
|
| 1097 |
+
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
|
| 1098 |
+
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
|
| 1099 |
+
_write_revocation_warning(rt, entry_id)
|
| 1100 |
+
return {"ok": True,
|
| 1101 |
+
"purged": {"tables": result["purged"], "rows": result["rows"],
|
| 1102 |
+
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
|
| 1103 |
+
"failed": result["failed"]},
|
| 1104 |
+
"residue": _revocation_residue(row["type"])}
|
| 1105 |
|
| 1106 |
|
| 1107 |
@router.post("/admin/keychain/{entry_id}/test")
|
|
|
|
| 1141 |
"shown": min(len(orphans), 50)}
|
| 1142 |
except Exception as e:
|
| 1143 |
return {"known": False, "count": None, "rows": [],
|
| 1144 |
+
"note": f"pool unavailable; {type(e).__name__}"}
|
| 1145 |
|
| 1146 |
|
| 1147 |
@router.get("/admin/connectors")
|
|
|
|
| 1153 |
# every colleague's personal connection on the screen next door to the one that hides them.
|
| 1154 |
entries = visible_entries(session.runtime, session.uname, bool(session.admin))
|
| 1155 |
# R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries β
|
| 1156 |
+
# mirrors TenantRuntime.odoo_source() exactly: the resolved keychain odoo connection (while
|
| 1157 |
+
# unlocked), else env for tenant #0 only, else nothing (fail closed β never another tenant's
|
| 1158 |
+
# environment).
|
| 1159 |
+
# ββ W54-T45 β THE WALK IS GONE AND THE RESOLUTION IS ASKED FOR. This was the third copy of
|
| 1160 |
+
# "the first odoo entry of this tenant", written out by hand beside a comment promising it
|
| 1161 |
+
# mirrors `odoo_source()`. Under R4 a tenant holds many Odoo connections, so a hand-rolled
|
| 1162 |
+
# first-entry walk here would light the ACTIVE badge on one client's connection while the
|
| 1163 |
+
# connector layer served another's β the two answers disagreeing on the one screen that
|
| 1164 |
+
# exists to show which is which. `keychain.resolve_connection_id` is the single resolver, it
|
| 1165 |
+
# decrypts nothing (this route runs on a locked keychain too), and it applies the same
|
| 1166 |
+
# personal-entry skip this comment already required.
|
| 1167 |
# β W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source
|
| 1168 |
# serves this workspace" is one fact for everybody, and deriving it from a per-USER list would
|
| 1169 |
# make the answer depend on who opened the pane. Personal entries are excluded for the same
|
| 1170 |
# reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source.
|
| 1171 |
+
serving_odoo = kc.resolve_connection_id(session.runtime, "odoo")
|
|
|
|
|
|
|
|
|
|
| 1172 |
# β W32-T10 β the env leg goes through `env_odoo_available` now, so this route and the
|
| 1173 |
# connectors DIRECTORY answer "does the environment serve this tenant?" with one function
|
| 1174 |
# instead of two spellings that agreed until a second tenant signed in.
|
| 1175 |
+
if serving_odoo and kc.unlocked():
|
| 1176 |
+
resolved = f"keychain:{serving_odoo}"
|
| 1177 |
elif env_odoo_available(session.runtime):
|
| 1178 |
resolved = "env"
|
| 1179 |
else:
|
|
|
|
| 1222 |
# so in as many words). So a paused connector plus a restarted server is exactly the
|
| 1223 |
# blank screen this sentence promised could not happen. A warning that over-promises
|
| 1224 |
# is worse than none: it is the sentence somebody quotes when the screen disagrees.
|
| 1225 |
+
"pausedNote": ("Pausing a connector never deletes data; notes, custom fields and "
|
| 1226 |
"views stay, and nothing reaches the source while it is paused. "
|
| 1227 |
"Anything this server has already read keeps showing: the customer "
|
| 1228 |
"workspace serves its pause-time snapshot, the product list serves "
|
| 1229 |
"the last copy read since startup, and measure columns keep answering "
|
| 1230 |
"from the mirror as it stood when you paused. What has NOT been read "
|
| 1231 |
+
"cannot be shown; a scope with no snapshot, or the product list after "
|
| 1232 |
"a restart, reports that the source is paused instead of showing "
|
| 1233 |
"figures. Resume to start reading live again.")}
|
| 1234 |
# ββ W32-T11 β ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to
|
|
|
|
| 1260 |
return rel
|
| 1261 |
|
| 1262 |
|
| 1263 |
+
def _odoo_entry(session, connection_id=None):
|
| 1264 |
"""The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing
|
| 1265 |
+
is). Business-scoped by construction β `TENANT_WIDE_TYPES` refuses a personal one.
|
| 1266 |
+
|
| 1267 |
+
ββ W54-T45 / R4 / C4 β IT ASKS THE KEYCHAIN WHICH CONNECTION SERVES RATHER THAN WALKING FOR
|
| 1268 |
+
THE FIRST ONE. `connection_id` names it; no id means the tenant's designated default, which is
|
| 1269 |
+
the lowest-`slot` odoo connection and, for every tenant that has exactly one, the same row this
|
| 1270 |
+
used to return. The walk this replaced was a second copy of the resolution
|
| 1271 |
+
`keychain.resolve_connection_id` owns, and a panel resolving one connection while the connector
|
| 1272 |
+
layer resolves another is precisely how an agency's screen comes to describe the wrong client.
|
| 1273 |
+
β A named connection that does not resolve is None, never a sibling and never the environment.
|
| 1274 |
+
"""
|
| 1275 |
kc = _kc()
|
| 1276 |
+
cid = kc.resolve_connection_id(session.runtime, "odoo", connection_id)
|
| 1277 |
+
if not cid:
|
| 1278 |
+
return None
|
| 1279 |
scopes = _scope_rows(session.runtime)
|
| 1280 |
for e in kc.list_entries(session.runtime):
|
| 1281 |
+
if str(e["connection_id"]) != str(cid) and str(e["id"]) != str(cid):
|
| 1282 |
+
continue
|
| 1283 |
+
# The personal skip belongs to the resolver above; re-asserted here because THIS module
|
| 1284 |
+
# owns the scope vocabulary and a divergence between the two buckets must fail closed.
|
| 1285 |
+
if entry_scope(session.runtime, e["id"], scopes)[0] == "personal":
|
| 1286 |
+
return None
|
| 1287 |
+
return e
|
| 1288 |
return None
|
| 1289 |
|
| 1290 |
|
|
|
|
| 1370 |
f"this connector has no grid called {', '.join(unknown)}")
|
| 1371 |
clean_grids = {str(k): bool(v) for k, v in grids.items()}
|
| 1372 |
if clean_grids and not any(clean_grids.get(k, True) for k in known):
|
| 1373 |
+
notes.append("every grid is switched off; nothing will be materialised on the next "
|
| 1374 |
"sync, and the databases you already have are left untouched")
|
| 1375 |
|
| 1376 |
every = body.get("syncEvery")
|
|
|
|
| 1384 |
# without a word is how a limit becomes invisible.
|
| 1385 |
clean_every = rel.DEFAULT_SYNC
|
| 1386 |
notes.append(f"{every!r} is not an interval this connector offers, and anything under "
|
| 1387 |
+
f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all; the "
|
| 1388 |
f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead")
|
| 1389 |
|
| 1390 |
server_db = body.get("serverDb")
|
|
|
|
| 1402 |
try:
|
| 1403 |
session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
| 1404 |
except Exception:
|
| 1405 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 1406 |
|
| 1407 |
if server_db:
|
| 1408 |
notes.append(_rewrite_server_db(session, server_db))
|
|
|
|
| 1422 |
if clean_every is not None:
|
| 1423 |
if not rel.is_royal(session.tenant):
|
| 1424 |
notes.append("this interval is saved, but the sync loop currently reads its schedule "
|
| 1425 |
+
"from one workspace for the whole deployment, so it will not change how "
|
| 1426 |
"often YOUR data refreshes until per-workspace scheduling ships")
|
| 1427 |
else:
|
| 1428 |
notes.append("this interval is saved and it is the one the deployment's sync loop "
|
| 1429 |
+
"uses; it changes the refresh rate for every workspace on this "
|
| 1430 |
"deployment, not only this one")
|
| 1431 |
if clean_every == "manual":
|
| 1432 |
notes.append("β 'manual' does not yet stop the background sync: the loop has no "
|
|
|
|
| 1444 |
So this is add-then-delete through the module's own doors, with the side rows (pause flag,
|
| 1445 |
scope) carried across because they are keyed by ENTRY ID.
|
| 1446 |
β THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the
|
| 1447 |
+
delete this tenant has TWO odoo entries. The alternative order can leave the workspace with no
|
| 1448 |
+
credential at all, which is worse than one stale read.
|
| 1449 |
+
β W54-T45 MADE THAT WINDOW DETERMINISTIC RATHER THAN AMBIGUOUS, as a side effect worth
|
| 1450 |
+
stating. It used to read "`odoo_creds` takes the first by id sort β so a resync landing inside
|
| 1451 |
+
that window COULD read the old database", i.e. a coin flip on random hex. The resolver now
|
| 1452 |
+
serves the tenant's designated default, the LOWEST slot, and the replacement entry is minted
|
| 1453 |
+
high-water: so the OLD connection serves for the whole window and the new one takes over the
|
| 1454 |
+
instant the old row is deleted. One stale read, not a coin flip.
|
| 1455 |
"""
|
| 1456 |
kc = _kc()
|
| 1457 |
entry = _odoo_entry(session)
|
|
|
|
| 1461 |
try:
|
| 1462 |
fields = kc.read_fields(session.runtime, entry["id"]) or {}
|
| 1463 |
except kc.KeychainLocked as e:
|
| 1464 |
+
raise err(503, "keychain_locked", f"the keychain is locked; {e}")
|
| 1465 |
if not fields:
|
| 1466 |
raise err(400, "bad_entry", "this credential could not be read back to be changed")
|
| 1467 |
if str(fields.get("db") or "") == server_db:
|
|
|
|
| 1471 |
new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname)
|
| 1472 |
except Exception:
|
| 1473 |
raise err(503, "store_unavailable",
|
| 1474 |
+
"the server database was not changed; the existing connection is untouched")
|
| 1475 |
# carry the side rows across, then retire the old entry
|
| 1476 |
try:
|
| 1477 |
flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {}
|
|
|
|
| 1484 |
_write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "")
|
| 1485 |
except Exception: # noqa: BLE001
|
| 1486 |
return (f"the connection now points at {server_db}, but the previous credential could "
|
| 1487 |
+
f"not be removed; delete it under Keychains")
|
| 1488 |
return f"the connection now points at the {server_db} database"
|
| 1489 |
|
| 1490 |
|
|
|
|
| 1549 |
try:
|
| 1550 |
session.runtime.update(rel.CONFIG_KEY, _up, flush="sync")
|
| 1551 |
except Exception:
|
| 1552 |
+
raise err(503, "store_unavailable", "nothing was disconnected; try again")
|
| 1553 |
|
| 1554 |
paused_mirror = False
|
| 1555 |
if flag_key:
|
|
|
|
| 1571 |
removed = entry["id"]
|
| 1572 |
except Exception:
|
| 1573 |
raise err(503, "store_unavailable",
|
| 1574 |
+
"the databases are frozen but the stored credential was not removed; "
|
| 1575 |
"delete it under Keychains")
|
| 1576 |
|
| 1577 |
# β THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the
|
|
|
|
| 1583 |
"still readable, and nothing is being refreshed.")
|
| 1584 |
if not paused_mirror and flag_key:
|
| 1585 |
note += (" β The live mirror could not be paused, so the two read-through databases "
|
| 1586 |
+
"(order lines and GL lines) may keep advancing; pause the Odoo connector under "
|
| 1587 |
"Keychains to stop them.")
|
| 1588 |
note += (" Reconnecting adds the key back and resumes into the same databases." if entry
|
| 1589 |
else " This workspace's Odoo credential comes from the deployment environment, so "
|
| 1590 |
+
"there was no stored key to remove; the databases are frozen and Reconnect "
|
| 1591 |
"resumes them into the same tables.")
|
| 1592 |
return {"frozen": True, "removedEntry": removed,
|
| 1593 |
# β ON THE WIRE, so the client and a gate can both see which half happened. A boolean
|
|
|
|
| 1609 |
try:
|
| 1610 |
_rel_reconnect(session.runtime)
|
| 1611 |
except Exception:
|
| 1612 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 1613 |
# ββ W33-T65 β AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only
|
| 1614 |
# `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the
|
| 1615 |
# two read-through grids would sit at the disconnect date while the panel said "connected
|
|
|
|
| 1632 |
return {"frozen": False, "connected": connected, "resumedMirror": resumed,
|
| 1633 |
"note": ("Odoo is connected again and the databases you already had will refresh in "
|
| 1634 |
"place." if connected else
|
| 1635 |
+
"The freeze is lifted, but there is no Odoo credential yet; add one under "
|
| 1636 |
"Keychains and the databases resume into the same tables.")}
|
| 1637 |
|
| 1638 |
|
|
|
|
| 1669 |
try:
|
| 1670 |
session.runtime.update(_CONNECTOR_FLAGS_KEY, _up)
|
| 1671 |
except Exception:
|
| 1672 |
+
raise err(503, "store_unavailable", "the change was not saved; try again")
|
| 1673 |
return {"key": key, "paused": paused, "snapshots": snapshots}
|
api/routes_nav.py
CHANGED
|
@@ -15,6 +15,7 @@ import time
|
|
| 15 |
from fastapi import APIRouter, Body, Depends
|
| 16 |
|
| 17 |
from deps import Session, err, perms, require_session
|
|
|
|
| 18 |
|
| 19 |
router = APIRouter(prefix="/api/v1")
|
| 20 |
|
|
@@ -422,6 +423,111 @@ def _granted_view_ids(session):
|
|
| 422 |
return set()
|
| 423 |
|
| 424 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
@router.get("/nav")
|
| 426 |
def nav(session: Session = Depends(require_session)):
|
| 427 |
"""`{pages: [{key,label,source?,chrome}], landing}`.
|
|
@@ -561,12 +667,23 @@ def nav(session: Session = Depends(require_session)):
|
|
| 561 |
# leak with no symptom until two tenants rename the same registry key. Copy the rows before
|
| 562 |
# merging on the day that invariant changes.
|
| 563 |
meta = _read_nav_meta(session.runtime)
|
|
|
|
|
|
|
| 564 |
for page in pages:
|
| 565 |
source = str(page.get("source") or "")
|
| 566 |
key = str(page.get("key") or "")
|
| 567 |
if key in {"customer_data", "product_data"} or source.lower() == "odoo" \
|
| 568 |
or key.startswith("ut_odoo_"):
|
|
|
|
|
|
|
|
|
|
| 569 |
page["sourceGroup"] = "Odoo"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 570 |
elif page.get("user_table") or source in {"", "Blank", "Custom"}:
|
| 571 |
page.setdefault("sourceGroup", "Custom")
|
| 572 |
# ββ W35-T43 (rulings R4/R7) β THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
|
@@ -1165,3 +1282,40 @@ def update_schema_map_setup(key: str, body: dict = Body(default=None),
|
|
| 1165 |
return {"map": {"canConfigure": True,
|
| 1166 |
"fields": map_payload.get("fields") or [],
|
| 1167 |
**active}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
from fastapi import APIRouter, Body, Depends
|
| 16 |
|
| 17 |
from deps import Session, err, perms, require_session
|
| 18 |
+
import application_registry
|
| 19 |
|
| 20 |
router = APIRouter(prefix="/api/v1")
|
| 21 |
|
|
|
|
| 423 |
return set()
|
| 424 |
|
| 425 |
|
| 426 |
+
#: ββ W54-T48 / OWNER RULING R4 / CONTRACT C4 β ONE CONNECTION, ONE FOLDER IN THE RAIL.
|
| 427 |
+
#:
|
| 428 |
+
#: R4, verbatim: *"under the View rail for each database that comes from the same connector, they
|
| 429 |
+
#: need to be put in the same Folder. Odoo 1, Odoo 2, and obviously user can rename this."* The
|
| 430 |
+
#: rail already folds the Data level by `sourceGroup` (`dataPath.ts::groupDataDatabases`), so the
|
| 431 |
+
#: grouping mechanism is not the missing piece β the NAME was. Every Odoo database used to answer
|
| 432 |
+
#: the literal `"Odoo"`, so an agency with two Odoo connections saw its two clients' invoices
|
| 433 |
+
#: nested under one folder, which is the exact confusion R4 exists to prevent.
|
| 434 |
+
#:
|
| 435 |
+
#: β THE RENAME IS THE CONNECTION'S RENAME AND THERE IS NO SECOND NAMING SCHEME. The folder shows
|
| 436 |
+
#: the keychain row's `label`, which `keychain.add_entry` defaults to `Odoo 1` / `Odoo 2` and
|
| 437 |
+
#: `keychain.rename_entry` changes (W54-T44). *"Obviously user can rename this"* is therefore
|
| 438 |
+
#: already shipped: renaming the connection renames its folder on the next nav read, and it
|
| 439 |
+
#: touches no TABLE KEY β those are namespaced on the immutable `connection_id` (W54-T46), never
|
| 440 |
+
#: on a label a person may change twice a day.
|
| 441 |
+
#:
|
| 442 |
+
#: ββ AND THE OVERRIDE ONLY FIRES WHEN THERE ARE TWO OR MORE CONNECTIONS TO TELL APART. With
|
| 443 |
+
#: exactly one Odoo connection β every tenant that exists today β the folder keeps the generic
|
| 444 |
+
#: `"Odoo"` it has always shown, unchanged, and nothing on anybody's rail moves.
|
| 445 |
+
#:
|
| 446 |
+
#: THE REASON IS R4 ITSELF, not caution. R4 asks for connections to be DISTINGUISHABLE: *"Odoo 1,
|
| 447 |
+
#: Odoo 2"*. With one connection there is one folder either way and nothing to distinguish it
|
| 448 |
+
#: from, so the label buys no clarity and costs the tenant a rail that suddenly reads differently.
|
| 449 |
+
#: Tenant #0's one connection is labelled "Odoo (migrated from this deployment)" by an August
|
| 450 |
+
#: migration, so an unconditional override renamed the owner's own Data folder to a truncated
|
| 451 |
+
#: migration artefact β a regression this ticket would have shipped in the name of a ruling about
|
| 452 |
+
#: agencies. The moment a SECOND connection is added the names appear on both folders together,
|
| 453 |
+
#: which is the only moment they mean anything.
|
| 454 |
+
#: β COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
|
| 455 |
+
#: of the reserved words below is still a second connection, and its sibling must still be named.
|
| 456 |
+
#:
|
| 457 |
+
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
|
| 458 |
+
#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
|
| 459 |
+
#: "Loopable" group, and the branch below puts user tables in "Custom". A connection someone named
|
| 460 |
+
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
|
| 461 |
+
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
|
| 462 |
+
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
|
| 463 |
+
|
| 464 |
+
#: The registry modules that ARE the designated default connection's data rather than a namespaced
|
| 465 |
+
#: copy of it: `odoo_relational.namespaced_fields` says so in as many words ("`customer_link`
|
| 466 |
+
#: targets the compiled registry module `customer_data`, which holds the DEFAULT connection's
|
| 467 |
+
#: customers"). They carry no namespace token to read backwards, so they are named here.
|
| 468 |
+
_DEFAULT_CONNECTION_PAGES = frozenset({"customer_data", "product_data"})
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def _odoo_connection_folders(runtime):
|
| 472 |
+
"""`({connection_id: folder label}, default_connection_id)` for this tenant's Odoo
|
| 473 |
+
connections β resolved ONCE per `/nav`, decrypting nothing, and `({}, None)` on any failure.
|
| 474 |
+
|
| 475 |
+
β ONCE PER REQUEST, NOT ONCE PER PAGE, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION.
|
| 476 |
+
`odoo_relational.connection_label` and `keychain.default_connection_id` each walk
|
| 477 |
+
`keychain.list_entries`, which is a store read; calling either inside the page loop would put
|
| 478 |
+
N of them back on the one route D-175 spent a wave reducing to a single read. The loop below
|
| 479 |
+
asks `connection_for_table_key` WITHOUT a runtime, which is pure string work, and maps its
|
| 480 |
+
`DEFAULT_CONNECTION` sentinel through the default resolved here.
|
| 481 |
+
β AND IT FAILS SOFT ON PURPOSE. A locked keychain, a store blip or a missing import leaves the
|
| 482 |
+
rail painting the generic "Odoo" folder; a rail that does not paint at all is far worse than a
|
| 483 |
+
rail whose folder carries a generic name.
|
| 484 |
+
β A TENANT WITH FEWER THAN TWO ODOO CONNECTIONS GETS `{}` β the same answer as a failure, and
|
| 485 |
+
deliberately so: both mean "there is nothing here to tell apart, keep the generic folder". The
|
| 486 |
+
count is taken over the CONNECTIONS, before the reserved-word filter, so a second connection
|
| 487 |
+
named `Blank` still switches its sibling on.
|
| 488 |
+
"""
|
| 489 |
+
if runtime is None:
|
| 490 |
+
return {}, None
|
| 491 |
+
try:
|
| 492 |
+
import core.keychain as keychain
|
| 493 |
+
import odoo_relational as odoo_rel
|
| 494 |
+
connections = 0
|
| 495 |
+
labels = {}
|
| 496 |
+
for row in keychain.list_entries(runtime):
|
| 497 |
+
if str(row.get("type") or "") != "odoo":
|
| 498 |
+
continue
|
| 499 |
+
connections += 1
|
| 500 |
+
cid = str(row.get("connection_id") or "")
|
| 501 |
+
label = " ".join(str(row.get("label") or "").split())
|
| 502 |
+
if cid and label and label.lower() not in _RESERVED_GROUP_LABELS:
|
| 503 |
+
labels[cid] = label
|
| 504 |
+
if connections < 2:
|
| 505 |
+
return {}, odoo_rel.default_connection_id(runtime)
|
| 506 |
+
return labels, odoo_rel.default_connection_id(runtime)
|
| 507 |
+
except Exception: # noqa: BLE001
|
| 508 |
+
return {}, None
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def _odoo_folder_label(key, labels, default_cid):
|
| 512 |
+
"""The folder name an Odoo page belongs under, or `""` to keep the generic fallback.
|
| 513 |
+
|
| 514 |
+
β `connection_for_table_key` is the ONLY sanctioned way to ask which connection owns a table,
|
| 515 |
+
because it is the inverse of the namespace `table_key_for` builds (C4). Parsing the key here
|
| 516 |
+
would be the second implementation that inverse exists to prevent.
|
| 517 |
+
"""
|
| 518 |
+
if not labels:
|
| 519 |
+
return ""
|
| 520 |
+
try:
|
| 521 |
+
import odoo_relational as odoo_rel
|
| 522 |
+
cid = (default_cid if key in _DEFAULT_CONNECTION_PAGES
|
| 523 |
+
else odoo_rel.connection_for_table_key(key))
|
| 524 |
+
if cid == odoo_rel.DEFAULT_CONNECTION:
|
| 525 |
+
cid = default_cid
|
| 526 |
+
except Exception: # noqa: BLE001
|
| 527 |
+
return ""
|
| 528 |
+
return labels.get(str(cid or "")) or ""
|
| 529 |
+
|
| 530 |
+
|
| 531 |
@router.get("/nav")
|
| 532 |
def nav(session: Session = Depends(require_session)):
|
| 533 |
"""`{pages: [{key,label,source?,chrome}], landing}`.
|
|
|
|
| 667 |
# leak with no symptom until two tenants rename the same registry key. Copy the rows before
|
| 668 |
# merging on the day that invariant changes.
|
| 669 |
meta = _read_nav_meta(session.runtime)
|
| 670 |
+
# W54-T48 (R4 / C4): the tenant's Odoo connections, read ONCE for the whole page loop.
|
| 671 |
+
_conn_labels, _conn_default = _odoo_connection_folders(session.runtime)
|
| 672 |
for page in pages:
|
| 673 |
source = str(page.get("source") or "")
|
| 674 |
key = str(page.get("key") or "")
|
| 675 |
if key in {"customer_data", "product_data"} or source.lower() == "odoo" \
|
| 676 |
or key.startswith("ut_odoo_"):
|
| 677 |
+
# β THIS LITERAL IS THE FALLBACK AND IT STAYS. It is what a tenant with ONE Odoo
|
| 678 |
+
# connection or none still sees β every tenant that exists today β and what a database
|
| 679 |
+
# that resolves to no connection at all keeps.
|
| 680 |
page["sourceGroup"] = "Odoo"
|
| 681 |
+
# β W54-T48 (R4 / C4): once the tenant has TWO connections to tell apart, each
|
| 682 |
+
# database is folded under its own connection's name, so two connections give two
|
| 683 |
+
# folders. `_odoo_connection_folders` answers `{}` below that threshold.
|
| 684 |
+
_folder = _odoo_folder_label(key, _conn_labels, _conn_default)
|
| 685 |
+
if _folder:
|
| 686 |
+
page["sourceGroup"] = _folder
|
| 687 |
elif page.get("user_table") or source in {"", "Blank", "Custom"}:
|
| 688 |
page.setdefault("sourceGroup", "Custom")
|
| 689 |
# ββ W35-T43 (rulings R4/R7) β THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
|
|
|
|
| 1282 |
return {"map": {"canConfigure": True,
|
| 1283 |
"fields": map_payload.get("fields") or [],
|
| 1284 |
**active}}
|
| 1285 |
+
|
| 1286 |
+
|
| 1287 |
+
@router.get("/nav/application-templates")
|
| 1288 |
+
def application_templates(session: Session = Depends(require_session)):
|
| 1289 |
+
"""The trusted template contracts the App creation flow may offer."""
|
| 1290 |
+
return {"templates": application_registry.template_contracts()}
|
| 1291 |
+
|
| 1292 |
+
|
| 1293 |
+
@router.post("/nav/applications", status_code=201)
|
| 1294 |
+
def create_nav_application(body: dict = Body(default=None),
|
| 1295 |
+
session: Session = Depends(require_session)):
|
| 1296 |
+
"""Create one tenant Application from the caller's permission-filtered database schema.
|
| 1297 |
+
|
| 1298 |
+
This is a thin nav/schema adapter over the existing Application registry writer. It does not
|
| 1299 |
+
create a View or code module, and it deliberately calls ``nav_schema`` before persistence so
|
| 1300 |
+
a field hidden from this caller cannot be smuggled into an Application definition.
|
| 1301 |
+
"""
|
| 1302 |
+
payload = body if isinstance(body, dict) else {}
|
| 1303 |
+
source = str(payload.get("sourceDatabase") or "").strip()
|
| 1304 |
+
if not source:
|
| 1305 |
+
raise err(400, "bad_application", "sourceDatabase is required")
|
| 1306 |
+
schema = nav_schema(source, session)
|
| 1307 |
+
fields = schema.get("fields") if isinstance(schema, dict) else None
|
| 1308 |
+
if not session.runtime.available():
|
| 1309 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.")
|
| 1310 |
+
try:
|
| 1311 |
+
application = application_registry.create(
|
| 1312 |
+
session.runtime, session.tenant, session.uname, payload,
|
| 1313 |
+
field_contracts={source: fields if isinstance(fields, list) else []},
|
| 1314 |
+
allowed_databases={source})
|
| 1315 |
+
except PermissionError as exc:
|
| 1316 |
+
raise err(403, "forbidden", str(exc))
|
| 1317 |
+
except application_registry.ApplicationError as exc:
|
| 1318 |
+
raise err(400, "bad_application", str(exc))
|
| 1319 |
+
except Exception:
|
| 1320 |
+
raise err(503, "store_unavailable", "the Application was not saved.")
|
| 1321 |
+
return {"application": application}
|
api/routes_oauth.py
CHANGED
|
@@ -43,7 +43,7 @@ def _redirect_uri(request: Request, provider: str) -> str:
|
|
| 43 |
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
-
Runbook: `
|
| 47 |
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
if not base:
|
| 49 |
base = f"{request.url.scheme}://{request.url.netloc}"
|
|
|
|
| 43 |
the user actually browsed plants the session on the wrong hostname and they return logged
|
| 44 |
out. Moving the app to a new hostname means moving this value AND re-registering the
|
| 45 |
resulting URI in the provider console; one without the other fails closed.
|
| 46 |
+
Runbook: `AutoWiki/research/loopable-domain-runbook.md`."""
|
| 47 |
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
|
| 48 |
if not base:
|
| 49 |
base = f"{request.url.scheme}://{request.url.netloc}"
|
api/routes_odoo_tables.py
CHANGED
|
@@ -24,10 +24,10 @@ import hmac
|
|
| 24 |
import json
|
| 25 |
import time
|
| 26 |
|
| 27 |
-
from fastapi import Body, Depends
|
| 28 |
from fastapi import APIRouter, Query
|
| 29 |
|
| 30 |
-
from deps import Session, err, require_session
|
| 31 |
|
| 32 |
router = APIRouter(prefix="/api/v1")
|
| 33 |
|
|
@@ -507,11 +507,19 @@ def _resync_state(cur):
|
|
| 507 |
# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as
|
| 508 |
# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]).
|
| 509 |
#
|
| 510 |
-
#
|
| 511 |
-
#
|
| 512 |
-
#
|
| 513 |
-
#
|
| 514 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 515 |
|
| 516 |
#: `ut_*` key -> how to read that grid straight out of the mirror.
|
| 517 |
#:
|
|
@@ -589,9 +597,12 @@ def _odoo_sources():
|
|
| 589 |
# β R6 / W30-T29 β TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops
|
| 590 |
# being a fact about them. `core` never imports up, so the declaration goes this way round.
|
| 591 |
# β ALL EIGHT, not just the read-through-bound ones β every row in these tables comes from
|
| 592 |
-
# Odoo, which is what R6 is about;
|
| 593 |
-
#
|
| 594 |
-
#
|
|
|
|
|
|
|
|
|
|
| 595 |
_ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES])
|
| 596 |
GRID_SOURCES = {} # the LOCAL registry this builder fills and returns
|
| 597 |
_s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)),
|
|
@@ -734,6 +745,186 @@ def _odoo_sources():
|
|
| 734 |
"THEN '1' ELSE '' END", _s),
|
| 735 |
},
|
| 736 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 737 |
return GRID_SOURCES
|
| 738 |
|
| 739 |
|
|
@@ -918,11 +1109,52 @@ def _degrade(expr, have_by_alias, default_alias=""):
|
|
| 918 |
|
| 919 |
|
| 920 |
def _source_for(cur, table_key):
|
| 921 |
-
"""The resolved spec for one connected grid, or None when this grid is not read-through yet.
|
| 922 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 923 |
if not spec:
|
| 924 |
return None
|
| 925 |
from harness import datastore
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 926 |
# β `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table
|
| 927 |
# spec declares none and its columns are bare, so it degrades against `table` as before.
|
| 928 |
aliases = dict(spec.get("tables") or {})
|
|
@@ -1284,6 +1516,301 @@ def _related_source_value(session, entry, record):
|
|
| 1284 |
return None
|
| 1285 |
|
| 1286 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1287 |
def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
|
| 1288 |
"""`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned.
|
| 1289 |
|
|
@@ -1309,6 +1836,30 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1309 |
rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`,
|
| 1310 |
so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same
|
| 1311 |
expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1312 |
"""
|
| 1313 |
import core.relations as relations
|
| 1314 |
|
|
@@ -1324,13 +1875,18 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1324 |
source = str(related.get("table") or "")
|
| 1325 |
record = related.get("record")
|
| 1326 |
via = related.get("via")
|
| 1327 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1328 |
raise err(400, "bad_relation",
|
| 1329 |
"a pinned read needs the record's database, the record id and the relation to "
|
| 1330 |
-
"follow, as {table, record, via}
|
|
|
|
| 1331 |
+ "/related for the relations this account may follow")
|
| 1332 |
|
| 1333 |
-
from routes_grid import _may_open, related_entries
|
| 1334 |
|
| 1335 |
if not _may_open(session, source):
|
| 1336 |
# β THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False
|
|
@@ -1346,8 +1902,20 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1346 |
# β RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be
|
| 1347 |
# offered is an edge you cannot pin" structural rather than remembered. `related_entries` is
|
| 1348 |
# the same function `GET /grid/{table}/related` answers with; there is no second resolution.
|
| 1349 |
-
|
| 1350 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1351 |
if entry is None:
|
| 1352 |
raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
|
| 1353 |
entry = dict(entry, __source__=source)
|
|
@@ -1361,12 +1929,33 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1361 |
if not entry.get("served", True):
|
| 1362 |
raise err(409, "relation_not_served", relations.unserved_target_refusal(entry))
|
| 1363 |
|
| 1364 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1365 |
# β `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value
|
| 1366 |
# this response returns: a relation must keep working when an administrator has hidden the id
|
| 1367 |
# column it runs over, and no cell of it is ever projected. The hidden-field closure still
|
| 1368 |
# governs everything that reaches the wire, one layer up.
|
| 1369 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1370 |
if not binding:
|
| 1371 |
raise err(409, "relation_not_served",
|
| 1372 |
f"'{table_key}' has no live expression for '{on}', which is the column this "
|
|
@@ -1374,14 +1963,23 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1374 |
f"this database directly, or ask for '{on}' to be added to its live connection")
|
| 1375 |
on_expr, on_cast = binding[0], binding[1]
|
| 1376 |
|
| 1377 |
-
|
| 1378 |
-
|
| 1379 |
-
|
| 1380 |
-
|
| 1381 |
-
|
| 1382 |
-
|
| 1383 |
-
|
| 1384 |
-
value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1385 |
|
| 1386 |
# ββ the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ββββββββββββββββ
|
| 1387 |
# β NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer
|
|
@@ -1391,22 +1989,27 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1391 |
# β THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can
|
| 1392 |
# produce is already narrowed to rows they may see, so the pin cannot be used to learn how many
|
| 1393 |
# rows exist beyond their boundary.
|
|
|
|
|
|
|
|
|
|
| 1394 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 1395 |
-
|
| 1396 |
-
|
| 1397 |
-
|
| 1398 |
-
|
| 1399 |
-
|
| 1400 |
-
|
|
|
|
|
|
|
|
|
|
| 1401 |
|
| 1402 |
-
notes = []
|
| 1403 |
# β STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited
|
| 1404 |
# here; the reader is told that this particular relation is answered by reading the source
|
| 1405 |
# database once per page, because it is keyed on a column rather than on the record id.
|
| 1406 |
-
if str((
|
| 1407 |
notes.append({
|
| 1408 |
"subject": "related", "effect": "slow",
|
| 1409 |
-
"cause": f"this relation is keyed on the '{
|
| 1410 |
f"'{source}' rather than on its record id, so every page of these rows is "
|
| 1411 |
f"resolved by reading that database first",
|
| 1412 |
"recommendation": "read this database on its own and filter it on "
|
|
@@ -1423,6 +2026,279 @@ def _related_pin(table_key, session, spec, base_where, wall_params, cur, related
|
|
| 1423 |
return sql, params, notes
|
| 1424 |
|
| 1425 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1426 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 1427 |
search=None, tree=_TREE_UNSET, related=None):
|
| 1428 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
|
@@ -1516,9 +2392,17 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 1516 |
|
| 1517 |
spec = _source_for(cur, table_key)
|
| 1518 |
if spec is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1519 |
raise err(409, "not_read_through",
|
| 1520 |
-
"this connected database
|
| 1521 |
-
"read
|
|
|
|
| 1522 |
|
| 1523 |
cols = _filter_columns(spec, fields)
|
| 1524 |
|
|
@@ -1636,8 +2520,20 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 1636 |
"recommendation": "ask an administrator to set the restriction on a whole number, "
|
| 1637 |
"or on a column that is not an amount"})
|
| 1638 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1639 |
# ββ the predicate, pushed down ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1640 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1641 |
missing = sorted(named - set(cols))
|
| 1642 |
if missing:
|
| 1643 |
limits.append({
|
|
@@ -1687,9 +2583,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
|
|
| 1687 |
|
| 1688 |
where, params = base_where, list(wall_params)
|
| 1689 |
try:
|
|
|
|
|
|
|
|
|
|
| 1690 |
pred = _fs().compile_filter_tree(
|
| 1691 |
tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols,
|
| 1692 |
-
today=_today())
|
| 1693 |
except ValueError as e:
|
| 1694 |
# β A REFUSAL IS AN ANSWER, NOT A CRASH β and it must not become "no filter". Ranking ops
|
| 1695 |
# ("top 10") have no SQL form in this compiler; saying so beats returning every row.
|
|
|
|
| 24 |
import json
|
| 25 |
import time
|
| 26 |
|
| 27 |
+
from fastapi import Body, Depends, HTTPException
|
| 28 |
from fastapi import APIRouter, Query
|
| 29 |
|
| 30 |
+
from deps import (Session, err, module_cache_rows, module_cache_source, require_session)
|
| 31 |
|
| 32 |
router = APIRouter(prefix="/api/v1")
|
| 33 |
|
|
|
|
| 507 |
# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as
|
| 508 |
# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]).
|
| 509 |
#
|
| 510 |
+
# ββ W53-T19 / RULING R5 β EVERY ODOO BUCKET IS BOUND HERE NOW, AND THAT IS THE WHOLE TICKET.
|
| 511 |
+
# R5: *"Every `ut_odoo_*` table becomes read-through bound. The stored-copy refusal is deleted,
|
| 512 |
+
# not hidden."* Before this the registry held four of the ten declared keys
|
| 513 |
+
# (orders, order lines, GL lines, accounts) and the other four DECLARED grains β invoices, agents,
|
| 514 |
+
# vendors, bills β answered `409 not_read_through` on the rows door, which is what every Linked
|
| 515 |
+
# database tab onto them showed a person. They are bound below, off the SAME SQL their readers in
|
| 516 |
+
# `odoo_relational` already run, so the differential oracle has something to hold them to.
|
| 517 |
+
#
|
| 518 |
+
# β BINDING IS NOT UN-MATERIALISING, and conflating the two would delete rows six link rollups
|
| 519 |
+
# still fold. `_unmaterialisable` is the separate, gated decision about who may STOP STORING rows,
|
| 520 |
+
# and none of the four newly bound grains becomes eligible by being bound: each is small enough to
|
| 521 |
+
# live in the document and each is folded by a link declared on another table. What changes is
|
| 522 |
+
# that the WINDOW door can now answer them, which is what a relation pin needs.
|
| 523 |
|
| 524 |
#: `ut_*` key -> how to read that grid straight out of the mirror.
|
| 525 |
#:
|
|
|
|
| 597 |
# β R6 / W30-T29 β TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops
|
| 598 |
# being a fact about them. `core` never imports up, so the declaration goes this way round.
|
| 599 |
# β ALL EIGHT, not just the read-through-bound ones β every row in these tables comes from
|
| 600 |
+
# Odoo, which is what R6 is about; where its rows are SERVED FROM is our conversion state, not
|
| 601 |
+
# a property of the data. `_sources`' own `register_connected` covers only the keys a provider
|
| 602 |
+
# RETURNS, so this call is not redundant with it and must not be folded into it.
|
| 603 |
+
# β W53-T19: the two sets now coincide (every `TABLES` key is bound below), and the call still
|
| 604 |
+
# stands rather than being deleted as redundant β a grain declared but not yet bound is the
|
| 605 |
+
# normal state of the next connector, and this is the line that keeps it uncapped meanwhile.
|
| 606 |
_ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES])
|
| 607 |
GRID_SOURCES = {} # the LOCAL registry this builder fills and returns
|
| 608 |
_s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)),
|
|
|
|
| 745 |
"THEN '1' ELSE '' END", _s),
|
| 746 |
},
|
| 747 |
}
|
| 748 |
+
|
| 749 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 750 |
+
# ββ W53-T19 / RULING R5 β THE LAST FOUR DECLARED GRAINS: invoices, agents, vendors, bills.
|
| 751 |
+
#
|
| 752 |
+
# Each one is the SQL its reader in `odoo_relational` already runs, restated as a window. That
|
| 753 |
+
# is a second statement of one question and it is gated rather than trusted: `verify_scopes`'s
|
| 754 |
+
# `section_unmaterialise` holds every binding against its own `*_fields()` declaration in BOTH
|
| 755 |
+
# directions (a `cols` key with no declaration is a silent no-cell, because `rows_from_pool`
|
| 756 |
+
# projects strictly; a declaration with no `cols` key is an INACTIVE filter leaf, which
|
| 757 |
+
# WIDENS), and `section_read_through` is the differential oracle shape for the row values.
|
| 758 |
+
#
|
| 759 |
+
# ββ NOT ONE `read_*` STATEMENT WAS TOUCHED TO MAKE THESE FIT, AND THAT IS A HARD RULE HERE.
|
| 760 |
+
# `verify_odoo_relational.FixtureCursor` DISPATCHES ON THE SELECT-CLAUSE HEAD, because
|
| 761 |
+
# `read_bills` reads `account_move` with a different projection and the opposite move types
|
| 762 |
+
# from `read_invoices`; aliasing either projection once already made the fixture recognise
|
| 763 |
+
# nothing and return a CONFIDENT EMPTY on every box with no mirror, six legs red. The specs
|
| 764 |
+
# below are new SQL beside those readers, never a rewrite of them.
|
| 765 |
+
#
|
| 766 |
+
# β THE PREDICATES ARE **IMPORTED**, NEVER RETYPED. `_POSTED_DOCS` and `_VENDOR_DOCS` are the
|
| 767 |
+
# only statement of which `account.move` documents belong to which grain, and a fifth copy of
|
| 768 |
+
# `move_type IN (...)` is how a grid and its reader start describing different populations.
|
| 769 |
+
# β They open with the bare column `state`, and they are left UNQUALIFIED on purpose: the one
|
| 770 |
+
# joined spec here (invoices) joins only a derived commission aggregate, which carries no
|
| 771 |
+
# `state`, `move_type` or `payment_state` column, so SQL resolution is unambiguous. Prefixing
|
| 772 |
+
# `am.` would qualify only the FIRST term of a two-term predicate, which is worse than not
|
| 773 |
+
# qualifying at all.
|
| 774 |
+
inv_key, ag_key = rel.INVOICES_KEY, rel.AGENTS_KEY
|
| 775 |
+
ven_key, bill_key = rel.VENDORS_KEY, rel.BILLS_KEY
|
| 776 |
+
_ut().register_connected(inv_key, ag_key, ven_key, bill_key)
|
| 777 |
+
|
| 778 |
+
# `checkbox` cells are the overlay's `'1'` / `''` convention (`odoo_relational._in_scope`), so
|
| 779 |
+
# a boolean column is coerced HERE rather than being CASEd in SQL: read as a bare column it
|
| 780 |
+
# goes through `_degrade`, which answers NULL when the mirror has not backfilled it, and NULL
|
| 781 |
+
# then lands as an honestly unticked cell instead of a Binder error.
|
| 782 |
+
_flag = (lambda v: "1" if v else "")
|
| 783 |
+
# β BLANK, NOT ZERO, AND `read_invoices` SAYS WHY: an invoice carrying more than one agent has
|
| 784 |
+
# NO single agent id, and `int(None or 0)` would render `0` β a plausible, wrong, clickable
|
| 785 |
+
# foreign key. The reader writes `""` for exactly that row and so does this.
|
| 786 |
+
_i_blank = (lambda v: "" if v in (None, "") else int(v))
|
| 787 |
+
_s_strip = (lambda v: str(v or "").strip())
|
| 788 |
+
|
| 789 |
+
# ββ INVOICES βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 790 |
+
# `read_invoices`' default projection: every POSTED customer document, the channel TAGGED
|
| 791 |
+
# rather than dropped. The commission aggregate is the reader's own subquery, verbatim, and it
|
| 792 |
+
# is the reason this grain needs `optional`: `account_invoice_line_agent` is an OCA module
|
| 793 |
+
# entity a mirror hydrated from an older seed can lack entirely, and the reader degrades to
|
| 794 |
+
# blank agent columns rather than refusing. So does this.
|
| 795 |
+
_AGENT_FOLD = (
|
| 796 |
+
"(SELECT invoice_id, count(DISTINCT agent_id) AS n_agents, "
|
| 797 |
+
" min(agent_id) AS one_agent, "
|
| 798 |
+
" string_agg(DISTINCT agent_name, ', ' ORDER BY agent_name) AS agent_names "
|
| 799 |
+
" FROM account_invoice_line_agent WHERE invoice_id IS NOT NULL GROUP BY invoice_id) ag")
|
| 800 |
+
GRID_SOURCES[inv_key] = {
|
| 801 |
+
"from_sql": f"(account_move am LEFT JOIN {_AGENT_FOLD} ON ag.invoice_id = am.id)",
|
| 802 |
+
# β `ag` IS ABSENT FROM THIS MAP ON PURPOSE: it is a derived alias, not a mirror table, so
|
| 803 |
+
# `_degrade` must leave its projections alone rather than look for a `PRAGMA table_info`
|
| 804 |
+
# that does not exist. `_source_for` refuses a spec whose declared tables are missing, and
|
| 805 |
+
# declaring `ag` here would turn a degradable absence into no binding at all.
|
| 806 |
+
"tables": {"am": "account_move"},
|
| 807 |
+
"where": f"{rel._POSTED_DOCS} AND partner_id IS NOT NULL",
|
| 808 |
+
"id": "am.id",
|
| 809 |
+
"needs_excluded": True,
|
| 810 |
+
"cols": {
|
| 811 |
+
"invoice_no": ("am.name", _s),
|
| 812 |
+
"odoo_id": ("am.id", _i),
|
| 813 |
+
"customer": ("am.partner_name", _s),
|
| 814 |
+
rel.JOIN_KEY: ("am.partner_id", _i),
|
| 815 |
+
"invoice_date": ("am.invoice_date", rel._as_date),
|
| 816 |
+
"due_date": ("am.invoice_date_due", rel._as_date),
|
| 817 |
+
# β THE SIGNED PAIR, exactly as the reader reads them: Odoo's `_signed` fields already
|
| 818 |
+
# carry a refund's direction, so nothing here re-derives a sign.
|
| 819 |
+
"residual": ("am.amount_residual_signed", _n),
|
| 820 |
+
"amount_untaxed": ("am.amount_untaxed_signed", _n),
|
| 821 |
+
"payment_state": ("am.payment_state", _s),
|
| 822 |
+
"move_type": ("am.move_type", _s),
|
| 823 |
+
"origin_order": ("am.invoice_origin", _s_strip),
|
| 824 |
+
"agent": ("ag.agent_names", _s),
|
| 825 |
+
rel.AGENT_JOIN_KEY: ("CASE WHEN ag.n_agents = 1 THEN ag.one_agent END", _i_blank),
|
| 826 |
+
"wholesale_scope": (
|
| 827 |
+
"CASE WHEN am.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s),
|
| 828 |
+
},
|
| 829 |
+
"optional": (
|
| 830 |
+
{"absent": (("account_invoice_line_agent",),),
|
| 831 |
+
"without": {"from_sql": "account_move am",
|
| 832 |
+
"cols": {"agent": ("''", _s),
|
| 833 |
+
rel.AGENT_JOIN_KEY: ("CAST(NULL AS BIGINT)", _i_blank)}}},
|
| 834 |
+
),
|
| 835 |
+
}
|
| 836 |
+
|
| 837 |
+
# ββ AGENTS βββββββββββοΏ½οΏ½οΏ½βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 838 |
+
# β THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES and `agent_fields`' docstring holds
|
| 839 |
+
# the measurement: 16 partners carry commission lines, 17 carry `res_partner.agent`, the union
|
| 840 |
+
# is 19. Either source alone silently drops real agents, so BOTH legs are here and each one is
|
| 841 |
+
# separately optional. The three `optional` entries are the three degraded mirrors, and the
|
| 842 |
+
# combined one is written LAST because entries apply in order and the last merge wins.
|
| 843 |
+
_COMMISSIONED = ("id IN (SELECT agent_id FROM account_invoice_line_agent "
|
| 844 |
+
" WHERE agent_id IS NOT NULL)")
|
| 845 |
+
GRID_SOURCES[ag_key] = {
|
| 846 |
+
"table": "res_partner",
|
| 847 |
+
"where": f"({_COMMISSIONED} OR COALESCE(agent, FALSE))",
|
| 848 |
+
"id": "id",
|
| 849 |
+
"cols": {
|
| 850 |
+
"agent": ("name", _s),
|
| 851 |
+
"odoo_id": ("id", _i),
|
| 852 |
+
rel.AGENT_JOIN_KEY: ("id", _i),
|
| 853 |
+
"flagged": ("agent", _flag),
|
| 854 |
+
"commissioned": (f"CASE WHEN {_COMMISSIONED} THEN '1' ELSE '' END", _s),
|
| 855 |
+
},
|
| 856 |
+
"optional": (
|
| 857 |
+
{"absent": (("account_invoice_line_agent",),),
|
| 858 |
+
"without": {"where": "COALESCE(agent, FALSE)",
|
| 859 |
+
"cols": {"commissioned": ("''", _s)}}},
|
| 860 |
+
{"absent": (("res_partner", "agent"),),
|
| 861 |
+
"without": {"where": _COMMISSIONED, "cols": {"flagged": ("''", _s)}}},
|
| 862 |
+
# β NEITHER SOURCE β NO POPULATION, SPELLED `1=0` RATHER THAN LEFT EMPTY. An empty
|
| 863 |
+
# `where` is EVERY partner, which is 3,649 customers wearing the Agents label; the
|
| 864 |
+
# reader answers `[]` on the same mirror and so must this.
|
| 865 |
+
{"absent": (("account_invoice_line_agent",), ("res_partner", "agent")),
|
| 866 |
+
"without": {"where": "1=0",
|
| 867 |
+
"cols": {"flagged": ("''", _s), "commissioned": ("''", _s)}}},
|
| 868 |
+
),
|
| 869 |
+
}
|
| 870 |
+
|
| 871 |
+
# ββ VENDORS ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 872 |
+
# β DERIVED FROM THE BILLS, exactly as `read_vendors` is, and the asymmetry with customers is
|
| 873 |
+
# deliberate there: a partner with no posted bill has no payable history to show. MEASURED in
|
| 874 |
+
# that reader's docstring: 393 vendors, 0 bills with a null partner, nothing dangles.
|
| 875 |
+
# β ALIASED `p` AND EVERY COLUMN QUALIFIED, matching the reader, because `zip` unqualified is
|
| 876 |
+
# a bare identifier a SQL dialect is free to read as something else.
|
| 877 |
+
GRID_SOURCES[ven_key] = {
|
| 878 |
+
"from_sql": "res_partner p",
|
| 879 |
+
"tables": {"p": "res_partner"},
|
| 880 |
+
"where": (f"p.id IN (SELECT partner_id FROM account_move WHERE {rel._VENDOR_DOCS} "
|
| 881 |
+
" AND partner_id IS NOT NULL)"),
|
| 882 |
+
"id": "p.id",
|
| 883 |
+
"cols": {
|
| 884 |
+
"vendor": ("p.name", _s),
|
| 885 |
+
"odoo_id": ("p.id", _i),
|
| 886 |
+
rel.VENDOR_JOIN_KEY: ("p.id", _i),
|
| 887 |
+
"country": ("p.country_name", _s),
|
| 888 |
+
"email": ("p.email", _s),
|
| 889 |
+
"phone": ("p.phone", _s),
|
| 890 |
+
"mobile": ("p.mobile", _s),
|
| 891 |
+
"website": ("p.website", _s),
|
| 892 |
+
"vat": ("p.vat", _s),
|
| 893 |
+
# β `vendor_ref` HERE, `ref` IN THE MIRROR, and `vendor_fields` explains the rename:
|
| 894 |
+
# the bills grid already spends `ref` on the vendor's own invoice number.
|
| 895 |
+
"vendor_ref": ("p.ref", _s),
|
| 896 |
+
"street": ("p.street", _s),
|
| 897 |
+
"street2": ("p.street2", _s),
|
| 898 |
+
"city": ("p.city", _s),
|
| 899 |
+
"zip": ("p.zip", _s),
|
| 900 |
+
},
|
| 901 |
+
# `account_move` is a SECONDARY dependency here (it defines the population, it projects
|
| 902 |
+
# nothing), so its absence degrades to an empty grid rather than to no binding. The reader
|
| 903 |
+
# answers `[]` on that mirror for the same reason.
|
| 904 |
+
"optional": ({"absent": (("account_move",),), "without": {"where": "1=0"}},),
|
| 905 |
+
}
|
| 906 |
+
|
| 907 |
+
# ββ BILLS ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 908 |
+
# β DOCUMENT GRAIN, and `bill_fields` records why it is the only one that fits: 6,538 bills
|
| 909 |
+
# against 154,917 expense GL lines. This is the payable; the line ledger is `ut_odoo_gl_lines`
|
| 910 |
+
# above, and they are different tables answering different questions.
|
| 911 |
+
GRID_SOURCES[bill_key] = {
|
| 912 |
+
"table": "account_move",
|
| 913 |
+
"where": f"{rel._VENDOR_DOCS} AND partner_id IS NOT NULL",
|
| 914 |
+
"id": "id",
|
| 915 |
+
"cols": {
|
| 916 |
+
"bill_no": ("name", _s),
|
| 917 |
+
"odoo_id": ("id", _i),
|
| 918 |
+
"vendor": ("partner_name", _s),
|
| 919 |
+
rel.VENDOR_JOIN_KEY: ("partner_id", _i),
|
| 920 |
+
"invoice_date": ("invoice_date", rel._as_date),
|
| 921 |
+
"due_date": ("invoice_date_due", rel._as_date),
|
| 922 |
+
"amount_untaxed": ("amount_untaxed_signed", _n),
|
| 923 |
+
"residual": ("amount_residual_signed", _n),
|
| 924 |
+
"payment_state": ("payment_state", _s),
|
| 925 |
+
"move_type": ("move_type", _s),
|
| 926 |
+
},
|
| 927 |
+
}
|
| 928 |
return GRID_SOURCES
|
| 929 |
|
| 930 |
|
|
|
|
| 1109 |
|
| 1110 |
|
| 1111 |
def _source_for(cur, table_key):
|
| 1112 |
+
"""The resolved spec for one connected grid, or None when this grid is not read-through yet.
|
| 1113 |
+
|
| 1114 |
+
ββ W53-T19 β `optional` RESOLVES A SPEC'S **SECONDARY** MIRROR DEPENDENCIES, and it exists
|
| 1115 |
+
because R5 asks every Odoo grain to be bound while two of them read a table or a column a
|
| 1116 |
+
mirror can honestly lack. `_degrade` already answers the missing-COLUMN case for a BARE
|
| 1117 |
+
projection; it cannot answer a missing TABLE named in the FROM or in the WHERE, and a SELECT
|
| 1118 |
+
naming an absent table is a DuckDB Binder error reaching a user as a bare 500 (D-107's shape).
|
| 1119 |
+
|
| 1120 |
+
An entry is `{"absent": ((table[, column]), ...), "without": {<partial spec>}}`: when EVERY
|
| 1121 |
+
requirement it names is absent from the mirror, its `without` is merged over the spec, with
|
| 1122 |
+
`cols` merged key by key rather than replacing the map. Entries apply IN DECLARATION ORDER, so
|
| 1123 |
+
a combined case (both of `read_agents`' two sources missing) is written last and wins.
|
| 1124 |
+
|
| 1125 |
+
β DEGRADE, NEVER REFUSE, which is the posture `odoo_relational.columns` already sets for the
|
| 1126 |
+
readers: without its commission table the invoices grid serves blank agent cells, not a 409,
|
| 1127 |
+
and `read_invoices` degrades the same way over the same absence. β THE **BASE** TABLE IS A
|
| 1128 |
+
DIFFERENT QUESTION and keeps its existing answer: `tables`/`table` must all resolve, and a
|
| 1129 |
+
spec whose base table the mirror lacks is no binding at all (None, and the caller refuses).
|
| 1130 |
+
"""
|
| 1131 |
+
key = str(table_key or "")
|
| 1132 |
+
# Registry-module pools are databases, but never mirror bindings. In particular,
|
| 1133 |
+
# `product_data` must be reached through its scoped product pool after a relation has resolved
|
| 1134 |
+
# target ids; letting it fall into `_sources()` makes a cache alias look like an absent Odoo
|
| 1135 |
+
# table and turns Products ordered into the wrong 404.
|
| 1136 |
+
if module_cache_source(key) is not None:
|
| 1137 |
+
return None
|
| 1138 |
+
spec = _sources().get(key)
|
| 1139 |
if not spec:
|
| 1140 |
return None
|
| 1141 |
from harness import datastore
|
| 1142 |
+
for opt in (spec.get("optional") or ()):
|
| 1143 |
+
wants = tuple(opt.get("absent") or ())
|
| 1144 |
+
if not wants:
|
| 1145 |
+
continue
|
| 1146 |
+
gone = True
|
| 1147 |
+
for need in wants:
|
| 1148 |
+
need = tuple(need)
|
| 1149 |
+
have = datastore.columns_of(str(need[0]), cur=cur)
|
| 1150 |
+
if have and (len(need) < 2 or str(need[1]).lower() in have):
|
| 1151 |
+
gone = False
|
| 1152 |
+
break
|
| 1153 |
+
if not gone:
|
| 1154 |
+
continue
|
| 1155 |
+
without = dict(opt.get("without") or {})
|
| 1156 |
+
merged = {**spec["cols"], **(without.pop("cols", None) or {})}
|
| 1157 |
+
spec = {**spec, **without, "cols": merged}
|
| 1158 |
# β `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table
|
| 1159 |
# spec declares none and its columns are bare, so it degrades against `table` as before.
|
| 1160 |
aliases = dict(spec.get("tables") or {})
|
|
|
|
| 1516 |
return None
|
| 1517 |
|
| 1518 |
|
| 1519 |
+
def _cache_relation_plan(session, source, via, target):
|
| 1520 |
+
"""The offered module link and its explicit bridge, or a cause-specific refusal.
|
| 1521 |
+
|
| 1522 |
+
A module/cache target is resolved from its module field declaration because the public
|
| 1523 |
+
relation entry intentionally carries only the edge needed by ordinary one-hop pins. The
|
| 1524 |
+
`through` metadata is not a second inferred graph: it is the exact link bag that declared
|
| 1525 |
+
Products ordered. Resolving against `related_entries` first keeps hidden link fields outside
|
| 1526 |
+
this door just as they are outside the drawer's tab list.
|
| 1527 |
+
"""
|
| 1528 |
+
import core.relations as relations
|
| 1529 |
+
from routes_grid import _may_open, related_entries
|
| 1530 |
+
|
| 1531 |
+
source = str(source or "")
|
| 1532 |
+
target = str(target or "")
|
| 1533 |
+
via = str(via or "")
|
| 1534 |
+
if relations.is_retired_table(target):
|
| 1535 |
+
raise err(404, "relation_target_retired",
|
| 1536 |
+
f"'{target}' is retired and is no longer a relation target in this workspace")
|
| 1537 |
+
if module_cache_source(target) is None:
|
| 1538 |
+
raise err(404, "relation_target_missing",
|
| 1539 |
+
f"'{target}' is not a current module/cache relation target in this workspace")
|
| 1540 |
+
if not _may_open(session, source):
|
| 1541 |
+
raise err(403, "not_permitted",
|
| 1542 |
+
f"'{source}' could not be opened for your account, so this relation cannot "
|
| 1543 |
+
"be followed")
|
| 1544 |
+
|
| 1545 |
+
offered = next((entry for entry in related_entries(session, source)
|
| 1546 |
+
if str(entry.get("key") or "") == via
|
| 1547 |
+
and str(entry.get("target") or "") == target
|
| 1548 |
+
and str((entry.get("via") or {}).get("direction") or "") == "forward"),
|
| 1549 |
+
None)
|
| 1550 |
+
if offered is None:
|
| 1551 |
+
raise err(400, "unknown_relation",
|
| 1552 |
+
f"'{via}' is not an available relation from '{source}' to '{target}'")
|
| 1553 |
+
|
| 1554 |
+
field = next((item for item in _rel().module_link_fields(source)
|
| 1555 |
+
if str(item.get("key") or "") == via), None)
|
| 1556 |
+
link = (field or {}).get("link") if isinstance(field, dict) else None
|
| 1557 |
+
if not isinstance(link, dict) or str(link.get("table") or "") != target:
|
| 1558 |
+
raise err(404, "relation_target_missing",
|
| 1559 |
+
f"the relation '{via}' no longer has a current target in this workspace")
|
| 1560 |
+
through = str(link.get("through") or "")
|
| 1561 |
+
through_on = str(link.get("throughOn") or "")
|
| 1562 |
+
through_to = str(link.get("throughTo") or "")
|
| 1563 |
+
on = str(link.get("on") or relations.IDENTITY_COL)
|
| 1564 |
+
if not through or not through_on or not through_to:
|
| 1565 |
+
raise err(409, "relation_target_unavailable",
|
| 1566 |
+
f"the relation '{via}' has no complete stored path to '{target}' right now")
|
| 1567 |
+
if relations.is_retired_table(through):
|
| 1568 |
+
raise err(404, "relation_target_retired",
|
| 1569 |
+
f"'{through}' is retired, so the relation to '{target}' is no longer available")
|
| 1570 |
+
if not _may_open(session, through):
|
| 1571 |
+
raise err(403, "forbidden_table",
|
| 1572 |
+
f"the relation to '{target}' crosses '{through}', which your account cannot "
|
| 1573 |
+
"open")
|
| 1574 |
+
if not _may_open(session, target):
|
| 1575 |
+
raise err(403, "forbidden_table",
|
| 1576 |
+
f"the relation ends at '{target}', which your account cannot open")
|
| 1577 |
+
return dict(offered, __source__=source), link, through, through_on, through_to, on
|
| 1578 |
+
|
| 1579 |
+
|
| 1580 |
+
def _cache_relation_source_value(session, entry, record):
|
| 1581 |
+
"""Read a cache-backed source value only from a row visible to this session."""
|
| 1582 |
+
import core.relations as relations
|
| 1583 |
+
|
| 1584 |
+
source = str(entry.get("__source__") or "")
|
| 1585 |
+
if module_cache_source(source) is None:
|
| 1586 |
+
return _related_source_value(session, entry, record)
|
| 1587 |
+
try:
|
| 1588 |
+
rows = module_cache_rows(session, source)
|
| 1589 |
+
except HTTPException:
|
| 1590 |
+
raise
|
| 1591 |
+
except Exception as exc: # noqa: BLE001
|
| 1592 |
+
raise err(503, "relation_source_unavailable",
|
| 1593 |
+
f"'{source}' could not be read right now; try this relation again") from exc
|
| 1594 |
+
row = next((item for item in rows if str(item.get("pid")) == str(record)), None)
|
| 1595 |
+
if row is None:
|
| 1596 |
+
raise err(404, "relation_source_missing",
|
| 1597 |
+
f"record {record} is no longer available in '{source}'")
|
| 1598 |
+
frm = str((entry.get("via") or {}).get("from") or relations.IDENTITY_COL)
|
| 1599 |
+
value = row.get("pid") if frm == relations.IDENTITY_COL else row.get(frm)
|
| 1600 |
+
return value if value not in (None, "") else None
|
| 1601 |
+
|
| 1602 |
+
|
| 1603 |
+
def _cache_relation_target_ids(session, source, record, via, target):
|
| 1604 |
+
"""Resolve one explicit source -> mirror bridge -> module-cache path to target row pids."""
|
| 1605 |
+
entry, _link, through, through_on, through_to, on = _cache_relation_plan(
|
| 1606 |
+
session, source, via, target)
|
| 1607 |
+
source_value = _cache_relation_source_value(session, entry, record)
|
| 1608 |
+
if source_value in (None, ""):
|
| 1609 |
+
raise err(400, "unknown_relation",
|
| 1610 |
+
f"record {record} has no readable source value for relation '{via}'")
|
| 1611 |
+
|
| 1612 |
+
compiled = _compiled_read(through, session)
|
| 1613 |
+
spec = compiled["spec"]
|
| 1614 |
+
left = (spec.get("cols") or {}).get(through_on)
|
| 1615 |
+
right = (spec.get("cols") or {}).get(through_to)
|
| 1616 |
+
if not left or not right:
|
| 1617 |
+
raise err(409, "relation_target_unavailable",
|
| 1618 |
+
f"the stored path through '{through}' is missing a required join column")
|
| 1619 |
+
try:
|
| 1620 |
+
source_value = left[1](source_value)
|
| 1621 |
+
except Exception: # noqa: BLE001
|
| 1622 |
+
source_value = str(source_value)
|
| 1623 |
+
|
| 1624 |
+
frm = spec.get("from_sql") or spec.get("table")
|
| 1625 |
+
where = str(compiled.get("where") or "").strip()
|
| 1626 |
+
params = list(compiled.get("params") or ())
|
| 1627 |
+
predicate = f"({where}) AND " if where else ""
|
| 1628 |
+
sql = (f"SELECT DISTINCT {right[0]} AS related_value FROM {frm} WHERE "
|
| 1629 |
+
f"{predicate}{left[0]} = ? AND {right[0]} IS NOT NULL")
|
| 1630 |
+
values = {str(row[0]) for row in compiled["cur"].execute(
|
| 1631 |
+
sql, params + [source_value]).fetchall() if row and row[0] not in (None, "")}
|
| 1632 |
+
|
| 1633 |
+
try:
|
| 1634 |
+
target_rows = module_cache_rows(session, target)
|
| 1635 |
+
except HTTPException:
|
| 1636 |
+
raise
|
| 1637 |
+
except Exception as exc: # noqa: BLE001
|
| 1638 |
+
raise err(503, "relation_target_unavailable",
|
| 1639 |
+
f"'{target}' could not be read right now; try this relation again") from exc
|
| 1640 |
+
ids = []
|
| 1641 |
+
seen = set()
|
| 1642 |
+
for row in target_rows:
|
| 1643 |
+
if str(row.get(on)) not in values:
|
| 1644 |
+
continue
|
| 1645 |
+
try:
|
| 1646 |
+
pid = int(row.get("pid"))
|
| 1647 |
+
except (TypeError, ValueError):
|
| 1648 |
+
continue
|
| 1649 |
+
if pid > 0 and pid not in seen:
|
| 1650 |
+
seen.add(pid)
|
| 1651 |
+
ids.append(pid)
|
| 1652 |
+
return {"ok": True, "sourceKind": "module_cache", "target": str(target),
|
| 1653 |
+
"ids": ids, "path": [str(source), through, str(target)]}
|
| 1654 |
+
|
| 1655 |
+
|
| 1656 |
+
@router.get("/odoo-tables/relation-targets")
|
| 1657 |
+
def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge=1),
|
| 1658 |
+
via: str = Query(...), target: str = Query(...),
|
| 1659 |
+
session: Session = Depends(require_session)):
|
| 1660 |
+
"""Target pids for a declared relation whose destination is a runtime-cache database."""
|
| 1661 |
+
return _cache_relation_target_ids(session, source, record, via, target)
|
| 1662 |
+
|
| 1663 |
+
|
| 1664 |
+
def _via_path(via):
|
| 1665 |
+
"""The list of EDGE KEYS a `via` names, or None when it is one edge's bag instead.
|
| 1666 |
+
|
| 1667 |
+
ββ W54-T40 / R12 β *"the client pins one by naming each edge (`via = ["orders",
|
| 1668 |
+
"order_lines"]`)"*. THREE SPELLINGS ARRIVE AND ALL THREE ARE THE SAME THING, which is why
|
| 1669 |
+
exactly one function decides:
|
| 1670 |
+
|
| 1671 |
+
* an ARRAY of keys, which is what `RecordDetail.tsx::relatedPinKey` sends for a two-hop tab
|
| 1672 |
+
and what `verify_wiring.py` pins that client literal to;
|
| 1673 |
+
* a bare STRING, C3's back-compat leg, which `relations.pick_path` already reads as a
|
| 1674 |
+
one-element path;
|
| 1675 |
+
* the two-hop entry's own `via` BAG, `{direction: 'path', key, path}`, which is what a client
|
| 1676 |
+
holding the whole entry naturally hands back. Reading `path` out of it costs one key and
|
| 1677 |
+
removes an entire class of "the shape was almost right" refusal.
|
| 1678 |
+
|
| 1679 |
+
β A ONE-HOP BAG STAYS A BAG AND ANSWERS None. `relations.pick` is the resolver for that shape
|
| 1680 |
+
and it matches on `direction` plus `key`; routing it here instead would resolve a one-hop pin
|
| 1681 |
+
against the depth-aware listing for no gain and change the cost of the six tabs that already
|
| 1682 |
+
worked. β AN EMPTY OR PARTLY-BLANK PATH IS **NOT** A PATH: it answers None and falls into the
|
| 1683 |
+
`bad_relation` guard, which is the honest refusal for `via = []`.
|
| 1684 |
+
"""
|
| 1685 |
+
if isinstance(via, str):
|
| 1686 |
+
return [via] if via.strip() else None
|
| 1687 |
+
if isinstance(via, (list, tuple)):
|
| 1688 |
+
keys = [str(v).strip() for v in via]
|
| 1689 |
+
return keys if keys and all(keys) else None
|
| 1690 |
+
if isinstance(via, dict) and isinstance(via.get("path"), (list, tuple)):
|
| 1691 |
+
keys = [str(v).strip() for v in via["path"]]
|
| 1692 |
+
return keys if keys and all(keys) else None
|
| 1693 |
+
return None
|
| 1694 |
+
|
| 1695 |
+
|
| 1696 |
+
def _related_path_legs(session, source, entry, entries):
|
| 1697 |
+
"""`(first, mid_key, last)` β the two ONE-HOP entries a two-hop composite is made of.
|
| 1698 |
+
|
| 1699 |
+
ββ EACH LEG IS RESOLVED FROM **ITS OWN TABLE'S** SCOPED LISTING, AND THAT IS R13 AT PIN TIME.
|
| 1700 |
+
An edge key is unique per SOURCE table and not across a connector (`relations`' own banner says
|
| 1701 |
+
so, with the ten-edges/seven-keys measurement behind it), so hop 2 is looked up in the
|
| 1702 |
+
intermediate's listing and nowhere else. Resolving both against the source's list would be the
|
| 1703 |
+
ambiguous-key bug the entry key exists to prevent, and it would also skip the intermediate's
|
| 1704 |
+
permission wall on the way past.
|
| 1705 |
+
|
| 1706 |
+
β HOP 1 COSTS NOTHING EXTRA: it is in `entries`, the very listing the composite was picked
|
| 1707 |
+
from, because a depth-2 enumeration returns the one-hop entries first. Only hop 2 needs a
|
| 1708 |
+
second listing, and it is asked at depth 1.
|
| 1709 |
+
"""
|
| 1710 |
+
import core.relations as relations
|
| 1711 |
+
from routes_grid import related_path_entries
|
| 1712 |
+
|
| 1713 |
+
hops = [str(k) for k in (entry.get("path") or ())]
|
| 1714 |
+
first = relations.pick_path(entries, hops[:1])
|
| 1715 |
+
if first is None:
|
| 1716 |
+
raise err(400, "unknown_relation", relations.unknown_via_refusal(source, hops[:1]))
|
| 1717 |
+
mid_key = str(first.get("target") or "")
|
| 1718 |
+
last = relations.pick_path(related_path_entries(session, mid_key, 1), hops[1:2])
|
| 1719 |
+
if last is None:
|
| 1720 |
+
raise err(400, "unknown_relation", relations.unknown_via_refusal(mid_key, hops[1:2]))
|
| 1721 |
+
return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key)
|
| 1722 |
+
|
| 1723 |
+
|
| 1724 |
+
def _related_hop_values(session, source, record, first, mid_key, last, notes):
|
| 1725 |
+
"""The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows.
|
| 1726 |
+
|
| 1727 |
+
ββ W54-T40 β THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE.
|
| 1728 |
+
Hop 1 turns the source record into a set of intermediate rows; this returns the value each of
|
| 1729 |
+
those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them.
|
| 1730 |
+
|
| 1731 |
+
ββ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR
|
| 1732 |
+
USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its
|
| 1733 |
+
administrator row wall and its hidden-field closure all run before a single id is produced, so
|
| 1734 |
+
a traversal cannot be used to learn about rows of the middle database a reader may not see. A
|
| 1735 |
+
second, lighter read here would be a second permission wall, and this file already carries the
|
| 1736 |
+
rule that a second copy of that block does not drift towards refusing.
|
| 1737 |
+
β AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled
|
| 1738 |
+
registry module with no read-through binding, so the one enumerated path that passes through it
|
| 1739 |
+
answers a sentence carrying the cause and a way round it, rather than an empty grid.
|
| 1740 |
+
|
| 1741 |
+
β NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back,
|
| 1742 |
+
however many there are, and the reader is told through `limits` that this tab resolves a second
|
| 1743 |
+
database on every page.
|
| 1744 |
+
"""
|
| 1745 |
+
import core.relations as relations
|
| 1746 |
+
from fastapi import HTTPException # noqa: PLC0415
|
| 1747 |
+
|
| 1748 |
+
value = _related_source_value(session, first, record)
|
| 1749 |
+
if value in (None, ""):
|
| 1750 |
+
raise err(400, "unknown_relation",
|
| 1751 |
+
relations.unresolvable_source_refusal(source, first))
|
| 1752 |
+
try:
|
| 1753 |
+
mid = _compiled_read(mid_key, session)
|
| 1754 |
+
except HTTPException as exc:
|
| 1755 |
+
why = ""
|
| 1756 |
+
if isinstance(getattr(exc, "detail", None), dict):
|
| 1757 |
+
why = str((exc.detail.get("error") or {}).get("message") or "")
|
| 1758 |
+
raise err(409, "relation_not_served",
|
| 1759 |
+
f"this relation reaches '{last.get('target')}' through '{mid_key}', and that "
|
| 1760 |
+
f"database could not be read here to follow it"
|
| 1761 |
+
+ (f" ({why})" if why else "")
|
| 1762 |
+
+ f". Open '{mid_key}' on its own, find the records this one is linked to, and "
|
| 1763 |
+
f"follow the next relation from there")
|
| 1764 |
+
mid_spec = mid["spec"]
|
| 1765 |
+
on1 = str((first.get("via") or {}).get("on") or "")
|
| 1766 |
+
step = (mid_spec.get("cols") or {}).get(on1)
|
| 1767 |
+
if not step:
|
| 1768 |
+
raise err(409, "relation_not_served",
|
| 1769 |
+
f"'{mid_key}' has no live expression for '{on1}', which is the column the first "
|
| 1770 |
+
f"hop of this relation joins on, so the traversal cannot be resolved. Open "
|
| 1771 |
+
f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection")
|
| 1772 |
+
# β THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is
|
| 1773 |
+
# not a column of the mirror spec, so it resolves to the spec's own identity expression, which
|
| 1774 |
+
# is the same one this route trusts everywhere else and is per table rather than a bare `id`.
|
| 1775 |
+
frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL)
|
| 1776 |
+
take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL
|
| 1777 |
+
else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0])
|
| 1778 |
+
if not take:
|
| 1779 |
+
raise err(409, "relation_not_served",
|
| 1780 |
+
f"'{mid_key}' has no live expression for '{frm2}', which is the value the second "
|
| 1781 |
+
f"hop of this relation is keyed on, so the traversal cannot be resolved. Open "
|
| 1782 |
+
f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection")
|
| 1783 |
+
try:
|
| 1784 |
+
value = step[1](value)
|
| 1785 |
+
except Exception: # noqa: BLE001
|
| 1786 |
+
value = str(value)
|
| 1787 |
+
mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"]
|
| 1788 |
+
parts = [p for p in (mid["base_where"], first.get("where")) if p]
|
| 1789 |
+
parts.append(f"{step[0]} = ?")
|
| 1790 |
+
pred = " AND ".join(f"({p})" for p in parts)
|
| 1791 |
+
rows = mid["cur"].execute(
|
| 1792 |
+
f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}",
|
| 1793 |
+
list(mid["wall_params"]) + [value]).fetchall()
|
| 1794 |
+
values = [r[0] for r in rows if r and r[0] is not None]
|
| 1795 |
+
# ββ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS**
|
| 1796 |
+
# SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one
|
| 1797 |
+
# fixed client phrase and falls through to the SERVER's own first sentence for an effect it has
|
| 1798 |
+
# not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking
|
| 1799 |
+
# every row above it"*, which is a true statement about a deep page and a FALSE one about a
|
| 1800 |
+
# seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the
|
| 1801 |
+
# cause below, which is what actually happened.
|
| 1802 |
+
# β THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full
|
| 1803 |
+
# stop and truncates past 120 characters, so the mechanism is stated before the count.
|
| 1804 |
+
notes.append({
|
| 1805 |
+
"subject": "related", "effect": "traversal",
|
| 1806 |
+
"cause": f"these rows are reached through '{mid_key}', so every page resolves that "
|
| 1807 |
+
f"database first. This record matched {len(values):,} of its records, and this "
|
| 1808 |
+
f"database is then narrowed to the rows those reach",
|
| 1809 |
+
"recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or "
|
| 1810 |
+
f"filter this database beside the relation to ask a smaller question"})
|
| 1811 |
+
return values
|
| 1812 |
+
|
| 1813 |
+
|
| 1814 |
def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
|
| 1815 |
"""`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned.
|
| 1816 |
|
|
|
|
| 1836 |
rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`,
|
| 1837 |
so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same
|
| 1838 |
expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY.
|
| 1839 |
+
|
| 1840 |
+
ββ W54-T40 / INSTRUCTION 22 / R12 β **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.**
|
| 1841 |
+
The owner's second quoted error was this function's own `bad_relation` sentence, and it fired
|
| 1842 |
+
on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming
|
| 1843 |
+
each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the
|
| 1844 |
+
guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and
|
| 1845 |
+
18 answered 400 on the first click.
|
| 1846 |
+
|
| 1847 |
+
ββ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT.
|
| 1848 |
+
`relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction
|
| 1849 |
+
is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So
|
| 1850 |
+
a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is
|
| 1851 |
+
then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values
|
| 1852 |
+
hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered
|
| 1853 |
+
`400 unknown_relation` and changed nothing the reader could see.
|
| 1854 |
+
|
| 1855 |
+
β EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The
|
| 1856 |
+
intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for
|
| 1857 |
+
an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a
|
| 1858 |
+
single id crosses into this table's predicate. A second, laxer read of the intermediate would
|
| 1859 |
+
be exactly the permission wall with a hole in it that `related_entries` exists to prevent.
|
| 1860 |
+
β AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty:
|
| 1861 |
+
`product_data` is a registry module with no read-through binding, so the one enumerated path
|
| 1862 |
+
that passes through it refuses by name with a cause and a recommendation.
|
| 1863 |
"""
|
| 1864 |
import core.relations as relations
|
| 1865 |
|
|
|
|
| 1875 |
source = str(related.get("table") or "")
|
| 1876 |
record = related.get("record")
|
| 1877 |
via = related.get("via")
|
| 1878 |
+
# ββ W54-T40 β `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the
|
| 1879 |
+
# one place that decides which. It answers None for the bag, which is what keeps the guard
|
| 1880 |
+
# below spelling the SAME refusal for a genuinely incomplete pin.
|
| 1881 |
+
path = _via_path(via)
|
| 1882 |
+
if not source or record in (None, "") or (path is None and not isinstance(via, dict)):
|
| 1883 |
raise err(400, "bad_relation",
|
| 1884 |
"a pinned read needs the record's database, the record id and the relation to "
|
| 1885 |
+
"follow, as {table, record, via}, where `via` is one relation or the array of "
|
| 1886 |
+
"edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}")
|
| 1887 |
+ "/related for the relations this account may follow")
|
| 1888 |
|
| 1889 |
+
from routes_grid import _may_open, related_entries, related_path_entries
|
| 1890 |
|
| 1891 |
if not _may_open(session, source):
|
| 1892 |
# β THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False
|
|
|
|
| 1902 |
# β RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be
|
| 1903 |
# offered is an edge you cannot pin" structural rather than remembered. `related_entries` is
|
| 1904 |
# the same function `GET /grid/{table}/related` answers with; there is no second resolution.
|
| 1905 |
+
# β W54-T40 β AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE
|
| 1906 |
+
# LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers
|
| 1907 |
+
# with, so a traversal this account is not offered is a traversal it cannot pin either.
|
| 1908 |
+
# β THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs
|
| 1909 |
+
# the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the
|
| 1910 |
+
# difference between roughly half a second and roughly two.
|
| 1911 |
+
if path is None:
|
| 1912 |
+
entries = related_entries(session, source)
|
| 1913 |
+
entry = relations.pick(entries, via)
|
| 1914 |
+
else:
|
| 1915 |
+
if len(path) > relations.MAX_DEPTH:
|
| 1916 |
+
raise err(400, "over_depth", relations.over_depth_refusal(source, path))
|
| 1917 |
+
entries = related_path_entries(session, source, len(path))
|
| 1918 |
+
entry = relations.pick_path(entries, path)
|
| 1919 |
if entry is None:
|
| 1920 |
raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
|
| 1921 |
entry = dict(entry, __source__=source)
|
|
|
|
| 1929 |
if not entry.get("served", True):
|
| 1930 |
raise err(409, "relation_not_served", relations.unserved_target_refusal(entry))
|
| 1931 |
|
| 1932 |
+
# ββ W54-T40 β THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is
|
| 1933 |
+
# not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no
|
| 1934 |
+
# `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates
|
| 1935 |
+
# onto the pair, because one of them may carry a predicate the other does not (the open-invoice
|
| 1936 |
+
# bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped
|
| 1937 |
+
# listing, and the last one is what this block below reads.
|
| 1938 |
+
notes = []
|
| 1939 |
+
first, mid_key, last = entry, "", entry
|
| 1940 |
+
if len(entry.get("path") or ()) > 1:
|
| 1941 |
+
first, mid_key, last = _related_path_legs(session, source, entry, entries)
|
| 1942 |
+
|
| 1943 |
+
on = str((last.get("via") or {}).get("on") or "")
|
| 1944 |
# β `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value
|
| 1945 |
# this response returns: a relation must keep working when an administrator has hidden the id
|
| 1946 |
# column it runs over, and no cell of it is ever projected. The hidden-field closure still
|
| 1947 |
# governs everything that reaches the wire, one layer up.
|
| 1948 |
+
# ββ W54-T40 β AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge
|
| 1949 |
+
# with no declared `on` to `pid`, which is the wire's name for the row identity and is never a
|
| 1950 |
+
# key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one
|
| 1951 |
+
# carries it). Every such edge therefore refused with "has no live expression for 'pid'" while
|
| 1952 |
+
# the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this
|
| 1953 |
+
# route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and
|
| 1954 |
+
# deliberately not a bare `id`, which is ambiguous on a join-shaped grain.
|
| 1955 |
+
if on == relations.IDENTITY_COL:
|
| 1956 |
+
binding = (spec["id"], int)
|
| 1957 |
+
else:
|
| 1958 |
+
binding = (spec.get("cols") or {}).get(on)
|
| 1959 |
if not binding:
|
| 1960 |
raise err(409, "relation_not_served",
|
| 1961 |
f"'{table_key}' has no live expression for '{on}', which is the column this "
|
|
|
|
| 1963 |
f"this database directly, or ask for '{on}' to be added to its live connection")
|
| 1964 |
on_expr, on_cast = binding[0], binding[1]
|
| 1965 |
|
| 1966 |
+
if mid_key:
|
| 1967 |
+
# β HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See
|
| 1968 |
+
# `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through
|
| 1969 |
+
# that database) and compiles to `1=0` two blocks down, never to "no pin".
|
| 1970 |
+
values = _related_hop_values(session, source, record, first, mid_key, last, notes)
|
| 1971 |
+
else:
|
| 1972 |
+
value = _related_source_value(session, entry, record)
|
| 1973 |
+
if value in (None, ""):
|
| 1974 |
+
raise err(400, "unknown_relation",
|
| 1975 |
+
relations.unresolvable_source_refusal(source, entry))
|
| 1976 |
+
values = [value]
|
| 1977 |
+
cast = []
|
| 1978 |
+
for one in values:
|
| 1979 |
+
try:
|
| 1980 |
+
cast.append(on_cast(one))
|
| 1981 |
+
except Exception: # noqa: BLE001
|
| 1982 |
+
cast.append(str(one))
|
| 1983 |
|
| 1984 |
# ββ the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ββββββββββββββββ
|
| 1985 |
# β NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer
|
|
|
|
| 1989 |
# β THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can
|
| 1990 |
# produce is already narrowed to rows they may see, so the pin cannot be used to learn how many
|
| 1991 |
# rows exist beyond their boundary.
|
| 1992 |
+
# β W54-T40 β `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept
|
| 1993 |
+
# BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The
|
| 1994 |
+
# many-value arm is only ever reached by the second hop of a traversal.
|
| 1995 |
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 1996 |
+
ids = []
|
| 1997 |
+
if cast:
|
| 1998 |
+
parts = [p for p in (base_where, last.get("where")) if p]
|
| 1999 |
+
parts.append(f"{on_expr} = ?" if len(cast) == 1
|
| 2000 |
+
else f"{on_expr} IN ({','.join('?' for _ in cast)})")
|
| 2001 |
+
pred = " AND ".join(f"({p})" for p in parts)
|
| 2002 |
+
ids = [int(r[0]) for r in cur.execute(
|
| 2003 |
+
f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}',
|
| 2004 |
+
list(wall_params) + list(cast)).fetchall() if r and r[0] is not None]
|
| 2005 |
|
|
|
|
| 2006 |
# β STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited
|
| 2007 |
# here; the reader is told that this particular relation is answered by reading the source
|
| 2008 |
# database once per page, because it is keyed on a column rather than on the record id.
|
| 2009 |
+
if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL:
|
| 2010 |
notes.append({
|
| 2011 |
"subject": "related", "effect": "slow",
|
| 2012 |
+
"cause": f"this relation is keyed on the '{first['via']['from']}' column of "
|
| 2013 |
f"'{source}' rather than on its record id, so every page of these rows is "
|
| 2014 |
f"resolved by reading that database first",
|
| 2015 |
"recommendation": "read this database on its own and filter it on "
|
|
|
|
| 2026 |
return sql, params, notes
|
| 2027 |
|
| 2028 |
|
| 2029 |
+
# ββ ββ W53-T20 / D-377 β THE VIEW-MEMBERSHIP LEAF, ON A SERVER-WINDOWED GRID βββββββββββββββββ
|
| 2030 |
+
#
|
| 2031 |
+
# D-377: *"`Where View is [x]` is unavailable on any server-windowed (read-through Odoo) grid"* β
|
| 2032 |
+
# W37-T26 shipped the leaf for client-windowed grids and the SQL compiler never learned it. T08
|
| 2033 |
+
# makes every Odoo grid server-windowed, so the latent gap became a visible one.
|
| 2034 |
+
#
|
| 2035 |
+
# ββ THE FIX IS A REWRITE, NOT A NEW SQL ARM, AND THAT IS THE WHOLE POINT.
|
| 2036 |
+
# `harness/filter_sql.py::_compile_cohort` ALREADY compiles "the row identity is in one of these
|
| 2037 |
+
# hand-supplied pid sets", and it already gets the three hard cases right: an empty set is `1=0`,
|
| 2038 |
+
# an ABSENT set is `1=0` **for `noneOf` too** (answering "everything" to `is none of [a set we
|
| 2039 |
+
# cannot see]` is the widening sin wearing a plausible face), and the membership test is wrapped in
|
| 2040 |
+
# `COALESCE(..., FALSE)` so a NULL row id does not make `NOT (id IN (...))` NULL and drop a row the
|
| 2041 |
+
# client would keep. A `__view__` leaf asks EXACTLY that question about a different kind of set. So
|
| 2042 |
+
# each leaf is rewritten into the `__cohort__` shape with the view's resolved pids supplied through
|
| 2043 |
+
# `cohort_sets=`, and `noneOf` is correct for free because it is the same compiler that already got
|
| 2044 |
+
# it right. `filter_sql.py` is not this lane's fence and does not need to be.
|
| 2045 |
+
#
|
| 2046 |
+
# β EXACTLY ONE PLACE LEARNS THE LEAF: `_compiled_read`, immediately before the ONE
|
| 2047 |
+
# `compile_filter_tree` call it makes. `/rows` and `/aggregates` are the same door (C1), so both
|
| 2048 |
+
# inherit it, and `totalUnfiltered` keeps counting the population with the reader's conditions
|
| 2049 |
+
# dropped because the rewrite happens to `tree` and never to `base_where`.
|
| 2050 |
+
|
| 2051 |
+
#: The two RESERVED pseudo-columns a filter tree may name. Neither is a column, and `_leaf_cols`
|
| 2052 |
+
#: cannot tell them apart from a real `colId` β it reads the key, not the vocabulary. Subtracting
|
| 2053 |
+
#: them is what keeps a membership leaf out of the `filter_ignored` sentence that explains a column
|
| 2054 |
+
#: with no mirror expression: that sentence would be FALSE here, and it reads as "your condition was
|
| 2055 |
+
#: dropped", which is the one thing a membership leaf must never be.
|
| 2056 |
+
_PSEUDO_COLS = frozenset(("__cohort__", "__view__"))
|
| 2057 |
+
|
| 2058 |
+
|
| 2059 |
+
def _active_cohort_leaves(nodes):
|
| 2060 |
+
"""Every ACTIVE `__cohort__` leaf a tree names, at any depth.
|
| 2061 |
+
|
| 2062 |
+
"Active" in `filter_sql.is_rule_active`'s own sense for this leaf: driven by the SET it names,
|
| 2063 |
+
so a leaf with a blank value is half-built and ignored on both engines. Inactive leaves are
|
| 2064 |
+
excluded deliberately β reporting one would refuse a read the client renders happily.
|
| 2065 |
+
"""
|
| 2066 |
+
out = []
|
| 2067 |
+
for n in (nodes or []):
|
| 2068 |
+
if not isinstance(n, dict):
|
| 2069 |
+
continue
|
| 2070 |
+
if isinstance(n.get("children"), list):
|
| 2071 |
+
out.extend(_active_cohort_leaves(n.get("children")))
|
| 2072 |
+
elif str(n.get("colId") or "") == "__cohort__" and _fs().parse_cohort_ids(n.get("value")):
|
| 2073 |
+
out.append(n)
|
| 2074 |
+
return out
|
| 2075 |
+
|
| 2076 |
+
|
| 2077 |
+
def _view_store(session, table_key):
|
| 2078 |
+
"""The workspace document a saved view of THIS database lives in.
|
| 2079 |
+
|
| 2080 |
+
β `table_store.find_view`, NOT a second resolver. `done-when` is a row-set EQUALITY between
|
| 2081 |
+
this path and the client-windowed one, so a second implementation is the one thing guaranteed
|
| 2082 |
+
to fail it: the client reads the views `/workspace` served out of this same document.
|
| 2083 |
+
"""
|
| 2084 |
+
import core.table_store as table_store
|
| 2085 |
+
return table_store.make(f"{table_key}_table_workspace", st=session.runtime)
|
| 2086 |
+
|
| 2087 |
+
|
| 2088 |
+
def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_params, cur, limits):
|
| 2089 |
+
"""Rewrite every `__view__` leaf into the `__cohort__` shape. `(tree, cohort_sets)`.
|
| 2090 |
+
|
| 2091 |
+
`cohort_sets` is **None** when the tree names no view, and that is load-bearing rather than
|
| 2092 |
+
tidy: `_compile_cohort` REFUSES when `cohort_sets` is missing, which is how a genuine cohort
|
| 2093 |
+
leaf keeps answering `400 filter_unsupported` on this door exactly as it does today instead of
|
| 2094 |
+
quietly compiling to "matches nothing".
|
| 2095 |
+
|
| 2096 |
+
HOW A VIEW IS RESOLVED, mirroring `CustomerGrid.tsx::viewSets` step for step, because the
|
| 2097 |
+
acceptance is that both paths answer the same rows:
|
| 2098 |
+
|
| 2099 |
+
* the view is found in this database's workspace and the caller may see it, or it is
|
| 2100 |
+
UNRESOLVABLE;
|
| 2101 |
+
* a view LOCKED to a cohort is unresolvable here (the lock is resolved from this tenant's
|
| 2102 |
+
records, not from the connection) and is reported;
|
| 2103 |
+
* `config.filters` is compiled through the SAME `compile_filter_tree` the reader's own
|
| 2104 |
+
conditions go through, with `member_ids=config.memberPids` β which reproduces the client's
|
| 2105 |
+
`curated.has(pid) || matchFilterTree(...)` union exactly;
|
| 2106 |
+
* no filters and a curated `memberPids` is the client's `else if (curated)` branch: the
|
| 2107 |
+
membership IS the view;
|
| 2108 |
+
* no filters and no members means the view holds every row in scope;
|
| 2109 |
+
* the ids are materialised in ONE `SELECT DISTINCT` under `base_where`, uncapped, the same
|
| 2110 |
+
way `_related_pin` does it, so the wall rides along and standing rule 1 is kept.
|
| 2111 |
+
|
| 2112 |
+
THE THREE ANSWERS, AND EVERY ONE OF THEM IS SPELLED IN THE COHORT VOCABULARY:
|
| 2113 |
+
resolved to ids -> `anyOf`/`noneOf` over a key PRESENT in `cohort_sets`
|
| 2114 |
+
holds every row -> `is` becomes `noneOf` over an EMPTY set (`NOT (1=0)`, true of every row)
|
| 2115 |
+
and `is not` becomes a key that is absent (`1=0`)
|
| 2116 |
+
unresolvable -> a key that is ABSENT, which `_compile_cohort` answers `1=0` for BOTH ops.
|
| 2117 |
+
That is `evalNode`'s `if (!set) return false;` byte for byte, and it is
|
| 2118 |
+
D-377's negative control: `is not [a view we cannot resolve]` must match
|
| 2119 |
+
NOTHING, never everything.
|
| 2120 |
+
"""
|
| 2121 |
+
import aios_grid
|
| 2122 |
+
|
| 2123 |
+
if not aios_grid.view_refs_of(tree):
|
| 2124 |
+
return tree, None
|
| 2125 |
+
|
| 2126 |
+
# β A GENUINE COHORT LEAF STILL REFUSES. Supplying `cohort_sets` for the views would otherwise
|
| 2127 |
+
# make a real `__cohort__` leaf beside them compile to `1=0` in silence, where today it answers
|
| 2128 |
+
# 400 with a cause. Narrowing is the safe direction, but an unanswerable condition that reports
|
| 2129 |
+
# nothing is how a reader ends up trusting an empty table.
|
| 2130 |
+
if _active_cohort_leaves(tree):
|
| 2131 |
+
raise err(400, "filter_unsupported",
|
| 2132 |
+
"a condition on cohort membership cannot be evaluated on this database's live "
|
| 2133 |
+
"connection, because a cohort is resolved from this tenant's own records rather "
|
| 2134 |
+
"than from the connection. Filter on a saved view instead, or open this "
|
| 2135 |
+
"database's stored copy")
|
| 2136 |
+
|
| 2137 |
+
store = _view_store(session, table_key)
|
| 2138 |
+
sets, memo, unresolved = {}, {}, []
|
| 2139 |
+
counter = [0]
|
| 2140 |
+
|
| 2141 |
+
def key_for():
|
| 2142 |
+
counter[0] += 1
|
| 2143 |
+
# β A SYNTHETIC KEY, NEVER THE VIEW ID. `parse_cohort_ids` splits `value` on commas and
|
| 2144 |
+
# truncates at 120 characters; a view id is free text and may carry either. A key we mint
|
| 2145 |
+
# cannot collide, cannot split, and cannot be mistaken for a cohort the tenant owns.
|
| 2146 |
+
return "__view_%d__" % counter[0]
|
| 2147 |
+
|
| 2148 |
+
def resolve(vid, seen):
|
| 2149 |
+
"""`('ids', [...])` / `('all', None)` / `(None, reason)` for one view id."""
|
| 2150 |
+
if vid in memo:
|
| 2151 |
+
return memo[vid]
|
| 2152 |
+
if vid in seen:
|
| 2153 |
+
# `viewSets`' `resolving` guard: a view re-entered while it is being resolved is left
|
| 2154 |
+
# ABSENT, so a stored loop degrades to an explained empty table, not a hung request.
|
| 2155 |
+
return None, "it refers to itself through another saved view"
|
| 2156 |
+
found = store.find_view(vid, prefer_owner=session.uname)
|
| 2157 |
+
if not found:
|
| 2158 |
+
got = (None, "it is no longer there")
|
| 2159 |
+
memo[vid] = got
|
| 2160 |
+
return got
|
| 2161 |
+
owner, view = found
|
| 2162 |
+
if owner != session.uname and not _may_see_view(session, vid, store):
|
| 2163 |
+
got = (None, "your account cannot open it")
|
| 2164 |
+
memo[vid] = got
|
| 2165 |
+
return got
|
| 2166 |
+
cfg = (view.get("config") or {}) if isinstance(view, dict) else {}
|
| 2167 |
+
if str(cfg.get("cohortLock") or "").strip():
|
| 2168 |
+
got = (None, "it is locked to a cohort, and a cohort is resolved from this tenant's "
|
| 2169 |
+
"own records rather than from this database's live connection")
|
| 2170 |
+
memo[vid] = got
|
| 2171 |
+
return got
|
| 2172 |
+
members = []
|
| 2173 |
+
for raw in (cfg.get("memberPids") or ()):
|
| 2174 |
+
try:
|
| 2175 |
+
members.append(int(raw))
|
| 2176 |
+
except (TypeError, ValueError): # noqa: PERF203
|
| 2177 |
+
continue
|
| 2178 |
+
raw_nodes = cfg.get("filters") or []
|
| 2179 |
+
if not isinstance(raw_nodes, list):
|
| 2180 |
+
raw_nodes = []
|
| 2181 |
+
if not raw_nodes:
|
| 2182 |
+
# `filters.length` is the client's own test, and it is the NODE COUNT rather than
|
| 2183 |
+
# "anything active": with no nodes at all the curated set IS the view, and with nodes
|
| 2184 |
+
# that all happen to be half-built the view holds everything. Two different answers
|
| 2185 |
+
# from one list, so the emptiness is checked before the compiler sees it.
|
| 2186 |
+
got = ("ids", members) if members else ("all", None)
|
| 2187 |
+
memo[vid] = got
|
| 2188 |
+
return got
|
| 2189 |
+
nodes = rewrite(raw_nodes, seen + (vid,))
|
| 2190 |
+
unknown = sorted((_leaf_cols(nodes) - _PSEUDO_COLS) - set(cols))
|
| 2191 |
+
if unknown:
|
| 2192 |
+
# β UNRESOLVABLE, NOT "IGNORE THAT LEAF". `compile_filter_tree` would treat a column it
|
| 2193 |
+
# cannot see as INACTIVE, which WIDENS the view's row set - and `cols` has the field
|
| 2194 |
+
# wall's hidden columns already stripped, so for a scoped reader that widening would be
|
| 2195 |
+
# the permission door leaking through the back. Reported below, never silently applied.
|
| 2196 |
+
got = (None, "it filters on " + ", ".join(unknown)
|
| 2197 |
+
+ ", which this database's live connection cannot read for your account")
|
| 2198 |
+
memo[vid] = got
|
| 2199 |
+
return got
|
| 2200 |
+
try:
|
| 2201 |
+
pred = _fs().compile_filter_tree(nodes, conj=("or" if cfg.get("filterConj") == "or"
|
| 2202 |
+
else "and"),
|
| 2203 |
+
columns=cols, member_ids=(members or None),
|
| 2204 |
+
id_sql=spec["id"], today=_today(), cohort_sets=sets)
|
| 2205 |
+
except ValueError as exc:
|
| 2206 |
+
got = (None, "it uses a condition this database's live connection cannot evaluate ("
|
| 2207 |
+
+ str(exc) + ")")
|
| 2208 |
+
memo[vid] = got
|
| 2209 |
+
return got
|
| 2210 |
+
if pred is None:
|
| 2211 |
+
# Every condition in it is half-built, so it matches every row - which is what
|
| 2212 |
+
# `matchFilterTree` answers for an inactive tree, and the `member_ids` union rides
|
| 2213 |
+
# inside that same `if (filters.length)` on both engines.
|
| 2214 |
+
got = ("all", None)
|
| 2215 |
+
memo[vid] = got
|
| 2216 |
+
return got
|
| 2217 |
+
if pred.uses_aggregate:
|
| 2218 |
+
got = (None, "it filters on an aggregate column, which this database's live connection "
|
| 2219 |
+
"cannot evaluate one row at a time")
|
| 2220 |
+
memo[vid] = got
|
| 2221 |
+
return got
|
| 2222 |
+
frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
|
| 2223 |
+
where = f"({base_where}) AND ({pred.sql})" if base_where else pred.sql
|
| 2224 |
+
ids = [int(r[0]) for r in cur.execute(
|
| 2225 |
+
f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {where}',
|
| 2226 |
+
list(wall_params) + list(pred.params)).fetchall() if r and r[0] is not None]
|
| 2227 |
+
got = ("ids", ids)
|
| 2228 |
+
memo[vid] = got
|
| 2229 |
+
return got
|
| 2230 |
+
|
| 2231 |
+
def rewrite(nodes, seen):
|
| 2232 |
+
out = []
|
| 2233 |
+
for n in (nodes or []):
|
| 2234 |
+
if not isinstance(n, dict):
|
| 2235 |
+
out.append(n)
|
| 2236 |
+
continue
|
| 2237 |
+
if isinstance(n.get("children"), list):
|
| 2238 |
+
out.append(dict(n, children=rewrite(n.get("children"), seen)))
|
| 2239 |
+
continue
|
| 2240 |
+
if str(n.get("colId") or "") != aios_grid.VIEW_FIELD:
|
| 2241 |
+
out.append(n)
|
| 2242 |
+
continue
|
| 2243 |
+
vid = str(n.get("value") or "").strip()[:aios_grid.MAX_VIEW_ID]
|
| 2244 |
+
if not vid:
|
| 2245 |
+
# Half-built, and half-built is IGNORED on both engines. Left as it stands: its
|
| 2246 |
+
# `colId` is not a column, so `is_rule_active` already answers False for it.
|
| 2247 |
+
out.append(n)
|
| 2248 |
+
continue
|
| 2249 |
+
# `normalizeViewOp`: anything that is not `notInView` reads as `inView`, so a leaf
|
| 2250 |
+
# carrying a corrupted op asks the narrower question rather than the wider one.
|
| 2251 |
+
not_in = str(n.get("op") or "") == "notInView"
|
| 2252 |
+
key = key_for()
|
| 2253 |
+
kind, got = resolve(vid, seen)
|
| 2254 |
+
if kind == "ids":
|
| 2255 |
+
sets[key] = got
|
| 2256 |
+
op = "noneOf" if not_in else "anyOf"
|
| 2257 |
+
if len(got) >= _PIN_REPORT_AT:
|
| 2258 |
+
limits.append({
|
| 2259 |
+
"subject": "view", "effect": "slow",
|
| 2260 |
+
"cause": f"the saved view this condition names holds {len(got):,} rows, "
|
| 2261 |
+
f"and each page of this database names every one of them in the "
|
| 2262 |
+
f"query that narrows it",
|
| 2263 |
+
"recommendation": "add a condition beside the view to ask a smaller "
|
| 2264 |
+
"question, or open the saved view itself"})
|
| 2265 |
+
elif kind == "all":
|
| 2266 |
+
if not_in:
|
| 2267 |
+
op = "anyOf" # key absent -> 1=0
|
| 2268 |
+
else:
|
| 2269 |
+
sets[key] = []
|
| 2270 |
+
op = "noneOf" # NOT (1=0) -> true of every row
|
| 2271 |
+
else:
|
| 2272 |
+
op = "anyOf" # key absent -> 1=0 for `is` AND for `is not`
|
| 2273 |
+
unresolved.append((vid, got))
|
| 2274 |
+
out.append({"colId": "__cohort__", "op": op, "value": key})
|
| 2275 |
+
return out
|
| 2276 |
+
|
| 2277 |
+
tree = rewrite(tree, ())
|
| 2278 |
+
if unresolved:
|
| 2279 |
+
limits.append({
|
| 2280 |
+
"subject": ", ".join(vid for vid, _why in unresolved),
|
| 2281 |
+
"effect": "filter_unresolved",
|
| 2282 |
+
"cause": "this condition asks whether a row is in a saved view that could not be "
|
| 2283 |
+
"resolved on this database's live connection ("
|
| 2284 |
+
+ "; ".join(f"{vid}: {why}" for vid, why in unresolved)
|
| 2285 |
+
+ "), so it matches no rows at all, for `is not` as well as for `is`",
|
| 2286 |
+
"recommendation": "pick a saved view you can open on this database, or remove the "
|
| 2287 |
+
"condition"})
|
| 2288 |
+
return tree, sets
|
| 2289 |
+
|
| 2290 |
+
|
| 2291 |
+
def _may_see_view(session, vid, store):
|
| 2292 |
+
"""May this caller open that saved view? The same predicate the view rail asks.
|
| 2293 |
+
|
| 2294 |
+
β NOT SKIPPED FOR BEING INCONVENIENT. A view a caller cannot see resolves to nothing on the
|
| 2295 |
+
client (it is not in the payload at all), so resolving it here would make this door answer a
|
| 2296 |
+
question the other door cannot, out of a row set somebody else curated.
|
| 2297 |
+
"""
|
| 2298 |
+
import core.shares as shares
|
| 2299 |
+
return shares.may_see("view", vid, session.uname, is_admin=session.admin, st=store.st)
|
| 2300 |
+
|
| 2301 |
+
|
| 2302 |
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
|
| 2303 |
search=None, tree=_TREE_UNSET, related=None):
|
| 2304 |
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
|
|
|
|
| 2392 |
|
| 2393 |
spec = _source_for(cur, table_key)
|
| 2394 |
if spec is None:
|
| 2395 |
+
# ββ W53-T19 / R5 β THE SENTENCE IS REWRITTEN AND THE REFUSAL IS NOT. R5 retires the
|
| 2396 |
+
# phrase, not the guard: `verify_scopes` drives `_source_for` to None as a negative control
|
| 2397 |
+
# and requires a 409 here rather than a grid quietly served from somewhere else, and every
|
| 2398 |
+
# Odoo grain is bound today only on a mirror that carries the tables they read. The cause
|
| 2399 |
+
# is now named as what it is, an absent binding, in the same words
|
| 2400 |
+
# `relations.unserved_target_refusal` uses for the same absence.
|
| 2401 |
+
# β NO EM DASH AND NO EN DASH: this string reaches a screen (standing rule 2).
|
| 2402 |
raise err(409, "not_read_through",
|
| 2403 |
+
"this connected database has no read-through binding on this deployment, so "
|
| 2404 |
+
"its rows cannot be read a window at a time here. Open it from the Data "
|
| 2405 |
+
"section, or ask for a read-through binding to be added for it")
|
| 2406 |
|
| 2407 |
cols = _filter_columns(spec, fields)
|
| 2408 |
|
|
|
|
| 2520 |
"recommendation": "ask an administrator to set the restriction on a whole number, "
|
| 2521 |
"or on a column that is not an amount"})
|
| 2522 |
|
| 2523 |
+
# ββ W53-T20 / D-377 β the view-membership leaf, rewritten into the cohort shape βββββββββββ
|
| 2524 |
+
# β HERE, AND NOWHERE ELSE. After the wall and the pin (so a view resolves inside the scope
|
| 2525 |
+
# the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
|
| 2526 |
+
# before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
|
| 2527 |
+
# no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
|
| 2528 |
+
tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
|
| 2529 |
+
base_where, wall_params, cur, limits)
|
| 2530 |
+
|
| 2531 |
# ββ the predicate, pushed down ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 2532 |
+
# β `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
|
| 2533 |
+
# are not columns and never will be in `cols`, so leaving them in would report every membership
|
| 2534 |
+
# condition as a column the mirror cannot read - a false sentence, and one that reads as "your
|
| 2535 |
+
# filter was dropped" beside a filter that was in fact applied.
|
| 2536 |
+
named = _leaf_cols(tree) - _PSEUDO_COLS
|
| 2537 |
missing = sorted(named - set(cols))
|
| 2538 |
if missing:
|
| 2539 |
limits.append({
|
|
|
|
| 2583 |
|
| 2584 |
where, params = base_where, list(wall_params)
|
| 2585 |
try:
|
| 2586 |
+
# β `cohort_sets` IS PASSED ONLY WHEN A VIEW LEAF WAS REWRITTEN (it is None otherwise), and
|
| 2587 |
+
# that is what keeps a genuine cohort leaf REFUSING here rather than compiling to `1=0` in
|
| 2588 |
+
# silence. `id_sql` rides along because `_compile_cohort` raises without it.
|
| 2589 |
pred = _fs().compile_filter_tree(
|
| 2590 |
tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols,
|
| 2591 |
+
id_sql=spec["id"], today=_today(), cohort_sets=cohort_sets)
|
| 2592 |
except ValueError as e:
|
| 2593 |
# β A REFUSAL IS AN ANSWER, NOT A CRASH β and it must not become "no filter". Ranking ops
|
| 2594 |
# ("top 10") have no SQL form in this compiler; saying so beats returning every row.
|
api/routes_platform_admin.py
CHANGED
|
@@ -537,7 +537,7 @@ def _aws_report(days=7):
|
|
| 537 |
"""
|
| 538 |
if not _AWS_REPORT.is_file():
|
| 539 |
return {"available": False, "text": "",
|
| 540 |
-
"note": ("The AWS usage report is not part of this deployment
|
| 541 |
"with the repository, not with the container image. Run "
|
| 542 |
"`python ops/aws_usage_report.py` locally for the live figures.")}
|
| 543 |
try:
|
|
@@ -547,7 +547,7 @@ def _aws_report(days=7):
|
|
| 547 |
cwd=str(_AWS_REPORT.parent.parent))
|
| 548 |
except subprocess.TimeoutExpired:
|
| 549 |
return {"available": False, "text": "",
|
| 550 |
-
"note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s
|
| 551 |
f"be unreachable from here. Nothing was assumed about usage.")}
|
| 552 |
except Exception as e: # noqa: BLE001
|
| 553 |
return {"available": False, "text": "",
|
|
@@ -556,7 +556,7 @@ def _aws_report(days=7):
|
|
| 556 |
if proc.returncode != 0 or not out:
|
| 557 |
detail = (proc.stderr or "").strip().splitlines()
|
| 558 |
return {"available": False, "text": out,
|
| 559 |
-
"note": ("AWS credentials are not available here, or boto3 is not installed
|
| 560 |
"no usage could be read, and none is guessed. "
|
| 561 |
+ (detail[-1][:200] if detail else ""))}
|
| 562 |
return {"available": True, "text": out, "note": ""}
|
|
@@ -655,7 +655,7 @@ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)):
|
|
| 655 |
"""
|
| 656 |
rows = _user_rows(tenant)
|
| 657 |
return {"users": rows, "count": len(rows),
|
| 658 |
-
"stampsNote": ("Login and activity stamps started with this release
|
| 659 |
"have not signed in since show no date rather than an invented one. "
|
| 660 |
"Activity is recorded at most once an hour per account.")}
|
| 661 |
|
|
@@ -700,7 +700,7 @@ def platform_connectors(tenant: str = "", session: Session = Depends(padmin_gate
|
|
| 700 |
out += rows
|
| 701 |
return {"connectors": out, "count": len(out), "keychainLocked": locked_any,
|
| 702 |
"errors": errors,
|
| 703 |
-
"note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY`
|
| 704 |
"credentials cannot be read, so a tenant's sources fail closed rather than "
|
| 705 |
"falling back to anyone else's.")}
|
| 706 |
|
|
|
|
| 537 |
"""
|
| 538 |
if not _AWS_REPORT.is_file():
|
| 539 |
return {"available": False, "text": "",
|
| 540 |
+
"note": ("The AWS usage report is not part of this deployment; `ops/` ships "
|
| 541 |
"with the repository, not with the container image. Run "
|
| 542 |
"`python ops/aws_usage_report.py` locally for the live figures.")}
|
| 543 |
try:
|
|
|
|
| 547 |
cwd=str(_AWS_REPORT.parent.parent))
|
| 548 |
except subprocess.TimeoutExpired:
|
| 549 |
return {"available": False, "text": "",
|
| 550 |
+
"note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s; CloudWatch may "
|
| 551 |
f"be unreachable from here. Nothing was assumed about usage.")}
|
| 552 |
except Exception as e: # noqa: BLE001
|
| 553 |
return {"available": False, "text": "",
|
|
|
|
| 556 |
if proc.returncode != 0 or not out:
|
| 557 |
detail = (proc.stderr or "").strip().splitlines()
|
| 558 |
return {"available": False, "text": out,
|
| 559 |
+
"note": ("AWS credentials are not available here, or boto3 is not installed; "
|
| 560 |
"no usage could be read, and none is guessed. "
|
| 561 |
+ (detail[-1][:200] if detail else ""))}
|
| 562 |
return {"available": True, "text": out, "note": ""}
|
|
|
|
| 655 |
"""
|
| 656 |
rows = _user_rows(tenant)
|
| 657 |
return {"users": rows, "count": len(rows),
|
| 658 |
+
"stampsNote": ("Login and activity stamps started with this release; accounts that "
|
| 659 |
"have not signed in since show no date rather than an invented one. "
|
| 660 |
"Activity is recorded at most once an hour per account.")}
|
| 661 |
|
|
|
|
| 700 |
out += rows
|
| 701 |
return {"connectors": out, "count": len(out), "keychainLocked": locked_any,
|
| 702 |
"errors": errors,
|
| 703 |
+
"note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY`; stored "
|
| 704 |
"credentials cannot be read, so a tenant's sources fail closed rather than "
|
| 705 |
"falling back to anyone else's.")}
|
| 706 |
|
api/routes_products.py
CHANGED
|
@@ -94,7 +94,7 @@ def _pool_for(rt, team_id):
|
|
| 94 |
if hit:
|
| 95 |
return hit[1]
|
| 96 |
raise err(503, "connector_paused",
|
| 97 |
-
"this data source is paused and nothing has been read since
|
| 98 |
"an admin can resume it under Settings β Connectors")
|
| 99 |
|
| 100 |
def build():
|
|
@@ -127,7 +127,7 @@ def scoped_pool(session: Session):
|
|
| 127 |
rows_src = _pool_for(session.runtime, team_id)
|
| 128 |
except Exception as e:
|
| 129 |
raise err(503, "pool_unavailable",
|
| 130 |
-
f"the product catalogue could not be built
|
| 131 |
# Image is a tenant-wide created field. Seed/repair its definition before
|
| 132 |
# the field wall runs so every reader receives the same metadata and the
|
| 133 |
# Leadership account owns the permission editor from the first request.
|
|
@@ -636,7 +636,7 @@ def patch_product(pid: int, body: dict = Body(default=None),
|
|
| 636 |
"pid": pid, "updates": updates}, ctx)
|
| 637 |
except grid_events.StoreUnavailable:
|
| 638 |
raise err(503, "store_unavailable",
|
| 639 |
-
"the tenant store is unavailable
|
| 640 |
|
| 641 |
# What actually landed, read back from the PRODUCT bucket rather than echoed from the
|
| 642 |
# request: a refused key or a truncated value must not be reported as accepted.
|
|
|
|
| 94 |
if hit:
|
| 95 |
return hit[1]
|
| 96 |
raise err(503, "connector_paused",
|
| 97 |
+
"this data source is paused and nothing has been read since; "
|
| 98 |
"an admin can resume it under Settings β Connectors")
|
| 99 |
|
| 100 |
def build():
|
|
|
|
| 127 |
rows_src = _pool_for(session.runtime, team_id)
|
| 128 |
except Exception as e:
|
| 129 |
raise err(503, "pool_unavailable",
|
| 130 |
+
f"the product catalogue could not be built; {str(e)[:160]}")
|
| 131 |
# Image is a tenant-wide created field. Seed/repair its definition before
|
| 132 |
# the field wall runs so every reader receives the same metadata and the
|
| 133 |
# Leadership account owns the permission editor from the first request.
|
|
|
|
| 636 |
"pid": pid, "updates": updates}, ctx)
|
| 637 |
except grid_events.StoreUnavailable:
|
| 638 |
raise err(503, "store_unavailable",
|
| 639 |
+
"the tenant store is unavailable; your change was not saved")
|
| 640 |
|
| 641 |
# What actually landed, read back from the PRODUCT bucket rather than echoed from the
|
| 642 |
# request: a refused key or a truncated value must not be reported as accepted.
|
api/routes_publish.py
CHANGED
|
@@ -301,12 +301,12 @@ def _may_publish_source_view(session: Session, table_key: str, view_id: str):
|
|
| 301 |
if cfg.get("cohortLock"):
|
| 302 |
raise err(400, "reader_scoped",
|
| 303 |
"this view is locked to a cohort, and a cohort's membership is resolved for "
|
| 304 |
-
"the person reading it
|
| 305 |
"no rows at all. Publish a copy without the cohort lock.")
|
| 306 |
if _needs_a_reader(cfg.get("filters")):
|
| 307 |
raise err(400, "reader_scoped",
|
| 308 |
"this view filters on a cohort or a measure rule, and each is resolved for the "
|
| 309 |
-
"person reading it
|
| 310 |
"the page would show no rows at all. Publish a copy filtered on columns.")
|
| 311 |
return view, mode
|
| 312 |
|
|
@@ -818,7 +818,7 @@ async def mint_view_link(request: Request, session: Session = Depends(require_se
|
|
| 818 |
# If the write cannot be read back, say so; do not describe a state nobody verified.
|
| 819 |
if not entry:
|
| 820 |
raise err(503, "not_saved",
|
| 821 |
-
"the link was minted but could not be read back
|
| 822 |
return _state_of(token, entry)
|
| 823 |
|
| 824 |
|
|
@@ -864,7 +864,7 @@ async def revoke_view_link(request: Request, session: Session = Depends(require_
|
|
| 864 |
def get_published(token: str, request: Request):
|
| 865 |
"""The published view. Read-only, unauthenticated, projected to the view's own columns."""
|
| 866 |
if not _rate_ok(_client_ip(request), time.time()):
|
| 867 |
-
raise err(429, "too_many_requests", "too many requests
|
| 868 |
found = _resolve(token)
|
| 869 |
if not found:
|
| 870 |
# ββ W33-T70 β AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS.
|
|
@@ -906,7 +906,7 @@ async def open_published(token: str, request: Request):
|
|
| 906 |
"""
|
| 907 |
now = time.time()
|
| 908 |
if not _rate_ok(_client_ip(request), now):
|
| 909 |
-
raise err(429, "too_many_requests", "too many requests
|
| 910 |
body = await _bounded_body(request)
|
| 911 |
found = _resolve(token)
|
| 912 |
if not found:
|
|
|
|
| 301 |
if cfg.get("cohortLock"):
|
| 302 |
raise err(400, "reader_scoped",
|
| 303 |
"this view is locked to a cohort, and a cohort's membership is resolved for "
|
| 304 |
+
"the person reading it; a public link has no reader, so the page would show "
|
| 305 |
"no rows at all. Publish a copy without the cohort lock.")
|
| 306 |
if _needs_a_reader(cfg.get("filters")):
|
| 307 |
raise err(400, "reader_scoped",
|
| 308 |
"this view filters on a cohort or a measure rule, and each is resolved for the "
|
| 309 |
+
"person reading it; a public link has no reader, so "
|
| 310 |
"the page would show no rows at all. Publish a copy filtered on columns.")
|
| 311 |
return view, mode
|
| 312 |
|
|
|
|
| 818 |
# If the write cannot be read back, say so; do not describe a state nobody verified.
|
| 819 |
if not entry:
|
| 820 |
raise err(503, "not_saved",
|
| 821 |
+
"the link was minted but could not be read back; reload and try again")
|
| 822 |
return _state_of(token, entry)
|
| 823 |
|
| 824 |
|
|
|
|
| 864 |
def get_published(token: str, request: Request):
|
| 865 |
"""The published view. Read-only, unauthenticated, projected to the view's own columns."""
|
| 866 |
if not _rate_ok(_client_ip(request), time.time()):
|
| 867 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 868 |
found = _resolve(token)
|
| 869 |
if not found:
|
| 870 |
# ββ W33-T70 β AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS.
|
|
|
|
| 906 |
"""
|
| 907 |
now = time.time()
|
| 908 |
if not _rate_ok(_client_ip(request), now):
|
| 909 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 910 |
body = await _bounded_body(request)
|
| 911 |
found = _resolve(token)
|
| 912 |
if not found:
|
api/routes_records.py
CHANGED
|
@@ -47,7 +47,7 @@ def _scope_or_400(raw):
|
|
| 47 |
if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
|
| 48 |
return "customer" if scope in _CUSTOMER_SCOPES else scope
|
| 49 |
raise err(400, "bad_scope",
|
| 50 |
-
"scope must be customer, cohort, product or a ut_ database
|
| 51 |
|
| 52 |
|
| 53 |
def _pool_or_refuse(session: Session, scope: str):
|
|
@@ -117,7 +117,7 @@ def _in_book(pid, session, scope):
|
|
| 117 |
# out loud rather than silently admitting: R6's second sentence is that a limit which
|
| 118 |
# cannot be removed gets REPORTED, and this is the one place the report has no envelope to
|
| 119 |
# ride in.
|
| 120 |
-
print(f"[records] {scope}: pid membership unresolved (read-through beyond one window)
|
| 121 |
f"admitting on the table wall alone, per D-72")
|
| 122 |
return
|
| 123 |
if pid not in pids:
|
|
@@ -129,7 +129,7 @@ def _unavailable():
|
|
| 129 |
return err(
|
| 130 |
503,
|
| 131 |
"store_unavailable",
|
| 132 |
-
"record comments are temporarily unavailable
|
| 133 |
)
|
| 134 |
|
| 135 |
|
|
|
|
| 47 |
if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
|
| 48 |
return "customer" if scope in _CUSTOMER_SCOPES else scope
|
| 49 |
raise err(400, "bad_scope",
|
| 50 |
+
"scope must be customer, cohort, product or a ut_ database; refusing to guess")
|
| 51 |
|
| 52 |
|
| 53 |
def _pool_or_refuse(session: Session, scope: str):
|
|
|
|
| 117 |
# out loud rather than silently admitting: R6's second sentence is that a limit which
|
| 118 |
# cannot be removed gets REPORTED, and this is the one place the report has no envelope to
|
| 119 |
# ride in.
|
| 120 |
+
print(f"[records] {scope}: pid membership unresolved (read-through beyond one window); "
|
| 121 |
f"admitting on the table wall alone, per D-72")
|
| 122 |
return
|
| 123 |
if pid not in pids:
|
|
|
|
| 129 |
return err(
|
| 130 |
503,
|
| 131 |
"store_unavailable",
|
| 132 |
+
"record comments are temporarily unavailable; no change was saved",
|
| 133 |
)
|
| 134 |
|
| 135 |
|
api/routes_sheet.py
CHANGED
|
@@ -35,6 +35,8 @@ while its surface is a 404.
|
|
| 35 |
(standing rule 2), and each names both the cause and the subject.
|
| 36 |
"""
|
| 37 |
|
|
|
|
|
|
|
| 38 |
import os
|
| 39 |
import tempfile
|
| 40 |
import threading
|
|
@@ -88,17 +90,30 @@ def _mirror(session: Session):
|
|
| 88 |
`RuntimeError` means the mirror is completing its first sync, so it is a 503 the caller may
|
| 89 |
retry. `DatastoreMismatch` SUBCLASSES `RuntimeError`, so the guard must run ABOVE the try, and
|
| 90 |
collapsing them would turn the loudest refusal in the system into a spinner.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
"""
|
| 92 |
import routes_odoo_tables as rot
|
| 93 |
import odoo_relational as rel
|
| 94 |
|
| 95 |
-
def _for(table_key):
|
| 96 |
fields = None
|
| 97 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 98 |
if key == table_key:
|
| 99 |
fields = mk_fields()
|
| 100 |
break
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
|
| 103 |
try:
|
| 104 |
cur, mismatch = rot._mirror_cur(session)
|
|
@@ -106,19 +121,389 @@ def _mirror(session: Session):
|
|
| 106 |
raise err(503, "store_not_ready", str(exc))
|
| 107 |
if mismatch:
|
| 108 |
raise err(409, "cross_tenant_store", mismatch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
spec = rot._source_for(cur, table_key)
|
| 110 |
if spec is None:
|
| 111 |
raise err(409, "not_read_through",
|
| 112 |
-
f"{table_key}
|
| 113 |
-
"through to it
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
|
| 116 |
return _for
|
| 117 |
|
| 118 |
|
| 119 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
"""`(nodes, conj)` for a saved view's filter, or a `view_missing` refusal that NAMES it.
|
| 121 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
ββ `prefer_owner=session.uname` IS LOAD-BEARING AND OMITTING IT IS A SILENT CROSS-USER READ.
|
| 123 |
The system view id `all-customers` is pinned on every database, and every user who has ever
|
| 124 |
touched that view's config holds their OWN record under the same id, so a bare first-hit scan
|
|
@@ -130,10 +515,42 @@ def _view_filter(session: Session, database: str, view_id: str, view_name: str =
|
|
| 130 |
β The refusal names the view, which is why `clean_binding` keeps `view_name` on the binding:
|
| 131 |
once the record is deleted there is nothing left to look a name up from.
|
| 132 |
"""
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
# β `TableStore(table_key, st=...)`, in that order. `st` is the STORE HANDLE, and passing the
|
| 135 |
# session's runtime is what makes the lookup read THIS tenant's bucket rather than tenant #0's.
|
| 136 |
-
store = table_store.TableStore(database
|
|
|
|
| 137 |
# ββ `include_trashed=True` IS R11, AND IT IS ONE KEYWORD BECAUSE THE DEFAULT MUST NOT MOVE
|
| 138 |
# (wave 51, W51-T26). A deleted view is MOVED into `<owner>.trash`, so the bare scan answered
|
| 139 |
# None and this door refused a cell whose view somebody deleted an hour ago and can still
|
|
@@ -144,11 +561,50 @@ def _view_filter(session: Session, database: str, view_id: str, view_name: str =
|
|
| 144 |
# permission-adjacent (shares, starred, publish, forms); widening there would open doors no
|
| 145 |
# ticket asked about, with every gate green.
|
| 146 |
found = store.find_view(view_id, prefer_owner=session.uname, include_trashed=True)
|
| 147 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
raise err(409, "view_missing", f"View {view_name or view_id} no longer exists.")
|
| 149 |
-
_owner, view = found
|
| 150 |
import routes_odoo_tables as rot
|
| 151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
|
| 153 |
|
| 154 |
#: β THE TWO CAP CODES ANSWER 413, NOT 400 (contract C3). A 400 says the request was malformed and
|
|
@@ -335,6 +791,23 @@ def _wire_index(index):
|
|
| 335 |
return out
|
| 336 |
|
| 337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
def _reresolved_for_reader(session: Session, book):
|
| 339 |
"""A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
|
| 340 |
|
|
@@ -350,14 +823,29 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 350 |
|
| 351 |
β `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of
|
| 352 |
literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
"""
|
|
|
|
| 354 |
import copy as _copy
|
| 355 |
fresh = _copy.deepcopy(book)
|
| 356 |
stamp = _now_iso()
|
| 357 |
seen_db = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
touched = 0
|
|
|
|
|
|
|
| 359 |
for tab in fresh.get("sheets") or ():
|
| 360 |
-
|
|
|
|
| 361 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 362 |
continue
|
| 363 |
touched += 1
|
|
@@ -377,8 +865,29 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 377 |
"message": f"You cannot read {db}."}
|
| 378 |
cell.pop("limits", None)
|
| 379 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
try:
|
| 381 |
-
out = _resolve_binding(session, binding)
|
| 382 |
except HTTPException as exc:
|
| 383 |
detail = getattr(exc, "detail", None) or {}
|
| 384 |
info = detail.get("error") if isinstance(detail, dict) else None
|
|
@@ -398,6 +907,15 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 398 |
cell["limits"] = out["limits"]
|
| 399 |
else:
|
| 400 |
cell.pop("limits", None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 401 |
if not touched:
|
| 402 |
return fresh
|
| 403 |
# β EVERY FORMULA IS RE-EVALUATED TOO, over the numbers this reader is entitled to. Leaving
|
|
@@ -611,8 +1129,7 @@ def _run_import(job_id, path, filename, runtime, uname):
|
|
| 611 |
stamp = _now_iso()
|
| 612 |
made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname)
|
| 613 |
saved = sheets.put_workbook(runtime, made["id"],
|
| 614 |
-
|
| 615 |
-
"refreshed_at": stamp, "rev": made["rev"]})
|
| 616 |
_job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
|
| 617 |
sentence=_import_sentence(report))
|
| 618 |
except sheet_import.ImportRefusal as exc:
|
|
@@ -777,7 +1294,8 @@ def read_sheet_tab(workbook_id: str, tab: str,
|
|
| 777 |
found = sheets.find_tab(served, tab)
|
| 778 |
except sheets.SheetRefusal as exc:
|
| 779 |
raise _refuse(exc)
|
| 780 |
-
out = {"id": book.get("id"), "rev": book.get("rev"), "tab": found
|
|
|
|
| 781 |
if served.get("refreshedFor"):
|
| 782 |
out["refreshedFor"] = served["refreshedFor"]
|
| 783 |
return out
|
|
@@ -806,6 +1324,12 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 806 |
`workbook_read_only`, not 409, because 409 names a problem they could fix by refetching and
|
| 807 |
this one they cannot fix at all. An editor's write falls straight through to the rev guard,
|
| 808 |
which is exactly what C5 asks for.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 809 |
"""
|
| 810 |
from core import sheets
|
| 811 |
raw = dict(body or {})
|
|
@@ -822,6 +1346,67 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
|
|
| 822 |
expect_rev=expect)
|
| 823 |
except sheets.SheetRefusal as exc:
|
| 824 |
raise _refuse(exc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
return out
|
| 826 |
|
| 827 |
|
|
@@ -976,8 +1561,11 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 976 |
forty cells to one database asks once; a reader who has lost access to that database gets a
|
| 977 |
refusal on every one of its cells rather than a stale number quietly repainting.
|
| 978 |
|
| 979 |
-
β Each distinct `view_id` is resolved ONCE
|
| 980 |
-
|
|
|
|
|
|
|
|
|
|
| 981 |
|
| 982 |
ββ C5 β THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED
|
| 983 |
HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed
|
|
@@ -999,6 +1587,12 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 999 |
|
| 1000 |
stamp = _now_iso()
|
| 1001 |
seen_db = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1002 |
for tab in book.get("sheets") or ():
|
| 1003 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1004 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
|
@@ -1019,7 +1613,7 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 1019 |
"message": f"You cannot read {db}."}
|
| 1020 |
continue
|
| 1021 |
try:
|
| 1022 |
-
out = _resolve_binding(session, binding)
|
| 1023 |
except HTTPException as exc:
|
| 1024 |
detail = getattr(exc, "detail", None) or {}
|
| 1025 |
info = detail.get("error") if isinstance(detail, dict) else None
|
|
@@ -1059,6 +1653,10 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
|
|
| 1059 |
"refreshed_at": stamp, "rev": book.get("rev")})
|
| 1060 |
except sheets.SheetRefusal as exc:
|
| 1061 |
raise _refuse(exc)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
return {"workbook": saved, "recalc_ms": recalc_ms}
|
| 1063 |
|
| 1064 |
|
|
@@ -1096,7 +1694,7 @@ def _tab_scope_names(tabs):
|
|
| 1096 |
return names
|
| 1097 |
|
| 1098 |
|
| 1099 |
-
def _recompute_formulas(book, fr, sheet_mod):
|
| 1100 |
"""Re-evaluate every formula cell across the WORKBOOK, in one memoized dependency pass.
|
| 1101 |
|
| 1102 |
ββ ONE PASS OVER EVERY TAB, NOT ONE PASS PER TAB (W51-T23). Per tab, `DATA!B12` was never in
|
|
@@ -1110,6 +1708,13 @@ def _recompute_formulas(book, fr, sheet_mod):
|
|
| 1110 |
A cell whose value is None because the database refused is handed to the engine as absent, so
|
| 1111 |
arithmetic over it stays None rather than silently counting it as nothing.
|
| 1112 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1113 |
tabs = [t for t in (book.get("sheets") or ()) if isinstance(t, dict)]
|
| 1114 |
names = _tab_scope_names(tabs)
|
| 1115 |
spec = {}
|
|
@@ -1190,7 +1795,7 @@ def _formula_message(code, subject=""):
|
|
| 1190 |
}.get(code, "This formula could not be calculated.")
|
| 1191 |
|
| 1192 |
|
| 1193 |
-
def _resolve_binding(session: Session, raw_binding, *, today=None):
|
| 1194 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 1195 |
|
| 1196 |
The order below is the contract, not a preference:
|
|
@@ -1198,6 +1803,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None):
|
|
| 1198 |
2. the mirror cursor, whose two failures mean opposite things;
|
| 1199 |
3. the ROW wall, fail-closed, refusing 409 when it names a column the mirror cannot answer;
|
| 1200 |
4. the binding's own scope, whose unanswerable columns are REPORTED rather than dropped.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1201 |
"""
|
| 1202 |
from modules import sheet as sheet_mod
|
| 1203 |
import core.perm_scope as perm_scope
|
|
@@ -1207,13 +1822,53 @@ def _resolve_binding(session: Session, raw_binding, *, today=None):
|
|
| 1207 |
except sheet_mod.SheetRefusal as exc:
|
| 1208 |
raise err(400, exc.code, exc.message)
|
| 1209 |
|
| 1210 |
-
|
| 1211 |
-
|
|
|
|
|
|
|
|
|
|
| 1212 |
|
| 1213 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1214 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 1215 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 1216 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1217 |
if wall_nodes:
|
| 1218 |
unknown = sorted(sheet_mod.filter_columns_named(wall_nodes) - set(cols))
|
| 1219 |
if unknown:
|
|
@@ -1228,12 +1883,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None):
|
|
| 1228 |
"you on a column this database carries")
|
| 1229 |
|
| 1230 |
# ββ the binding's own scope ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1231 |
-
|
| 1232 |
-
|
| 1233 |
-
|
| 1234 |
-
scope.get("view_name", ""))
|
| 1235 |
-
else:
|
| 1236 |
-
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 1237 |
|
| 1238 |
# β NOTHING IS DONE HERE ABOUT `needs_excluded`. `_source_for` has already resolved it into the
|
| 1239 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
|
@@ -1264,6 +1916,21 @@ def _resolve_binding(session: Session, raw_binding, *, today=None):
|
|
| 1264 |
|
| 1265 |
row = cur.execute(sql, params).fetchone()
|
| 1266 |
value = row[0] if row else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1267 |
return {"value": value, "limits": limits, "database": database, "binding": binding}
|
| 1268 |
|
| 1269 |
|
|
@@ -1288,6 +1955,182 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 1288 |
"limits": out["limits"]}
|
| 1289 |
|
| 1290 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1291 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 1292 |
def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
|
| 1293 |
"""`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
|
|
@@ -1330,8 +2173,12 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 1330 |
raise err(400, exc.code, exc.message)
|
| 1331 |
|
| 1332 |
database = _require_database(session, binding["database"])
|
| 1333 |
-
cur, spec, cols = _mirror(session)(database)
|
| 1334 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1335 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 1336 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 1337 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
|
@@ -1343,6 +2190,19 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 1343 |
else:
|
| 1344 |
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 1345 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1346 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 1347 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 1348 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
|
|
@@ -1360,3 +2220,156 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
|
|
| 1360 |
else:
|
| 1361 |
out["filter"] = scope.get("filter")
|
| 1362 |
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
(standing rule 2), and each names both the cause and the subject.
|
| 36 |
"""
|
| 37 |
|
| 38 |
+
import hashlib
|
| 39 |
+
import json
|
| 40 |
import os
|
| 41 |
import tempfile
|
| 42 |
import threading
|
|
|
|
| 90 |
`RuntimeError` means the mirror is completing its first sync, so it is a 503 the caller may
|
| 91 |
retry. `DatastoreMismatch` SUBCLASSES `RuntimeError`, so the guard must run ABOVE the try, and
|
| 92 |
collapsing them would turn the loudest refusal in the system into a spinner.
|
| 93 |
+
|
| 94 |
+
ββ W53-T13 β `_for` TAKES A `metric_window` NOW, AND EVERY EXISTING CALL SITE IS UNCHANGED.
|
| 95 |
+
It is keyword-only with a None default, so `_mirror(session)(database)` still means "leave
|
| 96 |
+
every Metric column at its own declared window". See `_registry_mirror` for what a window does.
|
| 97 |
"""
|
| 98 |
import routes_odoo_tables as rot
|
| 99 |
import odoo_relational as rel
|
| 100 |
|
| 101 |
+
def _for(table_key, *, metric_window=None, needed=None):
|
| 102 |
fields = None
|
| 103 |
for _bucket, key, _label, mk_fields in rel.TABLES:
|
| 104 |
if key == table_key:
|
| 105 |
fields = mk_fields()
|
| 106 |
break
|
| 107 |
+
# ββ W52-T11 β `TABLES` NAMES EIGHT ODOO GRIDS AND DELIBERATELY NOT THE TWO COMPILED
|
| 108 |
+
# REGISTRY GRIDS (W33-T44: `customer_data` / `product_data` present the same subjects and
|
| 109 |
+
# would duplicate them). `_for` used to check `TABLES` alone, so a cell bound to either of
|
| 110 |
+
# those two 404'd as "not a connected Odoo database" even though both plainly are one β
|
| 111 |
+
# it was consulting the wrong roster, not reporting a real refusal. `_REGISTRY_GRID_FIELDS`
|
| 112 |
+
# is the OTHER roster; a key found there is exactly as connected as one found in `TABLES`.
|
| 113 |
+
# β `TABLES` ITSELF IS UNTOUCHED β it has other consumers (`declared_schema_fields`,
|
| 114 |
+
# `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
|
| 115 |
+
is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
|
| 116 |
+
if fields is None and not is_registry_grid:
|
| 117 |
raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
|
| 118 |
try:
|
| 119 |
cur, mismatch = rot._mirror_cur(session)
|
|
|
|
| 121 |
raise err(503, "store_not_ready", str(exc))
|
| 122 |
if mismatch:
|
| 123 |
raise err(409, "cross_tenant_store", mismatch)
|
| 124 |
+
if is_registry_grid:
|
| 125 |
+
# ββ ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
|
| 126 |
+
# resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
|
| 127 |
+
# assembly rather than a `scoped_table` read: measured on the running app, 40 to 70
|
| 128 |
+
# seconds each. Forty cells on one database would rebuild that forty times and time the
|
| 129 |
+
# door out, which is a worse failure than the one this ticket fixes because it takes the
|
| 130 |
+
# whole workbook with it. `_view_filter`'s own `memo` argument exists for exactly this
|
| 131 |
+
# shape; the cache hangs off the SESSION because that object is built fresh by
|
| 132 |
+
# `require_session` on every request and dies with it, so nothing can outlive its own
|
| 133 |
+
# cursor or leak across users. The temp table it names lives in that same cursor.
|
| 134 |
+
#
|
| 135 |
+
# ββ W54-T36 β AND ONLY THE COLUMNS THE CELL ACTUALLY READS ARE MATERIALISED.
|
| 136 |
+
# MEASURED on tenant #0's `customer_data` (3,673 records, 47 columns), warm, through
|
| 137 |
+
# the real `drill_cell`: 8,437 ms total, of which `odoo_relational.registry_grid_spec`
|
| 138 |
+
# is 6,137 ms. That step turns every row into a tuple of 47 Python values and
|
| 139 |
+
# `executemany`s it, so its cost is per COLUMN, and a drill reads none of them: it
|
| 140 |
+
# counts records and lists ids. `needed` is the honest set (the aggregate's column,
|
| 141 |
+
# plus every column the cell's filter and the row wall name); `None` still means the
|
| 142 |
+
# whole grid, so a caller that has not worked out its set loses nothing.
|
| 143 |
+
#
|
| 144 |
+
# β THE MEMO WIDENS, IT NEVER NARROWS, and that is what keeps `refresh_sheet` safe.
|
| 145 |
+
# A slot already built for a wider set answers a narrower request unchanged; a request
|
| 146 |
+
# naming a column the slot does not carry rebuilds at the UNION, so a workbook whose
|
| 147 |
+
# cells read different columns converges instead of rebuilding per cell. A `None`
|
| 148 |
+
# entry means every column is present and is reusable by anything.
|
| 149 |
+
memo = getattr(session, "_sheet_mirrors", None)
|
| 150 |
+
if memo is None:
|
| 151 |
+
memo = {}
|
| 152 |
+
session._sheet_mirrors = memo
|
| 153 |
+
slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
|
| 154 |
+
want = None if needed is None else {str(k) for k in needed}
|
| 155 |
+
entry = memo.get(slot)
|
| 156 |
+
if entry is not None:
|
| 157 |
+
built, made = entry
|
| 158 |
+
if built is None or (want is not None and want <= built):
|
| 159 |
+
return made
|
| 160 |
+
if want is not None:
|
| 161 |
+
want = set(want) | built
|
| 162 |
+
made = _registry_mirror(session, cur, table_key, metric_window, needed=want)
|
| 163 |
+
memo[slot] = (None if want is None else set(want), made)
|
| 164 |
+
return made
|
| 165 |
spec = rot._source_for(cur, table_key)
|
| 166 |
if spec is None:
|
| 167 |
raise err(409, "not_read_through",
|
| 168 |
+
f"{table_key} has no read-through binding on this deployment, so a cell cannot "
|
| 169 |
+
"read through to it yet.")
|
| 170 |
+
# β THE FIELD LIST RIDES ON THE SPEC (W53-T13). `_resolve_binding` has to ask whether the
|
| 171 |
+
# bound column can be TOTALLED at all, and that question needs the field's own `type` and
|
| 172 |
+
# `rollup` bag, which `_filter_columns` reduces away. Copying rather than mutating is
|
| 173 |
+
# `_source_for`'s own idiom (it returns `{**spec, "cols": resolved}`) and matters for the
|
| 174 |
+
# same reason: the spec dict handed back by `_source_for` is not this call's to keep.
|
| 175 |
+
return cur, {**spec, "fields": fields}, rot._filter_columns(spec, fields)
|
| 176 |
|
| 177 |
return _for
|
| 178 |
|
| 179 |
|
| 180 |
+
def _registry_assembly(session: Session, table_key: str):
|
| 181 |
+
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
|
| 182 |
+
|
| 183 |
+
ββ W53-T13 β THIS IS THE FIX, AND IT IS A SECOND CALLER OF AN ESTABLISHED PATH RATHER THAN A
|
| 184 |
+
NEW ONE. `_offered_view` above already reaches for these same two assemblies. Measured on the
|
| 185 |
+
running app as `admin@royal-imports`: `/workspace?scope=customer` serves **47** fields, while
|
| 186 |
+
`perm_scope.scoped_fields` resolves through `routes_grid.py::_customer_rows`, which returns
|
| 187 |
+
`list(aios_grid.FIELDS)` - the **static 37-key contract**. So the Sheet was building its temp
|
| 188 |
+
table out of 37 of the tenant's 47 columns and `aggregate_sql` then refused the other 10 as
|
| 189 |
+
`field_unknown`, Metric and plain `custom_` alike. `count` was the only aggregate that worked
|
| 190 |
+
because it returns `COUNT(*)` before it ever looks at `columns`, which is exactly the owner's
|
| 191 |
+
report: *"It seems only Count works."*
|
| 192 |
+
|
| 193 |
+
ββ IT WIDENS NOTHING, AND THAT IS CHECKED RATHER THAN HOPED. `grid_assembly` applies
|
| 194 |
+
`perm_scope.apply_row_scope` (`routes_customers.py:358`) and `perm_scope.hidden_keys`
|
| 195 |
+
(`routes_customers.py:496`), narrowing the field list, the rows AND the overlay dict together;
|
| 196 |
+
`product_assembly` does the same. It is the wall this person's OWN grid goes through, so a
|
| 197 |
+
Sheet cell sourced from it can only ever see what they are already shown.
|
| 198 |
+
|
| 199 |
+
β `consume_corrections=False` IS NOT OPTIONAL, for `_offered_view`'s reason: those one-shot
|
| 200 |
+
field-name correction acks belong to the `/workspace` refresh that exists to show them to the
|
| 201 |
+
person who made the edit, and a bound cell resolving its scope must not eat them.
|
| 202 |
+
|
| 203 |
+
ββ W54-T36 β ONE ASSEMBLY PER DATABASE PER REQUEST, memoised on the SESSION exactly as
|
| 204 |
+
`_mirror` memoises the mirror and for the same lifetime reason: `require_session` builds that
|
| 205 |
+
object per request and it dies with the request, so nothing here outlives its own cursor or
|
| 206 |
+
reaches another account. MEASURED on tenant #0's `customer_data`, warm: 2,145 ms a call. It
|
| 207 |
+
already ran once per (database, window); it now also survives a mirror rebuilt at a wider
|
| 208 |
+
column set, and every caller BELOW reuses it. `_offered_view` reaches for `grid_assembly` and
|
| 209 |
+
`product_assembly` on its own and is a separate cost; see its own note.
|
| 210 |
+
|
| 211 |
+
β AN UNEXPECTED FAILURE REFUSES LOUDLY AND DOES NOT DEGRADE TO THE STATIC CONTRACT. Falling
|
| 212 |
+
back would answer the cell out of 37 columns while the person is looking at 47, which is the
|
| 213 |
+
defect this function exists to end, and it would do it with a 200.
|
| 214 |
+
"""
|
| 215 |
+
import core.perm_scope as perm_scope
|
| 216 |
+
memo = getattr(session, "_sheet_assemblies", None)
|
| 217 |
+
if memo is None:
|
| 218 |
+
memo = {}
|
| 219 |
+
session._sheet_assemblies = memo
|
| 220 |
+
if table_key in memo:
|
| 221 |
+
return memo[table_key]
|
| 222 |
+
try:
|
| 223 |
+
if table_key == "product_data":
|
| 224 |
+
from routes_products import product_assembly
|
| 225 |
+
memo[table_key] = product_assembly(session, scope="product",
|
| 226 |
+
consume_corrections=False)
|
| 227 |
+
return memo[table_key]
|
| 228 |
+
from routes_customers import grid_assembly
|
| 229 |
+
memo[table_key] = grid_assembly(session, scope="customer", consume_corrections=False)
|
| 230 |
+
return memo[table_key]
|
| 231 |
+
except perm_scope.Denied:
|
| 232 |
+
raise err(403, "database_unreadable", f"You cannot read {table_key}.")
|
| 233 |
+
except HTTPException:
|
| 234 |
+
raise
|
| 235 |
+
except Exception as exc: # noqa: BLE001
|
| 236 |
+
# β THE SENTENCE A PERSON READS SAYS WHAT HAPPENED; THE LOG SAYS WHY. Without this line
|
| 237 |
+
# the refusal is honest and undiagnosable at the same time: it names the database and the
|
| 238 |
+
# act, and nothing anywhere records which call inside a 300 line assembly failed.
|
| 239 |
+
# `harness.telemetry.error` is the same channel `routes_customers._measure_err` uses.
|
| 240 |
+
try:
|
| 241 |
+
import harness.telemetry as _tel
|
| 242 |
+
_tel.error("api:sheet-registry-assembly", exc)
|
| 243 |
+
except Exception: # noqa: BLE001
|
| 244 |
+
pass
|
| 245 |
+
raise err(503, "store_not_ready",
|
| 246 |
+
f"{table_key} could not be assembled for this cell, so the number is not being "
|
| 247 |
+
"shown rather than shown from an older set of columns. Try again in a moment.")
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
|
| 251 |
+
"""`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
|
| 252 |
+
|
| 253 |
+
ββ R7's SECOND HALF LIVES HERE: *"must also be bounded by date time lookup"*. `g["derived"]`
|
| 254 |
+
resolves each Metric column at ITS OWN declared window (`{kind: "ltm"}` on both of the live
|
| 255 |
+
ones), NOT at the cell's period, so a cell labelled one month would otherwise print a twelve
|
| 256 |
+
month number. When the binding carries a period, every Metric column is re-resolved at THAT
|
| 257 |
+
window through the SAME evaluator the grid renders from (`core/measure_resolve.py::
|
| 258 |
+
column_values` for the customer grain, `routes_products._measure_cells` for the product one),
|
| 259 |
+
so a Sheet cell and the grid cannot disagree about a number.
|
| 260 |
+
|
| 261 |
+
β A METRIC THAT CANNOT BE RESOLVED AT THE REQUESTED WINDOW GOES BLANK AND IS REPORTED. It
|
| 262 |
+
does NOT fall back to its own window: that would answer a wider question under a narrower
|
| 263 |
+
label, with nothing on screen saying so. The report is a `limits` row (standing rule 1).
|
| 264 |
+
|
| 265 |
+
β AND A METRIC THIS ACCOUNT MAY NOT SEE IS NEVER RE-RESOLVED. `grid_assembly` skips
|
| 266 |
+
`column_values` entirely when `may_metrics` is off, so a column with no value anywhere in
|
| 267 |
+
`g["derived"]` is one the capability wall has already answered for. Re-resolving it here would
|
| 268 |
+
be a second answer to that question, from the one door that never asked it.
|
| 269 |
+
|
| 270 |
+
ββ W54-T36 β `needed` IS THE COLUMN SET THIS CELL CAN POSSIBLY READ, AND `None` IS EVERY
|
| 271 |
+
COLUMN. It narrows what gets MATERIALISED, never what gets assembled: `g` is the same scoped
|
| 272 |
+
assembly, `rows` are the same scoped rows, and `spec["fields"]` below is still the FULL list
|
| 273 |
+
so a refusal can name a column the mirror did not carry.
|
| 274 |
+
|
| 275 |
+
ββ IT IS INTERSECTED WITH THE ASSEMBLY'S OWN FIELDS AND MUST STAY THAT WAY. `cols` is what
|
| 276 |
+
`_resolve_binding` subtracts a row wall's columns from to decide whether the wall can be
|
| 277 |
+
applied at all, and that check is FAIL CLOSED (409 `scope_not_applicable`). Materialising a
|
| 278 |
+
column just because a filter named it would invent an empty column for a name this database
|
| 279 |
+
does not have, and the wall would then compile against nothing and quietly stop restricting.
|
| 280 |
+
"""
|
| 281 |
+
import aios_grid
|
| 282 |
+
import odoo_relational as rel
|
| 283 |
+
import routes_odoo_tables as rot
|
| 284 |
+
from modules import sheet as sheet_mod
|
| 285 |
+
|
| 286 |
+
g = _registry_assembly(session, table_key)
|
| 287 |
+
fields = list(g.get("fields") or [])
|
| 288 |
+
derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
|
| 289 |
+
rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
|
| 290 |
+
derived=derived)
|
| 291 |
+
# The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
|
| 292 |
+
# `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
|
| 293 |
+
kept = (fields if needed is None
|
| 294 |
+
else [f for f in fields if str(f.get("key") or "") in needed])
|
| 295 |
+
# β THE TYPES ARE NORMALISED FOR THE MIRROR ONLY. See `modules/sheet.py::mirror_field_type`:
|
| 296 |
+
# `metric`, `rating` and a numeric `rollup` are in neither `_REGISTRY_COL_TYPE` nor
|
| 297 |
+
# `filter_sql.NUMERIC_TYPES`, so without this they land as VARCHAR and compare as strings.
|
| 298 |
+
# β THE SAME LIST GOES TO BOTH CALLS, deliberately. `registry_grid_spec` picks the SQL type and
|
| 299 |
+
# `_filter_columns` picks the compiler's type; handing them two different lists is how a column
|
| 300 |
+
# ends up stored as one thing and compared as another.
|
| 301 |
+
spec_fields = sheet_mod.mirror_fields(kept)
|
| 302 |
+
spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows)
|
| 303 |
+
# ββ THE MATERIALISED TABLE IS GIVEN THE WINDOW'S OWN NAME, AND WITHOUT THIS THE MEMO ABOVE
|
| 304 |
+
# SERVES ONE WINDOW'S ROWS UNDER ANOTHER WINDOW'S ANSWER (found by W53-T14, live).
|
| 305 |
+
# `odoo_relational.registry_grid_spec` names its temp table `_registry_grid_<table_key>` and
|
| 306 |
+
# `CREATE OR REPLACE`s it, so the name carries the DATABASE and not the PERIOD, while
|
| 307 |
+
# `_mirror`'s memo is keyed by `(table_key, window)`. Resolve one customer_data cell at no
|
| 308 |
+
# period and a second at `{month, 2026-08}` inside ONE request and the second build replaces
|
| 309 |
+
# the first one's table underneath the spec the memo is still holding; a third cell reading
|
| 310 |
+
# the first slot then runs its SQL against August's rows.
|
| 311 |
+
# β MEASURED, and it is silent: `avg` over `measure_sales_the_last_12_months_6hlkf` answered
|
| 312 |
+
# **138.3942281513749** in a batch that had already resolved that month (508,322.0 / 3,673)
|
| 313 |
+
# and **1710.679825755513** on its own, which is the all-time number the preview door and the
|
| 314 |
+
# grid both give. No refusal, no limits row, a plausible number under the wrong label.
|
| 315 |
+
# β IT IS FIXED HERE RATHER THAN IN `registry_grid_spec` BECAUSE THAT FILE BELONGS TO ANOTHER
|
| 316 |
+
# LANE THIS WAVE. The copy is one `CREATE TABLE AS SELECT` over rows already in memory, next
|
| 317 |
+
# to an assembly that costs 20 to 130 seconds, so it is not a cost worth conditioning on.
|
| 318 |
+
# β ALWAYS, INCLUDING THE `None` WINDOW: the unqualified name is REPLACED by every later
|
| 319 |
+
# build, so leaving the no-period case pointing at it is exactly the case measured above.
|
| 320 |
+
# ββ W54-T36 β THE COLUMN SET IS PART OF THAT NAME FOR THE SAME REASON THE WINDOW IS. Two
|
| 321 |
+
# cells on one database at one window may now be materialised over DIFFERENT columns, and a
|
| 322 |
+
# narrower later build would otherwise replace a wider earlier one under a spec the memo is
|
| 323 |
+
# still holding, so a query would address a column its own table no longer has.
|
| 324 |
+
slot = hashlib.sha1(
|
| 325 |
+
json.dumps([metric_window, None if needed is None else sorted(needed)],
|
| 326 |
+
sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12]
|
| 327 |
+
at_window = f'{spec["table"]}__at_{slot}'
|
| 328 |
+
cur.execute(f'CREATE OR REPLACE TEMP TABLE "{at_window}" AS '
|
| 329 |
+
f'SELECT * FROM "{spec["table"]}"')
|
| 330 |
+
spec = {**spec, "table": at_window}
|
| 331 |
+
# β `fields` HERE IS THE ASSEMBLY'S OWN LIST, NOT THE MIRROR-TYPED COPY. `aggregate_refusal`
|
| 332 |
+
# has to name WHY a column cannot be totalled, and the normalised copy has already thrown that
|
| 333 |
+
# away: a `concatenate` rollup reads as plain `text` there, so the sentence would say "it holds
|
| 334 |
+
# text" instead of "it folds its linked records into text rather than into a number".
|
| 335 |
+
spec = {**spec, "fields": fields, "unbounded": limits, "bounded": bounded}
|
| 336 |
+
return cur, spec, rot._filter_columns(spec, spec_fields)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def _metric_derived(session: Session, table_key: str, g, metric_window):
|
| 340 |
+
"""`(derived, bounded_keys, unbounded)` for one assembly at the cell's period.
|
| 341 |
+
|
| 342 |
+
`unbounded` is `{field key: limits row}` and is KEYED rather than listed on purpose: a mirror
|
| 343 |
+
is built once for a database and a cell reads one or two of its columns, so reporting every
|
| 344 |
+
Metric column that failed would bury the one the reader is actually looking at under a list
|
| 345 |
+
about columns their cell never touches. `_resolve_binding` picks the rows for the columns the
|
| 346 |
+
binding names.
|
| 347 |
+
|
| 348 |
+
With no period this is `g["derived"]` exactly as the grid uses it, no metric keys claimed as
|
| 349 |
+
bounded, and nothing to report. With one, every Metric column this account can already see is
|
| 350 |
+
recomputed at that window, ONE COLUMN AT A TIME so a failure can be attributed to the column
|
| 351 |
+
it belongs to rather than reported as "something went wrong".
|
| 352 |
+
|
| 353 |
+
β THE MEMO IS PER CALL. `core/measure_resolve.py`'s memo key carries a POOL STAMP this function
|
| 354 |
+
does not hold, and inventing one would park an answer under a stamp that never invalidates:
|
| 355 |
+
the cell would keep serving a number from before the last sync. A fresh dict costs one query
|
| 356 |
+
per Metric column and cannot go stale.
|
| 357 |
+
"""
|
| 358 |
+
import aios_grid
|
| 359 |
+
from modules import sheet as sheet_mod
|
| 360 |
+
|
| 361 |
+
derived = g.get("derived") or {}
|
| 362 |
+
mfields = aios_grid.measure_fields_of(g.get("fields") or [])
|
| 363 |
+
if not metric_window or not mfields:
|
| 364 |
+
return derived, (), {}
|
| 365 |
+
|
| 366 |
+
# Which Metric columns carry a value at all today: the capability wall's own answer, reused.
|
| 367 |
+
seen = set()
|
| 368 |
+
for cells in derived.values():
|
| 369 |
+
seen.update(cells or ())
|
| 370 |
+
live = [f for f in mfields if f.get("key") in seen]
|
| 371 |
+
|
| 372 |
+
out = {pid: dict(cells or {}) for pid, cells in derived.items()}
|
| 373 |
+
for f in mfields: # every Metric column starts blank at a new window
|
| 374 |
+
for cells in out.values():
|
| 375 |
+
cells.pop(f.get("key"), None)
|
| 376 |
+
|
| 377 |
+
bounded, unbounded = [], {}
|
| 378 |
+
for f in live:
|
| 379 |
+
key = f.get("key")
|
| 380 |
+
label = str(f.get("label") or key)
|
| 381 |
+
rewritten = dict(f, measure=dict(f.get("measure") or {}, window=metric_window))
|
| 382 |
+
try:
|
| 383 |
+
cells = _measure_cells_at(session, table_key, g, rewritten)
|
| 384 |
+
except Exception: # noqa: BLE001
|
| 385 |
+
cells = {}
|
| 386 |
+
got = False
|
| 387 |
+
for pid, values in (cells or {}).items():
|
| 388 |
+
if key in values:
|
| 389 |
+
out.setdefault(pid, {})[key] = values[key]
|
| 390 |
+
got = True
|
| 391 |
+
if got:
|
| 392 |
+
bounded.append(key)
|
| 393 |
+
else:
|
| 394 |
+
unbounded[key] = sheet_mod.metric_window_limit(
|
| 395 |
+
label,
|
| 396 |
+
"this column's metric could not be measured over the period this cell names, so "
|
| 397 |
+
"it was left blank rather than answered over its own wider window")
|
| 398 |
+
for f in mfields:
|
| 399 |
+
if f.get("key") not in seen:
|
| 400 |
+
# β THE CAUSE NAMES BOTH POSSIBILITIES because this door cannot tell them apart, and
|
| 401 |
+
# a sentence that picks one would be confidently wrong half the time. A column with no
|
| 402 |
+
# value anywhere in `derived` is either one the metrics capability withholds from this
|
| 403 |
+
# account or one whose own declared window already failed to resolve.
|
| 404 |
+
unbounded[f.get("key")] = sheet_mod.metric_window_limit(
|
| 405 |
+
str(f.get("label") or f.get("key")),
|
| 406 |
+
"this column has no value on this database for this account, either because the "
|
| 407 |
+
"metric behind it is not shared with you or because it could not be measured at "
|
| 408 |
+
"all, so there was nothing to bound to the period")
|
| 409 |
+
return out, tuple(bounded), unbounded
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def _measure_cells_at(session: Session, table_key: str, g, field):
|
| 413 |
+
"""`{pid: {field key: value}}` for ONE rewritten Metric column, through its grain's evaluator.
|
| 414 |
+
|
| 415 |
+
β TWO GRAINS, TWO EVALUATORS, AND NEITHER IS RE-IMPLEMENTED HERE. `core/measure_resolve.py`
|
| 416 |
+
is CUSTOMER grain (it resolves through `harness/measure_filter.py`, whose entity is the
|
| 417 |
+
partner); the product grain goes through `semantic.entity_measure_values`, which is what
|
| 418 |
+
`routes_products._measure_cells` wraps and which joins by SKU CODE rather than by pid. Calling
|
| 419 |
+
the wrong one would answer a column of blanks on one of the two grids.
|
| 420 |
+
"""
|
| 421 |
+
if table_key == "product_data":
|
| 422 |
+
import routes_products as rp
|
| 423 |
+
return rp._measure_cells([field], g.get("rows_src") or [], g.get("team_id"),
|
| 424 |
+
g.get("today"), "sheet-period", {},
|
| 425 |
+
rp.product_measures(session))
|
| 426 |
+
from core import measure_resolve
|
| 427 |
+
return measure_resolve.column_values([field], g.get("team_id"), g.get("pids") or frozenset(),
|
| 428 |
+
g.get("today"), "sheet-period", {})
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _offered_view(session: Session, database: str, view_id: str):
|
| 432 |
+
"""The view THIS database's own workspace assembly offers under `view_id`, or None.
|
| 433 |
+
|
| 434 |
+
ββ WAVE 53 FOLLOW-UP TO W53-T12 β `find_view` ALONE CANNOT SEE A PINNED VIEW NOBODY HAS
|
| 435 |
+
TOUCHED, AND THE BINDING EDITOR OFFERS EXACTLY THOSE. `all-customers` is pinned PER USER
|
| 436 |
+
(`core/grid_events.py:1652`: `_pinned_per_user = (_kind == 'system' or view_id in
|
| 437 |
+
('all-customers', _ag_overview_id()))`), so every user of a table gets their own copy of that
|
| 438 |
+
id, but only once they have touched it β an autosave, a column resize. An account that never
|
| 439 |
+
customised it has NO stored record, `find_view` answers None, and this door then refused, by
|
| 440 |
+
name, a view the `/workspace` door had just offered the same session. Measured on the running
|
| 441 |
+
app as `admin@royal-imports`: `/workspace?scope=customer` served exactly one view,
|
| 442 |
+
`all-customers`, and a cell bound to it answered `409 view_missing: View All customers no
|
| 443 |
+
longer exists.` That sentence is the owner's report of D-662's sibling, word for word.
|
| 444 |
+
|
| 445 |
+
ββ SO THE FALLBACK BUILDS THE SAME ASSEMBLY THE `/workspace` DOOR BUILDS AND PICKS THE VIEW
|
| 446 |
+
OUT OF IT. The mechanism is copied from `routes_alerts.py::_require_filtered_view`, including
|
| 447 |
+
its `_view_by_id` β IMPORTED rather than re-written, because two hand-rolled "find the view in
|
| 448 |
+
an assembly" walks is exactly the shape that let `g["views"] IS A LIST` raise on every call for
|
| 449 |
+
thirteen waves ([[one-question-two-normalizers]]).
|
| 450 |
+
|
| 451 |
+
β `consume_corrections=False` IS NOT OPTIONAL (`routes_alerts.py:498`). Those one-shot
|
| 452 |
+
field-name correction acks belong to the `/workspace` refresh that exists to show them to the
|
| 453 |
+
person who made the edit; a bound cell resolving its scope must not eat them.
|
| 454 |
+
|
| 455 |
+
β `with_rows=False` on the `ut_` leg for the reason W31-T20 built the flag: only `g["views"]`
|
| 456 |
+
is read here, and building the table would make a read-through grain refuse
|
| 457 |
+
`409 window_required` over rows nobody asked for.
|
| 458 |
+
|
| 459 |
+
β AN UNEXPECTED FAILURE ANSWERS None RATHER THAN ESCAPING. This function only ever runs after
|
| 460 |
+
`find_view` has already missed, so the cell was on its way to a refusal either way; letting a
|
| 461 |
+
raw exception out would turn one cell's bad luck into a 500 for the whole workbook refresh.
|
| 462 |
+
A refusal that already NAMES ITS OWN CAUSE (`HTTPException`) is let through instead, because
|
| 463 |
+
"the store is not ready" is a truer answer than "that view no longer exists".
|
| 464 |
+
"""
|
| 465 |
+
key = str(database or "")
|
| 466 |
+
try:
|
| 467 |
+
if key.startswith("ut_"):
|
| 468 |
+
from routes_tables import ut_assembly
|
| 469 |
+
g = ut_assembly(session, key,
|
| 470 |
+
storage_key=f"{session.tenant}:{key}:{session.uname}",
|
| 471 |
+
consume_corrections=False, with_rows=False)
|
| 472 |
+
elif key == "product_data":
|
| 473 |
+
from routes_products import product_assembly
|
| 474 |
+
g = product_assembly(session, scope="product", consume_corrections=False)
|
| 475 |
+
elif key == "customer_data":
|
| 476 |
+
from routes_customers import grid_assembly
|
| 477 |
+
g = grid_assembly(session, scope="customer", consume_corrections=False)
|
| 478 |
+
else:
|
| 479 |
+
return None
|
| 480 |
+
except HTTPException:
|
| 481 |
+
raise
|
| 482 |
+
except Exception: # noqa: BLE001
|
| 483 |
+
return None
|
| 484 |
+
from routes_alerts import _view_by_id
|
| 485 |
+
return _view_by_id(g, view_id)
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def _view_filter(session: Session, database: str, view_id: str, view_name: str = "",
|
| 489 |
+
memo=None):
|
| 490 |
"""`(nodes, conj)` for a saved view's filter, or a `view_missing` refusal that NAMES it.
|
| 491 |
|
| 492 |
+
ββ TWO LOOKUPS, CHEAPEST FIRST, AND THE ORDER IS THE CONTRACT (wave 53, follow-up to
|
| 493 |
+
W53-T12). `find_view` answers from the caller's OWN stored record and is authoritative when it
|
| 494 |
+
hits, trashed record included (R11). Only on a MISS does `_offered_view` rebuild the workspace
|
| 495 |
+
assembly, which is what resolves a pinned view this account has never customised. A miss in
|
| 496 |
+
BOTH still refuses by name: an id nobody can resolve must never quietly mean "every row", which
|
| 497 |
+
would serve a whole database under a scope nobody wrote.
|
| 498 |
+
|
| 499 |
+
β `memo` is a PER-REQUEST dict, keyed `(database, view_id)`, and it exists because the two
|
| 500 |
+
per-cell loops in this file resolve a binding CELL BY CELL. `refresh_sheet`'s own docstring
|
| 501 |
+
claimed "each distinct `view_id` is resolved ONCE" and the code never did that: `seen_db`
|
| 502 |
+
deduplicates the DATABASE WALL only, and the resolve underneath it ran per cell. That was
|
| 503 |
+
survivable while the whole lookup was one `find_view`; with an assembly behind the fallback,
|
| 504 |
+
forty cells on one view would rebuild it forty times. The memo caches the REFUSAL too, so a
|
| 505 |
+
workbook full of cells on one deleted view costs one lookup rather than forty.
|
| 506 |
+
|
| 507 |
ββ `prefer_owner=session.uname` IS LOAD-BEARING AND OMITTING IT IS A SILENT CROSS-USER READ.
|
| 508 |
The system view id `all-customers` is pinned on every database, and every user who has ever
|
| 509 |
touched that view's config holds their OWN record under the same id, so a bare first-hit scan
|
|
|
|
| 515 |
β The refusal names the view, which is why `clean_binding` keeps `view_name` on the binding:
|
| 516 |
once the record is deleted there is nothing left to look a name up from.
|
| 517 |
"""
|
| 518 |
+
slot = (str(database or ""), str(view_id or ""))
|
| 519 |
+
if memo is not None and slot in memo:
|
| 520 |
+
cached = memo[slot]
|
| 521 |
+
if isinstance(cached, Exception):
|
| 522 |
+
raise cached
|
| 523 |
+
return cached
|
| 524 |
+
try:
|
| 525 |
+
out = _view_filter_once(session, database, view_id, view_name)
|
| 526 |
+
except HTTPException as exc:
|
| 527 |
+
if memo is not None:
|
| 528 |
+
memo[slot] = exc
|
| 529 |
+
raise
|
| 530 |
+
if memo is not None:
|
| 531 |
+
memo[slot] = out
|
| 532 |
+
return out
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
def _view_filter_once(session: Session, database: str, view_id: str, view_name: str = ""):
|
| 536 |
+
"""`_view_filter` without the memo. Read that docstring first; this is only its body."""
|
| 537 |
+
from core import table_store, view_templates
|
| 538 |
+
# ββ THE BUCKET IS `<key>_table_workspace`, AND THIS DOOR WAS ASKING FOR THE BARE KEY (wave
|
| 539 |
+
# 53, follow-up to W53-T12). `view_templates.workspace_key` is the ONE normaliser between a
|
| 540 |
+
# database key and the store bucket its saved views live in β `customer_data` lives in
|
| 541 |
+
# `customer_table_workspace`, `product_data` in `product_table_workspace`, a `ut_` table in
|
| 542 |
+
# `<key>_table_workspace` β and every other reader in the app already goes through it or
|
| 543 |
+
# spells the suffix out (`routes_starred`, `routes_tables`, `routes_shares`, `routes_grid`,
|
| 544 |
+
# `routes_automation`). `TableStore("customer_data")` reads a bucket nothing writes, so this
|
| 545 |
+
# lookup could not hit for ANY database: `prefer_owner` and `include_trashed` below were both
|
| 546 |
+
# unreachable, and R11's promise that a trashed view keeps a cell working for 30 days was
|
| 547 |
+
# dead code. Fixing the key is what makes the cheap leg able to answer at all.
|
| 548 |
+
# β `workspace_key` answers None for a key that is not a table; the bare key is kept in that
|
| 549 |
+
# case so nothing that used to resolve stops resolving.
|
| 550 |
# β `TableStore(table_key, st=...)`, in that order. `st` is the STORE HANDLE, and passing the
|
| 551 |
# session's runtime is what makes the lookup read THIS tenant's bucket rather than tenant #0's.
|
| 552 |
+
store = table_store.TableStore(view_templates.workspace_key(database) or str(database or ""),
|
| 553 |
+
st=session.runtime)
|
| 554 |
# ββ `include_trashed=True` IS R11, AND IT IS ONE KEYWORD BECAUSE THE DEFAULT MUST NOT MOVE
|
| 555 |
# (wave 51, W51-T26). A deleted view is MOVED into `<owner>.trash`, so the bare scan answered
|
| 556 |
# None and this door refused a cell whose view somebody deleted an hour ago and can still
|
|
|
|
| 561 |
# permission-adjacent (shares, starred, publish, forms); widening there would open doors no
|
| 562 |
# ticket asked about, with every gate green.
|
| 563 |
found = store.find_view(view_id, prefer_owner=session.uname, include_trashed=True)
|
| 564 |
+
view = found[1] if found else None
|
| 565 |
+
if view is None:
|
| 566 |
+
# ββ THE SECOND LOOKUP, AND IT RUNS ONLY ON A MISS. See `_offered_view`: a pinned view
|
| 567 |
+
# this account has never touched has no stored record anywhere, so the workspace assembly
|
| 568 |
+
# is the only place it exists. A miss HERE too is a genuinely gone view and still refuses
|
| 569 |
+
# by name.
|
| 570 |
+
view = _offered_view(session, database, view_id)
|
| 571 |
+
if view is None:
|
| 572 |
raise err(409, "view_missing", f"View {view_name or view_id} no longer exists.")
|
|
|
|
| 573 |
import routes_odoo_tables as rot
|
| 574 |
+
# ββ D-662 β THE FILTER TREE LIVES AT `config.filters`, NOT AT THE RECORD'S TOP LEVEL.
|
| 575 |
+
# Reading `view["filters"]` answered None on every stored view, and `tree_parts(None)` is
|
| 576 |
+
# `([], "and")` β a legal EMPTY filter, so EVERY view-scoped cell resolved with no filter at
|
| 577 |
+
# all and reported the whole database, with nothing anywhere going red.
|
| 578 |
+
#
|
| 579 |
+
# ββ AND THE READER IS `core.sheets.adopted_scope_filter`, NOT A FOURTH HAND-ROLLED WALK.
|
| 580 |
+
# That function is this repo's canonical answer to "which rows does this view mean": its own
|
| 581 |
+
# docstring names THIS door as the one that got it wrong, and it already carries the three
|
| 582 |
+
# shapes a stored record comes in β `config.filters` as a `{conj, nodes}` tree, `config.filters`
|
| 583 |
+
# as a BARE LIST whose conjunction lives beside it in `config.filterConj` (a bare-list read
|
| 584 |
+
# through `tree_parts` alone would silently answer `and` for an OR view), and a LOCKED view,
|
| 585 |
+
# which is a curated ROW SET and adopts its COHORT leaf rather than the incidental filters its
|
| 586 |
+
# config happens to hold (`DESIGN.md` section 4, the three locks). The cohort leaf does not
|
| 587 |
+
# compile on the Sheet path and refuses by name, which is the honest answer: a locked view's
|
| 588 |
+
# membership is not a predicate, and inventing one here would put a plausible wrong number on
|
| 589 |
+
# a screen. It deep-copies, so nothing here can mutate the stored view.
|
| 590 |
+
from core import sheets as sheets_mod
|
| 591 |
+
return rot._fe().tree_parts(sheets_mod.adopted_scope_filter(view))
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def _materialize_copy_binding(raw_binding, filter_tree):
|
| 595 |
+
"""A copied saved-View binding with its own filter, without mutating either input.
|
| 596 |
+
|
| 597 |
+
R5 is deliberately a COPY-TIME action. The source cell keeps following its saved View, while
|
| 598 |
+
the copy receives a deep local shadow and can step a chosen leaf without editing shared View
|
| 599 |
+
state. A binding already scoped to a local filter passes through the normal cleaner unchanged.
|
| 600 |
+
"""
|
| 601 |
+
from modules import sheet as sheet_mod
|
| 602 |
+
import copy as _copy
|
| 603 |
+
|
| 604 |
+
binding = sheet_mod.clean_binding(raw_binding)
|
| 605 |
+
if not binding["scope"].get("view_id"):
|
| 606 |
+
return binding
|
| 607 |
+
return {**binding, "scope": {"filter": _copy.deepcopy(filter_tree)}}
|
| 608 |
|
| 609 |
|
| 610 |
#: β THE TWO CAP CODES ANSWER 413, NOT 400 (contract C3). A 400 says the request was malformed and
|
|
|
|
| 791 |
return out
|
| 792 |
|
| 793 |
|
| 794 |
+
def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
|
| 795 |
+
"""The scope hash `_resolve_binding` would compute under, for THIS caller, right now (R3,
|
| 796 |
+
W52-T12).
|
| 797 |
+
|
| 798 |
+
Mirrors the wall half of `_resolve_binding` without opening the mirror or running SQL, so a
|
| 799 |
+
cache hit can be decided before any database work runs. Both this function and
|
| 800 |
+
`_resolve_binding` derive `wall_tree` the same way and both feed `core.sheets.scope_hash`, the
|
| 801 |
+
ONE function that turns those pieces into a key, so a future change to the wall's shape cannot
|
| 802 |
+
make the two drift silently apart.
|
| 803 |
+
"""
|
| 804 |
+
from core import sheets
|
| 805 |
+
import core.perm_scope as perm_scope
|
| 806 |
+
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 807 |
+
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 808 |
+
return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
|
| 809 |
+
|
| 810 |
+
|
| 811 |
def _reresolved_for_reader(session: Session, book):
|
| 812 |
"""A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
|
| 813 |
|
|
|
|
| 823 |
|
| 824 |
β `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of
|
| 825 |
literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it.
|
| 826 |
+
|
| 827 |
+
ββ W52-T12 / R3: A CACHE HIT SKIPS `_resolve_binding` ENTIRELY, AND A MISS FILLS ONLY THE
|
| 828 |
+
CALLER'S OWN SLOT. The cache lives in a document `core.sheets` owns and keys by
|
| 829 |
+
`(database, wall, binding scope)` (`_binding_scope_key`), never by user id or role name, so
|
| 830 |
+
two readers who share a row restriction share an answer and a reader whose restriction differs
|
| 831 |
+
never reads the other's slot. Only `refresh_sheet` may ever clear it; a GET here only ever
|
| 832 |
+
fills the ONE slot the caller themselves is entitled to.
|
| 833 |
"""
|
| 834 |
+
from core import sheets
|
| 835 |
import copy as _copy
|
| 836 |
fresh = _copy.deepcopy(book)
|
| 837 |
stamp = _now_iso()
|
| 838 |
seen_db = {}
|
| 839 |
+
# β The same per-request memo as `refresh_sheet`, for the same reason and with the same
|
| 840 |
+
# lifetime: this loop resolves cell by cell too, and a reader's GET must not rebuild one
|
| 841 |
+
# database's workspace assembly once per bound cell.
|
| 842 |
+
view_memo = {}
|
| 843 |
touched = 0
|
| 844 |
+
store = _store(session)
|
| 845 |
+
wid = book.get("id") or ""
|
| 846 |
for tab in fresh.get("sheets") or ():
|
| 847 |
+
tab_ref = str(tab.get("id") or tab.get("name") or "")
|
| 848 |
+
for addr, cell in (tab.get("cells") or {}).items():
|
| 849 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
| 850 |
continue
|
| 851 |
touched += 1
|
|
|
|
| 865 |
"message": f"You cannot read {db}."}
|
| 866 |
cell.pop("limits", None)
|
| 867 |
continue
|
| 868 |
+
|
| 869 |
+
cell_key = tab_ref + "!" + addr
|
| 870 |
+
scope_key = _binding_scope_key(session, db, binding)
|
| 871 |
+
hit = None
|
| 872 |
+
if wid and scope_key:
|
| 873 |
+
try:
|
| 874 |
+
hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
|
| 875 |
+
except Exception: # noqa: BLE001
|
| 876 |
+
hit = None
|
| 877 |
+
if hit is not None:
|
| 878 |
+
cell["state"] = "fresh"
|
| 879 |
+
cell["value"] = hit.get("value")
|
| 880 |
+
cell["display"] = "" if hit.get("value") is None else str(hit.get("value"))
|
| 881 |
+
cell["refreshed_at"] = hit.get("refreshedAt")
|
| 882 |
+
cell.pop("refusal", None)
|
| 883 |
+
#: β NOT CLAIMED ON A CACHE HIT: `limits` describes THIS resolve's own truncation,
|
| 884 |
+
#: and a cached slot never recorded one, so a prior (possibly stale) value on the
|
| 885 |
+
#: cell is dropped rather than shown as if it described the answer just served.
|
| 886 |
+
cell.pop("limits", None)
|
| 887 |
+
continue
|
| 888 |
+
|
| 889 |
try:
|
| 890 |
+
out = _resolve_binding(session, binding, view_memo=view_memo)
|
| 891 |
except HTTPException as exc:
|
| 892 |
detail = getattr(exc, "detail", None) or {}
|
| 893 |
info = detail.get("error") if isinstance(detail, dict) else None
|
|
|
|
| 907 |
cell["limits"] = out["limits"]
|
| 908 |
else:
|
| 909 |
cell.pop("limits", None)
|
| 910 |
+
#: β FILLED AFTER THE CELL ABOVE IS DONE READING `out`, and best effort: a cache-write
|
| 911 |
+
#: failure only costs a future recompute, never a wrong number, so it never raises past
|
| 912 |
+
#: here.
|
| 913 |
+
if wid and scope_key:
|
| 914 |
+
try:
|
| 915 |
+
sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp,
|
| 916 |
+
out["database"])
|
| 917 |
+
except Exception: # noqa: BLE001
|
| 918 |
+
pass
|
| 919 |
if not touched:
|
| 920 |
return fresh
|
| 921 |
# β EVERY FORMULA IS RE-EVALUATED TOO, over the numbers this reader is entitled to. Leaving
|
|
|
|
| 1129 |
stamp = _now_iso()
|
| 1130 |
made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname)
|
| 1131 |
saved = sheets.put_workbook(runtime, made["id"],
|
| 1132 |
+
dict(book, refreshed_at=stamp, rev=made["rev"]))
|
|
|
|
| 1133 |
_job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
|
| 1134 |
sentence=_import_sentence(report))
|
| 1135 |
except sheet_import.ImportRefusal as exc:
|
|
|
|
| 1294 |
found = sheets.find_tab(served, tab)
|
| 1295 |
except sheets.SheetRefusal as exc:
|
| 1296 |
raise _refuse(exc)
|
| 1297 |
+
out = {"id": book.get("id"), "rev": book.get("rev"), "tab": found,
|
| 1298 |
+
"schemaVersion": book["schemaVersion"]}
|
| 1299 |
if served.get("refreshedFor"):
|
| 1300 |
out["refreshedFor"] = served["refreshedFor"]
|
| 1301 |
return out
|
|
|
|
| 1324 |
`workbook_read_only`, not 409, because 409 names a problem they could fix by refetching and
|
| 1325 |
this one they cannot fix at all. An editor's write falls straight through to the rev guard,
|
| 1326 |
which is exactly what C5 asks for.
|
| 1327 |
+
|
| 1328 |
+
ββ W52-T10 β A FORMULA EVALUATES WHEN YOU COMMIT IT. `refresh_sheet` used to be the ONLY
|
| 1329 |
+
caller of `_recompute_formulas`, so `=SUM(A1)` landed in the tab as the literal text and sat
|
| 1330 |
+
there until somebody pressed Refresh. `computed` now rides out on every accepted patch: the
|
| 1331 |
+
whole workbook is recomputed IN MEMORY (never a second write, never a second `rev`, see
|
| 1332 |
+
`_recomputed_tab_cells`) and every FORMULA cell on THIS tab is handed back with its answer.
|
| 1333 |
"""
|
| 1334 |
from core import sheets
|
| 1335 |
raw = dict(body or {})
|
|
|
|
| 1346 |
expect_rev=expect)
|
| 1347 |
except sheets.SheetRefusal as exc:
|
| 1348 |
raise _refuse(exc)
|
| 1349 |
+
accepted = out.pop("accepted", [])
|
| 1350 |
+
out["computed"] = _recomputed_tab_cells(session, workbook_id, tab, changes=accepted)
|
| 1351 |
+
return out
|
| 1352 |
+
|
| 1353 |
+
|
| 1354 |
+
def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, changes=None):
|
| 1355 |
+
"""W52-T10: every FORMULA cell on `tab_ref`, recomputed over the WHOLE workbook, IN MEMORY.
|
| 1356 |
+
|
| 1357 |
+
ββ THIS IS THE WHOLE FIX. The parser and the evaluator were already correct
|
| 1358 |
+
(`formula_runtime.py::_normal` upper-cases the function name and `evaluate_sheet` already
|
| 1359 |
+
resolves dependencies across every tab, W51-T23); the defect was that nothing on the commit
|
| 1360 |
+
path ever CALLED either one. This is that call.
|
| 1361 |
+
|
| 1362 |
+
β IN MEMORY, LIKE `_reresolved_for_reader`, AND FOR THE SAME REASON: this is a READ over what
|
| 1363 |
+
the PATCH above just persisted, not a second write. A caller that persisted the recomputed
|
| 1364 |
+
values would have to bump `rev` a second time for one commit, which is a revision the client
|
| 1365 |
+
never asked for and would have to guess at on its NEXT patch.
|
| 1366 |
+
|
| 1367 |
+
β A BOUND CELL NEEDS NOTHING EXTRA HERE. `_recompute_formulas` reads any non-formula cell's
|
| 1368 |
+
stored `value` (a literal's or a bound cell's, whichever it is) straight off the document, so
|
| 1369 |
+
`=SUM(A1)` sees A1's live database-bound number the moment A1 itself carries one, with no
|
| 1370 |
+
branch here that knows or cares which kind of cell A1 is.
|
| 1371 |
+
|
| 1372 |
+
β RECOMPUTED OVER THE WHOLE BOOK, RETURNED FOR ONE TAB. A cross-tab formula needs every tab's
|
| 1373 |
+
cells to resolve its reference (W51-T23), so the pass runs over the whole workbook; C3's
|
| 1374 |
+
client opens one tab at a time, so only the tab this commit landed on is worth sending back.
|
| 1375 |
+
A formula sitting on a tab nobody has open still shows its last answer until that tab is
|
| 1376 |
+
opened or the workbook is refreshed, exactly as a stale database-bound cell does.
|
| 1377 |
+
"""
|
| 1378 |
+
from core import sheets
|
| 1379 |
+
from modules import sheet as sheet_mod
|
| 1380 |
+
import formula_runtime as fr
|
| 1381 |
+
|
| 1382 |
+
try:
|
| 1383 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1384 |
+
found = sheets.find_tab(book, tab_ref)
|
| 1385 |
+
except sheets.SheetRefusal:
|
| 1386 |
+
return {}
|
| 1387 |
+
edits = None
|
| 1388 |
+
if changes is not None:
|
| 1389 |
+
tab_at = sheets._tab_index(book["sheets"], tab_ref)
|
| 1390 |
+
scope = _tab_scope_names(book["sheets"])[tab_at]
|
| 1391 |
+
edits = {fr.a1_key(str(addr).strip().upper(), scope):
|
| 1392 |
+
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
|
| 1393 |
+
_recompute_formulas(book, fr, sheet_mod,
|
| 1394 |
+
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
|
| 1395 |
+
found = sheets.find_tab(book, tab_ref)
|
| 1396 |
+
out = {}
|
| 1397 |
+
for addr, cell in (found.get("cells") or {}).items():
|
| 1398 |
+
if not isinstance(cell, dict) or (cell.get("kind") != "formula"
|
| 1399 |
+
and "spill" not in cell and "spillParent" not in cell):
|
| 1400 |
+
continue
|
| 1401 |
+
row = {"value": cell.get("value"),
|
| 1402 |
+
"display": "" if cell.get("value") is None else str(cell.get("display") or ""),
|
| 1403 |
+
"state": cell.get("state") or "fresh"}
|
| 1404 |
+
if cell.get("refusal"):
|
| 1405 |
+
row["refusal"] = cell["refusal"]
|
| 1406 |
+
for key in ("error", "style", "formulaMeta", "spill", "spillParent"):
|
| 1407 |
+
if key in cell:
|
| 1408 |
+
row[key] = cell[key]
|
| 1409 |
+
out[addr] = row
|
| 1410 |
return out
|
| 1411 |
|
| 1412 |
|
|
|
|
| 1561 |
forty cells to one database asks once; a reader who has lost access to that database gets a
|
| 1562 |
refusal on every one of its cells rather than a stale number quietly repainting.
|
| 1563 |
|
| 1564 |
+
β Each distinct `view_id` is resolved ONCE, and as of the wave-53 follow-up to W53-T12 that
|
| 1565 |
+
sentence is finally TRUE OF THE CODE. It was written of `seen_db`, which deduplicates the
|
| 1566 |
+
DATABASE WALL and nothing else: the resolve underneath it ran per cell, so a forty-cell
|
| 1567 |
+
workbook did forty `table_store.find_view` calls and forty legacy migrations with them.
|
| 1568 |
+
`view_memo` below is the dedupe this sentence describes.
|
| 1569 |
|
| 1570 |
ββ C5 β THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED
|
| 1571 |
HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed
|
|
|
|
| 1587 |
|
| 1588 |
stamp = _now_iso()
|
| 1589 |
seen_db = {}
|
| 1590 |
+
# β ONE MEMO FOR THE WHOLE REFRESH, keyed `(database, view_id)`. `seen_db` deduplicates the
|
| 1591 |
+
# WALL and never the scope, so forty cells on one saved view used to resolve that view forty
|
| 1592 |
+
# times. See `_view_filter`: the second lookup behind it builds a workspace assembly, which is
|
| 1593 |
+
# work no workbook should pay for per cell. Per REQUEST, never on the runtime: a view's filter
|
| 1594 |
+
# may be edited between two refreshes and a longer lived cache would serve the old rows.
|
| 1595 |
+
view_memo = {}
|
| 1596 |
for tab in book.get("sheets") or ():
|
| 1597 |
for addr, cell in (tab.get("cells") or {}).items():
|
| 1598 |
if not isinstance(cell, dict) or cell.get("kind") != "bound":
|
|
|
|
| 1613 |
"message": f"You cannot read {db}."}
|
| 1614 |
continue
|
| 1615 |
try:
|
| 1616 |
+
out = _resolve_binding(session, binding, view_memo=view_memo)
|
| 1617 |
except HTTPException as exc:
|
| 1618 |
detail = getattr(exc, "detail", None) or {}
|
| 1619 |
info = detail.get("error") if isinstance(detail, dict) else None
|
|
|
|
| 1653 |
"refreshed_at": stamp, "rev": book.get("rev")})
|
| 1654 |
except sheets.SheetRefusal as exc:
|
| 1655 |
raise _refuse(exc)
|
| 1656 |
+
# ββ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
|
| 1657 |
+
# VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
|
| 1658 |
+
# recomputes once, under their own scope, and fills its own slot again from there.
|
| 1659 |
+
sheets.clear_scope_cache(_store(session), workbook_id)
|
| 1660 |
return {"workbook": saved, "recalc_ms": recalc_ms}
|
| 1661 |
|
| 1662 |
|
|
|
|
| 1694 |
return names
|
| 1695 |
|
| 1696 |
|
| 1697 |
+
def _recompute_formulas(book, fr, sheet_mod, *, cache_key=None, changes=None):
|
| 1698 |
"""Re-evaluate every formula cell across the WORKBOOK, in one memoized dependency pass.
|
| 1699 |
|
| 1700 |
ββ ONE PASS OVER EVERY TAB, NOT ONE PASS PER TAB (W51-T23). Per tab, `DATA!B12` was never in
|
|
|
|
| 1708 |
A cell whose value is None because the database refused is handed to the engine as absent, so
|
| 1709 |
arithmetic over it stays None rather than silently counting it as nothing.
|
| 1710 |
"""
|
| 1711 |
+
# The document engine owns calculation, including spills and error traces. Pass the
|
| 1712 |
+
# whole workbook (names, epoch, formats and array metadata) and keep its in-place result.
|
| 1713 |
+
# This additive adapter becomes active when the engine ticket supplies the callable.
|
| 1714 |
+
recompute_document = getattr(fr, "recompute_document", None)
|
| 1715 |
+
if callable(recompute_document):
|
| 1716 |
+
return recompute_document(book, cache_key=cache_key, changes=changes)
|
| 1717 |
+
|
| 1718 |
tabs = [t for t in (book.get("sheets") or ()) if isinstance(t, dict)]
|
| 1719 |
names = _tab_scope_names(tabs)
|
| 1720 |
spec = {}
|
|
|
|
| 1795 |
}.get(code, "This formula could not be calculated.")
|
| 1796 |
|
| 1797 |
|
| 1798 |
+
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None):
|
| 1799 |
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
|
| 1800 |
|
| 1801 |
The order below is the contract, not a preference:
|
|
|
|
| 1803 |
2. the mirror cursor, whose two failures mean opposite things;
|
| 1804 |
3. the ROW wall, fail-closed, refusing 409 when it names a column the mirror cannot answer;
|
| 1805 |
4. the binding's own scope, whose unanswerable columns are REPORTED rather than dropped.
|
| 1806 |
+
|
| 1807 |
+
β W54-T36 β THE WALL AND THE SCOPE ARE NOW READ BEFORE STEP 2 SO THE MIRROR KNOWS WHICH
|
| 1808 |
+
COLUMNS TO MATERIALISE. Steps 3 and 4 still happen where they did: the wall is still checked
|
| 1809 |
+
against `cols` after the mirror exists, and it still refuses fail closed.
|
| 1810 |
+
|
| 1811 |
+
ββ W53-T13 / R7 β THE PERIOD IS RESOLVED BEFORE THE MIRROR IS OPENED, because it decides what
|
| 1812 |
+
the Metric columns of that mirror MEAN. `modules/sheet.py::period_window` turns the cell's
|
| 1813 |
+
`{unit, start}` into the custom window `harness/windows.py` speaks, and `_registry_mirror`
|
| 1814 |
+
re-resolves every Metric column at it. A period that bounds nothing is reported rather than
|
| 1815 |
+
left silent: see `period_limits`.
|
| 1816 |
"""
|
| 1817 |
from modules import sheet as sheet_mod
|
| 1818 |
import core.perm_scope as perm_scope
|
|
|
|
| 1822 |
except sheet_mod.SheetRefusal as exc:
|
| 1823 |
raise err(400, exc.code, exc.message)
|
| 1824 |
|
| 1825 |
+
# β C2's OTHER AGGREGATE ARM IS ANSWERED HERE, NOT LEFT TO FALL THROUGH TO `aggregate_sql`,
|
| 1826 |
+
# where it died as `field_unknown` with an empty subject. See `metric_key_refusal`.
|
| 1827 |
+
metric_refusal = sheet_mod.metric_key_refusal(binding["aggregate"])
|
| 1828 |
+
if metric_refusal is not None:
|
| 1829 |
+
raise err(400, metric_refusal.code, metric_refusal.message)
|
| 1830 |
|
| 1831 |
+
database = _require_database(session, binding["database"])
|
| 1832 |
+
window = sheet_mod.period_window(binding.get("period"))
|
| 1833 |
+
|
| 1834 |
+
# ββ W54-T36 β THE WALL AND THE SCOPE ARE READ BEFORE THE MIRROR IS OPENED, so the mirror can
|
| 1835 |
+
# be built over the columns this cell can possibly read instead of over the whole database.
|
| 1836 |
+
# Neither read touches the mirror: the wall comes from `perm_scope` and the scope from the
|
| 1837 |
+
# stored view document, which is why the order can move at all.
|
| 1838 |
+
# β WHAT MUST NOT MOVE IS THE WALL'S OWN CHECK. It subtracts the wall's columns from `cols`
|
| 1839 |
+
# and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
|
| 1840 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 1841 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 1842 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 1843 |
+
scope = binding["scope"]
|
| 1844 |
+
if scope.get("view_id"):
|
| 1845 |
+
scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
|
| 1846 |
+
scope.get("view_name", ""), memo=view_memo)
|
| 1847 |
+
else:
|
| 1848 |
+
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 1849 |
+
needed = (sheet_mod.filter_columns_named(scope_nodes)
|
| 1850 |
+
| sheet_mod.filter_columns_named(wall_nodes))
|
| 1851 |
+
if binding["aggregate"].get("field"):
|
| 1852 |
+
needed.add(str(binding["aggregate"]["field"]))
|
| 1853 |
+
|
| 1854 |
+
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
|
| 1855 |
+
|
| 1856 |
+
# β THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
|
| 1857 |
+
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
|
| 1858 |
+
# error and a `sum` over a browser-computed formula column reaches it as a confident 0.
|
| 1859 |
+
bound_field = next((f for f in (spec.get("fields") or [])
|
| 1860 |
+
if isinstance(f, dict)
|
| 1861 |
+
and f.get("key") == (binding["aggregate"].get("field") or None)), None)
|
| 1862 |
+
agg_refusal = sheet_mod.aggregate_refusal(binding["aggregate"], bound_field)
|
| 1863 |
+
if agg_refusal is not None:
|
| 1864 |
+
raise err(409, agg_refusal.code, agg_refusal.message)
|
| 1865 |
+
if bound_field and bound_field.get("measure") and not binding.get("period"):
|
| 1866 |
+
label = bound_field.get("label") or bound_field.get("key")
|
| 1867 |
+
raise err(409, "metric_period_required",
|
| 1868 |
+
f"{label} is measured over a period, so this cell needs one. "
|
| 1869 |
+
"Open the cell's Period section and choose a period.")
|
| 1870 |
+
|
| 1871 |
+
# ββ the ROW wall, fail-closed ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1872 |
if wall_nodes:
|
| 1873 |
unknown = sorted(sheet_mod.filter_columns_named(wall_nodes) - set(cols))
|
| 1874 |
if unknown:
|
|
|
|
| 1883 |
"you on a column this database carries")
|
| 1884 |
|
| 1885 |
# ββ the binding's own scope ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1886 |
+
# β W54-T36 β READ ABOVE, WITH THE WALL, because the mirror is now built over the columns the
|
| 1887 |
+
# two of them name. The contract's ORDER is unchanged: the wall is still applied and still
|
| 1888 |
+
# checked against `cols` before the scope reaches the compiler.
|
|
|
|
|
|
|
|
|
|
| 1889 |
|
| 1890 |
# β NOTHING IS DONE HERE ABOUT `needs_excluded`. `_source_for` has already resolved it into the
|
| 1891 |
# spec's column expressions; applying it a second time would double-count the exclusion and
|
|
|
|
| 1916 |
|
| 1917 |
row = cur.execute(sql, params).fetchone()
|
| 1918 |
value = row[0] if row else None
|
| 1919 |
+
# ββ EVERY WAY THIS ANSWER CAME OUT WIDER THAN THE BINDING ASKED FOR TRAVELS WITH IT
|
| 1920 |
+
# (standing rule 1's second sentence). Three sources, one list: a filter leaf the compiler
|
| 1921 |
+
# could not push down (`build_aggregate_query`), a Metric column that could not be measured at
|
| 1922 |
+
# the cell's period (`_metric_derived`), and a period that bounds nothing at all because the
|
| 1923 |
+
# bound column is stored per record rather than measured over time (`period_limits`).
|
| 1924 |
+
# β ONLY THE COLUMNS THIS BINDING DEPENDS ON. See `_metric_derived`: the mirror knows about
|
| 1925 |
+
# every Metric column on the database and a cell reads one or two of them, so the aggregate's
|
| 1926 |
+
# own column plus every column its filter names is the honest set.
|
| 1927 |
+
depends = set(sheet_mod.filter_columns_named(scope_nodes))
|
| 1928 |
+
if binding["aggregate"].get("field"):
|
| 1929 |
+
depends.add(str(binding["aggregate"]["field"]))
|
| 1930 |
+
limits = (list(limits)
|
| 1931 |
+
+ [row for key, row in (spec.get("unbounded") or {}).items() if key in depends]
|
| 1932 |
+
+ sheet_mod.period_limits(binding.get("period"), binding["aggregate"], bound_field,
|
| 1933 |
+
spec.get("bounded") or (), depends))
|
| 1934 |
return {"value": value, "limits": limits, "database": database, "binding": binding}
|
| 1935 |
|
| 1936 |
|
|
|
|
| 1955 |
"limits": out["limits"]}
|
| 1956 |
|
| 1957 |
|
| 1958 |
+
@router.post("/sheets/{workbook_id}/binding-shadows")
|
| 1959 |
+
def binding_shadows(workbook_id: str, body: dict = Body(default=None),
|
| 1960 |
+
session: Session = Depends(require_session)):
|
| 1961 |
+
"""Canonical saved-View filters needed before copied bindings step (C5, R5).
|
| 1962 |
+
|
| 1963 |
+
request: `{"bindings": [<binding>, ...]}`
|
| 1964 |
+
response: `{"shadows": [{database, viewId, filter}, ...]}`
|
| 1965 |
+
|
| 1966 |
+
The source bindings and saved Views are read only. The browser materialises each returned tree
|
| 1967 |
+
into the copied binding, commits the stale cells immediately, then schedules Save's evaluator.
|
| 1968 |
+
The editor wall is intentional: this door exists only to prepare a workbook write, and a
|
| 1969 |
+
viewer must never turn a reader-specific filter into the owner's stored document.
|
| 1970 |
+
"""
|
| 1971 |
+
from core import sheets
|
| 1972 |
+
from modules import sheet as sheet_mod
|
| 1973 |
+
|
| 1974 |
+
try:
|
| 1975 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 1976 |
+
_require_editor(session, book)
|
| 1977 |
+
except sheets.SheetRefusal as exc:
|
| 1978 |
+
raise _refuse(exc)
|
| 1979 |
+
|
| 1980 |
+
raw_bindings = (body or {}).get("bindings")
|
| 1981 |
+
if not isinstance(raw_bindings, list):
|
| 1982 |
+
raise err(400, "sheet_bindings_required",
|
| 1983 |
+
"Choose the database cells whose saved Views should be copied.")
|
| 1984 |
+
|
| 1985 |
+
shadows = []
|
| 1986 |
+
seen = set()
|
| 1987 |
+
view_memo = {}
|
| 1988 |
+
for raw in raw_bindings:
|
| 1989 |
+
try:
|
| 1990 |
+
binding = sheet_mod.clean_binding(raw)
|
| 1991 |
+
except sheet_mod.SheetRefusal as exc:
|
| 1992 |
+
raise err(400, exc.code, exc.message)
|
| 1993 |
+
scope = binding["scope"]
|
| 1994 |
+
view_id = str(scope.get("view_id") or "")
|
| 1995 |
+
if not view_id:
|
| 1996 |
+
continue
|
| 1997 |
+
database = _require_database(session, binding["database"])
|
| 1998 |
+
slot = (database, view_id)
|
| 1999 |
+
if slot in seen:
|
| 2000 |
+
continue
|
| 2001 |
+
seen.add(slot)
|
| 2002 |
+
nodes, conj = _view_filter(session, database, view_id, scope.get("view_name", ""),
|
| 2003 |
+
memo=view_memo)
|
| 2004 |
+
local = _materialize_copy_binding(
|
| 2005 |
+
binding, {"conj": conj, "nodes": nodes})
|
| 2006 |
+
shadows.append({"database": database, "viewId": view_id,
|
| 2007 |
+
"filter": local["scope"]["filter"]})
|
| 2008 |
+
return {"shadows": shadows}
|
| 2009 |
+
|
| 2010 |
+
|
| 2011 |
+
@router.post("/sheets/{workbook_id}/cells/{addr}/compute")
|
| 2012 |
+
def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
| 2013 |
+
session: Session = Depends(require_session)):
|
| 2014 |
+
"""W54-T32 (instruction 9): recompute ONE stored bound cell and persist just that cell.
|
| 2015 |
+
|
| 2016 |
+
ββ THE WHOLE OF THE TICKET IS THAT `refresh_sheet` WAS THE ONLY DOOR THAT COMPUTED. Saving a
|
| 2017 |
+
binding wrote a `stale` cell and nothing ever resolved it, so the shimmer that means "a number
|
| 2018 |
+
is coming" ran until somebody pressed Refresh. Refresh answers by recomputing EVERY bound cell
|
| 2019 |
+
and every formula in the workbook, which is why it is slow and why the save path must not call
|
| 2020 |
+
it: one cell over `ut_odoo_orders` measured 55 to 90 ms through `preview`, against a whole
|
| 2021 |
+
workbook pass plus a full document write.
|
| 2022 |
+
|
| 2023 |
+
β IT IS A WRITE DOOR, so `_require_editor` runs FIRST, exactly as it does on `refresh_sheet`
|
| 2024 |
+
and for the same C5 reason: this ends in a store write, and a viewer's numbers must never land
|
| 2025 |
+
in the owner's document. A viewer's own re-resolve still happens in memory on the READ path.
|
| 2026 |
+
|
| 2027 |
+
ββ THE STORED `binding` IS NEVER REWRITTEN HERE, AND THAT IS D-674's HALF OF THIS TICKET.
|
| 2028 |
+
Only `state`, `value`, `display`, `refreshed_at` and `refusal` are touched on a COPY of the
|
| 2029 |
+
cell; the binding object is carried through verbatim. So a binding that arrived without a
|
| 2030 |
+
`period` cannot acquire one by being computed, whatever a later editor does with the panel.
|
| 2031 |
+
|
| 2032 |
+
ββ AND IT FILLS THE ONE SCOPE-CACHE SLOT IT JUST COMPUTED, WHICH IS THE TRAP IN THIS FILE.
|
| 2033 |
+
`_reresolved_for_reader` serves a non-owner from `core.sheets.cached_scope_value`, keyed
|
| 2034 |
+
`(database, wall, binding scope)`; persisting a new number and leaving the old slot behind
|
| 2035 |
+
would make the very next GET serve the number this door was called to replace. There is no
|
| 2036 |
+
per-cell clear, and `clear_scope_cache` belongs to `refresh_sheet` alone under R3, so the
|
| 2037 |
+
surgical answer is `fill_scope_cache` on exactly the slot this caller is entitled to, which is
|
| 2038 |
+
the same call and the same key `_reresolved_for_reader` makes after its own resolve.
|
| 2039 |
+
|
| 2040 |
+
β NO FORMULA PASS RUNS HERE, DELIBERATELY. `_recomputed_tab_cells` recomputes the WHOLE
|
| 2041 |
+
workbook's formulas, which the prototype measured at 2.9 s as a FLOOR for the owner's 11,300
|
| 2042 |
+
formulas: fifty times the cost of the one number this door exists to deliver. The client
|
| 2043 |
+
evaluates its own formulas as the value lands (`sheetState.ts`), the ordinary cell PATCH
|
| 2044 |
+
already runs the server-side pass on the very next commit, and Refresh still runs it for
|
| 2045 |
+
everyone. A formula's STORED value therefore stays as last computed until one of those, which
|
| 2046 |
+
is what a stale bound cell already did before this ticket.
|
| 2047 |
+
|
| 2048 |
+
β `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
|
| 2049 |
+
its next PATCH would carry a revision the server has already moved past.
|
| 2050 |
+
"""
|
| 2051 |
+
from core import sheets
|
| 2052 |
+
|
| 2053 |
+
try:
|
| 2054 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2055 |
+
_require_editor(session, book)
|
| 2056 |
+
except sheets.SheetRefusal as exc:
|
| 2057 |
+
raise _refuse(exc)
|
| 2058 |
+
|
| 2059 |
+
key = str(addr or "").replace("$", "").upper()
|
| 2060 |
+
#: β THE TAB IS NAMED BY THE CALLER WHEN IT KNOWS ONE, and looked up by address when it does
|
| 2061 |
+
#: not, which is `drill_cell`'s behaviour. Two tabs may legally hold the same address, so a
|
| 2062 |
+
#: client that has a tab in hand should say which, or the first one wins.
|
| 2063 |
+
ref = str((body or {}).get("tab") or "").strip()
|
| 2064 |
+
tab, cell = None, None
|
| 2065 |
+
if ref:
|
| 2066 |
+
try:
|
| 2067 |
+
tab = sheets.find_tab(book, ref)
|
| 2068 |
+
except sheets.SheetRefusal as exc:
|
| 2069 |
+
raise _refuse(exc)
|
| 2070 |
+
cell = (tab.get("cells") or {}).get(key)
|
| 2071 |
+
else:
|
| 2072 |
+
for candidate in book.get("sheets") or ():
|
| 2073 |
+
if key in (candidate.get("cells") or {}):
|
| 2074 |
+
tab, cell = candidate, candidate["cells"][key]
|
| 2075 |
+
break
|
| 2076 |
+
if not isinstance(cell, dict):
|
| 2077 |
+
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
|
| 2078 |
+
if cell.get("kind") != "bound":
|
| 2079 |
+
raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.")
|
| 2080 |
+
|
| 2081 |
+
tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "")
|
| 2082 |
+
binding = cell.get("binding") or {}
|
| 2083 |
+
stamp = _now_iso()
|
| 2084 |
+
saved = dict(cell)
|
| 2085 |
+
limits, resolved = [], None
|
| 2086 |
+
try:
|
| 2087 |
+
resolved = _resolve_binding(session, binding)
|
| 2088 |
+
except HTTPException as exc:
|
| 2089 |
+
#: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door
|
| 2090 |
+
#: and a cell refused by that one read identically. It is PERSISTED rather than raised:
|
| 2091 |
+
#: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
|
| 2092 |
+
detail = getattr(exc, "detail", None) or {}
|
| 2093 |
+
info = detail.get("error") if isinstance(detail, dict) else None
|
| 2094 |
+
saved["state"] = "refused"
|
| 2095 |
+
saved["value"] = None
|
| 2096 |
+
saved["display"] = ""
|
| 2097 |
+
saved["refusal"] = {
|
| 2098 |
+
"code": (info or {}).get("code", "sheet_unresolved"),
|
| 2099 |
+
"message": (info or {}).get("message", "This cell could not be refreshed."),
|
| 2100 |
+
}
|
| 2101 |
+
else:
|
| 2102 |
+
limits = resolved["limits"]
|
| 2103 |
+
saved["state"] = "fresh"
|
| 2104 |
+
saved["value"] = resolved["value"]
|
| 2105 |
+
saved["display"] = "" if resolved["value"] is None else str(resolved["value"])
|
| 2106 |
+
saved["refreshed_at"] = stamp
|
| 2107 |
+
saved.pop("refusal", None)
|
| 2108 |
+
|
| 2109 |
+
try:
|
| 2110 |
+
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved})
|
| 2111 |
+
except sheets.SheetRefusal as exc:
|
| 2112 |
+
raise _refuse(exc)
|
| 2113 |
+
|
| 2114 |
+
if resolved is not None:
|
| 2115 |
+
#: Best effort, exactly as `_reresolved_for_reader` fills its own slot: a cache write that
|
| 2116 |
+
#: fails costs a future recompute and never a wrong number.
|
| 2117 |
+
try:
|
| 2118 |
+
sheets.fill_scope_cache(_store(session), workbook_id, tab_ref + "!" + key,
|
| 2119 |
+
_binding_scope_key(session,
|
| 2120 |
+
str(binding.get("database") or ""),
|
| 2121 |
+
binding),
|
| 2122 |
+
resolved["value"], stamp, resolved["database"])
|
| 2123 |
+
except Exception: # noqa: BLE001
|
| 2124 |
+
pass
|
| 2125 |
+
|
| 2126 |
+
wire = dict(saved)
|
| 2127 |
+
#: β `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
|
| 2128 |
+
#: such key: it describes THIS resolve's truncation, not the cell.
|
| 2129 |
+
if limits:
|
| 2130 |
+
wire["limits"] = limits
|
| 2131 |
+
return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key}
|
| 2132 |
+
|
| 2133 |
+
|
| 2134 |
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
|
| 2135 |
def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
|
| 2136 |
"""`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
|
|
|
|
| 2173 |
raise err(400, exc.code, exc.message)
|
| 2174 |
|
| 2175 |
database = _require_database(session, binding["database"])
|
|
|
|
| 2176 |
|
| 2177 |
+
# ββ W54-T36 β THE WALL AND THE SCOPE ARE READ FIRST, AND THE MIRROR IS BUILT OVER WHAT THEY
|
| 2178 |
+
# NAME. The drill probes `COUNT(*)` and lists ids, so on a cell whose scope is a plain view or
|
| 2179 |
+
# an empty filter the honest column set is EMPTY and only the id column is materialised.
|
| 2180 |
+
# MEASURED on tenant #0's `customer_data` (3,673 records, 47 columns), warm, through this
|
| 2181 |
+
# route: 8,437 ms, of which 6,137 ms was materialising columns no query here reads.
|
| 2182 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 2183 |
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 2184 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
|
|
|
| 2190 |
else:
|
| 2191 |
scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
|
| 2192 |
|
| 2193 |
+
# β THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
|
| 2194 |
+
# reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
|
| 2195 |
+
# different window would answer that condition against different values. The popup would then
|
| 2196 |
+
# list rows the cell's own number never counted.
|
| 2197 |
+
# β W54-T36 MEASURED WHAT THAT WINDOW COSTS, because the cheaper mirror was the obvious guess:
|
| 2198 |
+
# `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the
|
| 2199 |
+
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
|
| 2200 |
+
# the column set instead, where the time actually is.
|
| 2201 |
+
cur, spec, cols = _mirror(session)(
|
| 2202 |
+
database, metric_window=sheet_mod.period_window(binding.get("period")),
|
| 2203 |
+
needed=(sheet_mod.filter_columns_named(scope_nodes)
|
| 2204 |
+
| sheet_mod.filter_columns_named(wall_nodes)))
|
| 2205 |
+
|
| 2206 |
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
|
| 2207 |
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 2208 |
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
|
|
|
|
| 2220 |
else:
|
| 2221 |
out["filter"] = scope.get("filter")
|
| 2222 |
return out
|
| 2223 |
+
|
| 2224 |
+
|
| 2225 |
+
def _resolve_home_tab(book, tabs, names, ref):
|
| 2226 |
+
"""The tab an UNQUALIFIED address in a `resolve-bindings` request belongs to.
|
| 2227 |
+
|
| 2228 |
+
β `ref` MAY BE A TAB `id` OR ITS NAME, exactly as `GET /sheets/{id}/tabs/{tab}` accepts both,
|
| 2229 |
+
and a ref this workbook does not carry is a NAMED 404 rather than an empty answer. The flat
|
| 2230 |
+
`{addr: value}` body of contract C1 has no room to say "that tab is gone", so the only place
|
| 2231 |
+
that sentence can be told is the status line, and an answer of `{}` would read to the client
|
| 2232 |
+
as "none of those cells are bound" - a different problem with a different fix.
|
| 2233 |
+
|
| 2234 |
+
β NO `tab` AT ALL MEANS THE FIRST TAB, which is what `_tab_scope_names` already makes an
|
| 2235 |
+
unqualified reference mean everywhere else in this file.
|
| 2236 |
+
"""
|
| 2237 |
+
from core import sheets
|
| 2238 |
+
ref = str(ref or "").strip()
|
| 2239 |
+
if not ref:
|
| 2240 |
+
return names[0] if names else ""
|
| 2241 |
+
try:
|
| 2242 |
+
found = sheets.find_tab(book, ref)
|
| 2243 |
+
except sheets.SheetRefusal as exc:
|
| 2244 |
+
raise _refuse(exc)
|
| 2245 |
+
for tab, name in zip(tabs, names):
|
| 2246 |
+
if tab is found:
|
| 2247 |
+
return name
|
| 2248 |
+
return names[0] if names else ""
|
| 2249 |
+
|
| 2250 |
+
|
| 2251 |
+
def _asked_key(asked, home, fr):
|
| 2252 |
+
"""One requested address, as the `TAB!ADDR` key the whole-workbook pass already uses.
|
| 2253 |
+
|
| 2254 |
+
β A QUALIFIED REFERENCE KEEPS ITS OWN TAB and an unqualified one lands on `home`, which is
|
| 2255 |
+
the same rule `_recompute_formulas` gives a formula's own references. `'Sales data'!B4` is the
|
| 2256 |
+
quoted spelling a name with a space wears inside a formula, so the quotes are stripped here
|
| 2257 |
+
rather than making that cell unaddressable through this door.
|
| 2258 |
+
"""
|
| 2259 |
+
text = str(asked or "").strip()
|
| 2260 |
+
if "!" not in text:
|
| 2261 |
+
return fr.a1_key(text, home)
|
| 2262 |
+
sheet, _, addr = text.rpartition("!")
|
| 2263 |
+
sheet = sheet.strip()
|
| 2264 |
+
if len(sheet) >= 2 and sheet[0] == "'" and sheet[-1] == "'":
|
| 2265 |
+
sheet = sheet[1:-1].replace("''", "'")
|
| 2266 |
+
return fr.a1_key(addr, sheet or home)
|
| 2267 |
+
|
| 2268 |
+
|
| 2269 |
+
@router.post("/sheets/{workbook_id}/resolve-bindings")
|
| 2270 |
+
def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
|
| 2271 |
+
session: Session = Depends(require_session)):
|
| 2272 |
+
"""`{addr: value}` for the NAMED bound cells only (contract C1, ruling R2).
|
| 2273 |
+
|
| 2274 |
+
ββ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
|
| 2275 |
+
Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the
|
| 2276 |
+
client still needs the server for is the handful of DATABASE-BOUND cells that formula leans
|
| 2277 |
+
on. `POST /sheets/{id}/refresh` keeps recomputing the WHOLE workbook and is untouched: load
|
| 2278 |
+
and import still need it, and lane A still calls it for those two paths. What moved off the
|
| 2279 |
+
commit path is the loop, not the door.
|
| 2280 |
+
|
| 2281 |
+
request: `{"cells": ["B4", "'Sales data'!C9", ...], "tab": "<tab id or name>"}`
|
| 2282 |
+
`tab` is the tab an unqualified address belongs to and defaults to the first tab.
|
| 2283 |
+
|
| 2284 |
+
response: `{"B4": 6283327.0, "Sales data!C9": null}` - the flat map contract C1 names, keyed
|
| 2285 |
+
by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the
|
| 2286 |
+
reference it already holds and never has to re-derive this file's key spelling.
|
| 2287 |
+
|
| 2288 |
+
ββ A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS, AND THAT IS THE ONE
|
| 2289 |
+
DISTINCTION THE SHAPE CAN CARRY. An address that is empty, holds a typed literal or holds a
|
| 2290 |
+
formula is not this door's business - lane A's evaluator already has those values in hand -
|
| 2291 |
+
so asking for one costs nothing and answers nothing.
|
| 2292 |
+
|
| 2293 |
+
ββ AND HERE IS WHAT THE SHAPE CANNOT CARRY, STATED RATHER THAN HIDDEN: a bound cell whose
|
| 2294 |
+
binding was REFUSED just now (a database wall, a row wall, a scope condition this database
|
| 2295 |
+
cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching
|
| 2296 |
+
rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it.
|
| 2297 |
+
`POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full
|
| 2298 |
+
`{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is
|
| 2299 |
+
the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD
|
| 2300 |
+
amendment rather than fixed by widening the shape on one lane's authority, because lane A is
|
| 2301 |
+
the consumer and a second shape agreed by nobody is worse than a known gap.
|
| 2302 |
+
|
| 2303 |
+
β ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a
|
| 2304 |
+
refusal is caught per cell and the siblings still answer.
|
| 2305 |
+
|
| 2306 |
+
ββ THE BATCH PAYS FOR ITS ASSEMBLY ONCE, WHICH IS MOST OF THE REASON THIS DOOR TAKES A LIST
|
| 2307 |
+
AT ALL. A bound cell on a compiled registry grid (`customer_data`, `product_data`) costs a
|
| 2308 |
+
full workspace assembly - measured at 20 to 130 seconds on this box after W53-T13 - so N cells
|
| 2309 |
+
resolved one request each would cost N of them. Three memos make one request pay once:
|
| 2310 |
+
- `session._sheet_assemblies`, keyed by `table_key` inside `_registry_assembly` (W54-T36),
|
| 2311 |
+
so the assembly itself is built once per database however many mirrors are cut from it.
|
| 2312 |
+
- `session._sheet_mirrors`, keyed `(table_key, metric_window)` inside `_mirror`, so every
|
| 2313 |
+
cell on one database at one period shares one temp table. W54-T36 also gave each slot the
|
| 2314 |
+
COLUMN SET it was materialised over: a narrower cell reuses a wider slot, and a cell
|
| 2315 |
+
naming a column the slot lacks rebuilds at the union rather than per cell. Both hang off
|
| 2316 |
+
the Session, which `require_session` builds fresh per request and drops with it.
|
| 2317 |
+
- `view_memo`, keyed by `view_id`, so a saved-view scope is looked up once for the batch.
|
| 2318 |
+
β AND IDENTICAL BINDINGS RESOLVE ONCE. Two cells carrying the same database, aggregate, scope
|
| 2319 |
+
and period cannot disagree inside one request, so the second one reads the first one's answer
|
| 2320 |
+
rather than running the same SQL again. Per REQUEST only, for the same reason `view_memo` is:
|
| 2321 |
+
a saved view's filter may be edited between two calls.
|
| 2322 |
+
|
| 2323 |
+
β A READ DOOR, LIKE `preview_cell` AND `drill_cell`, NOT A WRITE ONE LIKE `refresh_sheet`.
|
| 2324 |
+
It calls neither `_require_editor` nor `put_workbook`, so C5's hazard - a viewer's narrower
|
| 2325 |
+
numbers being SAVED into the owner's document - cannot arise here: nothing this door computes
|
| 2326 |
+
is ever stored. The per-database wall still runs on every cell, inside `_resolve_binding`.
|
| 2327 |
+
|
| 2328 |
+
β NO CAP ON HOW MANY CELLS MAY BE ASKED FOR (standing rule 1). The client asks for the cells
|
| 2329 |
+
one commit actually touched; the memos above are what keep a large batch from multiplying
|
| 2330 |
+
database work, and a batch's cost is dominated by the number of distinct (database, period)
|
| 2331 |
+
pairs in it rather than by the number of cells.
|
| 2332 |
+
"""
|
| 2333 |
+
from core import sheets
|
| 2334 |
+
import formula_runtime as fr
|
| 2335 |
+
|
| 2336 |
+
try:
|
| 2337 |
+
book = sheets.get_workbook(_store(session), workbook_id)
|
| 2338 |
+
except sheets.SheetRefusal as exc:
|
| 2339 |
+
raise _refuse(exc)
|
| 2340 |
+
|
| 2341 |
+
payload = dict(body or {})
|
| 2342 |
+
tabs = [t for t in (book.get("sheets") or ()) if isinstance(t, dict)]
|
| 2343 |
+
names = _tab_scope_names(tabs)
|
| 2344 |
+
home = _resolve_home_tab(book, tabs, names, payload.get("tab"))
|
| 2345 |
+
|
| 2346 |
+
bound = {}
|
| 2347 |
+
for tab, name in zip(tabs, names):
|
| 2348 |
+
for addr, cell in (tab.get("cells") or {}).items():
|
| 2349 |
+
if isinstance(cell, dict) and cell.get("kind") == "bound":
|
| 2350 |
+
bound[fr.a1_key(addr, name)] = cell
|
| 2351 |
+
|
| 2352 |
+
view_memo = {}
|
| 2353 |
+
by_binding = {}
|
| 2354 |
+
answer = {}
|
| 2355 |
+
for asked in (payload.get("cells") or ()):
|
| 2356 |
+
label = str(asked or "")
|
| 2357 |
+
if not label:
|
| 2358 |
+
continue
|
| 2359 |
+
cell = bound.get(_asked_key(label, home, fr))
|
| 2360 |
+
if cell is None:
|
| 2361 |
+
continue
|
| 2362 |
+
binding = cell.get("binding") or {}
|
| 2363 |
+
slot = json.dumps(binding, sort_keys=True, default=str)
|
| 2364 |
+
if slot not in by_binding:
|
| 2365 |
+
try:
|
| 2366 |
+
by_binding[slot] = _resolve_binding(session, binding,
|
| 2367 |
+
view_memo=view_memo)["value"]
|
| 2368 |
+
except HTTPException:
|
| 2369 |
+
# β SWALLOWED HERE AND NOWHERE ELSE. See the docstring: the flat body has no
|
| 2370 |
+
# room for a cause, and failing the whole batch on one walled cell would take
|
| 2371 |
+
# every sibling's number down with it. The same refusal still travels in full on
|
| 2372 |
+
# the refresh and read doors, which is where the cell's own sentence comes from.
|
| 2373 |
+
by_binding[slot] = None
|
| 2374 |
+
answer[label] = by_binding[slot]
|
| 2375 |
+
return answer
|
api/routes_slack.py
CHANGED
|
@@ -18,7 +18,7 @@ live specimen of identity that LOOKS like a permission and is not.
|
|
| 18 |
`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per
|
| 19 |
principal. So item 10 is OUR engine projected onto a CHANNEL principal β see `agent_principal`
|
| 20 |
below, which is the whole projection and is nine lines. Study:
|
| 21 |
-
`
|
| 22 |
|
| 23 |
What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card`
|
| 24 |
below) and nothing else. Its TRANSPORT is AVOID wholesale β hosted CopilotKit Intelligence, a
|
|
@@ -450,7 +450,7 @@ def create_channel_agent(body: dict = Body(default=None),
|
|
| 450 |
for a in existing.values():
|
| 451 |
if isinstance(a, dict) and str(a.get("channel") or "") == channel:
|
| 452 |
raise err(409, "channel_taken",
|
| 453 |
-
"that channel already has an agent
|
| 454 |
aid = secrets.token_urlsafe(9)
|
| 455 |
rec = _clean_agent(dict(body, id=aid, createdBy=session.uname,
|
| 456 |
createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds")))
|
|
@@ -554,7 +554,7 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
|
|
| 554 |
if (fresh.get("perms") or {}) != cleaned:
|
| 555 |
# The one direction this must never fail in: telling an administrator the wall is up when
|
| 556 |
# it is not.
|
| 557 |
-
raise err(503, "store_unavailable", "the permissions were not saved
|
| 558 |
return {"id": str(agent_id), "perms": fresh.get("perms") or {},
|
| 559 |
"perms_v": int(fresh.get("perms_v") or 0)}
|
| 560 |
|
|
@@ -615,7 +615,7 @@ def _is_empty(value):
|
|
| 615 |
False is precisely the kind of write somebody needs to approve, and a naive `if not value`
|
| 616 |
drops both rows silently β the approver then reads a card that does not describe the write
|
| 617 |
they are approving. The MIT licence gate for this borrowing is recorded in
|
| 618 |
-
`
|
| 619 |
"""
|
| 620 |
return value is None or (isinstance(value, str) and value.strip() == "")
|
| 621 |
|
|
@@ -636,7 +636,7 @@ def approval_card(action, values, agent=None):
|
|
| 636 |
"action": str(action or "")[:80],
|
| 637 |
"rows": shown,
|
| 638 |
"truncated": dropped,
|
| 639 |
-
"note": (f"{dropped} more field(s) not shown
|
| 640 |
f"approving." if dropped else ""),
|
| 641 |
"agent": (agent or {}).get("label") or (agent or {}).get("channel") or "",
|
| 642 |
}
|
|
@@ -905,7 +905,7 @@ async def slack_events(request: Request):
|
|
| 905 |
|
| 906 |
team = str(payload.get("team_id") or slug)
|
| 907 |
if not _rate_ok(f"{slug}:{team}", now):
|
| 908 |
-
raise err(429, "too_many_requests", "too many requests
|
| 909 |
|
| 910 |
event = payload.get("event") if isinstance(payload.get("event"), dict) else {}
|
| 911 |
channel = str(event.get("channel") or "")
|
|
@@ -953,7 +953,7 @@ async def slack_interact(request: Request):
|
|
| 953 |
if not isinstance(payload, dict):
|
| 954 |
raise _refuse()
|
| 955 |
if not _rate_ok(f"{slug}:interact", now):
|
| 956 |
-
raise err(429, "too_many_requests", "too many requests
|
| 957 |
|
| 958 |
actions = payload.get("actions") if isinstance(payload.get("actions"), list) else []
|
| 959 |
choice = (actions[0] if actions and isinstance(actions[0], dict) else {})
|
|
@@ -974,7 +974,7 @@ async def slack_interact(request: Request):
|
|
| 974 |
"rejected": "Rejected. Nothing was changed.",
|
| 975 |
# β HONEST, not reassuring: the answering half that would register a committer is booked,
|
| 976 |
# so an approved card today performs nothing and says exactly that rather than "Approved."
|
| 977 |
-
"no_committer": "Approved
|
| 978 |
"failed": "That change could not be completed. Nothing was saved; ask the bot again.",
|
| 979 |
"committed": "Approved.",
|
| 980 |
}.get(verdict, "That approval is no longer available. Ask the bot again.")}
|
|
|
|
| 18 |
`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per
|
| 19 |
principal. So item 10 is OUR engine projected onto a CHANNEL principal β see `agent_principal`
|
| 20 |
below, which is the whole projection and is nine lines. Study:
|
| 21 |
+
`AutoWiki/research/opentag-adoption.md` (licence gate: MIT, PASSED).
|
| 22 |
|
| 23 |
What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card`
|
| 24 |
below) and nothing else. Its TRANSPORT is AVOID wholesale β hosted CopilotKit Intelligence, a
|
|
|
|
| 450 |
for a in existing.values():
|
| 451 |
if isinstance(a, dict) and str(a.get("channel") or "") == channel:
|
| 452 |
raise err(409, "channel_taken",
|
| 453 |
+
"that channel already has an agent; edit it instead of adding a second")
|
| 454 |
aid = secrets.token_urlsafe(9)
|
| 455 |
rec = _clean_agent(dict(body, id=aid, createdBy=session.uname,
|
| 456 |
createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds")))
|
|
|
|
| 554 |
if (fresh.get("perms") or {}) != cleaned:
|
| 555 |
# The one direction this must never fail in: telling an administrator the wall is up when
|
| 556 |
# it is not.
|
| 557 |
+
raise err(503, "store_unavailable", "the permissions were not saved; the agent is UNCHANGED")
|
| 558 |
return {"id": str(agent_id), "perms": fresh.get("perms") or {},
|
| 559 |
"perms_v": int(fresh.get("perms_v") or 0)}
|
| 560 |
|
|
|
|
| 615 |
False is precisely the kind of write somebody needs to approve, and a naive `if not value`
|
| 616 |
drops both rows silently β the approver then reads a card that does not describe the write
|
| 617 |
they are approving. The MIT licence gate for this borrowing is recorded in
|
| 618 |
+
`AutoWiki/research/opentag-adoption.md`.
|
| 619 |
"""
|
| 620 |
return value is None or (isinstance(value, str) and value.strip() == "")
|
| 621 |
|
|
|
|
| 636 |
"action": str(action or "")[:80],
|
| 637 |
"rows": shown,
|
| 638 |
"truncated": dropped,
|
| 639 |
+
"note": (f"{dropped} more field(s) not shown; open the record to review them before "
|
| 640 |
f"approving." if dropped else ""),
|
| 641 |
"agent": (agent or {}).get("label") or (agent or {}).get("channel") or "",
|
| 642 |
}
|
|
|
|
| 905 |
|
| 906 |
team = str(payload.get("team_id") or slug)
|
| 907 |
if not _rate_ok(f"{slug}:{team}", now):
|
| 908 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 909 |
|
| 910 |
event = payload.get("event") if isinstance(payload.get("event"), dict) else {}
|
| 911 |
channel = str(event.get("channel") or "")
|
|
|
|
| 953 |
if not isinstance(payload, dict):
|
| 954 |
raise _refuse()
|
| 955 |
if not _rate_ok(f"{slug}:interact", now):
|
| 956 |
+
raise err(429, "too_many_requests", "too many requests; wait a moment and try again")
|
| 957 |
|
| 958 |
actions = payload.get("actions") if isinstance(payload.get("actions"), list) else []
|
| 959 |
choice = (actions[0] if actions and isinstance(actions[0], dict) else {})
|
|
|
|
| 974 |
"rejected": "Rejected. Nothing was changed.",
|
| 975 |
# β HONEST, not reassuring: the answering half that would register a committer is booked,
|
| 976 |
# so an approved card today performs nothing and says exactly that rather than "Approved."
|
| 977 |
+
"no_committer": "Approved, but this workspace has nothing configured to carry it out yet.",
|
| 978 |
"failed": "That change could not be completed. Nothing was saved; ask the bot again.",
|
| 979 |
"committed": "Approved.",
|
| 980 |
}.get(verdict, "That approval is no longer available. Ask the bot again.")}
|
api/routes_tables.py
CHANGED
|
@@ -53,6 +53,119 @@ from deps import Session, err, require_session
|
|
| 53 |
router = APIRouter(prefix="/api/v1")
|
| 54 |
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
def _ut():
|
| 57 |
import core.user_tables as user_tables
|
| 58 |
return user_tables
|
|
|
|
| 53 |
router = APIRouter(prefix="/api/v1")
|
| 54 |
|
| 55 |
|
| 56 |
+
class _MetricReadRuntime:
|
| 57 |
+
"""Lend the tenant's read doors while refusing lazy schema/workspace migrations."""
|
| 58 |
+
def __init__(self, runtime):
|
| 59 |
+
self._runtime = runtime
|
| 60 |
+
|
| 61 |
+
def __getattr__(self, name):
|
| 62 |
+
if name in {"get", "get_projection", "revision", "available", "exists", "tenant", "pool_cache",
|
| 63 |
+
"key", "assert_datastore_matches", "odoo_paused", "odoo_source", "odoo_flag_key",
|
| 64 |
+
"datastore_path", "store_namespace"}:
|
| 65 |
+
return getattr(self._runtime, name)
|
| 66 |
+
if name in {"put", "update", "upload_bytes", "delete_path"}:
|
| 67 |
+
raise RuntimeError("Scoped Metrics require an existing, readable database definition.")
|
| 68 |
+
raise AttributeError(name)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _related_product_fields(session, rows):
|
| 72 |
+
"""Resolve authored Metric definitions through the existing workspace and field wall."""
|
| 73 |
+
import aios_grid
|
| 74 |
+
from core import grid_events, perm_scope, shared_overlay
|
| 75 |
+
from modules import product_data as pd
|
| 76 |
+
|
| 77 |
+
pids = frozenset(r["pid"] for r in rows)
|
| 78 |
+
ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[],
|
| 79 |
+
hidden_keys=set(), admin=session.admin, fallback_ws=None,
|
| 80 |
+
seen_ids={}, st=session.runtime, scope_key="product",
|
| 81 |
+
table=pd.table_ops(session.runtime))
|
| 82 |
+
ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False)
|
| 83 |
+
_, fields, _, _ = aios_grid.workspace_wire(
|
| 84 |
+
ws, session.uname, set(pids), defs={}, scope_key="product", with_cohorts=False,
|
| 85 |
+
fields_base=aios_grid.product_fields(), st=session.runtime)
|
| 86 |
+
have = {f["key"] for f in fields}
|
| 87 |
+
fields = list(fields) + [dict(f, source="overlay", shared=True)
|
| 88 |
+
for key, f in shared_overlay.fields(pd.TABLE_KEY, st=session.runtime).items()
|
| 89 |
+
if key not in have]
|
| 90 |
+
hidden = perm_scope.hidden_keys(session.user, "product_data", fields, st=session.runtime)
|
| 91 |
+
return [f for f in fields if f.get("measure") and f["key"] not in hidden]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@router.post("/tables/{table_key}/related-metrics")
|
| 95 |
+
def related_metrics(table_key: str, body: dict = Body(default=None),
|
| 96 |
+
session: Session = Depends(require_session)):
|
| 97 |
+
"""C6: a visible Products ordered edge, origin customer and real destination ids.
|
| 98 |
+
|
| 99 |
+
The request supplies only grid identities and a named relation. Fields, periods, source
|
| 100 |
+
identities and values are read from this tenant; nothing is authored by this endpoint.
|
| 101 |
+
"""
|
| 102 |
+
import copy
|
| 103 |
+
from core import perm_scope, relations
|
| 104 |
+
import routes_grid
|
| 105 |
+
import routes_products
|
| 106 |
+
import automation_engine
|
| 107 |
+
|
| 108 |
+
body = body if isinstance(body, dict) else {}
|
| 109 |
+
pin = body.get("related") or {}
|
| 110 |
+
pids = body.get("pids")
|
| 111 |
+
if (table_key != "product_data" or not isinstance(pin, dict)
|
| 112 |
+
or pin.get("table") != "customer_data"
|
| 113 |
+
or type(pin.get("record")) is not int or pin["record"] <= 0
|
| 114 |
+
or not isinstance(pids, list) or not pids
|
| 115 |
+
or any(type(pid) is not int or pid <= 0 for pid in pids)):
|
| 116 |
+
raise err(400, "bad_metric_scope", "Name the origin customer and related product records.")
|
| 117 |
+
viewer = copy.copy(session)
|
| 118 |
+
viewer.runtime = _MetricReadRuntime(session.runtime)
|
| 119 |
+
try:
|
| 120 |
+
# Both endpoints and the actual bridge are permission-bearing databases.
|
| 121 |
+
for key in ("customer_data", "product_data", "ut_odoo_order_lines"):
|
| 122 |
+
viewer.require(key)
|
| 123 |
+
entries = routes_grid.related_path_entries(viewer, "customer_data", 1)
|
| 124 |
+
via = pin.get("via")
|
| 125 |
+
edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via)
|
| 126 |
+
if (edge is None or edge.get("target") != "product_data"
|
| 127 |
+
or edge.get("via", {}).get("field") != "products_ordered"):
|
| 128 |
+
raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
|
| 129 |
+
source_fields = routes_grid._source_fields(viewer, "customer_data")
|
| 130 |
+
hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime)
|
| 131 |
+
if "products_ordered" in hidden:
|
| 132 |
+
raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
|
| 133 |
+
customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime)
|
| 134 |
+
if not any(r.get("pid") == pin["record"] for r in customers):
|
| 135 |
+
raise err(403, "metric_record_refused", "The origin record is not available to this account.")
|
| 136 |
+
products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime)
|
| 137 |
+
requested = set(pids)
|
| 138 |
+
selected = [r for r in products if r.get("pid") in requested]
|
| 139 |
+
if (len(selected) != len(requested)
|
| 140 |
+
or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)):
|
| 141 |
+
raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.")
|
| 142 |
+
# A bridge restriction that cannot be translated must never be silently dropped.
|
| 143 |
+
bridge_fields = perm_scope.scoped_fields(viewer.user, "ut_odoo_order_lines", st=viewer.runtime)
|
| 144 |
+
declared = perm_scope.entry(viewer.user, "ut_odoo_order_lines") or {}
|
| 145 |
+
if (perm_scope.row_scope_applies(viewer.user, "ut_odoo_order_lines")
|
| 146 |
+
or (not viewer.admin and declared.get("hiddenFields"))
|
| 147 |
+
or not {"partner_id", "product_id"}.issubset({f["key"] for f in bridge_fields})):
|
| 148 |
+
raise err(409, "metric_scope_unsupported", "Order-line permission restrictions cannot be applied to these Metrics yet.")
|
| 149 |
+
scopes = [perm_scope.derive_pool_scope(viewer.user, key)
|
| 150 |
+
for key in ("customer_data", "product_data", "ut_odoo_order_lines")]
|
| 151 |
+
teams = {team for team, _ in scopes if team is not None}
|
| 152 |
+
agents = {agent for _, agent in scopes if agent is not None}
|
| 153 |
+
if len(teams) > 1 or len(agents) > 1:
|
| 154 |
+
raise err(403, "metric_scope_conflict", "The related databases have incompatible permission scopes.")
|
| 155 |
+
fields = _related_product_fields(viewer, products)
|
| 156 |
+
values, reasons = automation_engine.related_product_metric_values(
|
| 157 |
+
viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer),
|
| 158 |
+
team_id=next(iter(teams), None), agent=next(iter(agents), None))
|
| 159 |
+
return {"ok": True, "values": values, "fields": [f["key"] for f in fields],
|
| 160 |
+
"scope": {"customer": pin["record"],
|
| 161 |
+
"products": {r["pid"]: r["product_id"] for r in selected}},
|
| 162 |
+
"reason": " ".join(reasons)}
|
| 163 |
+
except (perm_scope.Denied, perm_scope.UnknownTable):
|
| 164 |
+
raise err(403, "metric_record_refused", "The related records are not available to this account.")
|
| 165 |
+
except (perm_scope.Unresolvable, RuntimeError, ValueError):
|
| 166 |
+
raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.")
|
| 167 |
+
|
| 168 |
+
|
| 169 |
def _ut():
|
| 170 |
import core.user_tables as user_tables
|
| 171 |
return user_tables
|
api/routes_uploads.py
CHANGED
|
@@ -60,7 +60,7 @@ async def tabular_preview(file: UploadFile = File(...),
|
|
| 60 |
data = await file.read()
|
| 61 |
if len(data) > MAX_BYTES:
|
| 62 |
raise err(400, "too_large",
|
| 63 |
-
"that file is over 5 MB
|
| 64 |
if name.endswith(".csv"):
|
| 65 |
text = data.decode("utf-8-sig", errors="replace")
|
| 66 |
rows = iter(csv.reader(io.StringIO(text)))
|
|
@@ -70,7 +70,7 @@ async def tabular_preview(file: UploadFile = File(...),
|
|
| 70 |
wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
| 71 |
ws = wb.worksheets[0]
|
| 72 |
except Exception:
|
| 73 |
-
raise err(400, "bad_file", "that workbook could not be read
|
| 74 |
rows = ws.iter_rows(values_only=True)
|
| 75 |
else:
|
| 76 |
raise err(400, "bad_type",
|
|
|
|
| 60 |
data = await file.read()
|
| 61 |
if len(data) > MAX_BYTES:
|
| 62 |
raise err(400, "too_large",
|
| 63 |
+
"that file is over 5 MB; export just the column you need and retry")
|
| 64 |
if name.endswith(".csv"):
|
| 65 |
text = data.decode("utf-8-sig", errors="replace")
|
| 66 |
rows = iter(csv.reader(io.StringIO(text)))
|
|
|
|
| 70 |
wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
| 71 |
ws = wb.worksheets[0]
|
| 72 |
except Exception:
|
| 73 |
+
raise err(400, "bad_file", "that workbook could not be read; is it a real .xlsx?")
|
| 74 |
rows = ws.iter_rows(values_only=True)
|
| 75 |
else:
|
| 76 |
raise err(400, "bad_type",
|
api/test_odoo_agent_snapshot.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exercise the real invoice reader and its oracle on synthetic DuckDB snapshots."""
|
| 2 |
+
from contextlib import redirect_stdout
|
| 3 |
+
import io
|
| 4 |
+
import socket
|
| 5 |
+
import unittest
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
import duckdb
|
| 9 |
+
import verify_odoo_relational as gate
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AgentSnapshotTests(unittest.TestCase):
|
| 13 |
+
def setUp(self):
|
| 14 |
+
self.network = patch.object(socket.socket, "connect", side_effect=AssertionError("network forbidden"))
|
| 15 |
+
self.network_spy = self.network.start()
|
| 16 |
+
self.addCleanup(self.network.stop)
|
| 17 |
+
self.con = duckdb.connect(":memory:")
|
| 18 |
+
self.addCleanup(self.con.close)
|
| 19 |
+
self.con.execute("""
|
| 20 |
+
CREATE TABLE res_partner (id INTEGER, name VARCHAR, agent BOOLEAN);
|
| 21 |
+
INSERT INTO res_partner VALUES (10, 'Synthetic customer', false),
|
| 22 |
+
(70, 'Alpha', true), (80, 'Beta', true), (90, 'Gamma', true);
|
| 23 |
+
CREATE TABLE account_move (
|
| 24 |
+
id INTEGER, name VARCHAR, partner_id INTEGER, partner_name VARCHAR,
|
| 25 |
+
invoice_date DATE, invoice_date_due DATE, amount_untaxed_signed DOUBLE,
|
| 26 |
+
amount_residual_signed DOUBLE, payment_state VARCHAR, move_type VARCHAR,
|
| 27 |
+
state VARCHAR, invoice_origin VARCHAR);
|
| 28 |
+
INSERT INTO account_move VALUES
|
| 29 |
+
(1,'I1',10,'Synthetic customer','2026-09-01','2026-10-01',10,10,'not_paid','out_invoice','posted',''),
|
| 30 |
+
(2,'I2',10,'Synthetic customer','2026-09-01','2026-10-01',20,20,'not_paid','out_invoice','posted',''),
|
| 31 |
+
(3,'R3',10,'Synthetic customer','2026-09-01','2026-10-01',-5,-5,'not_paid','out_refund','posted',''),
|
| 32 |
+
(4,'Draft',10,'Synthetic customer','2026-09-01','2026-10-01',5,5,'not_paid','out_invoice','draft',''),
|
| 33 |
+
(5,'Bill',10,'Synthetic customer','2026-09-01','2026-10-01',5,5,'not_paid','in_invoice','posted',''),
|
| 34 |
+
(6,'No partner',NULL,'','2026-09-01','2026-10-01',5,5,'not_paid','out_invoice','posted','');
|
| 35 |
+
CREATE TABLE account_invoice_line_agent (invoice_id INTEGER, agent_id INTEGER, agent_name VARCHAR);
|
| 36 |
+
INSERT INTO account_invoice_line_agent VALUES
|
| 37 |
+
(1,70,'Alpha'), (1,70,'Alpha'), (2,80,'Beta'), (2,70,'Alpha'),
|
| 38 |
+
(4,90,'Gamma'), (5,90,'Gamma'), (6,90,'Gamma');
|
| 39 |
+
""")
|
| 40 |
+
self.saved_pass, self.saved_fail = gate.PASS[:], gate.FAIL[:]
|
| 41 |
+
self.addCleanup(self.restore_verdict)
|
| 42 |
+
|
| 43 |
+
def restore_verdict(self):
|
| 44 |
+
gate.PASS[:] = self.saved_pass
|
| 45 |
+
gate.FAIL[:] = self.saved_fail
|
| 46 |
+
|
| 47 |
+
def prove(self, mutate=None):
|
| 48 |
+
real = gate.rel.read_invoices
|
| 49 |
+
|
| 50 |
+
def read(cur, *args, **kwargs):
|
| 51 |
+
rows = real(cur, *args, **kwargs)
|
| 52 |
+
return mutate(rows) if cur is self.con and mutate else rows
|
| 53 |
+
|
| 54 |
+
gate.PASS.clear()
|
| 55 |
+
gate.FAIL.clear()
|
| 56 |
+
with patch.object(gate.rel, "read_invoices", side_effect=read), redirect_stdout(io.StringIO()):
|
| 57 |
+
gate._prove_invoice_agent_and_links(self.con)
|
| 58 |
+
self.network_spy.assert_not_called()
|
| 59 |
+
return gate.FAIL[:]
|
| 60 |
+
|
| 61 |
+
def test_healthy_new_snapshot_is_not_compared_to_august_counts(self):
|
| 62 |
+
self.assertEqual(self.prove(), [])
|
| 63 |
+
|
| 64 |
+
def test_growth_and_new_distinct_agent_still_reconcile(self):
|
| 65 |
+
self.con.execute("INSERT INTO account_move SELECT 7,'I7',partner_id,partner_name,invoice_date,invoice_date_due,amount_untaxed_signed,amount_residual_signed,payment_state,move_type,state,invoice_origin FROM account_move WHERE id=1")
|
| 66 |
+
self.con.execute("INSERT INTO account_invoice_line_agent VALUES (7,90,'Gamma')")
|
| 67 |
+
self.assertEqual(self.prove(), [])
|
| 68 |
+
|
| 69 |
+
def assert_row_parity_refuses(self, mutate):
|
| 70 |
+
failures = self.prove(mutate)
|
| 71 |
+
self.assertTrue(any("raw commission rows" in name for name in failures), failures)
|
| 72 |
+
|
| 73 |
+
def test_wrong_existing_agent_with_equal_counts_is_refused(self):
|
| 74 |
+
def mutate(rows):
|
| 75 |
+
return [dict(r, agent_id=80) if r['_id']=='1' else r for r in rows]
|
| 76 |
+
self.assert_row_parity_refuses(mutate)
|
| 77 |
+
|
| 78 |
+
def test_wrong_name_with_equal_counts_is_refused(self):
|
| 79 |
+
self.assert_row_parity_refuses(lambda rows: [dict(r, agent='Wrong') if r['_id']=='1' else r for r in rows])
|
| 80 |
+
|
| 81 |
+
def test_multi_agent_truncation_is_refused(self):
|
| 82 |
+
self.assert_row_parity_refuses(lambda rows: [dict(r, agent='Alpha', agent_id=70) if r['_id']=='2' else r for r in rows])
|
| 83 |
+
|
| 84 |
+
def test_dropped_invoice_is_refused(self):
|
| 85 |
+
self.assert_row_parity_refuses(lambda rows: [r for r in rows if r['_id']!='3'])
|
| 86 |
+
|
| 87 |
+
def test_duplicate_invoice_is_refused(self):
|
| 88 |
+
self.assert_row_parity_refuses(lambda rows: rows + [dict(rows[0])])
|
| 89 |
+
|
| 90 |
+
def test_missing_commission_source_is_a_failure(self):
|
| 91 |
+
self.con.execute('DROP TABLE account_invoice_line_agent')
|
| 92 |
+
self.assertTrue(any('raw commission oracle' in name for name in self.prove()))
|
| 93 |
+
|
| 94 |
+
def test_always_false_comparator_fails_its_positive_control(self):
|
| 95 |
+
with patch.object(gate, '_invoice_agents_match', return_value=False):
|
| 96 |
+
self.assertTrue(any('NC baseline:' in name for name in self.prove()))
|
| 97 |
+
|
| 98 |
+
def test_always_true_comparator_fails_all_three_negative_controls(self):
|
| 99 |
+
with patch.object(gate, '_invoice_agents_match', return_value=True):
|
| 100 |
+
failures = self.prove()
|
| 101 |
+
self.assertEqual(sum(name.startswith('NC: one ') for name in failures), 3)
|
| 102 |
+
|
| 103 |
+
def test_dated_measurement_is_preserved(self):
|
| 104 |
+
from copy import deepcopy
|
| 105 |
+
historical = deepcopy(gate.rel._INVOICE_COLUMN_FILL)
|
| 106 |
+
self.assertEqual(self.prove(), [])
|
| 107 |
+
self.assertEqual(gate.rel._INVOICE_COLUMN_FILL, historical)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
if __name__ == '__main__':
|
| 111 |
+
unittest.main()
|
api/test_odoo_customer_population.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Synthetic source/mirror tests of the real population oracle, never live proof."""
|
| 2 |
+
from contextlib import redirect_stdout
|
| 3 |
+
import io
|
| 4 |
+
import socket
|
| 5 |
+
import unittest
|
| 6 |
+
from unittest.mock import patch
|
| 7 |
+
|
| 8 |
+
import duckdb
|
| 9 |
+
import verify_odoo_relational as gate
|
| 10 |
+
import core.odoo as source
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class Source:
|
| 14 |
+
def __init__(self, rows):
|
| 15 |
+
self.rows = rows
|
| 16 |
+
self.calls = []
|
| 17 |
+
self.ignore_drop = False
|
| 18 |
+
self.count_calls = 0
|
| 19 |
+
|
| 20 |
+
def select(self, model, domain):
|
| 21 |
+
assert model == 'res.partner'
|
| 22 |
+
rows = self.rows
|
| 23 |
+
for field, op, value in domain or []:
|
| 24 |
+
predicates = {
|
| 25 |
+
'=': lambda item: item == value,
|
| 26 |
+
'>': lambda item: item > value,
|
| 27 |
+
'<=': lambda item: item <= value,
|
| 28 |
+
'in': lambda item: item in value,
|
| 29 |
+
'not in': lambda item: item not in value,
|
| 30 |
+
}
|
| 31 |
+
rows = [row for row in rows if predicates[op](row[field])]
|
| 32 |
+
return sorted(rows, key=lambda row: row['id'])
|
| 33 |
+
|
| 34 |
+
def search_count(self, model, domain=None):
|
| 35 |
+
self.count_calls += 1
|
| 36 |
+
self.calls.append(('count', domain))
|
| 37 |
+
# A deliberately broken provider double leaves the deletion invisible.
|
| 38 |
+
if self.ignore_drop and self.count_calls == 4:
|
| 39 |
+
return 1
|
| 40 |
+
return len(self.select(model, domain))
|
| 41 |
+
|
| 42 |
+
def search_read(self, model, domain=None, fields=None, limit=None, order=None):
|
| 43 |
+
self.calls.append(('read', domain, fields, limit))
|
| 44 |
+
return [{key: row[key] for key in fields}
|
| 45 |
+
for row in self.select(model, domain)[:limit]]
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class CustomerPopulationTests(unittest.TestCase):
|
| 49 |
+
def setUp(self):
|
| 50 |
+
self.network = patch.object(socket.socket, 'connect', side_effect=AssertionError('network forbidden'))
|
| 51 |
+
self.network_spy = self.network.start()
|
| 52 |
+
self.addCleanup(self.network.stop)
|
| 53 |
+
self.con = duckdb.connect(':memory:')
|
| 54 |
+
self.addCleanup(self.con.close)
|
| 55 |
+
self.con.execute("""
|
| 56 |
+
CREATE TABLE res_partner (id INTEGER, name VARCHAR, customer_rank INTEGER,
|
| 57 |
+
active BOOLEAN, write_date VARCHAR);
|
| 58 |
+
INSERT INTO res_partner VALUES
|
| 59 |
+
(90101,'Synthetic document partner',0,true,'2026-09-01 00:00:00'),
|
| 60 |
+
(90102,'Synthetic customer A',1,true,'2026-09-01 00:00:00'),
|
| 61 |
+
(90103,'Synthetic customer B',2,true,'2026-09-01 00:00:00');
|
| 62 |
+
CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR);
|
| 63 |
+
INSERT INTO sale_order VALUES (90101,'sale');
|
| 64 |
+
CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR);
|
| 65 |
+
CREATE TABLE _sync_state (entity VARCHAR, cursor_wd VARCHAR);
|
| 66 |
+
INSERT INTO _sync_state VALUES ('res_partner','2026-09-03 01:02:25');
|
| 67 |
+
""")
|
| 68 |
+
self.source = Source([
|
| 69 |
+
dict(id=90101, customer_rank=0, active=True, write_date='2026-09-01 00:00:00'),
|
| 70 |
+
dict(id=90102, customer_rank=1, active=True, write_date='2026-09-01 00:00:00'),
|
| 71 |
+
dict(id=90103, customer_rank=2, active=True, write_date='2026-09-01 00:00:00'),
|
| 72 |
+
])
|
| 73 |
+
self.saved_pass, self.saved_fail = gate.PASS[:], gate.FAIL[:]
|
| 74 |
+
self.addCleanup(self.restore)
|
| 75 |
+
|
| 76 |
+
def restore(self):
|
| 77 |
+
gate.PASS[:], gate.FAIL[:] = self.saved_pass, self.saved_fail
|
| 78 |
+
|
| 79 |
+
def prove(self, mutate=None):
|
| 80 |
+
real = gate.rel.read_customers
|
| 81 |
+
|
| 82 |
+
def read(cur, *args, **kwargs):
|
| 83 |
+
rows = real(cur, *args, **kwargs)
|
| 84 |
+
if cur is self.con:
|
| 85 |
+
# SQL has no ORDER BY: put a valid document-only row first.
|
| 86 |
+
rows.sort(key=lambda row: int(row[gate.rel.JOIN_KEY]))
|
| 87 |
+
if mutate:
|
| 88 |
+
rows = mutate(rows)
|
| 89 |
+
return rows
|
| 90 |
+
|
| 91 |
+
gate.PASS.clear()
|
| 92 |
+
gate.FAIL.clear()
|
| 93 |
+
output = io.StringIO()
|
| 94 |
+
with patch.object(source, 'get_odoo', return_value=self.source), \
|
| 95 |
+
patch.object(gate.rel, 'read_customers', side_effect=read), redirect_stdout(output):
|
| 96 |
+
gate._prove_customer_population(self.con)
|
| 97 |
+
self.network_spy.assert_not_called()
|
| 98 |
+
self.output = output.getvalue()
|
| 99 |
+
return gate.FAIL[:]
|
| 100 |
+
|
| 101 |
+
def test_document_only_first_row_does_not_break_healthy_control(self):
|
| 102 |
+
self.assertEqual(self.prove(), [])
|
| 103 |
+
|
| 104 |
+
def test_already_red_population_cannot_make_ineffective_drop_pass(self):
|
| 105 |
+
self.con.execute('DELETE FROM res_partner WHERE id=90103')
|
| 106 |
+
self.source.ignore_drop = True
|
| 107 |
+
failures = self.prove()
|
| 108 |
+
self.assertTrue(any('AS OF THE MIRROR' in name for name in failures))
|
| 109 |
+
self.assertTrue(any('NC:' in name and 'ONE in-scope' in name for name in failures), failures)
|
| 110 |
+
|
| 111 |
+
def assert_gap(self, marker, mutate=None):
|
| 112 |
+
failures = self.prove(mutate)
|
| 113 |
+
name = next(name for name in failures if 'AS OF THE MIRROR' in name)
|
| 114 |
+
self.assertIn('got=1, want=2', name)
|
| 115 |
+
self.assertIn(marker, name)
|
| 116 |
+
self.assertNotIn('90103', name)
|
| 117 |
+
self.assertNotIn('Synthetic', name)
|
| 118 |
+
reads = [call for call in self.source.calls if call[0] == 'read']
|
| 119 |
+
self.assertTrue(reads)
|
| 120 |
+
self.assertTrue(all(call[2] == ['id'] for call in reads))
|
| 121 |
+
|
| 122 |
+
def test_missing_raw_partner_is_classified_without_identity(self):
|
| 123 |
+
self.con.execute('DELETE FROM res_partner WHERE id=90103')
|
| 124 |
+
self.assert_gap('mirror_absent=1')
|
| 125 |
+
|
| 126 |
+
def test_stale_rank_is_distinguished_from_missing_partner(self):
|
| 127 |
+
self.con.execute('UPDATE res_partner SET customer_rank=0 WHERE id=90103')
|
| 128 |
+
self.assert_gap('mirror_rank_not_positive=1')
|
| 129 |
+
|
| 130 |
+
def test_stale_active_is_distinguished_from_missing_partner(self):
|
| 131 |
+
self.con.execute('UPDATE res_partner SET active=false WHERE id=90103')
|
| 132 |
+
self.assert_gap('mirror_inactive=1')
|
| 133 |
+
|
| 134 |
+
def test_eligible_raw_partner_omitted_by_reader_is_classified(self):
|
| 135 |
+
self.assert_gap('mirror_eligible_omitted=1',
|
| 136 |
+
lambda rows: [row for row in rows if row[gate.rel.JOIN_KEY] != 90103])
|
| 137 |
+
|
| 138 |
+
def test_newer_source_partner_stays_outside_both_counts(self):
|
| 139 |
+
self.source.rows.append(dict(id=90104, customer_rank=1, active=True,
|
| 140 |
+
write_date='2026-09-04 00:00:00'))
|
| 141 |
+
self.assertEqual(self.prove(), [])
|
| 142 |
+
self.assertEqual(sum(call[0] == 'read' for call in self.source.calls), 1)
|
| 143 |
+
|
| 144 |
+
def test_diagnostic_does_not_cap_the_missing_population(self):
|
| 145 |
+
self.con.execute('DELETE FROM res_partner WHERE id=90103')
|
| 146 |
+
self.prove()
|
| 147 |
+
diagnostic = [call for call in self.source.calls if call[0] == 'read'
|
| 148 |
+
and any(clause[1] == 'not in' for clause in call[1])]
|
| 149 |
+
self.assertEqual(len(diagnostic), 1)
|
| 150 |
+
self.assertIsNone(diagnostic[0][3])
|
| 151 |
+
|
| 152 |
+
def test_diagnostic_rejects_changed_or_incomplete_source_sets(self):
|
| 153 |
+
self.con.execute('DELETE FROM res_partner WHERE id=90103')
|
| 154 |
+
original = self.source.search_read
|
| 155 |
+
for response in ([], [{'id': 90103}, {'id': 90103}], [{'id': 90102}],
|
| 156 |
+
[{'id': '90103'}], [{'id': True}], [{'id': -1}]):
|
| 157 |
+
with self.subTest(response=response):
|
| 158 |
+
def read(model, domain, **kwargs):
|
| 159 |
+
if any(clause[1] == 'not in' for clause in domain):
|
| 160 |
+
return response
|
| 161 |
+
return original(model, domain, **kwargs)
|
| 162 |
+
with patch.object(self.source, 'search_read', side_effect=read):
|
| 163 |
+
failures = self.prove()
|
| 164 |
+
self.assertTrue(any('source_changed_or_incomplete' in name for name in failures))
|
| 165 |
+
|
| 166 |
+
def test_diagnostic_failure_preserves_red_and_hides_exception_contents(self):
|
| 167 |
+
self.con.execute('DELETE FROM res_partner WHERE id=90103')
|
| 168 |
+
original = self.source.search_read
|
| 169 |
+
|
| 170 |
+
def read(model, domain, **kwargs):
|
| 171 |
+
if any(clause[1] == 'not in' for clause in domain):
|
| 172 |
+
raise RuntimeError('Synthetic secret client details')
|
| 173 |
+
return original(model, domain, **kwargs)
|
| 174 |
+
|
| 175 |
+
with patch.object(self.source, 'search_read', side_effect=read):
|
| 176 |
+
failures = self.prove()
|
| 177 |
+
self.assertTrue(any('gap_diagnostic=unavailable' in name for name in failures))
|
| 178 |
+
self.assertNotIn('Synthetic secret client details', self.output)
|
| 179 |
+
|
| 180 |
+
def test_duplicate_mirror_identity_is_not_counted_as_a_valid_diagnosis(self):
|
| 181 |
+
self.con.execute('UPDATE res_partner SET customer_rank=0 WHERE id=90103')
|
| 182 |
+
self.con.execute('INSERT INTO res_partner SELECT * FROM res_partner WHERE id=90103')
|
| 183 |
+
self.assertTrue(any('duplicate_mirror_identity' in name for name in self.prove()))
|
| 184 |
+
|
| 185 |
+
def test_empty_reader_has_no_vacuous_negative_control(self):
|
| 186 |
+
failures = self.prove(lambda rows: [])
|
| 187 |
+
self.assertTrue(any('AS OF THE MIRROR' in name for name in failures))
|
| 188 |
+
self.assertTrue(any('NC: ONE in-scope' in name for name in failures))
|
| 189 |
+
|
| 190 |
+
def test_forged_witness_cannot_remove_an_unrelated_row(self):
|
| 191 |
+
with patch.object(self.source, 'search_read', return_value=[{'id': 99999}]):
|
| 192 |
+
failures = self.prove()
|
| 193 |
+
self.assertTrue(any('NC: ONE in-scope' in name for name in failures))
|
| 194 |
+
|
| 195 |
+
def test_one_control_cannot_hide_two_missing_partners(self):
|
| 196 |
+
self.con.execute('DELETE FROM res_partner WHERE id IN (90102,90103)')
|
| 197 |
+
failures = self.prove()
|
| 198 |
+
self.assertTrue(any('got=0, want=2' in name and 'mirror_absent=2' in name
|
| 199 |
+
for name in failures))
|
| 200 |
+
self.assertTrue(any('NC: ONE in-scope' in name for name in failures))
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
if __name__ == '__main__':
|
| 204 |
+
unittest.main()
|
api/test_odoo_rank_refresh.py
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exercise the real mirror writer with synthetic Odoo rows and temporary DuckDB only."""
|
| 2 |
+
from contextlib import ExitStack, redirect_stdout
|
| 3 |
+
import ast
|
| 4 |
+
import copy
|
| 5 |
+
import io
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import socket
|
| 9 |
+
import sys
|
| 10 |
+
import tempfile
|
| 11 |
+
from types import SimpleNamespace
|
| 12 |
+
import unittest
|
| 13 |
+
from unittest.mock import ANY, patch
|
| 14 |
+
|
| 15 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'platform'))
|
| 16 |
+
import duckdb
|
| 17 |
+
from harness import datastore as ds
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class RankRefreshTests(unittest.TestCase):
|
| 21 |
+
def setUp(self):
|
| 22 |
+
self.stack = ExitStack()
|
| 23 |
+
self.addCleanup(self.stack.close)
|
| 24 |
+
self.net = self.stack.enter_context(patch.object(
|
| 25 |
+
socket.socket, 'connect', side_effect=AssertionError('external network forbidden')))
|
| 26 |
+
self.temp = self.stack.enter_context(tempfile.TemporaryDirectory())
|
| 27 |
+
self.path = str(Path(self.temp) / 'synthetic.duckdb')
|
| 28 |
+
self.spec = copy.deepcopy(ds.ENTITIES['res_partner'])
|
| 29 |
+
self.spec['backfill_fields'] = []
|
| 30 |
+
self.spec['m2m'] = []
|
| 31 |
+
self.rows = [dict(id=pid, name='Synthetic partner', customer_rank=rank,
|
| 32 |
+
active=active, write_date='2026-09-01 00:00:00')
|
| 33 |
+
for pid, rank, active in [(1, 0, True), (2, 1, True), (3, 2, False)]]
|
| 34 |
+
with duckdb.connect(self.path) as con:
|
| 35 |
+
ds._ensure_table(con, 'res_partner', self.spec)
|
| 36 |
+
ds._upsert(con, 'res_partner', self.spec, self.rows)
|
| 37 |
+
con.execute('''CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY,
|
| 38 |
+
phase VARCHAR, last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)''')
|
| 39 |
+
ds._save_state(con, 'res_partner', 'live', 3, '2026-09-03 01:02:25')
|
| 40 |
+
self.calls = []
|
| 41 |
+
self.stack.enter_context(patch.object(ds, 'connect', side_effect=lambda: duckdb.connect(self.path)))
|
| 42 |
+
self.stack.enter_context(patch.dict(ds.ENTITIES, res_partner=self.spec))
|
| 43 |
+
self.stack.enter_context(patch.object(ds, 'source_paused', return_value=False))
|
| 44 |
+
self.stack.enter_context(patch.object(ds.O, 'search_read', side_effect=self.read))
|
| 45 |
+
|
| 46 |
+
def read(self, model, domain, fields, limit=None, order=None):
|
| 47 |
+
self.assertEqual(model, 'res.partner')
|
| 48 |
+
self.calls.append((copy.deepcopy(domain), list(fields), limit))
|
| 49 |
+
rows = self.rows
|
| 50 |
+
for field, op, value in domain:
|
| 51 |
+
if op == 'in':
|
| 52 |
+
rows = [r for r in rows if r[field] in value]
|
| 53 |
+
elif op == '>':
|
| 54 |
+
rows = [r for r in rows if r[field] > value]
|
| 55 |
+
elif op == '>=':
|
| 56 |
+
rows = [r for r in rows if r[field] >= value]
|
| 57 |
+
else:
|
| 58 |
+
self.fail('unexpected source domain')
|
| 59 |
+
return [{k: row.get(k) for k in ['id'] + list(fields)}
|
| 60 |
+
for row in sorted(rows, key=lambda r: r['id'])[:limit]]
|
| 61 |
+
|
| 62 |
+
def sync(self):
|
| 63 |
+
result = ds.sync_entity('res_partner', max_batches=1, log=lambda *_: None)
|
| 64 |
+
self.net.assert_not_called()
|
| 65 |
+
return result
|
| 66 |
+
|
| 67 |
+
def ranks(self):
|
| 68 |
+
with duckdb.connect(self.path) as con:
|
| 69 |
+
return con.execute('SELECT id, customer_rank FROM res_partner ORDER BY id').fetchall()
|
| 70 |
+
|
| 71 |
+
def report_and_rebuild_reason(self):
|
| 72 |
+
# Execute the production decision without importing main's app/startup effects.
|
| 73 |
+
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 74 |
+
decision = next(node for node in source.body if isinstance(node, ast.FunctionDef)
|
| 75 |
+
and node.name == '_resync_rebuild_reason')
|
| 76 |
+
age = next(node for node in source.body if isinstance(node, ast.Assign)
|
| 77 |
+
and any(isinstance(target, ast.Name)
|
| 78 |
+
and target.id == '_RESYNC_REBUILD_MAX_AGE_SECONDS'
|
| 79 |
+
for target in node.targets))
|
| 80 |
+
ns = {}
|
| 81 |
+
exec(compile(ast.Module(body=[age, decision], type_ignores=[]),
|
| 82 |
+
'main.py:resync-decision', 'exec'), ns)
|
| 83 |
+
report = ds.sync_all(entities=['res_partner'], max_batches=1, log=lambda *_: None)
|
| 84 |
+
self.net.assert_not_called()
|
| 85 |
+
return report, ns['_resync_rebuild_reason'](report, {}, 100, 101)
|
| 86 |
+
|
| 87 |
+
def test_rank_only_change_triggers_resident_projection_rebuild(self):
|
| 88 |
+
self.rows[0]['customer_rank'] = 4
|
| 89 |
+
report, reason = self.report_and_rebuild_reason()
|
| 90 |
+
self.assertEqual(self.ranks(), [(1, 4), (2, 1), (3, 2)])
|
| 91 |
+
self.assertEqual(reason, 'source-change')
|
| 92 |
+
self.assertEqual(report['res_partner']['pulled'], 1)
|
| 93 |
+
|
| 94 |
+
def observe_resident_recovery(self, failed_passes):
|
| 95 |
+
# Run the actual loop with a synthetic clock/source/store. No server is imported.
|
| 96 |
+
import harness
|
| 97 |
+
import odoo_relational as rel
|
| 98 |
+
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
| 99 |
+
functions = {'_store_resync_loop', '_resync_rebuild_reason',
|
| 100 |
+
'_meta_rebuild_force_reason'}
|
| 101 |
+
nodes = [node for node in source.body
|
| 102 |
+
if isinstance(node, ast.FunctionDef) and node.name in functions
|
| 103 |
+
or isinstance(node, ast.Assign) and any(
|
| 104 |
+
isinstance(target, ast.Name)
|
| 105 |
+
and target.id == '_RESYNC_REBUILD_MAX_AGE_SECONDS'
|
| 106 |
+
for target in node.targets)]
|
| 107 |
+
with duckdb.connect(self.path) as con:
|
| 108 |
+
con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
|
| 109 |
+
con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
|
| 110 |
+
cycle, current_pass = [0], [0]
|
| 111 |
+
attempts, projection, sleeps, reports, events = [], [], [], [], []
|
| 112 |
+
total = max(failed_passes, default=1) + 2
|
| 113 |
+
def sleep(seconds):
|
| 114 |
+
if len(reports) >= total:
|
| 115 |
+
raise StopIteration('bounded synthetic resident loop')
|
| 116 |
+
events.append(('sleep', seconds))
|
| 117 |
+
cycle[0] += 1
|
| 118 |
+
sleeps.append(seconds)
|
| 119 |
+
def sync(manual, passes):
|
| 120 |
+
events.append(('sync', passes))
|
| 121 |
+
self.assertFalse(manual)
|
| 122 |
+
self.assertEqual(passes, len(reports) + 1)
|
| 123 |
+
current_pass[0] = passes
|
| 124 |
+
if passes == 2:
|
| 125 |
+
self.rows[0]['customer_rank'] = 4
|
| 126 |
+
report = ds.sync_all(entities=['res_partner'], max_batches=1, log=lambda *_: None)
|
| 127 |
+
reports.append(report['res_partner']['pulled'])
|
| 128 |
+
return report, {}, None
|
| 129 |
+
def rebuild(why, previous, meta_state):
|
| 130 |
+
attempts.append(current_pass[0])
|
| 131 |
+
if current_pass[0] in failed_passes:
|
| 132 |
+
return False
|
| 133 |
+
with duckdb.connect(self.path) as con:
|
| 134 |
+
projection[:] = sorted(r[rel.JOIN_KEY] for r in rel.read_customers(con))
|
| 135 |
+
return True
|
| 136 |
+
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + current_pass[0] * 60,
|
| 137 |
+
strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
|
| 138 |
+
ns = dict(RESYNC_STATE={}, _RESYNC_NOTICE=[None],
|
| 139 |
+
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
|
| 140 |
+
tenant='synthetic', tenantSource='test', tenantSeconds=60,
|
| 141 |
+
envSeconds=60, disagreement=None),
|
| 142 |
+
_resync_mirror_pass=sync, _pull_meta=lambda *_: True,
|
| 143 |
+
_rebuild_odoo_relational=rebuild,
|
| 144 |
+
_rebuild_meta_relational=lambda *_: {},
|
| 145 |
+
_rebuild_amazon_relational=lambda *_: {})
|
| 146 |
+
exec(compile(ast.Module(body=nodes, type_ignores=[]),
|
| 147 |
+
'main.py:resident-loop', 'exec'), ns)
|
| 148 |
+
runtime = SimpleNamespace(get_runtime=lambda *_: object())
|
| 149 |
+
with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \
|
| 150 |
+
patch.object(harness, 'runtime', runtime, create=True), \
|
| 151 |
+
redirect_stdout(io.StringIO()):
|
| 152 |
+
with self.assertRaisesRegex(StopIteration, 'bounded synthetic'):
|
| 153 |
+
ns['_store_resync_loop']()
|
| 154 |
+
self.net.assert_not_called()
|
| 155 |
+
self.resident_events = events
|
| 156 |
+
return attempts, projection, sleeps, reports
|
| 157 |
+
|
| 158 |
+
def test_resident_loop_runs_first_pass_before_cadence_sleep(self):
|
| 159 |
+
self.observe_resident_recovery(set())
|
| 160 |
+
self.assertEqual(self.resident_events[0], ('sync', 1))
|
| 161 |
+
|
| 162 |
+
def test_trusted_gate_refreshes_only_its_marked_isolated_copy(self):
|
| 163 |
+
import core.odoo as odoo
|
| 164 |
+
import verify_odoo_relational as gate
|
| 165 |
+
gate.PASS.clear()
|
| 166 |
+
gate.FAIL.clear()
|
| 167 |
+
path = Path(self.path)
|
| 168 |
+
no_client = object()
|
| 169 |
+
previous_client = getattr(odoo._tlocal, 'client', no_client)
|
| 170 |
+
with patch.dict(os.environ, {'AIOS_TRUSTED_GATE_SOURCE_ISOLATED': '1'}), \
|
| 171 |
+
patch.object(ds, 'path_for', return_value=path), \
|
| 172 |
+
patch.object(ds, 'sync_entity', return_value=('live', 2)) as sync, \
|
| 173 |
+
patch.object(ds, 'use_path') as use_path, \
|
| 174 |
+
patch.object(odoo, 'OdooClient', return_value=object()):
|
| 175 |
+
self.assertTrue(gate._prepare_isolated_trusted_mirror())
|
| 176 |
+
sync.assert_called_once_with('res_partner', log=ANY)
|
| 177 |
+
use_path.assert_called_once_with(path)
|
| 178 |
+
self.assertEqual(gate.FAIL, [])
|
| 179 |
+
if previous_client is no_client:
|
| 180 |
+
self.assertFalse(hasattr(odoo._tlocal, 'client'))
|
| 181 |
+
else:
|
| 182 |
+
self.assertIs(odoo._tlocal.client, previous_client)
|
| 183 |
+
self.net.assert_not_called()
|
| 184 |
+
|
| 185 |
+
def test_failed_trusted_gate_refresh_still_releases_writer_and_client(self):
|
| 186 |
+
import core.odoo as odoo
|
| 187 |
+
import verify_odoo_relational as gate
|
| 188 |
+
gate.PASS.clear()
|
| 189 |
+
gate.FAIL.clear()
|
| 190 |
+
path = Path(self.path)
|
| 191 |
+
no_client = object()
|
| 192 |
+
previous_client = getattr(odoo._tlocal, 'client', no_client)
|
| 193 |
+
with patch.dict(os.environ, {'AIOS_TRUSTED_GATE_SOURCE_ISOLATED': '1'}), \
|
| 194 |
+
patch.object(ds, 'path_for', return_value=path), \
|
| 195 |
+
patch.object(ds, 'sync_entity', side_effect=RuntimeError('synthetic failure')), \
|
| 196 |
+
patch.object(ds, 'use_path') as use_path, \
|
| 197 |
+
patch.object(odoo, 'OdooClient', return_value=object()):
|
| 198 |
+
self.assertFalse(gate._prepare_isolated_trusted_mirror())
|
| 199 |
+
use_path.assert_called_once_with(path)
|
| 200 |
+
self.assertEqual(gate.PASS, [])
|
| 201 |
+
self.assertEqual(gate.FAIL, [
|
| 202 |
+
'trusted isolated mirror completed the product customer-rank refresh before live oracles'
|
| 203 |
+
])
|
| 204 |
+
if previous_client is no_client:
|
| 205 |
+
self.assertFalse(hasattr(odoo._tlocal, 'client'))
|
| 206 |
+
else:
|
| 207 |
+
self.assertIs(odoo._tlocal.client, previous_client)
|
| 208 |
+
self.net.assert_not_called()
|
| 209 |
+
|
| 210 |
+
def test_unmarked_gate_never_writes_a_mirror(self):
|
| 211 |
+
import verify_odoo_relational as gate
|
| 212 |
+
with patch.dict(os.environ, {}, clear=True), \
|
| 213 |
+
patch.object(ds, 'sync_entity') as sync:
|
| 214 |
+
self.assertTrue(gate._prepare_isolated_trusted_mirror())
|
| 215 |
+
sync.assert_not_called()
|
| 216 |
+
|
| 217 |
+
def test_failed_rank_rebuild_retries_on_next_unchanged_pass(self):
|
| 218 |
+
attempts, projection, sleeps, reports = self.observe_resident_recovery({2})
|
| 219 |
+
self.assertEqual(reports, [0, 1, 0, 0])
|
| 220 |
+
self.assertEqual(attempts, [1, 2, 3])
|
| 221 |
+
self.assertEqual(projection, [1, 2])
|
| 222 |
+
self.assertEqual(sleeps, [60] * 3)
|
| 223 |
+
|
| 224 |
+
def test_repeated_rank_rebuild_failure_retries_at_configured_cadence(self):
|
| 225 |
+
attempts, projection, sleeps, reports = self.observe_resident_recovery({2, 3})
|
| 226 |
+
self.assertEqual(reports, [0, 1, 0, 0, 0])
|
| 227 |
+
self.assertEqual(attempts, [1, 2, 3, 4])
|
| 228 |
+
self.assertEqual(projection, [1, 2])
|
| 229 |
+
self.assertEqual(sleeps, [60] * 4)
|
| 230 |
+
|
| 231 |
+
def test_successful_rank_rebuild_still_skips_unchanged_pass(self):
|
| 232 |
+
attempts, projection, sleeps, reports = self.observe_resident_recovery(set())
|
| 233 |
+
self.assertEqual(reports, [0, 1, 0])
|
| 234 |
+
self.assertEqual(attempts, [1, 2])
|
| 235 |
+
self.assertEqual(projection, [1, 2])
|
| 236 |
+
self.assertEqual(sleeps, [60] * 2)
|
| 237 |
+
|
| 238 |
+
def test_unchanged_refresh_remains_a_confirmed_noop(self):
|
| 239 |
+
for _ in range(2):
|
| 240 |
+
report, reason = self.report_and_rebuild_reason()
|
| 241 |
+
self.assertEqual(report['res_partner']['pulled'], 0)
|
| 242 |
+
self.assertIsNone(reason)
|
| 243 |
+
|
| 244 |
+
def test_cleared_and_null_ranks_report_only_changed_existing_rows(self):
|
| 245 |
+
self.rows[0]['customer_rank'] = False # Odoo's unset numeric becomes SQL NULL.
|
| 246 |
+
self.rows[1]['customer_rank'] = 0
|
| 247 |
+
self.rows.append(dict(id=4, customer_rank=9, active=True,
|
| 248 |
+
name='Absent mirror partner', write_date='2026-09-01 00:00:00'))
|
| 249 |
+
report, reason = self.report_and_rebuild_reason()
|
| 250 |
+
self.assertEqual(self.ranks(), [(1, None), (2, 0), (3, 2)])
|
| 251 |
+
self.assertEqual(reason, 'source-change')
|
| 252 |
+
self.assertEqual(report['res_partner']['pulled'], 2)
|
| 253 |
+
report, reason = self.report_and_rebuild_reason()
|
| 254 |
+
self.assertEqual(report['res_partner']['pulled'], 0)
|
| 255 |
+
self.assertIsNone(reason)
|
| 256 |
+
|
| 257 |
+
def test_refresh_and_delta_both_reach_the_existing_report(self):
|
| 258 |
+
self.rows[0]['customer_rank'] = 5
|
| 259 |
+
self.rows[1].update(name='Delta source name', write_date='2026-09-04 00:00:00')
|
| 260 |
+
report, reason = self.report_and_rebuild_reason()
|
| 261 |
+
self.assertEqual(reason, 'source-change')
|
| 262 |
+
self.assertEqual(report['res_partner']['pulled'], 2)
|
| 263 |
+
with duckdb.connect(self.path) as con:
|
| 264 |
+
self.assertEqual(con.execute('SELECT name FROM res_partner WHERE id=2').fetchone(),
|
| 265 |
+
('Delta source name',))
|
| 266 |
+
|
| 267 |
+
def test_bounded_refresh_reports_each_changed_page_then_zero(self):
|
| 268 |
+
original = self.read
|
| 269 |
+
def small_page(model, domain, fields, limit=None, order=None):
|
| 270 |
+
return original(model, domain, fields, limit=1, order=order)
|
| 271 |
+
self.rows[0]['customer_rank'] = 5
|
| 272 |
+
self.rows[2]['customer_rank'] = 6
|
| 273 |
+
with patch.object(ds.O, 'search_read', side_effect=small_page), \
|
| 274 |
+
patch.object(ds, 'COL_BACKFILL_MAX_BATCHES', 1):
|
| 275 |
+
for expected in (1, 0, 1, 0):
|
| 276 |
+
report, reason = self.report_and_rebuild_reason()
|
| 277 |
+
self.assertEqual(report['res_partner']['pulled'], expected)
|
| 278 |
+
self.assertEqual(reason, 'source-change' if expected else None)
|
| 279 |
+
|
| 280 |
+
def test_lost_refresh_signal_mutant_is_detected_at_resident_consumer(self):
|
| 281 |
+
actual = ds._backfill_columns
|
| 282 |
+
def suppress(*args, **kwargs):
|
| 283 |
+
actual(*args, **kwargs)
|
| 284 |
+
return 0
|
| 285 |
+
with patch.object(ds, '_backfill_columns', side_effect=suppress):
|
| 286 |
+
with self.assertRaises(AssertionError):
|
| 287 |
+
self.test_rank_only_change_triggers_resident_projection_rebuild()
|
| 288 |
+
|
| 289 |
+
def test_false_refresh_signal_mutant_is_detected_by_noop_control(self):
|
| 290 |
+
actual = ds._backfill_columns
|
| 291 |
+
def inflate(*args, **kwargs):
|
| 292 |
+
result = actual(*args, **kwargs)
|
| 293 |
+
return 1 if kwargs.get('repeat') else result
|
| 294 |
+
with patch.object(ds, '_backfill_columns', side_effect=inflate):
|
| 295 |
+
with self.assertRaises(AssertionError):
|
| 296 |
+
self.test_unchanged_refresh_remains_a_confirmed_noop()
|
| 297 |
+
|
| 298 |
+
def test_rank_change_behind_cursor_reaches_customer_reader(self):
|
| 299 |
+
self.rows[0]['customer_rank'] = 4
|
| 300 |
+
self.assertEqual(self.sync()[0], 'live')
|
| 301 |
+
self.assertEqual(self.ranks(), [(1, 4), (2, 1), (3, 2)])
|
| 302 |
+
import odoo_relational as rel
|
| 303 |
+
with duckdb.connect(self.path) as con:
|
| 304 |
+
con.execute('CREATE TABLE sale_order (partner_id INTEGER, state VARCHAR)')
|
| 305 |
+
con.execute('CREATE TABLE account_move (partner_id INTEGER, state VARCHAR, move_type VARCHAR)')
|
| 306 |
+
self.assertEqual({r[rel.JOIN_KEY] for r in rel.read_customers(con)}, {1, 2})
|
| 307 |
+
self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
|
| 308 |
+
|
| 309 |
+
def test_completed_refresh_repeats_and_clears_rank(self):
|
| 310 |
+
self.sync()
|
| 311 |
+
self.rows[1]['customer_rank'] = 0
|
| 312 |
+
self.rows[2]['customer_rank'] = 7
|
| 313 |
+
self.sync()
|
| 314 |
+
self.assertEqual(self.ranks(), [(1, 0), (2, 0), (3, 7)])
|
| 315 |
+
refresh = [call for call in self.calls if 'write_date' not in call[1]]
|
| 316 |
+
self.assertTrue(refresh)
|
| 317 |
+
self.assertTrue(all(call[1] == ['customer_rank'] for call in refresh))
|
| 318 |
+
self.assertTrue(all(('active', 'in', [True, False]) in call[0] for call in refresh))
|
| 319 |
+
|
| 320 |
+
def test_bounded_refresh_resumes_then_starts_another_cycle(self):
|
| 321 |
+
original = self.read
|
| 322 |
+
def small_page(model, domain, fields, limit=None, order=None):
|
| 323 |
+
return original(model, domain, fields, limit=1, order=order)
|
| 324 |
+
for row in self.rows:
|
| 325 |
+
row['customer_rank'] += 10
|
| 326 |
+
with patch.object(ds.O, 'search_read', side_effect=small_page), \
|
| 327 |
+
patch.object(ds, 'COL_BACKFILL_MAX_BATCHES', 1):
|
| 328 |
+
self.sync()
|
| 329 |
+
self.assertEqual(self.ranks(), [(1, 10), (2, 1), (3, 2)])
|
| 330 |
+
self.sync()
|
| 331 |
+
self.assertEqual(self.ranks(), [(1, 10), (2, 11), (3, 2)])
|
| 332 |
+
self.sync()
|
| 333 |
+
self.assertEqual(self.ranks(), [(1, 10), (2, 11), (3, 12)])
|
| 334 |
+
self.sync() # Empty source page proves this cycle finished.
|
| 335 |
+
self.rows[0]['customer_rank'] = 20
|
| 336 |
+
self.sync()
|
| 337 |
+
self.assertEqual(self.ranks()[0], (1, 20))
|
| 338 |
+
|
| 339 |
+
def test_undeclared_entity_does_not_add_recurring_source_work(self):
|
| 340 |
+
self.spec.pop('refresh_fields', None)
|
| 341 |
+
self.sync()
|
| 342 |
+
self.assertEqual(len(self.calls), 1)
|
| 343 |
+
self.assertIn('write_date', self.calls[0][1])
|
| 344 |
+
|
| 345 |
+
def test_completed_schema_backfill_does_not_suppress_rank_refresh(self):
|
| 346 |
+
self.spec['backfill_fields'] = ['customer_rank']
|
| 347 |
+
marker = 'res_partner.cols.customer_rank'
|
| 348 |
+
with duckdb.connect(self.path) as con:
|
| 349 |
+
con.execute('INSERT INTO _sync_state VALUES (?,?,?,?,?,?)',
|
| 350 |
+
[marker, 'done', 3, None, 3, '2026-09-02 00:00:00'])
|
| 351 |
+
self.rows[0]['customer_rank'] = 8
|
| 352 |
+
self.sync()
|
| 353 |
+
self.assertEqual(self.ranks()[0], (1, 8))
|
| 354 |
+
with duckdb.connect(self.path) as con:
|
| 355 |
+
self.assertEqual(con.execute('SELECT * FROM _sync_state WHERE entity=?', [marker]).fetchone(),
|
| 356 |
+
(marker, 'done', 3, None, 3, '2026-09-02 00:00:00'))
|
| 357 |
+
|
| 358 |
+
def test_failed_page_resumes_without_advancing_delta_cursor(self):
|
| 359 |
+
original = self.read
|
| 360 |
+
connections = []
|
| 361 |
+
def connect():
|
| 362 |
+
con = duckdb.connect(self.path)
|
| 363 |
+
connections.append(con)
|
| 364 |
+
return con
|
| 365 |
+
def interrupted(model, domain, fields, limit=None, order=None):
|
| 366 |
+
if ('id', '>', 1) in domain:
|
| 367 |
+
raise RuntimeError('synthetic source interruption')
|
| 368 |
+
return original(model, domain, fields, limit=1, order=order)
|
| 369 |
+
self.rows[0]['customer_rank'] = 5
|
| 370 |
+
self.rows[1]['customer_rank'] = 6
|
| 371 |
+
with patch.object(ds, 'connect', side_effect=connect), \
|
| 372 |
+
patch.object(ds.O, 'search_read', side_effect=interrupted):
|
| 373 |
+
with self.assertRaisesRegex(RuntimeError, 'synthetic source interruption'):
|
| 374 |
+
self.sync()
|
| 375 |
+
with self.assertRaises(duckdb.ConnectionException):
|
| 376 |
+
connections[0].execute('SELECT 1')
|
| 377 |
+
with duckdb.connect(self.path) as con:
|
| 378 |
+
self.assertEqual(ds._state(con, 'res_partner')['cursor_wd'], '2026-09-03 01:02:25')
|
| 379 |
+
self.assertEqual(con.execute("SELECT phase,last_id FROM _sync_state "
|
| 380 |
+
"WHERE entity='res_partner.refresh.customer_rank'").fetchone(),
|
| 381 |
+
('running', 1))
|
| 382 |
+
self.calls.clear()
|
| 383 |
+
self.sync()
|
| 384 |
+
self.assertEqual(self.ranks(), [(1, 5), (2, 6), (3, 2)])
|
| 385 |
+
self.assertIn(('id', '>', 1), self.calls[0][0])
|
| 386 |
+
|
| 387 |
+
def test_only_rank_changes_and_missing_rows_are_not_invented(self):
|
| 388 |
+
self.rows[0].update(customer_rank=9, name='Changed source name', active=False)
|
| 389 |
+
self.rows.append(dict(id=4, customer_rank=9, active=True,
|
| 390 |
+
name='Absent mirror partner', write_date='2026-09-01 00:00:00'))
|
| 391 |
+
self.sync()
|
| 392 |
+
self.assertEqual(self.ranks(), [(1, 9), (2, 1), (3, 2)])
|
| 393 |
+
with duckdb.connect(self.path) as con:
|
| 394 |
+
self.assertEqual(con.execute('SELECT name,active,write_date FROM res_partner WHERE id=1').fetchone(),
|
| 395 |
+
('Synthetic partner', True, '2026-09-01 00:00:00'))
|
| 396 |
+
|
| 397 |
+
def test_refresh_removal_mutant_is_detected_by_customer_membership(self):
|
| 398 |
+
self.spec.pop('refresh_fields', None)
|
| 399 |
+
with self.assertRaises(AssertionError):
|
| 400 |
+
self.test_rank_change_behind_cursor_reaches_customer_reader()
|
| 401 |
+
|
| 402 |
+
def test_pause_never_opens_mirror_or_source(self):
|
| 403 |
+
with patch.object(ds, 'source_paused', return_value=True), \
|
| 404 |
+
patch.object(ds, 'connect') as connect:
|
| 405 |
+
self.assertEqual(self.sync(), (ds.PAUSED_PHASE, 0))
|
| 406 |
+
connect.assert_not_called()
|
| 407 |
+
self.assertEqual(self.calls, [])
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
if __name__ == '__main__':
|
| 411 |
+
unittest.main()
|
api/test_odoo_relational_verdict.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hermetic tests of relational gate verdicts; these never prove source parity."""
|
| 2 |
+
from contextlib import ExitStack, redirect_stdout
|
| 3 |
+
import io
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import tempfile
|
| 6 |
+
from types import SimpleNamespace
|
| 7 |
+
import unittest
|
| 8 |
+
from unittest.mock import Mock, patch
|
| 9 |
+
|
| 10 |
+
import verify_odoo_relational as gate
|
| 11 |
+
from harness import datastore
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class RelationalVerdictTests(unittest.TestCase):
|
| 15 |
+
def setUp(self):
|
| 16 |
+
self.original_pass = gate.PASS[:]
|
| 17 |
+
self.original_fail = gate.FAIL[:]
|
| 18 |
+
self.original_taken = gate._MIRROR_TAKEN[:]
|
| 19 |
+
gate.PASS.clear()
|
| 20 |
+
gate.FAIL.clear()
|
| 21 |
+
gate._MIRROR_TAKEN[0] = False
|
| 22 |
+
self.capture = io.StringIO()
|
| 23 |
+
self.output = redirect_stdout(self.capture)
|
| 24 |
+
self.output.__enter__()
|
| 25 |
+
|
| 26 |
+
def tearDown(self):
|
| 27 |
+
self.output.__exit__(None, None, None)
|
| 28 |
+
gate.PASS[:] = self.original_pass
|
| 29 |
+
gate.FAIL[:] = self.original_fail
|
| 30 |
+
gate._MIRROR_TAKEN[:] = self.original_taken
|
| 31 |
+
|
| 32 |
+
def main_verdict(self, cursor_result):
|
| 33 |
+
# Run the real gate orchestration. Source oracles are disabled only in
|
| 34 |
+
# this unit test; the prerequisite check and final exit stay real.
|
| 35 |
+
with ExitStack() as stack:
|
| 36 |
+
for name in vars(gate):
|
| 37 |
+
if name.startswith("_prove_"):
|
| 38 |
+
stack.enter_context(patch.object(gate, name, return_value=None))
|
| 39 |
+
stack.enter_context(patch.object(gate, "run", return_value={}))
|
| 40 |
+
stack.enter_context(patch.object(gate, "real_cursor", return_value=cursor_result))
|
| 41 |
+
stack.enter_context(patch.object(datastore, "ready", return_value=False))
|
| 42 |
+
return gate._main(SimpleNamespace(negative_control=False))
|
| 43 |
+
|
| 44 |
+
def test_absent_mirror_cannot_pass_readability(self):
|
| 45 |
+
self.assertEqual(self.main_verdict((None, "missing")), 1)
|
| 46 |
+
self.assertTrue(any("[missing]" in name for name in gate.FAIL))
|
| 47 |
+
|
| 48 |
+
def test_unopenable_mirror_cannot_pass_readability(self):
|
| 49 |
+
self.assertEqual(self.main_verdict((None, "open-failed")), 1)
|
| 50 |
+
self.assertTrue(any("[open-failed]" in name for name in gate.FAIL))
|
| 51 |
+
|
| 52 |
+
def test_copy_failure_remains_fatal_even_if_source_can_open(self):
|
| 53 |
+
gate._MIRROR_TAKEN[0] = True
|
| 54 |
+
self.assertEqual(self.main_verdict((Mock(), "synthetic")), 1)
|
| 55 |
+
|
| 56 |
+
def test_readable_mirror_passes_and_releases_probe_cursor(self):
|
| 57 |
+
cursor = Mock()
|
| 58 |
+
self.assertEqual(self.main_verdict((cursor, "synthetic")), 0)
|
| 59 |
+
cursor.close.assert_called_once_with()
|
| 60 |
+
|
| 61 |
+
def test_probe_releases_real_duckdb_before_different_configuration(self):
|
| 62 |
+
import duckdb
|
| 63 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 64 |
+
path = Path(tmp) / "synthetic.duckdb"
|
| 65 |
+
duckdb.connect(str(path)).close()
|
| 66 |
+
with patch.object(datastore, "path_for", return_value=path):
|
| 67 |
+
self.assertTrue(gate._check_mirror_admission())
|
| 68 |
+
with duckdb.connect(str(path), read_only=False) as con:
|
| 69 |
+
self.assertEqual(con.execute("SELECT 42").fetchone(), (42,))
|
| 70 |
+
|
| 71 |
+
def test_probe_close_failure_remains_red(self):
|
| 72 |
+
cursor = Mock()
|
| 73 |
+
cursor.close.side_effect = RuntimeError("PRIVATE_SENTINEL")
|
| 74 |
+
self.assertEqual(self.main_verdict((cursor, "synthetic")), 1)
|
| 75 |
+
self.assertTrue(any("[close-failed]" in name for name in gate.FAIL))
|
| 76 |
+
self.assertNotIn("PRIVATE_SENTINEL", self.capture.getvalue())
|
| 77 |
+
|
| 78 |
+
def test_missing_path_has_safe_machine_readable_reason(self):
|
| 79 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 80 |
+
with patch.object(datastore, "path_for", return_value=Path(tmp) / "absent.duckdb"):
|
| 81 |
+
self.assertEqual(gate.real_cursor(), (None, "missing"))
|
| 82 |
+
|
| 83 |
+
def test_open_error_does_not_disclose_exception_or_path(self):
|
| 84 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 85 |
+
path = Path(tmp) / "synthetic.duckdb"
|
| 86 |
+
path.touch()
|
| 87 |
+
with patch.object(datastore, "path_for", return_value=path), patch(
|
| 88 |
+
"duckdb.connect", side_effect=RuntimeError("PRIVATE_SENTINEL /private/source")
|
| 89 |
+
):
|
| 90 |
+
result = gate.real_cursor()
|
| 91 |
+
self.assertEqual(result, (None, "open-failed"))
|
| 92 |
+
self.assertNotIn("PRIVATE_SENTINEL", repr(result))
|
| 93 |
+
|
| 94 |
+
def control_verdict(self, *, background=False, detect=True):
|
| 95 |
+
def observed_run(corrupt=None):
|
| 96 |
+
gate.check("synthetic ordinary passing assertion", True)
|
| 97 |
+
if background:
|
| 98 |
+
gate.check("synthetic source parity failure", False)
|
| 99 |
+
if corrupt is not None and detect:
|
| 100 |
+
gate.check("synthetic corruption detected", False)
|
| 101 |
+
with patch.object(gate, "run", side_effect=observed_run):
|
| 102 |
+
return gate._negative_control()
|
| 103 |
+
|
| 104 |
+
def test_negative_controls_preserve_prior_failures(self):
|
| 105 |
+
gate.check("synthetic prerequisite refused", False)
|
| 106 |
+
self.assertEqual(self.control_verdict(), 0)
|
| 107 |
+
self.assertIn("synthetic prerequisite refused", gate.FAIL)
|
| 108 |
+
self.assertNotIn("synthetic corruption detected", gate.FAIL)
|
| 109 |
+
|
| 110 |
+
def test_uncorrupted_control_baseline_failure_survives(self):
|
| 111 |
+
self.assertEqual(self.control_verdict(background=True), 0)
|
| 112 |
+
self.assertIn("synthetic source parity failure", gate.FAIL)
|
| 113 |
+
self.assertNotIn("synthetic corruption detected", gate.FAIL)
|
| 114 |
+
|
| 115 |
+
def test_control_replays_do_not_inflate_passing_population(self):
|
| 116 |
+
gate.check("synthetic original passing assertion", True)
|
| 117 |
+
self.assertEqual(self.control_verdict(), 0)
|
| 118 |
+
self.assertEqual(gate.PASS.count("synthetic ordinary passing assertion"), 0)
|
| 119 |
+
self.assertIn("synthetic original passing assertion", gate.PASS)
|
| 120 |
+
self.assertFalse(gate.FAIL)
|
| 121 |
+
|
| 122 |
+
def test_background_red_cannot_kill_an_undetected_mutation(self):
|
| 123 |
+
self.assertEqual(self.control_verdict(background=True, detect=False), 1)
|
| 124 |
+
self.assertIn("synthetic source parity failure", gate.FAIL)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
if __name__ == "__main__":
|
| 128 |
+
unittest.main()
|
platform/aios_grid.py
CHANGED
|
@@ -1594,7 +1594,7 @@ def cohort_field(label='Locked views'):
|
|
| 1594 |
'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
|
| 1595 |
'default': False, 'filterable': False, 'derived': True, 'multi': True,
|
| 1596 |
'note': 'The locked views this customer is in, newest first. A locked view holds a SET, '
|
| 1597 |
-
'so grouping by this column puts a customer under EVERY view they are in
|
| 1598 |
'group counts therefore add up to more than the record count, which stays the '
|
| 1599 |
'number of distinct customers. Read-only: membership changes only by adding or '
|
| 1600 |
'removing customers on the locked view itself.',
|
|
@@ -3940,10 +3940,10 @@ def scope_counts(shown, matched, total):
|
|
| 3940 |
"""
|
| 3941 |
shown, matched, total = int(shown), int(matched), int(total)
|
| 3942 |
if matched > total:
|
| 3943 |
-
raise ValueError(f"matched ({matched}) exceeds total ({total})
|
| 3944 |
f"more rows than the scope holds")
|
| 3945 |
if shown > matched:
|
| 3946 |
-
raise ValueError(f"shown ({shown}) exceeds matched ({matched})
|
| 3947 |
f"more rows than the filter matched")
|
| 3948 |
return {"shown": shown, "matched": matched, "total": total, "windowed": True}
|
| 3949 |
|
|
|
|
| 1594 |
'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
|
| 1595 |
'default': False, 'filterable': False, 'derived': True, 'multi': True,
|
| 1596 |
'note': 'The locked views this customer is in, newest first. A locked view holds a SET, '
|
| 1597 |
+
'so grouping by this column puts a customer under EVERY view they are in; the '
|
| 1598 |
'group counts therefore add up to more than the record count, which stays the '
|
| 1599 |
'number of distinct customers. Read-only: membership changes only by adding or '
|
| 1600 |
'removing customers on the locked view itself.',
|
|
|
|
| 3940 |
"""
|
| 3941 |
shown, matched, total = int(shown), int(matched), int(total)
|
| 3942 |
if matched > total:
|
| 3943 |
+
raise ValueError(f"matched ({matched}) exceeds total ({total}); a filter cannot match "
|
| 3944 |
f"more rows than the scope holds")
|
| 3945 |
if shown > matched:
|
| 3946 |
+
raise ValueError(f"shown ({shown}) exceeds matched ({matched}); the window cannot hold "
|
| 3947 |
f"more rows than the filter matched")
|
| 3948 |
return {"shown": shown, "matched": matched, "total": total, "windowed": True}
|
| 3949 |
|
platform/aios_grid_fields.json
CHANGED
|
@@ -118,6 +118,22 @@
|
|
| 118 |
"default": true,
|
| 119 |
"description": "A Google Maps link to this customer's address."
|
| 120 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
{
|
| 122 |
"key": "customer_since",
|
| 123 |
"label": "Customer since",
|
|
@@ -166,17 +182,6 @@
|
|
| 166 |
"default": true,
|
| 167 |
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 168 |
},
|
| 169 |
-
{
|
| 170 |
-
"_note": "filterable:false β DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule β see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 171 |
-
"key": "est_missed",
|
| 172 |
-
"label": "Est. missed $",
|
| 173 |
-
"type": "currency",
|
| 174 |
-
"source": "odoo",
|
| 175 |
-
"default": true,
|
| 176 |
-
"agg": "sum",
|
| 177 |
-
"filterable": false,
|
| 178 |
-
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 179 |
-
},
|
| 180 |
{
|
| 181 |
"_note": "wave 21 R1 β KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 182 |
"key": "ar_open",
|
|
|
|
| 118 |
"default": true,
|
| 119 |
"description": "A Google Maps link to this customer's address."
|
| 120 |
},
|
| 121 |
+
{
|
| 122 |
+
"key": "lat",
|
| 123 |
+
"label": "Latitude",
|
| 124 |
+
"type": "text",
|
| 125 |
+
"source": "odoo",
|
| 126 |
+
"default": false,
|
| 127 |
+
"description": "Latitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address."
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"key": "lon",
|
| 131 |
+
"label": "Longitude",
|
| 132 |
+
"type": "text",
|
| 133 |
+
"source": "odoo",
|
| 134 |
+
"default": false,
|
| 135 |
+
"description": "Longitude of this customer's address, from Odoo when it has one and otherwise from our geocode of the address."
|
| 136 |
+
},
|
| 137 |
{
|
| 138 |
"key": "customer_since",
|
| 139 |
"label": "Customer since",
|
|
|
|
| 182 |
"default": true,
|
| 183 |
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 184 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
{
|
| 186 |
"_note": "wave 21 R1 β KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
|
| 187 |
"key": "ar_open",
|
platform/core/auth_google.py
CHANGED
|
@@ -11,8 +11,8 @@ DESIGN (recorded now so the full build has a contract to meet):
|
|
| 11 |
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
-
- The password gate stays
|
| 15 |
-
|
| 16 |
|
| 17 |
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
|
@@ -50,7 +50,7 @@ def auth_url(state: str, code_challenge: str) -> str:
|
|
| 50 |
session. Template guard: raises until the TODO checklist lands.
|
| 51 |
"""
|
| 52 |
raise NotImplementedError(
|
| 53 |
-
"Google login is a template
|
| 54 |
"auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
|
| 55 |
"code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
|
| 56 |
)
|
|
@@ -61,7 +61,7 @@ def exchange_code(code: str, code_verifier: str) -> dict:
|
|
| 61 |
Template guard: raises until implemented (requests.post with client_id/secret,
|
| 62 |
redirect_uri, grant_type=authorization_code, code, code_verifier)."""
|
| 63 |
raise NotImplementedError(
|
| 64 |
-
"Google login is a template
|
| 65 |
)
|
| 66 |
|
| 67 |
|
|
@@ -70,7 +70,7 @@ def verify_id_token(id_token: str) -> dict:
|
|
| 70 |
return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
|
| 71 |
email_verified=False. Template guard: raises until implemented."""
|
| 72 |
raise NotImplementedError(
|
| 73 |
-
"Google login is a template
|
| 74 |
)
|
| 75 |
|
| 76 |
|
|
|
|
| 11 |
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
+
- The password gate stays beside it. APP_PASSWORD may seed a fresh registry, but it has no
|
| 15 |
+
authentication bypass; removing stored-password auth is a separate, owner-approved step.
|
| 16 |
|
| 17 |
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
|
|
|
| 50 |
session. Template guard: raises until the TODO checklist lands.
|
| 51 |
"""
|
| 52 |
raise NotImplementedError(
|
| 53 |
+
"Google login is a template; complete docs/GOOGLE_LOGIN_TODO.md, then implement "
|
| 54 |
"auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
|
| 55 |
"code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
|
| 56 |
)
|
|
|
|
| 61 |
Template guard: raises until implemented (requests.post with client_id/secret,
|
| 62 |
redirect_uri, grant_type=authorization_code, code, code_verifier)."""
|
| 63 |
raise NotImplementedError(
|
| 64 |
+
"Google login is a template; implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md."
|
| 65 |
)
|
| 66 |
|
| 67 |
|
|
|
|
| 70 |
return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
|
| 71 |
email_verified=False. Template guard: raises until implemented."""
|
| 72 |
raise NotImplementedError(
|
| 73 |
+
"Google login is a template; implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md."
|
| 74 |
)
|
| 75 |
|
| 76 |
|
platform/core/connector_directory.py
CHANGED
|
@@ -18,7 +18,7 @@ it, and there is nothing to find. Measured against the live Monid API on 2026-09
|
|
| 18 |
tags, categories, metrics, hints`, and none of those carries a sidebar name.
|
| 19 |
|
| 20 |
So the 14 categories are reachable only from Monid's WEB UI SIDEBAR, which is what
|
| 21 |
-
`
|
| 22 |
regenerated by re-reading that file, not by calling an endpoint.
|
| 23 |
|
| 24 |
β AND THE CATALOGUE IS A FLOOR, NOT A CENSUS. `discover` returns providers absent from the proto
|
|
@@ -87,7 +87,7 @@ MONID_CATEGORY_ENDPOINTS = {
|
|
| 87 |
#: ββ THE AUTHORED CROSSWALK: (provider name, endpoint count, the categories it is filed under).
|
| 88 |
#:
|
| 89 |
#: REGENERATION, because there is no generator: re-read the `PROVIDERS` line of
|
| 90 |
-
#: `
|
| 91 |
#: the middle dot. The NAME and the COUNT come from there unchanged and `verify_catalog.py` proves
|
| 92 |
#: it, pair by pair, so a typo here goes red rather than quiet. The CATEGORIES are the authored
|
| 93 |
#: half: a judgement about which sidebar rows a provider sits under, made once, here.
|
|
@@ -228,6 +228,13 @@ MONID_PROVIDERS = (
|
|
| 228 |
#: NO category, appears only under `All`, and `verify_catalog.py` names it and goes red.
|
| 229 |
NATIVE_CATEGORIES = {
|
| 230 |
"odoo": ("Data", "Finance", "E-commerce"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
"webhooks": ("Utilities", "Web Tools"),
|
| 232 |
"scraper": ("Scrapers", "Social Media"),
|
| 233 |
"stripe": ("Finance", "E-commerce"),
|
|
@@ -300,6 +307,12 @@ MONID_ACTION_STATE = "available"
|
|
| 300 |
#: state with the neutral tone by construction (the wave-9 law: no client union over a server
|
| 301 |
#: vocabulary), so the client renders it without a change. Any gate holding a HAND-TYPED copy of
|
| 302 |
#: the five-word set has to learn the sixth.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
CATALOGUE_STATE = "catalogue"
|
| 304 |
|
| 305 |
#: ββ W49-T14 Β· RULINGS R21 + R24 β WHICH CONNECTORS HAVE A FIELD CATALOGUE, AUTHORED IN CODE.
|
|
@@ -320,10 +333,26 @@ CATALOGUE_STATE = "catalogue"
|
|
| 320 |
#: behind them to introspect. Membership is by KEY, and every catalogue key wears
|
| 321 |
#: `MONID_KEY_PREFIX`, so the two sets cannot overlap by construction.
|
| 322 |
#:
|
| 323 |
-
#: β
|
| 324 |
-
#:
|
| 325 |
-
#:
|
| 326 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
|
| 328 |
|
| 329 |
def is_introspectable(connector_key) -> bool:
|
|
@@ -334,6 +363,38 @@ def is_introspectable(connector_key) -> bool:
|
|
| 334 |
return str(connector_key or "") in INTROSPECTABLE_CONNECTORS
|
| 335 |
|
| 336 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
#: β THE KEYS A ROW CARRIES FOR US AND NEVER FOR THE CLIENT (C4). One tuple, read by
|
| 338 |
#: `to_client_row` and by the gate's negative control, so "which keys are server-only" has one
|
| 339 |
#: answer wherever it is asked.
|
|
@@ -455,8 +516,17 @@ def to_client_row(row):
|
|
| 455 |
catalogue's, we reason with it on the server, and R27's design rule is that it never reaches a
|
| 456 |
screen in any form. Stripping it here rather than at each composer means there is ONE place to
|
| 457 |
read to know what the wire carries, and one place for the gate to break.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
"""
|
| 459 |
-
|
|
|
|
|
|
|
| 460 |
|
| 461 |
|
| 462 |
def to_client_rows(rows):
|
|
|
|
| 18 |
tags, categories, metrics, hints`, and none of those carries a sidebar name.
|
| 19 |
|
| 20 |
So the 14 categories are reachable only from Monid's WEB UI SIDEBAR, which is what
|
| 21 |
+
`AutoWiki/waves/wave49/proto/monid-catalogue.md` captured by hand. `MONID_PROVIDERS` is
|
| 22 |
regenerated by re-reading that file, not by calling an endpoint.
|
| 23 |
|
| 24 |
β AND THE CATALOGUE IS A FLOOR, NOT A CENSUS. `discover` returns providers absent from the proto
|
|
|
|
| 87 |
#: ββ THE AUTHORED CROSSWALK: (provider name, endpoint count, the categories it is filed under).
|
| 88 |
#:
|
| 89 |
#: REGENERATION, because there is no generator: re-read the `PROVIDERS` line of
|
| 90 |
+
#: `AutoWiki/waves/wave49/proto/monid-catalogue.md`, which is `name count` pairs separated by
|
| 91 |
#: the middle dot. The NAME and the COUNT come from there unchanged and `verify_catalog.py` proves
|
| 92 |
#: it, pair by pair, so a typo here goes red rather than quiet. The CATEGORIES are the authored
|
| 93 |
#: half: a judgement about which sidebar rows a provider sits under, made once, here.
|
|
|
|
| 228 |
#: NO category, appears only under `All`, and `verify_catalog.py` names it and goes red.
|
| 229 |
NATIVE_CATEGORIES = {
|
| 230 |
"odoo": ("Data", "Finance", "E-commerce"),
|
| 231 |
+
# ββ W52-T01 β FOUND UNFILED WHILE BUILDING THE `spawns_tables` TIER, AND FIXED HERE RATHER
|
| 232 |
+
# THAN LEFT: `amazon_sp` joined `TOKEN_CONNECTORS` (routes_connectors.py) in wave 31 and this
|
| 233 |
+
# dict never gained a matching row, so `verify_catalog.py`'s C4 leg
|
| 234 |
+
# (`every-native-connector-is-filed`) was red before this ticket touched the file. Filed the
|
| 235 |
+
# same way Odoo is: it reaches orders, fees and listings, which is Data plus the two shelves
|
| 236 |
+
# Odoo itself sits under for the commerce side of the same story.
|
| 237 |
+
"amazon_sp": ("Data", "E-commerce", "Finance"),
|
| 238 |
"webhooks": ("Utilities", "Web Tools"),
|
| 239 |
"scraper": ("Scrapers", "Social Media"),
|
| 240 |
"stripe": ("Finance", "E-commerce"),
|
|
|
|
| 307 |
#: state with the neutral tone by construction (the wave-9 law: no client union over a server
|
| 308 |
#: vocabulary), so the client renders it without a change. Any gate holding a HAND-TYPED copy of
|
| 309 |
#: the five-word set has to learn the sixth.
|
| 310 |
+
#:
|
| 311 |
+
#: β W53-T31 (R11) β TELEGRAM IS ONE OF THE 117, AND IT STAYS ONE. `MONID_PROVIDERS` carries
|
| 312 |
+
#: `("Telegram", 7, ...)`: a vendor listing, not a connector we have built or planned. Wave 53
|
| 313 |
+
#: adopted only its CREDENTIAL SHAPE on paper and built no runtime path, so the row keeps this
|
| 314 |
+
#: word and Telegram is deliberately absent from `routes_connectors.PLANNED`. The build plan,
|
| 315 |
+
#: with the two keychain steps it needs first, is `AutoWiki/research/ideation/telegram-bot.md`.
|
| 316 |
CATALOGUE_STATE = "catalogue"
|
| 317 |
|
| 318 |
#: ββ W49-T14 Β· RULINGS R21 + R24 β WHICH CONNECTORS HAVE A FIELD CATALOGUE, AUTHORED IN CODE.
|
|
|
|
| 333 |
#: behind them to introspect. Membership is by KEY, and every catalogue key wears
|
| 334 |
#: `MONID_KEY_PREFIX`, so the two sets cannot overlap by construction.
|
| 335 |
#:
|
| 336 |
+
#: β ADDING AN ENTRY IS A REAL DECISION. A connector named here promises the Databases tab a
|
| 337 |
+
#: field catalogue; whether it also gets FIELD TOGGLES is a separate flag, because a toggle needs
|
| 338 |
+
#: a write route behind it and one that always 409s is R8's fake affordance.
|
| 339 |
+
#:
|
| 340 |
+
#: ββ W52 Β· R6 AMENDS R24, AND `amazon_sp` IS WHY. R24 restricted this screen to *introspectable*
|
| 341 |
+
#: connectors, meaning a live field catalogue β a `fields_get`. Owner ruling R6 (wave 52):
|
| 342 |
+
#: **the test is now "is there a real schema artifact", not "is there a `fields_get`"**. SP-API
|
| 343 |
+
#: has NO introspection endpoint at any price, and Amazon publishes versioned, vendor-authored
|
| 344 |
+
#: JSON Schemas on GitHub which satisfy the ruling's purpose better than a probe would: they are
|
| 345 |
+
#: pinned by blob sha and cannot drift under us between reads.
|
| 346 |
+
#:
|
| 347 |
+
#: β AND AMAZON GETS THE SCREEN WITHOUT THE TOGGLES, DELIBERATELY. The field-toggle door is
|
| 348 |
+
#: `PATCH /connectors/odoo/schema/{field_key}`; there is no Amazon equivalent and this wave does
|
| 349 |
+
#: not build one, so `_detail_introspection` answers `canToggleFields: False` for it. Naming it
|
| 350 |
+
#: here buys the DISCLOSURE and nothing more, which is exactly what R6 asked for.
|
| 351 |
+
#:
|
| 352 |
+
#: β Its catalogue is honestly INCOMPLETE and says so: only two of the seven tables have a
|
| 353 |
+
#: published artifact. The other five measure their columns from the report's own header at first
|
| 354 |
+
#: sync, so before that they have none β see `harness/amazon_schema.PROVENANCE`.
|
| 355 |
+
INTROSPECTABLE_CONNECTORS = ("odoo", "amazon_sp")
|
| 356 |
|
| 357 |
|
| 358 |
def is_introspectable(connector_key) -> bool:
|
|
|
|
| 363 |
return str(connector_key or "") in INTROSPECTABLE_CONNECTORS
|
| 364 |
|
| 365 |
|
| 366 |
+
#: ββ W52-T01 Β· RULING R1 / CONTRACT C1 β WHICH CONNECTORS SPAWN A RELATIONAL DATABASE OF THEIR
|
| 367 |
+
#: OWN, THE TEST THAT TIERS THE CONNECTORS PAGE.
|
| 368 |
+
#:
|
| 369 |
+
#: R1, verbatim: *"A connector is a DATABASE connector when it spawns `ut_*` tables browsable
|
| 370 |
+
#: under Data. Odoo, Amazon and Meta Ads all qualify, and a future connector qualifies
|
| 371 |
+
#: automatically the day it spawns tables."*
|
| 372 |
+
#:
|
| 373 |
+
#: ββ THIS IS A DIFFERENT QUESTION FROM `INTROSPECTABLE_CONNECTORS`, AND REUSING THAT TUPLE WAS
|
| 374 |
+
#: THE TRAP THE TICKET NAMES. `INTROSPECTABLE_CONNECTORS` answers "does this connector publish a
|
| 375 |
+
#: LIVE FIELD CATALOGUE" (R24) and Meta Ads is deliberately absent from it, because Meta has no
|
| 376 |
+
#: `fields_get` to disclose. But Meta Ads DOES spawn `ut_meta_*` tables
|
| 377 |
+
#: (`aios-web/api/meta_relational.py`) even though it publishes no schema, so a Databases tier
|
| 378 |
+
#: computed from `INTROSPECTABLE_CONNECTORS` would silently drop Meta Ads from R1's own example.
|
| 379 |
+
#:
|
| 380 |
+
#: Ground truth, read from the relational spawn modules directly rather than guessed:
|
| 381 |
+
#: `odoo_relational.py`, `amazon_relational.py` and `meta_relational.py` each carry a `# THE SPAWN`
|
| 382 |
+
#: section registering a `ut_<key>_*` prefix through `user_tables.register_connected_prefix`
|
| 383 |
+
#: (mirrored for the row-cap question in `routes_connectors.CONNECTED_PREFIXES`, a different
|
| 384 |
+
#: subsystem answering a different question β see that constant's own header). A future connector
|
| 385 |
+
#: joins this tuple in the same change that ships its relational spawn, exactly as C1 says.
|
| 386 |
+
SPAWNS_TABLES_CONNECTORS = ("odoo", "amazon_sp", "meta_ads")
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def spawns_tables(connector_key) -> bool:
|
| 390 |
+
"""Does this connector spawn a `ut_<key>_*` relational database, browsable under Data (R1)?
|
| 391 |
+
|
| 392 |
+
Pure and total: an unknown key does not spawn tables, the fail-closed direction β a connector
|
| 393 |
+
is tiered into "Databases" only once its spawn is real, never by a guessed default.
|
| 394 |
+
"""
|
| 395 |
+
return str(connector_key or "") in SPAWNS_TABLES_CONNECTORS
|
| 396 |
+
|
| 397 |
+
|
| 398 |
#: β THE KEYS A ROW CARRIES FOR US AND NEVER FOR THE CLIENT (C4). One tuple, read by
|
| 399 |
#: `to_client_row` and by the gate's negative control, so "which keys are server-only" has one
|
| 400 |
#: answer wherever it is asked.
|
|
|
|
| 516 |
catalogue's, we reason with it on the server, and R27's design rule is that it never reaches a
|
| 517 |
screen in any form. Stripping it here rather than at each composer means there is ONE place to
|
| 518 |
read to know what the wire carries, and one place for the gate to break.
|
| 519 |
+
|
| 520 |
+
ββ W52-T01 / C1 β AND THIS IS ALSO WHERE `spawns_tables` IS STAMPED, for the same reason.
|
| 521 |
+
Every row that reaches this function carries a `key`, native or catalogue alike, so computing
|
| 522 |
+
the tier HERE rather than at each composer means there is one place to read for "does this
|
| 523 |
+
card belong under Databases" and one place for `verify_catalog.py` to break if it stops being
|
| 524 |
+
true. A catalogue row's key is never in `SPAWNS_TABLES_CONNECTORS` (they all wear
|
| 525 |
+
`MONID_KEY_PREFIX`), so every one of the 100+ listings answers `False` by construction.
|
| 526 |
"""
|
| 527 |
+
out = {k: v for k, v in row.items() if k not in SERVER_ONLY_KEYS}
|
| 528 |
+
out["spawns_tables"] = spawns_tables(row.get("key"))
|
| 529 |
+
return out
|
| 530 |
|
| 531 |
|
| 532 |
def to_client_rows(rows):
|
platform/core/connector_spec.py
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""connector_spec.py β ONE shared schema spec for every connector's tables, carrying the vendor's
|
| 2 |
+
own name for each of them.
|
| 3 |
+
|
| 4 |
+
ββ **THIS FILE IS THE SINGLE SOURCE OF TRUTH FOR A CONNECTOR'S TABLE SHAPE FROM NOW ON.** The
|
| 5 |
+
shape already existed twice before it had a name: `aios-web/api/amazon_relational.py` and
|
| 6 |
+
`aios-web/api/meta_relational.py` each carry loose `SOURCE` / `LABELS` / `PINNED` / `IDENTITY` /
|
| 7 |
+
`LINKS` module dicts saying the same six things about their tables in two different spellings, and
|
| 8 |
+
`odoo_relational.py` carries a third. Nobody called it a spec, so nothing could read it generically
|
| 9 |
+
and every new consumer (the relation engine, the connector detail door, the Databases tab) had to
|
| 10 |
+
learn each connector's dialect. This module generalises those dicts; it does not invent a new model.
|
| 11 |
+
|
| 12 |
+
ConnectorTableSpec one table: key, native, source, label, pinned, identity, links
|
| 13 |
+
register(connector, ...) add a connector's tables to the registry
|
| 14 |
+
tables_for(connector) that connector's specs, in declaration order
|
| 15 |
+
spec_for(ut_key) one table's spec, or None
|
| 16 |
+
connector_of(ut_key) which connector owns a ut_* key, or None
|
| 17 |
+
all_specs() flat ut_key -> spec across every connector
|
| 18 |
+
connectors() the connector keys, in registration order
|
| 19 |
+
edges_for(key) the relation edges of a CONNECTOR or of ONE table
|
| 20 |
+
|
| 21 |
+
β **THE FIFTH KEY IS `native`, AND IT IS THE ONE THAT DID NOT EXIST BEFORE.** `SOURCE` is the
|
| 22 |
+
MIRROR table β `amzn_orders`, `meta_campaigns` β which is OUR name for the copy we hold. The
|
| 23 |
+
vendor's own identifier for the same thing (`GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL`,
|
| 24 |
+
`AdAccount`, `res.partner`) lived only in comments, so no screen and no gate could reach it.
|
| 25 |
+
|
| 26 |
+
ββ **R9 β `label` IS A DEFAULT, `native` IS NEVER EDITABLE.** The `label` here is our SHIPPED
|
| 27 |
+
default display name and nothing else. A tenant's override lives in `core/schema_overlay.py`, is
|
| 28 |
+
stored per tenant, and **a connector re-sync must never write it**; renaming a database from the
|
| 29 |
+
navigation rail is that same override. Nothing in this module is a place to store an override, and
|
| 30 |
+
a caller that writes one here has put tenant state in shipped code.
|
| 31 |
+
|
| 32 |
+
β **LAYERING: `core` NEVER IMPORTS UP.** `aios-web/api/*` is above `platform/core/`, so this file
|
| 33 |
+
must not import `amazon_relational`, `meta_relational` or `odoo_relational`, and it does not import
|
| 34 |
+
`harness` either. Every value below is therefore written out LITERALLY, transcribed from the modules
|
| 35 |
+
named beside it.
|
| 36 |
+
|
| 37 |
+
β **THE TRANSCRIPTION IS HELD TRUE BY A GATE, NOT BY A COMMENT.** `aios-web/api/verify_meta.py`
|
| 38 |
+
(`section_connector_spec`) asserts, per table, that this module's `source` / `label` / `pinned` /
|
| 39 |
+
`identity` / `links` agree exactly with the live `amazon_relational` and `meta_relational` dicts.
|
| 40 |
+
That gate lives up in the api layer precisely because it may import both sides. It is the codebase's
|
| 41 |
+
own idiom: `core/relations.py::compiled_pin_matches_filter_sql` proves two SQL spellings agree by a
|
| 42 |
+
gate rather than by a comment, and `relations.OPEN_INVOICE_WHERE` is held against
|
| 43 |
+
`odoo_relational._AR_WHERE` the same way.
|
| 44 |
+
β **WHEN W53-T37 LANDS** and `amazon_relational` / `meta_relational` READ their dicts from here,
|
| 45 |
+
those parity checks become tautological and this note should be revisited β but **the checks stay**,
|
| 46 |
+
because a tautology is exactly what catches a hand edit that re-introduces a literal on either side.
|
| 47 |
+
|
| 48 |
+
β **ODOO IS NOT HERE YET.** W53-T38 writes Odoo's spec in its own file fence. There is deliberately
|
| 49 |
+
no `"odoo"` entry and no stub for one; `edges_for("odoo")` returns `[]` until that ticket lands.
|
| 50 |
+
"""
|
| 51 |
+
from dataclasses import dataclass, replace
|
| 52 |
+
import hashlib
|
| 53 |
+
import re
|
| 54 |
+
|
| 55 |
+
#: β THE CARDINALITY VOCABULARY, AS A CONSTANT SO EVERY CONSUMER CHECKS MEMBERSHIP RATHER THAN
|
| 56 |
+
#: SPELLING. The relation wire (W53-T17) and the parity gate both read this set; a fifth value
|
| 57 |
+
#: added by hand in a link tuple goes red instead of travelling silently to a screen.
|
| 58 |
+
CARDINALITIES = ("many_to_one", "one_to_many", "one_to_one", "many_to_many")
|
| 59 |
+
|
| 60 |
+
# The discoverable Odoo catalogue is runtime data, because two API accounts can read different
|
| 61 |
+
# installed modules. Only its stable naming rule belongs in this tenant-independent spec layer.
|
| 62 |
+
# A digest closes the otherwise-real collision between model names such as ``x.a_b`` and
|
| 63 |
+
# ``x_a.b`` after punctuation is normalised for a ``ut_*`` key.
|
| 64 |
+
ODOO_MODEL_TABLE_PREFIX = "ut_odoo_model_"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def odoo_model_table_key(model):
|
| 68 |
+
"""Stable ``ut_*`` key for one vendor model name; blank/invalid input is refused."""
|
| 69 |
+
native = str(model or "").strip().lower()
|
| 70 |
+
if not native or not re.fullmatch(r"[a-z0-9_.]+", native):
|
| 71 |
+
raise ValueError("an Odoo model name may contain only lowercase letters, digits, dots and underscores")
|
| 72 |
+
# 18 + the 14-character prefix + separator + digest leaves room for the connector's
|
| 73 |
+
# ``__k_<12 hex>`` namespace under user_tables' 60-character key ceiling.
|
| 74 |
+
stem = re.sub(r"[^a-z0-9]+", "_", native).strip("_")[:18] or "model"
|
| 75 |
+
digest = hashlib.sha256(native.encode("utf-8")).hexdigest()[:10]
|
| 76 |
+
return f"{ODOO_MODEL_TABLE_PREFIX}{stem}_{digest}"
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def odoo_model_label(model, vendor_label=""):
|
| 80 |
+
"""Our default database label, while preserving Odoo's own label when it supplied one."""
|
| 81 |
+
label = " ".join(str(vendor_label or "").split())
|
| 82 |
+
if label:
|
| 83 |
+
return label[:80]
|
| 84 |
+
return str(model or "").replace(".", " ").replace("_", " ").strip().capitalize()[:80]
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
@dataclass(frozen=True)
|
| 88 |
+
class ConnectorTableSpec:
|
| 89 |
+
"""One connector table. β EXACTLY SEVEN FIELDS, IN THIS ORDER β contract C2 says "exactly",
|
| 90 |
+
and `verify_meta.section_connector_spec` reads `dataclasses.fields` to make that word
|
| 91 |
+
mechanical. Grouping by connector is done by the two-level `SPECS` registry below, NOT by an
|
| 92 |
+
eighth field.
|
| 93 |
+
"""
|
| 94 |
+
key: str #: our `ut_*` key β the grid a person opens
|
| 95 |
+
native: str #: the vendor's OWN identifier. β NEVER editable (R9)
|
| 96 |
+
source: str #: the mirror table `harness/*_store.py` writes
|
| 97 |
+
label: str #: our shipped DEFAULT display name. A DEFAULT, never an override (R9)
|
| 98 |
+
pinned: str = None #: the title column, or None when the table has no natural one
|
| 99 |
+
identity: tuple = () #: the id columns, in FALLING preference
|
| 100 |
+
#: β A TUPLE OF 4-TUPLES `((from_col, target_key, to_col, cardinality), ...)`, AND THE TUPLE IS
|
| 101 |
+
#: DELIBERATE: a `list` default on a frozen dataclass is a mutable shared default and defeats
|
| 102 |
+
#: the freeze. C2's word "LIST" is about `links` being a SEQUENCE per key β several links from
|
| 103 |
+
#: one table, which is Amazon's shape β rather than Meta's single scalar 3-tuple; a tuple
|
| 104 |
+
#: satisfies that. **Do not "fix" this back to a list.** `register()` normalises a caller's
|
| 105 |
+
#: list into a tuple, so passing either is safe and only tuples are ever stored.
|
| 106 |
+
links: tuple = ()
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
#: β THE REGISTRY IS TWO-LEVEL: `{connector_key: {ut_key: ConnectorTableSpec}}`. That is what makes
|
| 110 |
+
#: a connector groupable without an eighth dataclass field. Insertion order is declaration order, so
|
| 111 |
+
#: `tables_for` and `edges_for` return a connector's tables in the order its store syncs them.
|
| 112 |
+
SPECS = {}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _normalise_links(links):
|
| 116 |
+
"""A caller's list-of-lists becomes a tuple-of-4-tuples. Explicit, so `links` is ONE type
|
| 117 |
+
everywhere downstream and the gate can assert 4-tuple-ness without a shape apology."""
|
| 118 |
+
return tuple(tuple(entry) for entry in (links or ()))
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def register(connector, specs):
|
| 122 |
+
"""Add one connector's tables. `specs` may be a mapping of `ut_key -> spec` or any iterable of
|
| 123 |
+
specs; `links` is normalised to tuples on the way in. Re-registering a connector REPLACES it,
|
| 124 |
+
which is what lets a gate's negative control swap the registry and restore it in a `finally`.
|
| 125 |
+
"""
|
| 126 |
+
rows = list(specs.values()) if isinstance(specs, dict) else list(specs)
|
| 127 |
+
SPECS[connector] = {s.key: replace(s, links=_normalise_links(s.links)) for s in rows}
|
| 128 |
+
return SPECS[connector]
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def connectors():
|
| 132 |
+
"""The registered connector keys, in registration order."""
|
| 133 |
+
return tuple(SPECS.keys())
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def tables_for(connector):
|
| 137 |
+
"""One connector's specs, in declaration order. Unknown connector -> `()`."""
|
| 138 |
+
return tuple(SPECS.get(connector, {}).values())
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def all_specs():
|
| 142 |
+
"""Flat `ut_key -> spec` across every connector."""
|
| 143 |
+
out = {}
|
| 144 |
+
for tables in SPECS.values():
|
| 145 |
+
out.update(tables)
|
| 146 |
+
return out
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def spec_for(ut_key):
|
| 150 |
+
"""One table's spec, or None. Never raises."""
|
| 151 |
+
for tables in SPECS.values():
|
| 152 |
+
if ut_key in tables:
|
| 153 |
+
return tables[ut_key]
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def connector_of(ut_key):
|
| 158 |
+
"""Which connector owns a `ut_*` key, or None."""
|
| 159 |
+
for connector, tables in SPECS.items():
|
| 160 |
+
if ut_key in tables:
|
| 161 |
+
return connector
|
| 162 |
+
return None
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _label(col):
|
| 166 |
+
"""`amazon_order_id` -> `Amazon order id`. β THE SAME RULE BOTH CONNECTORS ALREADY USE
|
| 167 |
+
(`amazon_relational._label`, `meta_relational._label`), copied because `core` cannot import
|
| 168 |
+
them. The parity gate holds the two spellings together."""
|
| 169 |
+
return col.replace("_", " ").strip().capitalize()
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def edges_for(key):
|
| 173 |
+
"""The relation edges of a CONNECTOR key (`"amazon"`) or of ONE `ut_*` table key.
|
| 174 |
+
|
| 175 |
+
β **ONE FUNCTION, TWO NAMESPACES, AND THEY CANNOT COLLIDE:** every table key is `ut_`-prefixed
|
| 176 |
+
and no connector key is, so `edges_for("amazon")` (every edge of the connector, in table
|
| 177 |
+
declaration order) and `edges_for("ut_amzn_order_items")` (just that table's) are never
|
| 178 |
+
ambiguous. W53-T38's `edges_for("odoo")` is the connector form.
|
| 179 |
+
|
| 180 |
+
Each edge is `{"key", "label", "source", "target", "from_col", "to_col", "cardinality"}` where
|
| 181 |
+
`source` is the `ut_*` table the edge starts FROM β needed because the connector form mixes
|
| 182 |
+
tables β and:
|
| 183 |
+
Β· `key` is `f"{target}_link__{from_col}"`, Amazon's shape. β THE SOURCE COLUMN IS IN THE
|
| 184 |
+
KEY ON PURPOSE: two links from one table to the same target (orders by id and by sku) would
|
| 185 |
+
otherwise collide on one key and the second would silently overwrite the first. Meta's older
|
| 186 |
+
`f"{target}_link"` cannot express that, which is why Amazon's convention is the shared one.
|
| 187 |
+
Β· `label` is `f"{target_label} by {_label(from_col)}"`.
|
| 188 |
+
|
| 189 |
+
β **NEVER RAISES.** An unknown key returns `[]`, and a link whose target is not registered still
|
| 190 |
+
emits its edge, labelled off the raw target key. A lenient reader is what keeps a partially
|
| 191 |
+
ported connector set from blowing up in a caller's hands; the GATE is what asserts that every
|
| 192 |
+
target resolves, and that is the right place for that judgement.
|
| 193 |
+
"""
|
| 194 |
+
specs = all_specs()
|
| 195 |
+
if key in SPECS:
|
| 196 |
+
rows = list(SPECS[key].values())
|
| 197 |
+
elif key in specs:
|
| 198 |
+
rows = [specs[key]]
|
| 199 |
+
else:
|
| 200 |
+
return []
|
| 201 |
+
out = []
|
| 202 |
+
for spec in rows:
|
| 203 |
+
for from_col, target, to_col, cardinality in spec.links:
|
| 204 |
+
target_spec = specs.get(target)
|
| 205 |
+
target_label = target_spec.label if target_spec else target
|
| 206 |
+
out.append({
|
| 207 |
+
"key": f"{target}_link__{from_col}",
|
| 208 |
+
"label": f"{target_label} by {_label(from_col)}",
|
| 209 |
+
"source": spec.key,
|
| 210 |
+
"target": target,
|
| 211 |
+
"from_col": from_col,
|
| 212 |
+
"to_col": to_col,
|
| 213 |
+
"cardinality": cardinality,
|
| 214 |
+
})
|
| 215 |
+
return out
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 219 |
+
# AMAZON β transcribed from `aios-web/api/amazon_relational.py` + `harness/amazon_client.py`
|
| 220 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 221 |
+
#: β `native` IS THE SP-API REPORT TYPE, from `harness/amazon_client.py::REPORT_TYPES[t]["report"]`
|
| 222 |
+
#: β the identifier Amazon itself names the data by. β `orders` and `order_items` share one report
|
| 223 |
+
#: on purpose: the flat-file order report is already one row per order ITEM, so orders is the
|
| 224 |
+
#: deduplicated head and order_items the lines (see `amazon_store.TABLES`). Two of our tables, one
|
| 225 |
+
#: vendor object, which is exactly why `native` and `source` had to be separate keys.
|
| 226 |
+
#: β `identity` is `amazon_relational.IDENTITY` verbatim: a flat-file report has no `id` column the
|
| 227 |
+
#: way Graph does, so each table names the columns that identify one of its rows, in falling
|
| 228 |
+
#: preference.
|
| 229 |
+
#: β R12 β AMAZON JOINS ONLY TO AMAZON. Nothing here targets `ut_odoo_*`, `product_data` or the
|
| 230 |
+
#: hand-maintained ASIN mastersheet.
|
| 231 |
+
#: β EVERY LINK BELOW IS `many_to_one`, and the reason is uniform: each one is a CHILD row naming
|
| 232 |
+
#: its PARENT (a line names its order, a pricing row names its listing). Amazon's `LINKS` carries
|
| 233 |
+
#: 3-tuples today with the cardinality left implicit; this is where it becomes explicit.
|
| 234 |
+
_AMZN = "ut_amzn_"
|
| 235 |
+
|
| 236 |
+
AMAZON_TABLES = (
|
| 237 |
+
ConnectorTableSpec(
|
| 238 |
+
key=_AMZN + "orders",
|
| 239 |
+
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 240 |
+
source="amzn_orders",
|
| 241 |
+
label="Amazon orders",
|
| 242 |
+
pinned="amazon_order_id",
|
| 243 |
+
identity=("amazon_order_id",),
|
| 244 |
+
links=(
|
| 245 |
+
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 246 |
+
),
|
| 247 |
+
),
|
| 248 |
+
ConnectorTableSpec(
|
| 249 |
+
key=_AMZN + "order_items",
|
| 250 |
+
native="GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
|
| 251 |
+
source="amzn_order_items",
|
| 252 |
+
label="Amazon order items",
|
| 253 |
+
pinned="amazon_order_id",
|
| 254 |
+
identity=("order_item_id", "amazon_order_id"),
|
| 255 |
+
links=(
|
| 256 |
+
("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 257 |
+
# β the order report carries `sku`; listings key on `seller_sku`. Same value, two
|
| 258 |
+
# spellings β the reason a link names BOTH columns rather than one shared name.
|
| 259 |
+
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 260 |
+
),
|
| 261 |
+
),
|
| 262 |
+
ConnectorTableSpec(
|
| 263 |
+
key=_AMZN + "listings",
|
| 264 |
+
native="GET_MERCHANT_LISTINGS_ALL_DATA",
|
| 265 |
+
source="amzn_listings",
|
| 266 |
+
label="Amazon listings",
|
| 267 |
+
pinned="seller_sku",
|
| 268 |
+
identity=("seller_sku", "asin1", "asin"),
|
| 269 |
+
links=(), # the product spine's HEAD: everything points at it, it points nowhere
|
| 270 |
+
),
|
| 271 |
+
ConnectorTableSpec(
|
| 272 |
+
key=_AMZN + "settlements",
|
| 273 |
+
native="GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
|
| 274 |
+
source="amzn_settlements",
|
| 275 |
+
label="Amazon settlements",
|
| 276 |
+
pinned="settlement_id",
|
| 277 |
+
identity=("settlement_id",),
|
| 278 |
+
# β THE MONEY-TO-ORDER BRIDGE β what turns "an order happened" into "this is what we were
|
| 279 |
+
# actually paid for it". The V2 settlement report carries the order id per line.
|
| 280 |
+
links=(
|
| 281 |
+
("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
|
| 282 |
+
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 283 |
+
),
|
| 284 |
+
),
|
| 285 |
+
ConnectorTableSpec(
|
| 286 |
+
key=_AMZN + "inventory",
|
| 287 |
+
native="GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 288 |
+
source="amzn_inventory",
|
| 289 |
+
label="Amazon inventory",
|
| 290 |
+
pinned="asin",
|
| 291 |
+
identity=("asin", "fnsku", "msku"),
|
| 292 |
+
# β THE LEDGER KEYS ON `msku`, NOT `sku` β FBA calls the seller's own SKU the MSKU. Same
|
| 293 |
+
# value, a third spelling.
|
| 294 |
+
links=(
|
| 295 |
+
("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 296 |
+
("asin", _AMZN + "listings", "asin1", "many_to_one"),
|
| 297 |
+
),
|
| 298 |
+
),
|
| 299 |
+
ConnectorTableSpec(
|
| 300 |
+
key=_AMZN + "pricing",
|
| 301 |
+
native="GET_SALES_AND_TRAFFIC_REPORT",
|
| 302 |
+
source="amzn_pricing",
|
| 303 |
+
label="Amazon pricing and Buy Box",
|
| 304 |
+
pinned="parent_asin",
|
| 305 |
+
identity=("child_asin", "parent_asin"),
|
| 306 |
+
links=(
|
| 307 |
+
("child_asin", _AMZN + "listings", "asin1", "many_to_one"),
|
| 308 |
+
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
|
| 309 |
+
),
|
| 310 |
+
),
|
| 311 |
+
ConnectorTableSpec(
|
| 312 |
+
key=_AMZN + "brand_analytics",
|
| 313 |
+
native="GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
|
| 314 |
+
source="amzn_brand_analytics",
|
| 315 |
+
label="Amazon brand analytics",
|
| 316 |
+
pinned="search_term",
|
| 317 |
+
identity=("search_term",),
|
| 318 |
+
# β keyed by SEARCH TERM, not by product, so it joins to nothing by default. Its ASIN
|
| 319 |
+
# columns are the top-clicked ASINs for a term, a different relationship from "this row is
|
| 320 |
+
# about this product", and a link would be misleading.
|
| 321 |
+
links=(),
|
| 322 |
+
),
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 327 |
+
# META ADS β transcribed from `aios-web/api/meta_relational.py`
|
| 328 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 329 |
+
#: β `native` IS THE GRAPH API NODE NAME β `AdAccount`, `Campaign`, `AdSet`, `Ad`, `AdCreative`,
|
| 330 |
+
#: `AdsInsights`. Meta's own vocabulary for the objects `meta_store` mirrors.
|
| 331 |
+
#: ββ **`identity` WAS DECLARED HERE FOR THE FIRST TIME, NOT TRANSCRIBED.** It is honest rather
|
| 332 |
+
#: than invented: `meta_store._read` keys every row on the object's own Graph `id`, so `("id",)` is
|
| 333 |
+
#: what the five object tables are already identified by in practice. `ut_meta_insights` has no
|
| 334 |
+
#: single `id` β the daily insights grain is one row per ad per day β so it takes
|
| 335 |
+
#: `("ad_id", "date_start")`.
|
| 336 |
+
#: β W53-T37 AMENDMENT, 2026-09-05 β TWO SENTENCES HERE WERE TRUE FOR ONE TICKET AND ARE NOW
|
| 337 |
+
#: FALSE, so they are corrected rather than left to be falsified in one grep.
|
| 338 |
+
#: * *"`meta_relational` has NO `IDENTITY` dict at all"* β it has one now, DERIVED FROM THIS
|
| 339 |
+
#: DECLARATION, so the value is no longer unverifiable against the live module; what is true
|
| 340 |
+
#: instead is that nothing in `meta_relational` READS it yet (`_read` keys rows on the Graph
|
| 341 |
+
#: `id` inline), so it is reachable by name for the caller that will ask.
|
| 342 |
+
#: * *"Meta's `LINKS` is a SCALAR 3-tuple per key today and its generated edge key is
|
| 343 |
+
#: `f"{target}_link"`"* β W53-T37 made it a SEQUENCE keyed by `edges_for`, which is what
|
| 344 |
+
#: retired the collision described below. It is no longer latent, it is fixed.
|
| 345 |
+
#: The sequence-plus-source-column shape is still the reason this file holds links the way it
|
| 346 |
+
#: does: two edges to one target collide on both key and label the moment a scalar spelling
|
| 347 |
+
#: declares them, and Meta was one declaration away from that.
|
| 348 |
+
_META = "ut_meta_"
|
| 349 |
+
|
| 350 |
+
META_TABLES = (
|
| 351 |
+
ConnectorTableSpec(
|
| 352 |
+
key=_META + "ad_accounts",
|
| 353 |
+
native="AdAccount",
|
| 354 |
+
source="meta_ad_accounts",
|
| 355 |
+
label="Meta ad accounts",
|
| 356 |
+
pinned="name",
|
| 357 |
+
identity=("id",),
|
| 358 |
+
links=(), # the hierarchy's HEAD: account -> campaign -> ad set -> ad
|
| 359 |
+
),
|
| 360 |
+
ConnectorTableSpec(
|
| 361 |
+
key=_META + "campaigns",
|
| 362 |
+
native="Campaign",
|
| 363 |
+
source="meta_campaigns",
|
| 364 |
+
label="Meta campaigns",
|
| 365 |
+
pinned="name",
|
| 366 |
+
identity=("id",),
|
| 367 |
+
links=(
|
| 368 |
+
("account_id", _META + "ad_accounts", "account_id", "many_to_one"),
|
| 369 |
+
),
|
| 370 |
+
),
|
| 371 |
+
ConnectorTableSpec(
|
| 372 |
+
key=_META + "adsets",
|
| 373 |
+
native="AdSet",
|
| 374 |
+
source="meta_adsets",
|
| 375 |
+
label="Meta ad sets",
|
| 376 |
+
pinned="name",
|
| 377 |
+
identity=("id",),
|
| 378 |
+
links=(
|
| 379 |
+
("campaign_id", _META + "campaigns", "id", "many_to_one"),
|
| 380 |
+
),
|
| 381 |
+
),
|
| 382 |
+
ConnectorTableSpec(
|
| 383 |
+
key=_META + "ads",
|
| 384 |
+
native="Ad",
|
| 385 |
+
source="meta_ads",
|
| 386 |
+
label="Meta ads",
|
| 387 |
+
pinned="name",
|
| 388 |
+
identity=("id",),
|
| 389 |
+
links=(
|
| 390 |
+
("adset_id", _META + "adsets", "id", "many_to_one"),
|
| 391 |
+
),
|
| 392 |
+
),
|
| 393 |
+
ConnectorTableSpec(
|
| 394 |
+
key=_META + "creatives",
|
| 395 |
+
native="AdCreative",
|
| 396 |
+
source="meta_creatives",
|
| 397 |
+
label="Meta creatives",
|
| 398 |
+
pinned="name",
|
| 399 |
+
identity=("id",),
|
| 400 |
+
# a creative is referenced BY an ad rather than naming its parent, so it declares no link
|
| 401 |
+
# of its own β the same posture as `ut_amzn_listings`.
|
| 402 |
+
links=(),
|
| 403 |
+
),
|
| 404 |
+
ConnectorTableSpec(
|
| 405 |
+
key=_META + "insights",
|
| 406 |
+
native="AdsInsights",
|
| 407 |
+
source="meta_insights_daily",
|
| 408 |
+
label="Meta insights (daily)",
|
| 409 |
+
pinned="date_start",
|
| 410 |
+
identity=("ad_id", "date_start"),
|
| 411 |
+
links=(
|
| 412 |
+
("ad_id", _META + "ads", "id", "many_to_one"),
|
| 413 |
+
),
|
| 414 |
+
),
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
register("amazon", AMAZON_TABLES)
|
| 419 |
+
register("meta", META_TABLES)
|
| 420 |
+
|
| 421 |
+
__all__ = ["CARDINALITIES", "ConnectorTableSpec", "SPECS", "AMAZON_TABLES", "META_TABLES",
|
| 422 |
+
"ODOO_MODEL_TABLE_PREFIX", "odoo_model_table_key", "odoo_model_label",
|
| 423 |
+
"register", "connectors", "tables_for", "all_specs", "spec_for", "connector_of",
|
| 424 |
+
"edges_for"]
|