diff --git a/Dockerfile b/Dockerfile
index 56fc23ab2036fd1bd3331714aba19be683f2764b..1e5fd4fc3ca95337034cfe37d02474c92f82994f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -98,6 +98,18 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
# '*' assumes the container is unreachable except through that trusted proxy, which must replace
# client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual
# proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener.
-# Suppress the server banner and release idle keep-alive sockets after five seconds. W59-T36
-# owns the API posture read-back and application host/header policy; these flags are its launcher seam.
-CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 5"]
+# Suppress the server banner. W59-T36 owns the API posture read-back and application host/header
+# policy; these flags are its launcher seam.
+#
+# SDLC 67 close (2026-09-29): `--timeout-keep-alive 5` was the suspected cause of a measured
+# 4-12% 502 rate at the HF proxy -- uvicorn closes an idle keep-alive socket after 5s, and if the
+# proxy in front of it still believes that connection is reusable and forwards a request onto it,
+# the client sees a 502 rather than a fresh connection (the classic reverse-proxy keep-alive race:
+# https://adamcrowder.net/posts/node-express-api-and-aws-alb-502/ describes the same shape against
+# an ALB). The fix is always the same direction -- make the BACKEND's idle timeout longer than
+# whatever the front proxy holds a connection for, never shorter -- because only the side with the
+# longer timeout can be the one to close first. HF's Spaces proxy timeout is not published; 75s is
+# nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
+# proxies in this class use, so it is the safest available floor without over-fitting to a number
+# we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
+CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
diff --git a/RELEASES.json b/RELEASES.json
index 9f3d2c794c78edefa7f74e4141cfdea1d5206a2c..ae813781c71ccd6ebd20b84ab905848eb3167bb8 100644
--- a/RELEASES.json
+++ b/RELEASES.json
@@ -1,4 +1,4 @@
{
- "current": "37854d8928902b38cc4f5ddab2e2ddb0c9605671 (37854d8)",
+ "current": "be8fb2f (be8fb2f)",
"releases": []
}
\ No newline at end of file
diff --git a/VERSION b/VERSION
index b6b032fa8cbcfb4f6e0c1121e3665e38338a3a30..52ea473424f2fa1565fa0621bd6004c1eb154ddf 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-37854d8928902b38cc4f5ddab2e2ddb0c9605671 (37854d8)
+be8fb2f (be8fb2f)
diff --git a/api/ai_enrich.py b/api/ai_enrich.py
index dbeb6153ad53eaab15f1ff8a38c460c0aa2164e6..e1ac9a95a74857f14dd1b0df3ff4484a94d1f266 100644
--- a/api/ai_enrich.py
+++ b/api/ai_enrich.py
@@ -269,12 +269,15 @@ _SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: n
'exactly: UNKNOWN')
-def _ask(provider, model, prompt, max_tokens, timeout):
+def _ask(provider, model, prompt, max_tokens, timeout, system=None):
"""One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
ceiling instead of a hardcoded 300, and they read `usage` back.
+ S67-T41: `system` replaces the one-cell instruction for a caller that is not filling a cell
+ (Content Studio's drafts); every existing caller passes nothing and keeps `_SYSTEM`.
"""
+ system = system or _SYSTEM
key = (os.environ.get(provider['env']) or '').strip()
if not key:
return '', None, f"{provider['name']} has no key on this deployment"
@@ -284,7 +287,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
provider['url'], timeout=timeout,
headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
'content-type': 'application/json'},
- json={'model': model, 'max_tokens': max_tokens, 'system': _SYSTEM,
+ json={'model': model, 'max_tokens': max_tokens, 'system': system,
'messages': [{'role': 'user', 'content': prompt}]})
if r.status_code >= 400:
return '', None, f"{provider['name']} answered {r.status_code}"
@@ -300,7 +303,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
provider['url'], timeout=timeout,
headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
- 'messages': [{'role': 'system', 'content': _SYSTEM},
+ 'messages': [{'role': 'system', 'content': system},
{'role': 'user', 'content': prompt}]})
if r.status_code >= 400:
return '', None, f"{provider['name']} answered {r.status_code}"
diff --git a/api/amazon_relational.py b/api/amazon_relational.py
index fd47fcd38d081ead163082978398eb80a37da345..141997c6c039779a438d490417a84eabfa3f6ae4 100644
--- a/api/amazon_relational.py
+++ b/api/amazon_relational.py
@@ -28,10 +28,8 @@ AMAZON TABLES: order lines to their order, pricing to its listing. **Nothing her
removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
converted to an Amazon unit, because the two never meet.
-⚠ MATERIALISED, WITH THE CEILING REPORTED RATHER THAN ENFORCED. Same posture as Meta's: a table
-over `MAX_ROWS` is REFUSED with its cause and its fix named, never truncated. For Amazon that is
-standing rule 1's second half, and it is the likeliest table to hit it — an order-item history is
-one row per line, not per order.
+History stays in the mirror. Small snapshots may be materialised; a row or byte budget switches
+them to read-through too. The shared document holds their schemas, never a truncated population.
"""
import hashlib
import json
@@ -45,7 +43,7 @@ if str(_API.parent.parent / "platform") not in sys.path:
import odoo_relational as _odoo # noqa: E402
from core import connector_spec # noqa: E402
-from harness import amazon_schema, amazon_store, datastore # noqa: E402
+from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
Refused = _odoo.Refused
@@ -119,7 +117,7 @@ def source_attribution(key):
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
- "report_date", "start_date", "end_date", "_time")
+ "report_date", "start_date", "end_date", "_snapshot_date", "month", "_time")
#: ⛔ MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
#: and an int column would round every row of a settlement. The grid shows what the report said and
@@ -130,6 +128,10 @@ _MONEY = {"item_price", "item_tax", "shipping_price", "shipping_tax", "gift_wrap
"your_price", "landed_price", "listing_price", "open_date_price",
"sales_by_asin_ordered_product_sales_amount",
"sales_by_asin_ordered_product_sales_b2_b_amount"}
+_MONEY.update({"estimated_fee_total", "estimated_referral_fee_per_unit",
+ "estimated_variable_closing_fee", "estimated_order_handling_fee_per_order",
+ "estimated_pick_pack_fee_per_unit", "estimated_weight_handling_fee_per_unit",
+ "expected_fulfillment_fee_per_unit", "estimated_monthly_storage_fee"})
#: Percentages and rates are fractional too, but they are not money. `currency` is the only
#: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
@@ -150,7 +152,7 @@ def _ftype(col, duck_type):
"""
if any(col == s or col.endswith(s) for s in _DATEISH):
return "date"
- if col in _MONEY or any(col.endswith(s) for s in _RATE_SUFFIX):
+ if col in _MONEY or col.endswith(("_amount", "_fee")) or any(col.endswith(s) for s in _RATE_SUFFIX):
return "currency"
base = str(duck_type or "").upper().split("(")[0]
if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
@@ -168,6 +170,116 @@ def mirror_columns(cur, table):
return []
+#: ⭐⭐ S67-T30 (instr 13, R29 / O23) — THE PRODUCT LOOKUPS. Every Amazon database with an ASIN
+#: shows its "Parent ASIN" and "SKU" beside it, and every one with a SKU its "ASIN" and "Parent
+#: ASIN", with no configuration. They are PLAIN TEXT fields whose value the SERVER derives, not
+#: link fields, on purpose: a link renders as a count bubble, is not groupable
+#: (`types.ts:isGroupableField`) and has no SQL binding for a filter (`routes_odoo_tables.
+#: _filter_columns` and `_group_pin` read `spec["cols"]` only). A text field with a `cols`
+#: expression groups, filters and sorts through the machinery every other column already uses.
+#: ⚠ The ASIN / SKU column of a table is the `from_col` of its declared listings link (on `asin1`
+#: / `seller_sku`), so the spec's links are the one statement of which column is which. A table
+#: that carries its own `parent_asin` (pricing, economics) gains no derived Parent ASIN; one that
+#: carries its own SKU or ASIN gains no duplicate of it; listings gains Parent ASIN only.
+#: ⛔ Parent comes from `amazon_store.asin_parents_sql` ALONE (latest catalog, else economics, else
+#: the ASIN itself), never from `amzn_catalog_items` directly: that table keeps older snapshot days
+#: (D-3031) and five own ASINs have no catalog row. SKU and ASIN come from the LATEST listings day
+#: only, because listings keeps one copy per snapshot date.
+LISTINGS = PREFIX + "listings"
+LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN = "lookup_parent_asin", "lookup_sku", "lookup_asin"
+LOOKUP_KEYS = frozenset((LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN))
+_LOOKUP_LABELS = {LOOKUP_PARENT: "Parent ASIN", LOOKUP_SKU: "SKU", LOOKUP_ASIN: "ASIN"}
+_LOOKUP_DESCRIPTIONS = {
+ LOOKUP_PARENT: ("The parent ASIN of this row's product, from Amazon's catalog, else the "
+ "economics report, else the ASIN itself. Click it to open the listing."),
+ LOOKUP_SKU: ("Your seller SKUs for this row's ASIN in the latest listings report. "
+ "Click it to open the listing."),
+ LOOKUP_ASIN: ("The ASIN of this row's SKU in the latest listings report. "
+ "Click it to open the listing."),
+}
+
+
+def _product_columns(key):
+ """`(asin_col, sku_col, asin_link_key, sku_link_key)` as the spec declares them for `key`."""
+ if key == LISTINGS:
+ return "asin1", "seller_sku", None, None # the row IS the listing: Parent ASIN only
+ asin = sku = asin_via = sku_via = None
+ for edge in connector_spec.edges_for(key):
+ if edge["target"] != LISTINGS:
+ continue
+ if edge["to_col"] == "asin1" and asin is None:
+ asin, asin_via = edge["from_col"], edge["key"]
+ elif edge["to_col"] == "seller_sku" and sku is None:
+ sku, sku_via = edge["from_col"], edge["key"]
+ return asin, sku, asin_via, sku_via
+
+
+def _latest_listings_sql(listing_names):
+ """`SELECT sku, asin` over the LATEST listings snapshot day, or None without both columns."""
+ if not {"seller_sku", "asin1"} <= set(listing_names):
+ return None
+ table = SOURCE[LISTINGS]
+ latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
+ if "_snapshot_date" in listing_names else "")
+ return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
+ f"TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM {table} "
+ f"WHERE seller_sku IS NOT NULL AND asin1 IS NOT NULL{latest}) "
+ "WHERE sku <> '' AND asin <> ''")
+
+
+def lookups(cur, key, names=None):
+ """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
+
+ `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
+ (every reader selects `FROM
` unaliased), so it binds to the outer row and never to the
+ resolver's own `asin`. DuckDB decorrelates it into a single hash join per query, which is what
+ keeps a grouped page of a read-through grid set-based rather than per-row. No match is NULL,
+ which every reader serialises as a blank cell. Nothing is capped (standing rule 1).
+ """
+ if names is None:
+ names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
+ asin, sku, asin_via, sku_via = _product_columns(key)
+ asin = asin if asin in names else None
+ sku = sku if sku in names else None
+ if not asin and not sku:
+ return []
+ listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
+ parents = None if "parent_asin" in names else amazon_store.asin_parents_sql(cur)
+ latest = _latest_listings_sql(listing_names)
+ table = SOURCE[key]
+ out = []
+
+ def add(lookup_key, after, expr, via):
+ if lookup_key in names:
+ return
+ out.append({"key": lookup_key, "label": _LOOKUP_LABELS[lookup_key], "after": after,
+ "expr": expr, "opens": {"via": via} if via else None,
+ "description": _LOOKUP_DESCRIPTIONS[lookup_key]})
+
+ if asin:
+ ref = f"TRIM(CAST({table}.{_quoted(asin)} AS VARCHAR))"
+ via = asin_via if "asin1" in listing_names else None
+ if parents:
+ add(LOOKUP_PARENT, asin,
+ f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via)
+ if latest and not sku:
+ add(LOOKUP_SKU, asin,
+ f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
+ f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via)
+ elif sku and latest:
+ ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
+ via = sku_via if "seller_sku" in listing_names else None
+ add(LOOKUP_ASIN, sku,
+ f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
+ f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via)
+ if parents:
+ add(LOOKUP_PARENT, sku,
+ f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
+ f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
+ f"WHERE _l.sku = {ref})", via)
+ return out
+
+
def _fields_for(cur, key):
"""One preset field per mirror column — the whole schema, derived. See the header."""
cols = mirror_columns(cur, SOURCE[key])
@@ -175,14 +287,31 @@ def _fields_for(cur, key):
return []
names = {c[0] for c in cols}
pinned = PINNED.get(key) if PINNED.get(key) in names else None
+ # S67-T30: each lookup lands IMMEDIATELY after the column it derives from, shown by default,
+ # and the ASIN / SKU column it sits beside is shown by default too (inventory's `msku` was not).
+ derived = {}
+ for lookup in lookups(cur, key, names):
+ derived.setdefault(lookup["after"], []).append(lookup)
+ product = {c for c in _product_columns(key)[:2] if c in names}
out = []
- for name, dtype in cols:
+ # The identity column goes FIRST, the rest keep the report's own order. A record reached
+ # through a link is titled by its first field (the record drawer's frame label), and a
+ # report's first column is whatever Amazon put there (`item_name`, `_data_origin`), so a
+ # listing opened from an order must lead with its seller SKU, as Odoo's pinned fields do.
+ for name, dtype in sorted(cols, key=lambda col: col[0] != pinned):
out.append(_odoo._preset({
"key": name, "label": _label(name), "type": _ftype(name, dtype),
"source": "overlay",
- "default": name == pinned or name in _DEFAULT_COLUMNS,
+ "default": name == pinned or name in _DEFAULT_COLUMNS or name in product,
"pinned": name == pinned,
}, flow="amazon_relational"))
+ for lookup in derived.get(name, ()):
+ field = {"key": lookup["key"], "label": lookup["label"], "type": "text",
+ "source": "overlay", "default": True,
+ "description": lookup["description"]}
+ if lookup["opens"]:
+ field["opens"] = dict(lookup["opens"])
+ out.append(_odoo._preset(field, flow="amazon_relational"))
# ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
# known until it syncs, so a link to a table that has never run would name a column that does
# not exist. **Absent is a normal state here, not a failure** — which is exactly what lets the
@@ -205,7 +334,8 @@ def _fields_for(cur, key):
out.append(_odoo._preset({
"key": edge["key"], "label": edge["label"],
"type": "link", "source": "overlay", "default": True,
- "link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"]},
+ "link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"],
+ "single": edge["cardinality"] == "many_to_one"},
}, flow="amazon_relational"))
out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
return out
@@ -222,8 +352,128 @@ _DEFAULT_COLUMNS = {
}
+#: The original seven used first-field identities. Retained for legacy-id migration
+#: and for distinguishing history from the original small snapshots.
+_LEGACY_IDENTITY_TABLES = frozenset(("orders", "order_items", "listings", "settlements",
+ "inventory", "pricing", "brand_analytics"))
+
+# The original seven do not declare snapshot/window metadata. Keep their four small
+# tables materialised; every newer non-snapshot report/API is history, including monthly ones.
+READ_THROUGH = frozenset(
+ PREFIX + table for table, spec in amazon_client.REPORT_TYPES.items()
+ if table in {"settlements", "orders", "order_items", "transactions", "economics", "storage_fees"}
+ or (table not in _LEGACY_IDENTITY_TABLES and not spec.get("snapshot")
+ and (spec.get("report") or spec.get("api") or spec.get("monthly"))))
+
+
+def _quoted(name):
+ return '"' + name.replace('"', '""') + '"'
+
+
+#: ⛔⛔ HOTFIX 2026-09-27 (D-3284's cost): an Amazon snapshot table larger than this is served
+#: READ-THROUGH, never stored in the tenant document. The per-table `MAX_TABLE_BYTES` (48 MB) let
+#: catalog items (32.9 MB for 2,317 rows), pricing (11.7 MB) and FBA inventory (7.5 MB) in once
+#: connector databases stopped counting toward MAX_TABLES, and `user_tables` doubled to 118 MB.
+#: Every whole-document read and write doubled with it: the 2 GB DigitalOcean staging node ran
+#: out of memory and PostgreSQL restarted (2026-09-27 16:36Z), and on Neon each read is billed at
+#: the document's text size. Small tables stay stored; nothing is capped (standing rule 1).
+AMAZON_STORED_BYTES = 4 * 1024 * 1024
+
+
+def _read_through(cur, key, count):
+ """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
+ ut = _odoo._ut()
+ if key in READ_THROUGH or count > ut.MAX_ROWS:
+ return True
+ if not count:
+ return False
+ names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])]
+ sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
+ # Include the stored id, refreshed stamp and row-object punctuation, with the same
+ # ASCII escaping as the document writer. The largest sampled row is conservative.
+ size = max((len(json.dumps(dict(zip(names, row)), ensure_ascii=True,
+ default=str).encode("utf-8")) + 96 for row in sample), default=0)
+ return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
+
+
+def missing_source(key):
+ return {"missing": (
+ f"The Amazon mirror in this container has not loaded Amazon "
+ f"{key.removeprefix(PREFIX).replace('_', ' ')} yet. "
+ "The next Amazon sync loads it. Nothing is lost.")}
+
+
+def _text(value):
+ return "" if value is None else str(value)
+
+
+def _number(value):
+ return None if value in (None, "") else float(value)
+
+
+def _integer(value):
+ return None if value in (None, "") else int(value)
+
+
+def grid_source(cur, key):
+ """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
+ columns = mirror_columns(cur, SOURCE[key])
+ if not columns:
+ return missing_source(key)
+ names = {name for name, _dtype in columns}
+ identity = [name for name in IDENTITY[key] if name in names]
+ coordinates = ["_report_document_id", "_report_row_number"]
+ if all(name in names for name in coordinates):
+ identity = list(dict.fromkeys(identity + coordinates))
+ if (not identity or identity == ["_report_row_number"]
+ or (key in {PREFIX + "settlements", PREFIX + "order_items"}
+ and not all(name in names for name in coordinates))):
+ identity = sorted(names)
+ # Report coordinates qualify line numbers by their document. If the declared identity
+ # is incomplete/null, hash the whole row instead of collapsing several unknown lines.
+ def digest(keys):
+ values = ", ".join(f"CAST({_quoted(name)} AS VARCHAR)" for name in keys)
+ return f"sha256(to_json(list_value({values})))"
+ complete = " AND ".join(f"NULLIF(CAST({_quoted(name)} AS VARCHAR), '') IS NOT NULL"
+ for name in identity)
+ digest_sql = f"CASE WHEN {complete} THEN {digest(identity)} ELSE {digest(sorted(names))} END"
+ # The shared grid, relation and selection contracts all use numeric pids. Thirteen
+ # hex digits retain a stable 52-bit identity that JavaScript represents exactly.
+ ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
+ cols = {}
+ for name, dtype in columns:
+ kind = _ftype(name, dtype)
+ expr = _quoted(name)
+ if kind in ("currency", "int"):
+ expr = f"TRY_CAST({expr} AS {'DOUBLE' if kind == 'currency' else 'BIGINT'})"
+ cols[name] = (expr, {"currency": _number, "int": _integer}.get(kind, _text))
+ # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
+ # group (count by parent) and a sort compile against them in SQL with no per-row Python.
+ for lookup in lookups(cur, key, names):
+ cols[lookup["key"]] = (lookup["expr"], _text)
+ return {"table": SOURCE[key], "where": "", "id": ident,
+ "needs_excluded": False, "cols": cols}
+
+
+def grid_sources():
+ """Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
+ cur = datastore.ro_cursor()
+ try:
+ present = {key for key, table in SOURCE.items() if mirror_columns(cur, table)}
+ if not present:
+ return {}
+ keys = set(READ_THROUGH)
+ for key in present - keys:
+ count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
+ if _read_through(cur, key, count):
+ keys.add(key)
+ return {key: grid_source(cur, key) for key in SOURCE if key in keys}
+ finally:
+ cur.close()
+
+
def _row_id(row, key):
- """A STABLE id for one mirror row.
+ """The legacy stored identity, retained to migrate existing snapshot rows.
⭐ STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
@@ -232,6 +482,24 @@ def _row_id(row, key):
Falls back to a content hash rather than a counter for exactly that reason: a settlement line
with no id column still hashes to the same value next week if its content has not changed.
"""
+ if key.removeprefix(PREFIX) not in _LEGACY_IDENTITY_TABLES:
+ # These tables retain multiple days/items per SKU/transaction. First-field identity
+ # would collapse them in apply_plan even though all rows survived the raw mirror.
+ # S66-T40 (lead, observed live): a report whose header spells an identity column
+ # differently (the referral fee preview says `seller_sku`, not `sku`) made
+ # `row_identity_for` raise, and because `apply_plan` is ONE write for every Amazon table,
+ # one such row cost all of them. A row missing a component falls through to the full
+ # content hash below, which is still stable and never merges two different rows.
+ try:
+ identity = amazon_schema.row_identity_for(key.removeprefix(PREFIX), row)
+ except ValueError:
+ identity = None
+ if identity is not None:
+ return hashlib.sha256(json.dumps(identity, ensure_ascii=True,
+ default=str).encode()).hexdigest()[:32]
+ payload = json.dumps({k: row.get(k) for k in sorted(row)}, sort_keys=True,
+ ensure_ascii=True, separators=(",", ":"), default=str)
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
for col in IDENTITY.get(key, ()):
val = row.get(col)
if val not in (None, ""):
@@ -242,22 +510,76 @@ def _row_id(row, key):
def _read(cur, key):
- """[(row dict)] straight off the mirror, each carrying a stable `_id`."""
+ """Small snapshots use the same numeric identity as their read-through grid.
+
+ A SKU or hex digest used as the stored row key is silently skipped by
+ perm_scope._read_user_table, and cannot be opened by the record drawer.
+ """
cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
if not cols:
return []
quoted = ", ".join(f'"{c}"' for c in cols)
- rows = cur.execute(f"SELECT {quoted} FROM {SOURCE[key]}").fetchall()
+ spec = grid_source(cur, key)
+ # S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
+ # expressions the read-through binding uses, in this one SELECT (never per row).
+ derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
+ extra = "".join(f", {expr}" for _k, expr in derived)
+ rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]}').fetchall()
+ names = cols + [k for k, _expr in derived]
out = []
for r in rows:
row = {}
- for name, val in zip(cols, r):
- row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val))
- row["_id"] = _row_id(row, key)
+ for name, val in zip(names, r[1:]):
+ # Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
+ row[name] = "" if val is None else (val if isinstance(val, (int, float))
+ else str(val).replace("\x00", ""))
+ row["_id"] = str(r[0])
out.append(row)
+ def read_targets(target, column, values):
+ target_spec = grid_source(cur, target)
+ expr = _quoted(column)
+ found = cur.execute(
+ f'SELECT {target_spec["id"]}, {expr} FROM {SOURCE[target]} '
+ f'WHERE {expr} IN ({",".join("?" for _ in values)})', list(values)).fetchall()
+ return [{"pid": pid, column: value} for pid, value in found]
+
+ fill_link_cells(out, _fields_for(cur, key), read_targets)
+ return out
+
+
+def link_edges(fields):
+ """`[(field, target, from, on)]` for every derived Amazon link `fill_link_cells` fills.
+
+ HOTFIX 2026-09-28: the ONE predicate, so the HTTP reader can see up front every column it
+ will be asked for on a target and read a STORED target once rather than once per edge."""
+ out = []
+ for field in fields:
+ bag = field.get("link") or {}
+ target, frm, on = bag.get("table"), bag.get("from"), bag.get("on")
+ if (field.get("type") != "link" or target not in SOURCE or not frm or not on
+ or bag.get("inverse") or bag.get("through") or bag.get("selectionMode") == "manual"):
+ continue
+ out.append((field, target, frm, on))
return out
+def fill_link_cells(rows, fields, read_targets):
+ """Populate derived Amazon links with target pids, with one read per edge.
+
+ The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
+ target reader. Neither path stores history rows or caps the matching target set.
+ """
+ for field, target, frm, on in link_edges(fields):
+ values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
+ matches = {}
+ for row in read_targets(target, on, values) if values else ():
+ if row.get(on) not in (None, ""):
+ matches.setdefault(str(row[on]), set()).add(int(row["pid"]))
+ for row in rows:
+ row[field["key"]] = ",".join(str(pid) for pid in sorted(
+ matches.get(str(row.get(frm)), ())))
+
+
def available(tenant_key="royal-imports"):
"""Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
try:
@@ -268,10 +590,8 @@ def available(tenant_key="royal-imports"):
def plan(cur, rt=None):
- """{bucket_key: rows} plus `problems`. No store write; every cap checked BEFORE any commit."""
+ """Rows for small snapshots, schema/count stubs for history or oversized tables."""
built = {"problems": []}
- ut = _odoo._ut()
- cap = getattr(ut, "MAX_ROWS", 60_000)
for key in SOURCE:
# ⛔ COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
# the cap afterwards means the one table too big to keep is the one built in memory first.
@@ -279,17 +599,8 @@ def plan(cur, rt=None):
n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
except Exception: # noqa: BLE001
continue # absent = never synced
- if n > cap:
- # ⛔ REPORTED WITH ITS CAUSE AND ITS FIX, NEVER TRUNCATED. Standing rule 1's second
- # half, and `ut_amzn_order_items` is the table most likely to reach it: an order
- # history is one row per LINE, so a mid-size seller passes 60,000 in a year.
- built["problems"].append(
- f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was "
- f"truncated and nothing was written for this table. Cause: this seller's history "
- f"is larger than a materialised ut_* table can hold. Fix: bind it read-through "
- f"off the mirror as `ut_odoo_gl_lines` is "
- f"(odoo_relational.READ_THROUGH_KEYS), which `routes_connected_tables` "
- f"already serves.")
+ if _read_through(cur, key, n):
+ built[key] = {"readThrough": True, "fields": _fields_for(cur, key), "count": n}
continue
rows = _read(cur, key)
if rows:
@@ -306,6 +617,9 @@ def plan_fingerprint(cur, built):
"""
tables = []
for key in SOURCE:
+ if isinstance(built.get(key), dict) and built[key].get("readThrough"):
+ tables.append({"key": key, **built[key]})
+ continue
rows = built.get(key) or []
canonical = sorted(rows, key=lambda row: json.dumps(
row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
@@ -315,6 +629,62 @@ def plan_fingerprint(cur, built):
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+def _place_lookups(stored, fields):
+ """An EXISTING table's fields with each NEW product lookup placed right after its source.
+
+ `_ensure_table_inplace` only appends, so a lookup first planned for a table stored before
+ S67-T30 would land at the far end of the grid instead of beside the ASIN or SKU it derives
+ from. A lookup already stored keeps its place (a person may have moved it) and only has its
+ `opens` bag brought up to date, which `_ensure_table_inplace` does not forward-migrate.
+ """
+ out = list(stored)
+ have = {f.get("key"): i for i, f in enumerate(out)}
+ anchor = None
+ for field in fields:
+ fkey = field.get("key")
+ if fkey not in LOOKUP_KEYS:
+ anchor = fkey
+ continue
+ if fkey in have:
+ old = out[have[fkey]]
+ if (old.get("automation") or {}).get("flowId") == "amazon_relational" \
+ and not _odoo._ut().user_edited(old):
+ if field.get("opens"):
+ old["opens"] = dict(field["opens"])
+ else:
+ old.pop("opens", None)
+ continue
+ keys = [f.get("key") for f in out]
+ if anchor not in keys:
+ continue # no source column stored yet: the append places it
+ at = keys.index(anchor) + 1
+ while at < len(keys) and keys[at] in LOOKUP_KEYS:
+ at += 1 # after any lookup already beside the same source
+ out.insert(at, dict(field))
+ have = {f.get("key"): i for i, f in enumerate(out)}
+ anchor = fkey
+ return out
+
+
+def _lead_with_identity(stored, report, pinned):
+ """An EXISTING table's fields with its identity column moved first, while the order is still
+ the report's own.
+
+ `_ensure_table_inplace` keeps a stored table's field order and only appends, so the
+ identity-first order `_fields_for` gives a NEW table never reaches one stored before it. The
+ move happens only when the stored mirror columns sit exactly in the report's order: any other
+ order is a person's arrangement, and theirs wins.
+ """
+ keys = [f.get("key") for f in stored]
+ if not pinned or pinned not in keys or keys[0] == pinned:
+ return stored
+ in_report = set(report)
+ if [k for k in keys if k in in_report] != [k for k in report if k in set(keys)]:
+ return stored
+ return ([f for f in stored if f.get("key") == pinned]
+ + [f for f in stored if f.get("key") != pinned])
+
+
def apply_plan(rt, built, username="amazon", today=None):
"""Create-or-merge every table in `built`, in ONE `flush="sync"` write.
@@ -327,24 +697,103 @@ def apply_plan(rt, built, username="amazon", today=None):
raise Refused("; ".join(built["problems"]))
stamp = today or _odoo._iso_today()
written = {}
- cur_mirror = datastore.ro_con()
- plans = [(key, LABELS[key], _fields_for(cur_mirror, key), built[key])
- for key in SOURCE if key in built]
+ cur_mirror = datastore.ro_cursor()
+ try:
+ plans = [(key, LABELS[key], (built[key]["fields"] if isinstance(built[key], dict)
+ else _fields_for(cur_mirror, key)), built[key])
+ for key in SOURCE if key in built]
+ legacy_ids = {}
+ report_order = {key: [name for name, _dtype in mirror_columns(cur_mirror, SOURCE[key])]
+ for key, _label, _fields, _rows in plans}
+ for key, _label, _fields, rows in plans:
+ if isinstance(rows, dict):
+ continue
+ names = report_order[key]
+ for row in rows:
+ raw = {name: row.get(name) for name in names}
+ payload = json.dumps(raw, sort_keys=True, ensure_ascii=True,
+ separators=(",", ":"), default=str)
+ legacy_ids[(key, row["_id"])] = (
+ _row_id(raw, key), hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
+ finally:
+ cur_mirror.close()
+
+ not_stored = {}
def _apply_all(doc):
doc = doc if isinstance(doc, dict) else {}
+ not_stored.clear()
for key, label, fields, rows in plans:
- written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
+ # ⭐⭐ D-3284 (owner ruling 2026-09-27): S66-T42/T43 created a NEW Amazon table only while
+ # the workspace had room under MAX_TABLES and reported the rest `not_stored`; ten Amazon
+ # databases stayed missing that way. Connector databases now take no MAX_TABLES slot
+ # (`user_tables.counts_toward_table_cap`), so every planned Amazon table is created.
+ # `not_stored` stays as bookkeeping: empty, it clears any earlier record.
+ # Corrected/removed measured joins retire their machine-owned old fields.
+ # Preserve fields a person created or explicitly took over.
+ current = doc.get(key) or {}
+ wanted = {f["key"] for f in fields}
+ # S67-T30: a product lookup the mirror no longer supports retires the same way.
+ current["fields"] = _lead_with_identity(
+ [f for f in current.get("fields", [])
+ if not ((f.get("type") == "link" or f.get("key") in LOOKUP_KEYS)
+ and (f.get("automation") or {}).get("flowId") == "amazon_relational"
+ and not _odoo._ut().user_edited(f)
+ and f.get("key") not in wanted)],
+ report_order.get(key) or [], PINNED.get(key))
+ if current["fields"]:
+ current["fields"] = _place_lookups(current["fields"], fields)
+ stub = isinstance(rows, dict) and rows.get("readThrough") is True
+ if not stub:
+ stored = current.get("rows") or {}
+ for row in rows:
+ if row["_id"] in stored:
+ continue
+ for old_id in legacy_ids.get((key, row["_id"]), ()):
+ if old_id in stored:
+ # Carry authored cells along when replacing an unusable string id.
+ stored[row["_id"]] = stored.pop(old_id)
+ break
+ removed = len((doc.get(key) or {}).get("rows") or {}) if stub else 0
+ written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, None if stub else rows,
username, stamp)
+ if stub:
+ doc[key]["readThrough"] = True
+ doc[key]["rows"] = {}
+ doc[key].pop(_odoo._ut().OVERSIZE_FIELD, None)
+ written[key].update(rows=rows["count"], removed=removed, readThrough=True)
+ else:
+ doc[key].pop("readThrough", None)
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
# mutable display label or from the process having imported this module.
doc[key]["connector"] = source_attribution(key)
return doc
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
+ _record_not_stored({key for key, *_rest in plans}, dict(not_stored))
return written
+def _record_not_stored(planned, not_stored):
+ """Durable per-table marker in the Amazon mirror, read by `connectors_amazon.sync_state`.
+
+ A table refused at the store's table cap gets `not_stored` with its reason; a table that was
+ stored clears any earlier marker. Best effort: the store write above already happened, and a
+ marker failure must not turn it into an error.
+ """
+ try:
+ con = datastore.connect()
+ for key in planned:
+ table = key[len(PREFIX):]
+ entity = amazon_store.entity_key("not_stored_" + table)
+ if key in not_stored:
+ amazon_store._record_state(con, entity, "not_stored", 0, not_stored[key])
+ else:
+ con.execute("DELETE FROM _sync_state WHERE entity = ?", [entity])
+ except Exception: # noqa: BLE001
+ pass
+
+
def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
"""Open, plan and conditionally apply the Amazon mirror.
@@ -357,7 +806,14 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
path = datastore.path_for(tenant)
if Path(datastore.DB_PATH) != Path(path):
datastore.use_path(path)
- cur = datastore.ro_con()
+ cur = datastore.ro_cursor()
+ try:
+ return _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state)
+ finally:
+ cur.close()
+
+
+def _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state):
if not any(mirror_columns(cur, t) for t in SOURCE.values()):
log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
@@ -381,6 +837,8 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
state["changed"] = True
state["written"] = apply_plan(rt, applicable)
state["applied"] = True
+ import routes_odoo_tables
+ routes_odoo_tables.invalidate_sources()
for key, counts in state["written"].items():
log(f"[amazon_relational] {why}: {key:<28} {counts}")
return state
diff --git a/api/application_registry.py b/api/application_registry.py
index 56f880fa04026c7da219319fb4b4a816234b9492..2a4ca9c78f3cc62e670b9c680864b830f5e7f833 100644
--- a/api/application_registry.py
+++ b/api/application_registry.py
@@ -16,6 +16,11 @@ APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
FINANCE_TEMPLATE_KIND = "finance"
APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
+# S67-T40: Content Studio creates its OWN databases (content_studio.py), so it is never offered to
+# the five-step wizard either; its catalogue POST provisions References + Posts and binds Posts.
+# Appended on its own line so the wizard / Finance separation above reads exactly as S65-T10 pins it.
+CONTENT_STUDIO_TEMPLATE_KIND = "content_studio"
+APPLICATION_TEMPLATE_KINDS = APPLICATION_TEMPLATE_KINDS + (CONTENT_STUDIO_TEMPLATE_KIND,)
_KEY = "applications_v1"
_MAX = 500
@@ -65,6 +70,17 @@ _TEMPLATE_CONTRACTS = {
"fieldRoles": [],
"defaultPlacement": {"section": "app", "order": 0},
},
+ # S67-T40 (R17, R21, O3: one declarative row, no new type). S67-T41 owner correction: the App
+ # opens on its OWN surface in the App module (`content_studio.app_link`), like Catalog and
+ # Route, so the governed type only has to be a real one; its calendar is time.
+ "content_studio": {
+ "kind": "content_studio",
+ "label": "Content Studio",
+ "applicationType": "timeseries",
+ "displayMode": "timeseries",
+ "fieldRoles": [],
+ "defaultPlacement": {"section": "app", "order": 0},
+ },
}
@@ -363,6 +379,18 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
"navPlacement": {"section": "app", "order": order}}
+ if kind == CONTENT_STUDIO_TEMPLATE_KIND:
+ # Created server-side by content_studio.create right after it made the Posts database, so
+ # the binding is the one database it just created; the link can only open that database.
+ source = str(payload.get("sourceDatabase") or "").strip()[:120]
+ if not source.startswith("ut_"):
+ raise ApplicationError("Content Studio binds the Posts database it created.")
+ if allowed_databases is not None and source not in set(allowed_databases):
+ raise PermissionError("an Application cannot bind a database you cannot open")
+ # S67-T41 owner correction: no stored link. `_public` derives the App's own address from
+ # its id, so every Studio, including one created before the correction, opens in App.
+ return {"templateKind": kind, "sourceDatabase": source,
+ "navPlacement": {"section": "app", "order": 0}}
source = str(payload.get("sourceDatabase") or "").strip()[:120]
if not source:
raise ApplicationError("sourceDatabase is required")
@@ -470,12 +498,49 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
"edit", "share", "delete", "duplicate", "star", "default")})
row["starred"] = False
row["isDefault"] = False
+ if row.get("templateKind") == CONTENT_STUDIO_TEMPLATE_KIND:
+ # S67-T41 owner correction: a Studio opens on its own App surface, never in Data. The
+ # database-bound edit, share, duplicate and default doors cannot reach an App bound to a
+ # user table, so they are off; star and delete stay, routed to `/content-studio/apps`.
+ row["deepLink"] = f"/app?studio={row['id']}"
+ row["capabilities"].update({key: False for key in ("edit", "share", "duplicate", "default")})
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
"database":target_database}
return row
+#: `core.table_store.RESERVED_MEMBERS`: workspace members that are registries, not users.
+_RESERVED_WORKSPACE_MEMBERS = ("__shared__", "__assigned__", "__audit__")
+
+
+def dependent_view_counts(runtime, database):
+ """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
+ user's Views in that database's workspace. The Delete App dialog names this count before
+ confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
+ """
+ from core.view_templates import workspace_key
+ key = workspace_key(database)
+ if not key:
+ return {}
+ try:
+ doc = runtime.get(key)
+ except Exception:
+ return None
+ counts = {}
+ for member, bucket in (doc.items() if isinstance(doc, dict) else ()):
+ if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict):
+ continue
+ views = bucket.get("views")
+ for view in (views.values() if isinstance(views, dict) else ()):
+ config = view.get("config") if isinstance(view, dict) else None
+ ref = config.get("application") if isinstance(config, dict) else None
+ app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
+ if app_id:
+ counts[app_id] = counts.get(app_id, 0) + 1
+ return counts
+
+
def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
allowed_databases=None):
doc = load(runtime)
@@ -491,6 +556,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
target_database=database,
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
+ # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
+ # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
+ deletable = [row for row in rows if row["capabilities"].get("delete")]
+ if database and deletable:
+ counts = dependent_view_counts(runtime, database)
+ for row in deletable:
+ row["dependentViews"] = None if counts is None else counts.get(row["id"], 0)
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
diff --git a/api/assistant_mirror.py b/api/assistant_mirror.py
index 1054ee6c03d0f99bc454a9406d0658ff97b48776..333f119639893e4622055eff1c7b8a985a9489b1 100644
--- a/api/assistant_mirror.py
+++ b/api/assistant_mirror.py
@@ -28,6 +28,15 @@ that need them. A module-level import either way is a cycle at startup.
"""
+#: ⭐ S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what
+#: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window):
+#: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an
+#: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is
+#: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's
+#: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631).
+ASSISTANT_READ_MAX = 200_000
+
+
class MirrorRefusal(Exception):
"""A named, fail-closed refusal with the status and code `deps` should raise it as.
@@ -49,9 +58,14 @@ def _rot():
return routes_odoo_tables
-def open_cursor(session):
+def open_cursor(session, table_key=None):
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
+ ⭐ S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's
+ `_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its
+ own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so
+ the assistant no longer answers 503 for Amazon while Odoo is still backfilling.
+
⛔ THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
@@ -61,7 +75,7 @@ def open_cursor(session):
"""
rot = _rot()
try:
- cur, mismatch = rot._mirror_cur(session)
+ cur, mismatch = rot._mirror_cur(session, table_key)
except RuntimeError as exc:
return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
except Exception as exc: # noqa: BLE001
@@ -118,8 +132,26 @@ def is_bound(cur, table_key):
"""
from harness import datastore
try:
- spec = _rot()._sources().get(str(table_key or ""))
- if not spec:
+ rot = _rot()
+ key = str(table_key or "")
+ spec = rot._sources().get(key)
+ if not spec and key.startswith("ut_amzn_"):
+ # S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key.
+ # `_source_for` and `_materialised_table` both learn them on demand; so must this.
+ rot._learn_amazon_sources()
+ spec = rot.GRID_SOURCES.get(key)
+ if not spec:
+ # A dynamically sized Amazon snapshot is admitted per key by the grid door, not by
+ # the provider build, so the registry can lack a key the grid serves (measured on
+ # staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the
+ # provider's own per-key resolver, exactly as `require_amazon_mirror` does, without
+ # admitting it (the read path admits it through that same door).
+ import amazon_relational as amazon
+ if key not in amazon.SOURCE:
+ return False
+ current = amazon.grid_source(cur, key) or {}
+ return bool(current) and not current.get("missing")
+ if not spec or spec.get("missing"):
return False
aliases = dict(spec.get("tables") or {})
if spec.get("table"):
@@ -129,8 +161,13 @@ def is_bound(cur, table_key):
return False
-def declared_fields(table_key):
- """`mk_fields()` for one connected grid, or None when this key is not a connected Odoo grid.
+def declared_fields(table_key, definition=None):
+ """`mk_fields()` for one connected grid, or None when this key is not a connected grid.
+
+ ⭐ S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that
+ is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid,
+ a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before
+ this, those databases were listed as answerable and then refused 409 on every read.
⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
each table's stored `fields` from this same function, so reading it directly is reading the
@@ -142,10 +179,20 @@ def declared_fields(table_key):
if key == str(table_key or ""):
return [dict(field) for field in (mk_fields() or [])
if isinstance(field, dict) and field.get("key")]
- return None
+ key = str(table_key or "")
+ if not isinstance(definition, dict) or not key.startswith("ut_"):
+ return None
+ try:
+ if _rot()._materialised_table(key):
+ return None
+ except Exception: # noqa: BLE001
+ return None
+ fields = [dict(field) for field in (definition.get("fields") or [])
+ if isinstance(field, dict) and field.get("key")]
+ return fields or None
-def answerable_fields(table_key, spec):
+def answerable_fields(table_key, spec, fields=None):
"""The declared field contract INTERSECTED with the columns the mirror can actually express.
⛔ THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
@@ -155,7 +202,8 @@ def answerable_fields(table_key, spec):
refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
for it with a loud `assistant_unknown_field`.
"""
- fields = declared_fields(table_key)
+ if fields is None:
+ fields = declared_fields(table_key)
if fields is None:
return None
cols = set((spec or {}).get("cols") or ())
@@ -197,30 +245,14 @@ def _compiled(nodes, conj, columns, *, code, subject, rot):
return pred
-def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
- filter_keys=None):
- """Every permitted row of a mirror-served grid, or a refusal that names its cause.
-
- Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
- (`{pid, **cells}`), sorted by pid.
+def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys):
+ """`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34.
- ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
- connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
- with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
- states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
- every aggregate from those same records, so a short set does not produce a smaller answer, it
- produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
- already settled this policy in the same words: REFUSE, NEVER TRIM.
-
- ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
- the field projection in Python AFTER this returns. Hand it a row missing a column the wall
- names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
- pass would quietly disagree with the pushdown it is meant to double-check.
+ Extracted verbatim from `read_rows` so the row walk, the page read and the server-side
+ aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler.
+ Three readers with one predicate cannot disagree about which rows the caller may see.
"""
- from harness import datastore
- rot = _rot()
fe = rot._fe()
-
cols = rot._filter_columns(spec, fields)
# ── the administrator's wall, compiled FIRST and pushed down ──────────────────────────────
@@ -267,23 +299,191 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
where = f"({where}) AND {pred.sql}" if where else pred.sql
params.extend(pred.params)
+ return where, params, cols
+
+
+def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
+ filter_keys=None, defn=None):
+ """Every permitted row of a mirror-served grid, or a refusal that names its cause.
+
+ Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
+ (`{pid, **cells}`), sorted by pid.
+
+ ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
+ connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
+ with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
+ states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
+ every aggregate from those same records, so a short set does not produce a smaller answer, it
+ produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
+ already settled this policy in the same words: REFUSE, NEVER TRIM.
+
+ ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
+ the field projection in Python AFTER this returns. Hand it a row missing a column the wall
+ names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
+ pass would quietly disagree with the pushdown it is meant to double-check.
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+
# ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
# spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
- win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
- where=where, params=tuple(params), order_by=spec["id"],
- limit=datastore.WINDOW_MAX, cur=cur, **frm)
- if win["total"] > len(win["rows"]):
+ # ⭐⭐ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX`
+ # pass this replaced refused any connected database past 5,000 matching rows, which is a cap
+ # on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages,
+ # KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs
+ # one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a
+ # user condition: every condition above still comes from the grid's compiler.
+ # ⛔ STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that
+ # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
+ first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
+ where=where, params=tuple(params), order_by=spec["id"],
+ limit=datastore.WINDOW_MAX, cur=cur, **frm)
+ total = int(first["total"])
+ if total > ASSISTANT_READ_MAX:
raise MirrorRefusal(
409, "assistant_source_too_large",
- f"this database matches {win['total']:,} rows for that question, and the assistant "
- f"reads a connected database in one pass of at most "
- f"{datastore.WINDOW_MAX:,} rows, so answering now would mean counting only part of "
- f"them and reporting the total as if it were whole. Narrow it with a filter and ask "
- f"again")
+ f"this database matches {total:,} rows for that question, and one assistant answer "
+ f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question "
+ f"cannot exhaust the server's memory. Answering over part of them would report a "
+ f"wrong total, so it is refused instead. Narrow it with a filter (for example a date "
+ f"range) and ask again, or ask for totals as aggregates, which the server computes "
+ f"over every row")
+ got = list(first["rows"])
+ page = list(first["rows"])
+ while len(got) < total and len(page) >= datastore.WINDOW_MAX:
+ last = page[-1][0]
+ more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
+ nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
+ where=more_where, params=tuple(params) + (last,),
+ order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm)
+ page = list(nxt["rows"])
+ if not page:
+ break
+ got.extend(page)
+ if len(got) != total:
+ raise MirrorRefusal(
+ 409, "assistant_source_changed",
+ f"this database changed while it was being read ({len(got):,} rows read against "
+ f"{total:,} counted), so the answer would not describe one consistent set of rows. "
+ f"Ask again once its connector sync has finished")
+ win = {**first, "rows": got, "total": total}
+
+ # S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does
+ # not re-read the whole tenant document (measured about 3 s a question on staging).
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
+ defn=defn if isinstance(defn, dict) else None)
+ rows = rot._pool_from_window(spec, win, stamp=stamp)
+ rows.sort(key=lambda row: row["pid"])
+ return rows, stamp, int(win["total"])
+
- stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None))
+#: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`).
+_GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"}
+
+
+def _json_value(rot, value):
+ """A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a
+ date or timestamp as its ISO text, anything else as is. Never a substituted 0."""
+ if value is None:
+ return None
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ return rot._agg_number(value)
+ iso = getattr(value, "isoformat", None)
+ if callable(iso):
+ return iso()
+ try:
+ from decimal import Decimal
+ if isinstance(value, Decimal):
+ return rot._agg_number(value)
+ except Exception: # noqa: BLE001
+ pass
+ return value
+
+
+def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
+ filter_tree=None, filter_keys=None):
+ """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
+
+ Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the
+ count of rows the permitted predicate matches, from `datastore.window`'s own count statement
+ over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract).
+
+ ⛔⛔ THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall
+ and the caller's filter with the grid's compiler, so a sum over a million-row table runs over
+ exactly the rows this caller may read and no row is pulled into Python to compute it. There
+ is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size.
+
+ ⛔ AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and
+ a field outside it has no expression here at all, so a sum over a hidden column cannot be
+ built (the caller's `_normalise_aggregates` has already refused it by name).
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+ visible = set(filter_keys) if filter_keys is not None else set(cols)
+ fs = rot._fs()
+ proj, slots = [], []
+ for item in specs or ():
+ fn, field = item.get("fn"), item.get("field")
+ if field is None:
+ slots.append(None) # count(*): the window's own total
+ continue
+ col = cols.get(field) if field in visible else None
+ if col is None:
+ raise MirrorRefusal(400, "assistant_unknown_field",
+ "every requested field must be readable in this database")
+ if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES:
+ raise MirrorRefusal(400, "query_aggregate_type",
+ "sum and avg require a readable numeric field.")
+ if fn in _GRID_AGG:
+ expr = fs.aggregate_sql(col, _GRID_AGG[fn])
+ elif fn in {"min", "max"}:
+ expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES
+ else f"{fn}({col['sql']})")
+ elif fn == "count":
+ expr = f"count({col['sql']})"
+ else:
+ raise MirrorRefusal(400, "query_aggregates_invalid",
+ "That aggregate is not available.")
+ slots.append(len(proj))
+ proj.append(f'{expr} AS "a{len(proj)}"')
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
+ win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params),
+ order_by="1", limit=1, cur=cur, **frm)
+ row = (win["rows"] or [()])[0]
+ total = int(win["total"])
+ out = []
+ for item, slot in zip(specs or (), slots):
+ value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row)
+ else None)
+ out.append({**item, "value": value})
+ return out, total
+
+
+def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
+ filter_tree=None, filter_keys=None, defn=None):
+ """ONE page of the permitted rows, paged IN SQL, in pid order. S67-T34 (rule 1).
+
+ Returns `(rows, stamp, total)` in `read_rows`'s shape. This is how a table past
+ `ASSISTANT_READ_MAX` still answers its rows: the page is `LIMIT/OFFSET` over the same
+ predicate `read_rows` runs, so only `limit` rows ever reach this process, and `total` is the
+ population that predicate matches. It carries no sort other than the pid; a caller asking for
+ one past the ceiling is refused by name upstream, because sorting a million rows in Python
+ is the memory failure the ceiling exists to prevent.
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+ select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
+ win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
+ params=tuple(params), order_by=spec["id"], offset=int(offset),
+ limit=int(limit), cur=cur, **frm)
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
+ defn=defn if isinstance(defn, dict) else None)
rows = rot._pool_from_window(spec, win, stamp=stamp)
rows.sort(key=lambda row: row["pid"])
return rows, stamp, int(win["total"])
diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py
index f86d6565a81ffd185b345dd764359b0454484051..c0b4b766dfb4082c9a4d50030bafef6afe11b3f0 100644
--- a/api/connectors_amazon.py
+++ b/api/connectors_amazon.py
@@ -49,11 +49,80 @@ class AmazonStatusSafetyError(ValueError):
"""A status response contains a forbidden credential shape."""
-def status_payload_has_secret(serialized_payload):
+class AmazonStatusRowsRefused(AmazonStatusSafetyError):
+ """S67-T31: some status ROWS were refused; the rest are carried, already guarded.
+
+ Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
+ refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
+ neither carries the refused content, so a caller may degrade those rows and keep the others.
+ """
+
+ def __init__(self, safe_status, refused):
+ super().__init__('Amazon status contains a forbidden credential shape.')
+ self.safe_status = safe_status
+ self.refused = tuple(refused)
+
+
+#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
+STATUS_SCALAR_LIMIT = 512
+STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
+
+
+def trim_status(value, limit=STATUS_SCALAR_LIMIT):
+ """Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
+
+ S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
+ characters) used to trip the length branch of the guard and take the whole connector modal
+ down with a 500. ⛔ Call this only AFTER the credential-shape scan of the UNTRIMMED value
+ (`screen_status_rows`): cutting first could slice a token shape out of a string and accept
+ the rest, which would trim a refusal into acceptance.
+ """
+ if isinstance(value, dict):
+ return {key: trim_status(item, limit) for key, item in value.items()}
+ if isinstance(value, list):
+ return [trim_status(item, limit) for item in value]
+ if isinstance(value, tuple):
+ return tuple(trim_status(item, limit) for item in value)
+ if isinstance(value, str) and len(value) > limit:
+ head = value[:limit - 3]
+ cut = head.rsplit(" ", 1)[0] if " " in head else head
+ return cut.rstrip(" ,;:") + "..."
+ return value
+
+
+def screen_status_rows(status):
+ """`(safe, refused)` for a `{table: row}` status: each row judged on its own.
+
+ A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
+ form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
+ is only too long is trimmed, then held to the full guard again.
+ """
+ import json
+
+ safe, refused = {}, []
+ for name, row in (status or {}).items():
+ try:
+ whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
+ if status_payload_has_secret(whole, max_length=None):
+ refused.append(name)
+ continue
+ trimmed = trim_status(row)
+ if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
+ refused.append(name)
+ continue
+ except (TypeError, ValueError, RecursionError):
+ refused.append(name)
+ continue
+ safe[name] = trimmed
+ return safe, refused
+
+
+def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
The credential key roster is the same one the connector accepts at connect time.
+ `max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
"""
import json
import re
@@ -73,10 +142,11 @@ def status_payload_has_secret(serialized_payload):
elif isinstance(value, list):
pending.extend(value)
elif isinstance(value, str):
- if (len(value) > 512 or 'Atzr|' in value
+ if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
return True
- elif value is not None and len(str(value)) > 512:
+ elif (value is not None and max_length is not None
+ and len(str(value)) > max_length):
return True
return False
@@ -132,8 +202,51 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
from datetime import datetime, timezone
import duckdb
from harness import amazon_store as az
+ from harness.amazon_client import DECLARED_DATASETS
from routes_odoo_tables import _sync_entities, stuck_verdict
+ # S66-T46 (lead): an unreadable deletion ledger is flagged once per row as
+ # `purgeScheduleUnknown`, never written into every table's cause, which would push out the
+ # reason each table already names (Amazon's refusal, the empty period, the declared reason).
+ purge_unknown = False
+ try:
+ purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
+ purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
+ except Exception:
+ purge_plan, purge_message, purge_unknown = None, "", True
+
+ def finish(status):
+ for row in status.values():
+ # A key only when there is something to say: a declared dataset's row stays exactly
+ # its declaration while no purge is scheduled (the group (a) status contract).
+ if purge_plan:
+ row["purgeScheduledAt"] = purge_plan["purgeAt"]
+ if purge_unknown and not row.get("declared"):
+ row["purgeScheduleUnknown"] = True
+ if purge_message:
+ cause = row.get("unknownCause") or ""
+ # Status scalars have a 512-character safety ceiling. Keep the full existing
+ # cause separately if the deletion sentence needs a shorter screen excerpt.
+ budget = 512 - len(purge_message) - 1
+ if len(cause) > budget:
+ row["syncCause"] = cause
+ cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
+ row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
+ # S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
+ # `search_query_performance.unknownCause` at 935 characters); a credential shape still
+ # refuses, now naming only its rows so the caller degrades those and keeps the rest.
+ safe, refused = screen_status_rows(status)
+ if refused:
+ raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
+ return assert_safe_status(safe)
+
+ declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
+ for name, spec in DECLARED_DATASETS.items()}
+ unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False,
+ "unknownCause": "This container has no readable sync status. Sync this database to load its rows."}
+ for table in az.TABLES}
+ unavailable.update(declared)
+
def stamp(value):
if not value:
return None
@@ -150,7 +263,7 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
try:
path = az._tenant_mirror_path(tenant_key, rt=rt)
if not path.is_file():
- return assert_safe_status(None)
+ return finish(unavailable)
# ⛔⛔ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
# `ConnectionException: Can't open a connection to same database file with a different
# configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
@@ -174,9 +287,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
mirrored = _mirror_counts(cur, az)
except Exception: # noqa: BLE001
# No exception text: paths, provider payloads and credentials never become status copy.
- return assert_safe_status(None)
+ return finish(unavailable)
if entities is None:
- return assert_safe_status(None)
+ return finish(unavailable)
clock = stamp(now) if now is not None else datetime.now(timezone.utc)
states, bad_ticks = {}, set()
@@ -209,12 +322,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
- answer = {}
+ answer = dict(declared)
for table in az.TABLES:
landed = entities.get(az.entity_key(table)) or {}
tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
- rows = landed.get("rows")
+ rows = mirrored.get(table, landed.get("rows"))
if type(rows) is not int or rows < 0:
rows = None
# Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
@@ -222,7 +335,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
# own count is the measured answer.
if rows is None and table in mirrored:
rows = mirrored[table]
- successes = [when for marker in (landed, tick, watermark)
+ # A refused pass closes its tick (S66-T42); only the watermark can then prove a success.
+ refused_now = landed.get("phase") in ("refused", "failed", "throttled")
+ successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark))
if marker.get("phase") == "done"
and (when := stamp(marker.get("updatedAt"))) is not None]
stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
@@ -237,8 +352,20 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
# a promise about when a later resident tick will select this table.
next_due = stamp(reason.partition("next due ")[2]) if reason else None
unknown = [load_unknown] if load_unknown else []
+ proof = entities.get(az.entity_key("reconcile_" + table)) or {}
+ try:
+ period = json.loads(proof.get("cursor") or "{}")
+ if not isinstance(period, dict):
+ period = {}
+ except (ValueError, TypeError):
+ period = {}
if rows is None:
- unknown.append("This container has no readable row count for this table.")
+ unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.")
+ if landed.get("phase") in ("refused", "failed", "throttled"):
+ unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")]
+ elif rows == 0 or period.get("count") == 0:
+ span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k)))
+ unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".")
# Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
# and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
# server log, so the owner saw an empty database and no reason.
@@ -246,10 +373,68 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
if over is not None and over > _stored_table_cap():
unknown.append(
f"The mirror holds {over:,} rows, more than a stored table can hold "
- f"({_stored_table_cap():,}), so this database shows none yet. Nothing was "
- "truncated. Serving it straight from the mirror is scheduled for SDLC 66.")
- if plan is None:
+ f"({_stored_table_cap():,}). This table is served read-through from the mirror "
+ "with no row cap.")
+ # S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the
+ # workspace is at the store's table cap says so here, with the cause and the fix.
+ not_stored = entities.get(az.entity_key("not_stored_" + table)) or {}
+ if not_stored.get("phase") == "not_stored":
+ unknown.append(str(not_stored.get("cursor") or
+ "This table is not in this workspace yet because the workspace is at "
+ "its database limit. Its rows are kept in the Amazon mirror."))
+ if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"):
unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
+ limit = None
+ if table in az.GROUP_A_TABLES and table != "catalog_items":
+ marker = entities.get(az.entity_key("limit_" + table)) or {}
+ try:
+ limit = json.loads(marker.get("cursor") or "{}").get("reason")
+ except (ValueError, TypeError, AttributeError):
+ limit = "History progress is unreadable. Retry sync to refresh its saved date."
+ if limit:
+ unknown.append(limit)
+ if table == "catalog_items":
+ marker = entities.get(az.entity_key("limit_" + table)) or {}
+ try:
+ limit = json.loads(marker.get("cursor") or "{}").get("reason")
+ except (ValueError, TypeError, AttributeError):
+ limit = "Catalog request progress is unreadable. Sync Amazon listings and retry."
+ limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. "
+ "Every own ASIN and page is followed; sync Amazon listings first.")
+ unknown.append(limit)
+ if table == "listing_items":
+ limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. "
+ "Date ranges are split and every page followed. Keep the connection enabled.")
+ unknown.append(limit)
+ if table == az.SQP_TABLE:
+ marker = entities.get(az.SQP_LIMIT_KEY) or {}
+ if marker.get("cursor"):
+ try:
+ detail = json.loads(marker["cursor"])
+ limit = az.sqp_limit_sentence(detail)
+ unknown.append(limit)
+ # S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18
+ # ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and
+ # every refused ASIN is still named on screen.
+ by_reason = {}
+ for asin, refusal in detail.get("refusedAsins", {}).items():
+ by_reason.setdefault(str(refusal), []).append(str(asin))
+ for refusal, asins in by_reason.items():
+ unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1
+ else f"Amazon refused ASINs {', '.join(sorted(asins))} for "
+ f"Search Query Performance: {refusal}")
+ unknown.extend(detail.get("errors", []))
+ if detail.get("deferredReason"):
+ reason = detail["deferredReason"]
+ if detail.get("partialMonthReason"):
+ unknown.append(detail["partialMonthReason"])
+ except (ValueError, TypeError, KeyError, AttributeError):
+ unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.")
+ if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512:
+ # Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule
+ # diagnostics can otherwise overflow the status guard and hide every database.
+ unknown = [part for part in unknown if part != load_unknown
+ and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))]
answer[table] = {
"rowsLoaded": rows,
"lastSuccess": iso(max(successes)) if successes else None,
@@ -260,7 +445,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
"stuck": stuck, "stuckCause": stuck_cause,
"unknownCause": " ".join(unknown),
}
- return assert_safe_status(answer)
+ if table == az.SQP_TABLE or table in az.GROUP_A_TABLES:
+ answer[table]["limit"] = limit
+ return finish(answer)
#: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
diff --git a/api/content_studio.py b/api/content_studio.py
new file mode 100644
index 0000000000000000000000000000000000000000..10696a42b7bc4d9428277b20602836918a6ff839
--- /dev/null
+++ b/api/content_studio.py
@@ -0,0 +1,638 @@
+"""Content Studio: an App template that creates its own References and Posts databases.
+
+S67-T40 (owner instruction 1; rulings R17, R18, R21, R11). Plan-only (round 1 R5): nothing here
+publishes to any network. The template creates two TENANT-OWNED databases and binds the App to
+Posts, opening on a calendar of posts. The App is delete-safe (rule 13, ARCHITECTURE section 24):
+deleting it removes only the App record, so References and Posts and every row in them outlive it.
+
+This module is the App's own code. Shared files reach it only through a guarded import
+(`routes_templates.py`), so deleting this file removes the template and nothing else breaks.
+"""
+from __future__ import annotations
+
+TEMPLATE_KEY = "content_studio"
+LABEL = "Content Studio"
+DESCRIPTION = "Plan posts from your references. Creates References and Posts databases."
+
+#: R4: the channels a post is planned for. Channel length limits are ADVISED, never enforced (O4).
+CHANNELS = ["LinkedIn", "X", "Instagram", "TikTok", "YouTube", "Reddit"]
+NETWORKS = CHANNELS + ["Other"]
+STATUSES = ["Idea", "Draft", "Ready", "Posted", "Skipped"]
+CALENDAR_VIEW_ID = "cs_calendar"
+
+#: R15 (structure, not words): the three fields a Reference is drafted FROM are extracted from its
+#: pasted text by the per-row AI field machinery (`ai_enrich`), re-run when that text changes.
+_EXTRACT = {
+ "hook": ("Hook", "In one sentence, describe the opening move this post uses to earn attention "
+ "(for example a surprising number, a contrarian claim, a question). Describe "
+ "the technique; do not quote the post. Post: {pasted_text}"),
+ "structure": ("Structure", "List the post's structure as 3 to 6 short numbered beats, each "
+ "naming what that part does (setup, proof, turn, ask). Describe, "
+ "never quote. Post: {pasted_text}"),
+ "angle": ("Angle", "In one sentence, name the point of view or argument the post takes and "
+ "who it is for. Do not quote the post. Post: {pasted_text}"),
+}
+
+
+#: The platform ladder leads with a REASONING model, which spends a few hundred tokens thinking
+#: before it answers; at 300 the column's ceiling was reached with no answer ("the model had
+#: nothing to answer from"), so each extraction gets room for thought plus a short answer.
+EXTRACT_MAX_TOKENS = 1500
+
+
+def reference_fields():
+ """References: URL, network, author, pasted text, tags, hook, structure, angle (T40 done-when)."""
+ fields = [
+ {"key": "title", "label": "Title", "type": "text"},
+ {"key": "url", "label": "URL", "type": "url"},
+ {"key": "network", "label": "Network", "type": "select", "options": list(NETWORKS)},
+ {"key": "author", "label": "Author", "type": "text"},
+ {"key": "pasted_text", "label": "Pasted text", "type": "text"},
+ {"key": "tags", "label": "Tags", "type": "multiselect", "options": []},
+ ]
+ for key, (label, prompt) in _EXTRACT.items():
+ fields.append({"key": key, "label": label, "type": "ai_enrich",
+ "aiEnrich": {"prompt": prompt, "maxTokens": EXTRACT_MAX_TOKENS,
+ "trigger": {"mode": "on_change"}, "overwrite": "always"}})
+ return fields
+
+
+def post_fields(references_key):
+ """Posts: caption, channel, scheduled, status, media, campaign, post URL, References link,
+ attribution (T40 done-when). `media` is an `image` field: the store has no attachment type."""
+ return [
+ {"key": "caption", "label": "Caption", "type": "text"},
+ {"key": "channel", "label": "Channel", "type": "select", "options": list(CHANNELS)},
+ {"key": "scheduled", "label": "Scheduled", "type": "date"},
+ {"key": "status", "label": "Status", "type": "select", "options": list(STATUSES)},
+ {"key": "media", "label": "Media", "type": "image"},
+ {"key": "campaign", "label": "Campaign", "type": "text"},
+ {"key": "post_url", "label": "Post URL", "type": "url"},
+ {"key": "references", "label": "References", "type": "link",
+ "link": {"table": references_key, "selectionMode": "manual"}},
+ {"key": "attribution", "label": "Attribution", "type": "text"},
+ ]
+
+
+class ContentStudioError(RuntimeError):
+ """A refusal the route turns into a plain sentence; nothing half-created is left behind."""
+
+
+def _create_table(label, username, fields, st):
+ """`user_tables.create` refuses a duplicate label, so a second Studio gets "Posts 2" and so on."""
+ from core import user_tables
+ for attempt in range(1, 10):
+ name = label if attempt == 1 else f"{label} {attempt}"
+ key = user_tables.create(name, username, fields=fields, st=st)
+ if key:
+ return key
+ raise ContentStudioError(f"The {label} database could not be created. "
+ "Rename or delete an older one, then try again.")
+
+
+def create(session, name=""):
+ """Create References + Posts (tenant-owned), a calendar View on Posts, and the App bound to
+ Posts. Returns `{address, application, databases}`; on a failure part-way, the databases this
+ call created are deleted again, so a refusal leaves the workspace as it was."""
+ import application_registry
+ from core import user_tables, view_templates
+ from core import table_store
+
+ st = session.runtime
+ user = session.uname
+ app_name = " ".join(str(name or "").split())[:120] or LABEL
+ created = []
+ try:
+ references = _create_table("References", user, reference_fields(), st)
+ created.append(references)
+ posts = _create_table("Posts", user, post_fields(references), st)
+ created.append(posts)
+ ops = table_store.make(view_templates.workspace_key(posts), st=st)
+ ops.save_view(user, {"id": CALENDAR_VIEW_ID, "name": "Calendar", "createdBy": user,
+ "config": {"filters": [], "sorts": [],
+ "display": {"mode": "calendar", "dateField": "scheduled",
+ "titleField": "caption"}}})
+ application = application_registry.create(
+ st, session.tenant, user,
+ {"templateKind": TEMPLATE_KEY, "name": app_name, "sourceDatabase": posts,
+ "navPlacement": {"section": "app", "order": 0}},
+ allowed_databases=None)
+ except Exception:
+ for key in reversed(created):
+ try:
+ user_tables.delete(key, st=st)
+ except Exception:
+ pass
+ raise
+ # S67-T41 owner correction: the App opens on its own surface in the App module, not in Data.
+ return {"address": "#" + app_link(application["id"]), "application": application,
+ "databases": {"references": references, "posts": posts}}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T41 (R13, R14, R15, R19, R20, R5, O4): DRAFTING. One Draft row per chosen channel, from
+# several selected References plus a one-line topic, written from their extracted STRUCTURE
+# (hook, structure, angle), never their words, with every author credited. Plan-only: a draft
+# is a Posts row with status Draft; nothing is published anywhere.
+# ---------------------------------------------------------------------------------------------
+import re
+
+from fastapi import APIRouter, Body, Depends
+
+from deps import Session, err, require_session
+
+router = APIRouter(prefix="/content-studio")
+
+#: O4: channel limits are ADVICE in the prompt and on screen, never a truncation.
+CHANNEL_ADVICE = {
+ "LinkedIn": "LinkedIn post: up to about 3,000 characters, short paragraphs, no hashtag wall.",
+ "X": "X post: aim for 280 characters or fewer.",
+ "Instagram": "Instagram caption: up to about 2,200 characters; a strong first line.",
+ "TikTok": "TikTok caption: short, about 150 characters.",
+ "YouTube": "YouTube: a title line, then a two or three sentence description.",
+ "Reddit": "Reddit: a title line, then the body; plain, specific, no marketing tone.",
+}
+DRAFT_SYSTEM = (
+ "You draft ONE original social media post for the channel named. You are given a one-line "
+ "topic and the STRUCTURE of several reference posts: the technique of each hook, its beats "
+ "and its angle. Borrow those structural moves; write entirely new sentences. Never copy or "
+ "closely paraphrase any sentence from anywhere. Answer with the post text only: no preamble, "
+ "no quotation marks around it, no markdown heading. Never use an em dash or an en dash; use "
+ "a comma, a colon or a full stop.")
+LEDGER_SURFACE = "field_agent"
+DRAFT_MAX_TOKENS = 3000 # a reasoning model thinks first; see EXTRACT_MAX_TOKENS
+DRAFT_TIMEOUT_S = 60
+#: A shared run of this many words is a copied sentence, whatever surrounds it (R15).
+VERBATIM_WORDS = 8
+
+
+def _studio(session, app_id):
+ """(app, posts_key, references_key) for a Content Studio App this session may open."""
+ import application_registry
+ from core import user_tables
+ doc = application_registry.load(session.runtime)
+ app = doc["applications"].get(str(app_id or ""))
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KEY
+ or not application_registry.may_read(app, session.uname, session.admin)):
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
+ posts = str(app.get("sourceDatabase") or "")
+ defn = user_tables.get(posts, session.runtime) or {}
+ link = next((f.get("link") for f in defn.get("fields") or ()
+ if isinstance(f, dict) and f.get("key") == "references"), None)
+ references = str((link or {}).get("table") or "")
+ if not posts or not references:
+ raise err(409, "content_studio_incomplete",
+ "This Content Studio has lost its Posts or References database.")
+ return app, posts, references
+
+
+def _reference_rows(session, references):
+ from core import user_tables
+ defn = user_tables.get(references, session.runtime) or {}
+ rows = defn.get("rows") or {}
+ return {str(rid): (row if isinstance(row, dict) else {}) for rid, row in rows.items()}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T42 (R35, standing rule 1): THE MONTHLY DRAFTING ALLOWANCE. Drafting runs on Loopable's
+# platform keys, so each tenant gets a monthly count of drafts (one Draft row = one draft). Past
+# it, Draft pauses with a sentence naming the reset date; the platform admin raises it per tenant
+# (`PUT /content-studio/allowance`). A tenant admin cannot: that would defeat R35.
+# One per-tenant bucket, owned by this module: {"allowance": int (absent = default),
+# "months": {"YYYY-MM": drafts made}}. Month and reset are UTC.
+# ---------------------------------------------------------------------------------------------
+ALLOWANCE_KEY = "content_studio_usage"
+DEFAULT_MONTHLY_DRAFTS = 100
+ALLOWANCE_KEEP_MONTHS = 12
+_MONTH_NAMES = ("January", "February", "March", "April", "May", "June", "July", "August",
+ "September", "October", "November", "December")
+
+
+def _now():
+ from datetime import datetime, timezone
+ return datetime.now(timezone.utc)
+
+
+def _month_key(now):
+ return f"{now.year:04d}-{now.month:02d}"
+
+
+def _reset_on(now):
+ """The first day of next month (UTC): the day this month's count stops counting."""
+ from datetime import date
+ return date(now.year + 1, 1, 1) if now.month == 12 else date(now.year, now.month + 1, 1)
+
+
+def _spoken_date(day):
+ return f"{_MONTH_NAMES[day.month - 1]} {day.day}, {day.year}"
+
+
+def _limit(doc):
+ raw = (doc or {}).get("allowance")
+ if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
+ return DEFAULT_MONTHLY_DRAFTS
+ return raw
+
+
+def allowance_state(doc, now=None):
+ """{used, limit, remaining, resetsOn, paused, message}: the ONE reading of month and reset
+ that both the panel and the enforcement use."""
+ now = now or _now()
+ doc = doc if isinstance(doc, dict) else {}
+ months = doc.get("months") if isinstance(doc.get("months"), dict) else {}
+ try:
+ used = max(0, int(months.get(_month_key(now)) or 0))
+ except (TypeError, ValueError):
+ used = 0
+ limit = _limit(doc)
+ remaining = max(0, limit - used)
+ reset = _reset_on(now)
+ paused = remaining <= 0
+ message = (f"This workspace has used this month's drafting allowance. "
+ f"It resets on {_spoken_date(reset)}.") if paused else ""
+ return {"used": used, "limit": limit, "remaining": remaining,
+ "resetsOn": reset.isoformat(), "paused": paused, "message": message}
+
+
+def _short_message(remaining):
+ drafts = "1 draft" if remaining == 1 else f"{remaining} drafts"
+ channels = "1 channel" if remaining == 1 else f"up to {remaining} channels"
+ return f"This workspace has {drafts} left this month. Choose {channels}."
+
+
+def _refusal(state, wanted):
+ """The 429 a draft of `wanted` drafts earns against `state`, or None when it fits."""
+ if state["remaining"] <= 0:
+ return err(429, "drafting_allowance_used", state["message"])
+ if state["remaining"] < wanted:
+ return err(429, "drafting_allowance_short", _short_message(state["remaining"]))
+ return None
+
+
+def _read_allowance(st):
+ doc = st.get(ALLOWANCE_KEY)
+ return doc if isinstance(doc, dict) else {}
+
+
+def _count(st, month, delta):
+ """Add `delta` drafts to `month` in one read-modify-write; keeps the last 12 months."""
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
+ months[month] = max(0, int(months.get(month) or 0) + int(delta))
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
+ months.pop(stale, None)
+ doc["months"] = months
+ return doc
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+
+
+def reserve_drafts(st, wanted, now=None):
+ """Check AND reserve `wanted` drafts in ONE store update, so two concurrent drafts cannot both
+ take the last slot. Raises the 429 (no write) when it does not fit; returns the month key the
+ reservation was booked to, for `release_drafts` on a failure."""
+ now = now or _now()
+ try:
+ current = _read_allowance(st)
+ except Exception: # noqa: BLE001
+ raise err(503, "allowance_unavailable",
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
+ refusal = _refusal(allowance_state(current, now), wanted)
+ if refusal is not None: # fast path: a paused workspace writes nothing
+ raise refusal
+ month = _month_key(now)
+ verdict = {}
+
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ refused = _refusal(allowance_state(doc, now), wanted)
+ if refused is not None:
+ verdict["refusal"] = refused
+ return doc
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
+ months[month] = max(0, int(months.get(month) or 0)) + int(wanted)
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
+ months.pop(stale, None)
+ doc["months"] = months
+ return doc
+
+ try:
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+ except Exception: # noqa: BLE001
+ raise err(503, "allowance_unavailable",
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
+ if "refusal" in verdict:
+ raise verdict["refusal"]
+ return month
+
+
+def release_drafts(st, month, n):
+ """Hand back a reservation whose drafts were never written (AI failure, copy refusal, store)."""
+ try:
+ _count(st, month, -int(n))
+ except Exception: # noqa: BLE001
+ pass
+
+
+@router.put("/allowance")
+def set_allowance(body: dict = Body(default=None), session: Session = Depends(require_session)):
+ """The owner raises (or resets) one tenant's monthly drafting allowance. Platform admin only."""
+ import core.platform_admin as platform_admin
+ from harness import runtime
+ if not platform_admin.is_platform_admin(session.user):
+ raise err(403, "forbidden", "your account does not have access to this surface")
+ body = body if isinstance(body, dict) else {}
+ slug = str(body.get("tenant") or "").strip().lower()
+ monthly = body.get("monthly")
+ if monthly is not None and (isinstance(monthly, bool) or not isinstance(monthly, int)
+ or monthly < 0):
+ raise err(400, "bad_allowance",
+ "The monthly allowance must be a whole number of drafts, 0 or more.")
+ if not slug or slug not in runtime.known_tenants():
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
+ try:
+ st = runtime.get_runtime(slug)
+ except KeyError:
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
+
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ if monthly is None:
+ doc.pop("allowance", None)
+ else:
+ doc["allowance"] = int(monthly)
+ return doc
+
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+ return {"tenant": slug, "allowance": allowance_state(_read_allowance(st))}
+
+
+@router.get("/references")
+def list_references(app: str = "", session: Session = Depends(require_session)):
+ """The References a Draft can use: id, title, author, network, and whether its structure is in.
+ Also this workspace's drafting allowance, so the panel reports the limit before it bites."""
+ _app, _posts, references = _studio(session, app)
+ rows = _reference_rows(session, references)
+ try:
+ allowance = allowance_state(_read_allowance(session.runtime))
+ except Exception: # noqa: BLE001
+ allowance = None # unreadable: the draft route still enforces it
+ return {"references": [
+ {"id": rid, "title": str(row.get("title") or row.get("url") or "Untitled reference")[:160],
+ "author": str(row.get("author") or "")[:120], "network": str(row.get("network") or ""),
+ "extracted": bool(str(row.get("structure") or "").strip())}
+ for rid, row in rows.items()],
+ "channels": list(CHANNELS), "advice": dict(CHANNEL_ADVICE), "allowance": allowance}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an
+# App ... it should act like Catalog / Route where it just uses the data module as what its
+# building on top." So the App opens in the App module on its OWN surface (`#/app?studio=`,
+# `apps/contentStudio/ContentStudioApp.tsx`): a calendar and a board over Posts, the Draft door,
+# and the two databases underneath. Data still holds References and Posts; the App never sends
+# anyone there to use it.
+# ⚠ The shared `/applications` doors (`routes_assets.py`) admit only topic-backed databases, so
+# they refuse an App bound to a user table; the read, star and delete this App needs live here,
+# on the registry's own writers, behind the same `_studio` + `may_read` / `may_edit` walls.
+# ---------------------------------------------------------------------------------------------
+def app_link(app_id):
+ """The one address a Content Studio App opens at (App module, its own surface)."""
+ return f"/app?studio={app_id}"
+
+
+#: The Posts fields the surface draws, in the grid's own Field shape so `viewModes.CalendarView`
+#: (by Scheduled) and `KanbanView` (by Status) render them unchanged.
+SURFACE_FIELDS = ("caption", "channel", "scheduled", "status", "attribution", "campaign")
+
+
+def _surface_field(defn):
+ kind = {"date": "date", "select": "select"}.get(defn.get("type"), "text")
+ field = {"key": defn["key"], "label": str(defn.get("label") or defn["key"]), "type": kind,
+ "source": "overlay"}
+ if kind == "select":
+ field["options"] = [str(o) for o in (defn.get("options") or ())]
+ return field
+
+
+def _studio_or_missing(session, app_id):
+ """`_studio`, answering an absent App with the code the App home already reads as 'gone'."""
+ try:
+ return _studio(session, app_id)
+ except Exception as exc: # noqa: BLE001
+ if getattr(exc, "status_code", None) == 404:
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
+ raise
+
+
+@router.get("/studio")
+def studio(app: str = "", session: Session = Depends(require_session)):
+ """Everything the App surface draws: the App, Posts (surface fields + rows), References'
+ count, the statuses in board order, and the drafting allowance."""
+ from core import user_tables
+ application, posts, references = _studio(session, app)
+ if not user_tables.may_open(posts, session.uname, session.admin, st=session.runtime):
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
+ posts_def = user_tables.get(posts, session.runtime) or {}
+ refs_def = user_tables.get(references, session.runtime) or {}
+ by_key = {f.get("key"): f for f in posts_def.get("fields") or () if isinstance(f, dict)}
+ fields = [_surface_field(by_key[key]) for key in SURFACE_FIELDS if key in by_key]
+ rows = []
+ for rid, row in (posts_def.get("rows") or {}).items():
+ try:
+ pid = int(rid)
+ except (TypeError, ValueError):
+ continue
+ row = row if isinstance(row, dict) else {}
+ rows.append({"pid": pid, **{key: row.get(key) for key in SURFACE_FIELDS}})
+ rows.sort(key=lambda r: r["pid"])
+ try:
+ allowance = allowance_state(_read_allowance(session.runtime))
+ except Exception: # noqa: BLE001
+ allowance = None
+ return {"app": {"id": application["id"], "name": application.get("name") or LABEL},
+ "posts": {"database": posts, "label": str(posts_def.get("label") or "Posts"),
+ "fields": fields, "rows": rows},
+ "references": {"database": references,
+ "label": str(refs_def.get("label") or "References"),
+ "count": len(refs_def.get("rows") or {})},
+ "statuses": list(STATUSES), "allowance": allowance}
+
+
+@router.patch("/apps/{app_id}")
+def star_app(app_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Star or unstar a Content Studio on the App home (the registry's own star writer)."""
+ import application_registry
+ _app, posts, _references = _studio_or_missing(session, app_id)
+ body = body if isinstance(body, dict) else {}
+ if set(body) != {"starred"} or not isinstance(body.get("starred"), bool):
+ raise err(400, "bad_patch", "Only the star of a Content Studio changes here.")
+ try:
+ landed = application_registry.mutate(session.runtime, app_id, session.uname, session.admin,
+ {"starred": body["starred"]}, allowed_databases=[posts])
+ except PermissionError:
+ raise err(403, "forbidden", "You cannot change this App.")
+ except application_registry.ApplicationError:
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
+ return {"application": landed}
+
+
+@router.delete("/apps/{app_id}")
+def delete_app(app_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Delete the App and nothing else (rule 13): References and Posts, every row in them, and
+ every View on them outlive it. The name is confirmed inside the registry's own update."""
+ import application_registry
+ application, posts, _references = _studio_or_missing(session, app_id)
+ confirm = (body or {}).get("confirmName") if isinstance(body, dict) else None
+ if not isinstance(confirm, str) or confirm != application.get("name"):
+ raise err(400, "confirm_name", "Enter the exact App name to confirm.")
+
+ class _NameConfirmed:
+ def update(self, key, delete):
+ def confirmed(raw):
+ deleted = delete(raw)
+ if (raw.get("applications") or {}).get(app_id, {}).get("name") != confirm:
+ raise application_registry.ApplicationError("the App name changed")
+ return deleted
+ return session.runtime.update(key, confirmed)
+
+ try:
+ return application_registry.delete(_NameConfirmed(), app_id, session.uname, session.admin,
+ allowed_databases=[posts])
+ except PermissionError:
+ raise err(403, "forbidden", "You no longer have permission to delete this App.")
+ except application_registry.ApplicationError:
+ raise err(400, "confirm_name", "The App name may have changed. Close this dialog and reload to retry.")
+
+
+def _words(text):
+ return re.findall(r"[a-z0-9']+", str(text or "").lower())
+
+
+def copied_sentence(draft, sources):
+ """The first run of VERBATIM_WORDS consecutive words a draft shares with any source, or ''."""
+ grams = set()
+ for source in sources:
+ words = _words(source)
+ grams.update(" ".join(words[i:i + VERBATIM_WORDS])
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)))
+ words = _words(draft)
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)):
+ gram = " ".join(words[i:i + VERBATIM_WORDS])
+ if gram in grams:
+ return gram
+ return ""
+
+
+def attribution(rows):
+ """Every selected reference's author, credited in plain words (R14 traceability)."""
+ names = list(dict.fromkeys(
+ str(row.get("author") or row.get("title") or "an unnamed source").strip() for row in rows))
+ joined = names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
+ return f"Structure drawn from posts by {joined}."
+
+
+def draft_prompt(topic, channel, rows):
+ parts = [f"Channel: {channel}. {CHANNEL_ADVICE.get(channel, '')}", f"Topic: {topic}", ""]
+ for n, row in enumerate(rows, 1):
+ parts.append(f"Reference {n} ({row.get('network') or 'post'}):")
+ for key, label in (("hook", "Hook technique"), ("structure", "Structure"), ("angle", "Angle")):
+ value = " ".join(str(row.get(key) or "").split())[:900]
+ if value:
+ parts.append(f" {label}: {value}")
+ parts.append("")
+ parts.append("Write the post now.")
+ return "\n".join(parts)
+
+
+def draft_posts(session, app_id, reference_ids, topic, channels, *, ask=None):
+ """Generate EVERY channel first; write the Posts rows only if all succeeded (no half row)."""
+ topic = " ".join(str(topic or "").split())[:200]
+ ids = [str(x) for x in dict.fromkeys(reference_ids or ())][:50]
+ wanted = [c for c in dict.fromkeys(channels or ()) if c in CHANNELS]
+ if not ids:
+ raise err(400, "no_references", "Select at least one reference to draft from.")
+ if not topic:
+ raise err(400, "no_topic", "Type a one-line topic for the draft.")
+ if not wanted:
+ raise err(400, "no_channels", "Choose at least one channel.")
+ _app, posts, references = _studio(session, app_id)
+ rows_by_id = _reference_rows(session, references)
+ missing = [rid for rid in ids if rid not in rows_by_id]
+ if missing:
+ raise err(409, "references_missing", "A selected reference no longer exists. Reload and try again.")
+ chosen = [rows_by_id[rid] for rid in ids]
+ if not any(str(row.get("structure") or "").strip() for row in chosen):
+ raise err(409, "references_not_extracted",
+ "The selected references have no extracted structure yet. Paste their text first.")
+ # R35: the allowance is checked and reserved BEFORE any provider is resolved or called. One
+ # draft per channel; the reservation is handed back unless every row is written.
+ month = reserve_drafts(session.runtime, len(wanted))
+ try:
+ return _generate_and_save(session, posts, ids, topic, wanted, chosen, ask=ask)
+ except BaseException:
+ release_drafts(session.runtime, month, len(wanted))
+ raise
+
+
+def _generate_and_save(session, posts, ids, topic, wanted, chosen, *, ask=None):
+ import ai_enrich
+ import ai_review
+ import usage_ledger
+ from core import user_tables
+
+ live = ai_review.ladder()
+ if not live and ask is None:
+ raise err(503, "ai_unavailable", "No AI provider is configured here, so nothing was drafted.")
+ provider = (live or [{"name": "injected", "model": "injected"}])[0]
+ model = provider["model"]
+ caller = ask or ai_enrich._ask
+ sources = [row.get("pasted_text") or "" for row in chosen]
+ drafts, tokens, calls = [], 0, 0
+ try:
+ for channel in wanted:
+ prompt = draft_prompt(topic, channel, chosen)
+ text = ""
+ for attempt in range(2):
+ answer, used, problem = caller(provider, model, prompt, DRAFT_MAX_TOKENS,
+ DRAFT_TIMEOUT_S, system=DRAFT_SYSTEM)
+ calls += 1
+ tokens += int(used or 0)
+ if problem or not str(answer or "").strip():
+ raise err(502, "draft_failed", f"The {channel} draft could not be written "
+ f"({problem or 'the model returned nothing'}). Nothing was saved.")
+ text = ai_enrich.no_dashes(str(answer).strip())
+ copied = copied_sentence(text, sources)
+ if not copied:
+ break
+ prompt += f"\n\nYour last draft reused this wording from a reference: \"{copied}\". Rewrite it in new words."
+ text = ""
+ if not text:
+ raise err(502, "draft_copied", f"The {channel} draft kept repeating a reference's "
+ "wording, so it was not saved. Nothing was saved.")
+ drafts.append({"channel": channel, "caption": text})
+ finally:
+ if calls:
+ # `usage_ledger.SURFACES` names four surfaces and refuses any other, so drafts are
+ # booked as `field_agent` (AI-written cells of a user table) until a `content_studio`
+ # surface exists there (a finding for the ledger's owner; this module cannot add it).
+ usage_ledger.record(LEDGER_SURFACE, provider["name"], model, total=tokens,
+ calls=calls, st=session.runtime, user=session.uname)
+ credit = attribution(chosen)
+ new_rows = [{"caption": d["caption"], "channel": d["channel"], "status": "Draft",
+ "references": ",".join(ids), "attribution": credit} for d in drafts]
+ created = user_tables.add_rows(posts, new_rows, username=session.uname, st=session.runtime)
+ if not created:
+ raise err(503, "store_unavailable", "The drafts could not be saved. Nothing was saved.")
+ return {"created": list(created) if isinstance(created, (list, tuple)) else created,
+ "drafts": drafts, "attribution": credit, "tokens": tokens, "calls": calls,
+ "provider": provider["name"], "model": model}
+
+
+@router.post("/draft")
+def draft(body: dict = Body(default=None), session: Session = Depends(require_session)):
+ body = body if isinstance(body, dict) else {}
+ return draft_posts(session, body.get("app"), body.get("references"), body.get("topic"),
+ body.get("channels"))
diff --git a/api/deps.py b/api/deps.py
index 19f4fc91afbbb27014eee560230ac435957cf04d..bcb27545f37c62d376975e77f2fffbae144d2844 100644
--- a/api/deps.py
+++ b/api/deps.py
@@ -232,6 +232,28 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
else ("product_pool", team_id))
cache = getattr(session.runtime, "pool_cache", None)
entry = cache.get(cache_key) if isinstance(cache, dict) else None
+ if not isinstance(entry, tuple) and isinstance(cache, dict):
+ # ⭐⭐ S67-T33 / D-778: A DATABASE NEVER OPENED ANSWERS. The snapshot used to exist only
+ # once somebody opened the grid in this process, so after every restart the assistant
+ # refused a database its caller demonstrably holds. The miss is now filled by the GRID'S
+ # OWN pool builder, for the SAME scope key the grid would fill (`derive_pool_scope`
+ # above), so this is the grid's first open, not a second reader. It is read-only on Odoo
+ # (rule 4) and honours the connector pause exactly as the grid does (503 when paused
+ # with nothing cached). Permission and field checks above have already run.
+ try:
+ if database == "customer_data":
+ import routes_customers
+ routes_customers._pool_for(session.runtime, team_id, agent)
+ else:
+ import routes_products
+ routes_products._pool_for(session.runtime, team_id)
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ _assistant_refuse(409, "assistant_snapshot_unavailable",
+ f"this database's snapshot could not be built for your current "
+ f"scope ({type(exc).__name__}); try again shortly")
+ entry = cache.get(cache_key)
if (not isinstance(entry, tuple) or len(entry) < 2
or not isinstance(entry[1], (list, tuple))
or any(not isinstance(row, dict) for row in entry[1])):
@@ -247,6 +269,44 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
_ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
+def _assistant_nav_opens(session: Session, database: str, lent, definition=None) -> bool:
+ """Does the NAVIGATION offer this `ut_*` database to this caller? S67-T33 / R33.
+
+ ⭐⭐ THE AI CONNECTOR'S LIST IS THE NAV'S LIST, BY CONSTRUCTION RATHER THAN BY AGREEMENT.
+ `routes_nav.nav` admits a user table when `user_tables.nav_entries` lists it (which is
+ `may_open` plus the route-system exclusion), it is not a retired key, and
+ `perm_scope.nav_may_open` agrees (an administrator's explicit untick denies). The assistant
+ used to ask `may_open` alone, so a database the member's navigation did not show could still
+ be listed and read through the connector. This asks the nav's own predicates, in the nav's
+ order; it is not a second permission rule.
+ ⚠ `session.admin` short-circuits leg 1 of `nav_may_open` exactly as the nav's own admin
+ session does; `nav_may_open` itself returns True for an administrator.
+ """
+ import core.perm_scope as perm_scope
+ from core import route_system, user_tables
+
+ key = str(database or "")
+ try:
+ from odoo_relational import RETIRED_KEYS as retired
+ except Exception: # noqa: BLE001
+ retired = ()
+ if key in retired:
+ return False
+ if not user_tables.may_open(key, session.uname, session.admin, st=lent):
+ return False
+ if isinstance(definition, dict) and (route_system.is_managed(definition)
+ or route_system.is_managed(key)):
+ return False
+ if session.admin:
+ return True
+ user = getattr(session, "user", None)
+ if not isinstance(user, dict):
+ # No account record means no explicit per-key deny can exist, and `nav_may_open` leg 3
+ # then defers to `may_open`, which has already answered above.
+ return True
+ return bool(perm_scope.nav_may_open(user, key, st=lent))
+
+
def _assistant_table_defn(session: Session, database: str):
"""The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
@@ -276,7 +336,9 @@ def _assistant_table_defn(session: Session, database: str):
if not isinstance(definition, dict):
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
- if not user_tables.may_open(database, session.uname, session.admin, st=definition_runtime):
+ # S67-T33: the NAV's predicate, not `may_open` alone, so a database the caller's navigation
+ # does not show is refused here too (403), not merely left off the list.
+ if not _assistant_nav_opens(session, database, definition_runtime, definition):
# Deliberately no label/owner in this response: knowing a key must not become an oracle
# for somebody else's database.
_assistant_refuse(403, "assistant_forbidden",
@@ -346,7 +408,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
- *, defn=None, lent=None):
+ *, defn=None, lent=None, mode=None, extra=None):
"""Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
@@ -382,11 +444,24 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
# THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
# A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
# about it would open a cursor to learn nothing.
- if assistant_mirror.declared_fields(database) is None:
+ if database.startswith("ut_amzn_"):
+ # ⭐ S67-T33: THE GRID'S OWN ADMISSION FIRST. A dynamically sized Amazon snapshot enters
+ # the source registry only through `require_amazon_mirror`, so without it a database
+ # never opened in this process had no binding here (measured on staging for
+ # `ut_amzn_fba_inventory`). It refuses by name (503) while the Amazon mirror is unloaded.
+ import routes_odoo_tables
+ routes_odoo_tables.require_amazon_mirror(database, session, definition=definition)
+ # ⭐ S67-T33: the stored definition rides along, so an Amazon grid or a generated
+ # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
+ declared = assistant_mirror.declared_fields(database, definition)
+ if declared is None:
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database does not store its rows and it is not one of the "
"connected databases the assistant can read from the mirror")
- cur, refusal = assistant_mirror.open_cursor(session)
+ # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
+ # first sync. Passed only for an Amazon key, which is the one family it changes.
+ cur, refusal = (assistant_mirror.open_cursor(session, database)
+ if database.startswith("ut_amzn_") else assistant_mirror.open_cursor(session))
if refusal is not None:
_assistant_refuse(refusal.status, refusal.code, refusal.message)
spec = assistant_mirror.binding(cur, database)
@@ -394,7 +469,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database is served live from its connector and this deployment "
"has no read-through binding for it yet, so it holds no rows to read")
- fields = assistant_mirror.answerable_fields(database, spec)
+ fields = assistant_mirror.answerable_fields(database, spec, declared)
if not fields:
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database's connector mirror carries none of its declared columns "
@@ -422,12 +497,36 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
if grant is None:
wall_tree = None
+ filter_keys = [field["key"] for field in visible
+ if isinstance(field, dict) and field.get("key")]
+ # ⭐⭐ S67-T34: THREE READS BEHIND THE ONE PREFLIGHT ABOVE, chosen by `mode`, all compiling the
+ # SAME wall and filter (`assistant_mirror._predicate`). "schema" reads and counts no row
+ # (D-2630: describe answers the field list of a million-row table without counting it);
+ # ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
+ # ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
try:
- rows, stamp, total = assistant_mirror.read_rows(
- session, database, cur, spec, fields,
- wall_tree=wall_tree, filter_tree=filter_tree,
- filter_keys=[field["key"] for field in visible
- if isinstance(field, dict) and field.get("key")])
+ if mode == "schema":
+ return [], fields, {"read": "connector mirror"}, definition.get("label") or database
+ if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
+ values, total = assistant_mirror.aggregate(
+ session, database, cur, spec, fields, mode[1],
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
+ if isinstance(extra, dict):
+ extra["aggregates"], extra["total"] = values, total
+ return ([], fields, {"read": "connector mirror", "rows": total},
+ definition.get("label") or database)
+ if isinstance(mode, tuple) and mode and mode[0] == "page":
+ rows, stamp, total = assistant_mirror.read_page(
+ session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
+ defn=definition)
+ if isinstance(extra, dict):
+ extra["total"] = total
+ else:
+ rows, stamp, total = assistant_mirror.read_rows(
+ session, database, cur, spec, fields,
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
+ defn=definition)
except assistant_mirror.MirrorRefusal as exc:
_assistant_refuse(exc.status, exc.code, exc.message)
raise # unreachable; see above
@@ -459,7 +558,33 @@ def _assistant_requested_fields(fields, visible_fields):
return out
-def assistant_read_scope(session: Session, database, fields=None, filters=None):
+def _assistant_oversize_reason(database: str, definition, lent) -> str:
+ """The named cause an oversize stored table carries, or "". S67-T33 / rule 1.
+
+ ⛔ AN EVICTED TABLE KEEPS ITS DEFINITION AND LOSES ITS ROWS, so the materialised reader used
+ to answer it as a complete table of 0 rows. `user_tables.limit_report` is the one evaluator
+ of that stamp (the grid serves the same sentence), so its cause and recommendation are read
+ from there rather than re-worded here.
+ """
+ from core import user_tables
+
+ if not user_tables.oversize_of(database, defn=definition if isinstance(definition, dict)
+ else {}):
+ return ""
+ report = user_tables.limit_report(database, st=lent) or {}
+ label = str((definition or {}).get("label") or database)
+ return (f"{label} is not stored: {report.get('cause') or 'its rows were not stored'}. "
+ f"{str(report.get('recommendation') or '').rstrip('.')}. The assistant refuses "
+ f"rather than answering 0 rows").replace("\u2014", ",").replace("\u2013", "-")
+
+
+def _assistant_refuse_oversize(database: str, definition, lent) -> None:
+ reason = _assistant_oversize_reason(database, definition, lent)
+ if reason:
+ _assistant_refuse(409, "assistant_source_too_large", reason)
+
+
+def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None):
"""Return a detached, permissioned snapshot for one Assistant tool call.
Signature (the C8 contract for E):
@@ -468,6 +593,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
permission_scope_applied, fields, filters, records
}
+ ⭐ S67-T34: `mode` changes only HOW a mirror-served database is read, never who may read it
+ (every wall below runs unchanged): "schema" returns the field contract with no row read,
+ ("aggregate", specs) adds `server_aggregates` and `total_rows` computed in SQL over every
+ permitted row, ("page", offset, limit) returns one SQL page plus `total_rows`. Every other
+ source kind ignores it and answers as before.
+
`database` is exactly one source. `fields` is an explicit list of visible field keys (or
``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
@@ -478,6 +609,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
from harness import filter_eval
key = str(database or "").strip()
+ extra = {}
if key in _ASSISTANT_BUILT_INS:
rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
label = _ASSISTANT_BUILT_INS[key]["label"]
@@ -494,12 +626,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
from core import user_tables
_defn, _lent = _assistant_table_defn(session, key)
if user_tables.materialises(key, st=_lent, defn=_defn):
+ # S67-T33: a stored table whose rows were evicted refuses by name, never answers 0.
+ # Only on the STORED reader: a read-through table's rows come from the mirror.
+ _assistant_refuse_oversize(key, _defn, _lent)
rows, canonical_fields, source_version, label = _assistant_user_table(
session, key, fields, filters, defn=_defn, lent=_lent)
source_kind = "materialised_user_table"
else:
rows, canonical_fields, source_version, label = _assistant_mirror_table(
- session, key, fields, filters, defn=_defn, lent=_lent)
+ session, key, fields, filters, defn=_defn, lent=_lent, mode=mode, extra=extra)
source_kind = "mirror_user_table"
else:
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
@@ -556,7 +691,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
records.append({"pid": copy.deepcopy(row["pid"]),
**{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
- return {
+ out = {
"database": key,
"label": str(label),
"source_kind": source_kind,
@@ -568,6 +703,11 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
"filters": copy.deepcopy(requested_filter),
"records": records,
}
+ if "aggregates" in extra:
+ out["server_aggregates"] = copy.deepcopy(extra["aggregates"])
+ if "total" in extra:
+ out["total_rows"] = int(extra["total"])
+ return out
def assistant_permitted_sources(session: Session, databases):
@@ -672,6 +812,19 @@ def assistant_source_status(session: Session, databases=None):
mirror_cur = None
return mirror_cur
+ amazon_cur = {}
+
+ def _amazon_mirror(key):
+ """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
+ if "cur" not in amazon_cur:
+ try:
+ import assistant_mirror
+ cur, refusal = assistant_mirror.open_cursor(session, key)
+ amazon_cur["cur"] = None if refusal is not None else cur
+ except Exception: # noqa: BLE001
+ amazon_cur["cur"] = None
+ return amazon_cur["cur"]
+
def _definitions():
nonlocal definitions, read_definitions
if not read_definitions:
@@ -752,6 +905,10 @@ def assistant_source_status(session: Session, databases=None):
entry = cache.get(cache_key) if isinstance(cache, dict) else None
if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
row["answerable"] = True
+ elif isinstance(cache, dict):
+ # ⭐ S67-T33 / D-778: a miss is no longer a refusal; the read builds the snapshot
+ # through the grid's own pool builder, so this source is answerable now.
+ row["answerable"] = True
else:
row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
continue
@@ -762,10 +919,16 @@ def assistant_source_status(session: Session, databases=None):
if not isinstance(definition, dict):
continue
lent = user_tables.lend(session.runtime, user_tables=defs)
- if not user_tables.may_open(key, session.uname, session.admin, st=lent):
+ # ⭐ S67-T33 / R33: the NAV's predicate, so this list and the navigation are one set.
+ if not _assistant_nav_opens(session, key, lent, definition):
continue
row["permitted"] = True
if user_tables.materialises(key, st=lent, defn=definition):
+ oversize = _assistant_oversize_reason(key, definition, lent)
+ if oversize:
+ # Rule 1's second sentence: listed, with its cause, never answered as 0 rows.
+ row["reason"] = oversize
+ continue
row["answerable"] = True
continue
# ⭐⭐ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
@@ -778,7 +941,9 @@ def assistant_source_status(session: Session, databases=None):
# is transient and retrying works, while a grid this deployment has no read-through
# binding for will not fix itself by waiting. One sentence for both would make the first
# look permanent, which is the failure mode the ruling exists to prevent.
- cur = _mirror()
+ # ⭐ S67-T33: an Amazon grid asks ITS OWN mirror (`_table_cursor`), never the Odoo
+ # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
+ cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
if cur is None:
row["reason"] = ("this database is read live from its connector mirror, and that "
"mirror is not open on this deployment right now; try again shortly")
@@ -801,6 +966,49 @@ def assistant_source_status(session: Session, databases=None):
return out
+#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
+#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
+#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
+#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
+#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
+#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
+#: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
+#: answers 503 when the store is unreadable); a cached registry is never served in its place.
+#: The key carries the identity of the reader and the store it reads, so a swapped store or a
+#: patched `users.registry` / `store.get` misses instead of replaying another registry.
+_SESSION_REGISTRY = {"key": None, "reg": None}
+_SESSION_REGISTRY_LOCK = __import__("threading").Lock()
+
+
+def _session_registry():
+ """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed.
+
+ ⛔ POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`),
+ and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
+ fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
+ memo on HF), and its `get()` already holds the bucket for the life of the process, so the
+ re-read this memo removes does not exist there: it reads straight through.
+ """
+ key = None
+ try:
+ backend = store.backend()
+ token = (store.revision("users") or {}).get("token") if backend == "pg" else None
+ if token not in (None, ""):
+ where = os.environ.get("AIOS_TENANT") or "royal-imports"
+ key = (backend, where, id(users.registry), id(store.get), str(token))
+ except Exception: # noqa: BLE001 -- fail-closed below
+ key = None
+ if key is not None:
+ with _SESSION_REGISTRY_LOCK:
+ if _SESSION_REGISTRY["key"] == key:
+ return _SESSION_REGISTRY["reg"]
+ reg = users.registry() or {}
+ if key is not None:
+ with _SESSION_REGISTRY_LOCK:
+ _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
+ return reg
+
+
def _user_for(claims):
"""The account named by a verified cookie, or None — with the epoch check that makes a
stateless cookie revocable.
@@ -829,11 +1037,16 @@ def _user_for(claims):
if (qa and claim_tenant == qa.get("tenant")
and int(claims.get("e") or 0) == int(qa.get("epoch") or 0)):
return qa
+ # ⛔ HOTFIX 2026-09-28: AN UNREADABLE REGISTRY IS A 503, NEVER "NO SUCH ACCOUNT". This caught
+ # `Exception` into `{}`, so a store `PoolTimeout` on staging answered a valid cookie with 401
+ # `invalid_session` and the client signed the owner out. Still fail-closed: nobody is admitted.
try:
- reg = users.registry() or {}
- except Exception:
- reg = {}
- rec = reg.get(uname)
+ reg = _session_registry()
+ except Exception as exc:
+ print(f"[aios-api] session account store unavailable: {type(exc).__name__}: {exc}")
+ raise err(503, "session_store_unavailable",
+ "your account could not be checked just now. Try again in a moment")
+ rec = copy.deepcopy(reg.get(uname)) # the registry may be the held copy: never mutate it
if (rec and rec.get("active", True)
and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
pub = users._public(uname, rec)
diff --git a/api/formula_runtime.py b/api/formula_runtime.py
index a306d38c44d0bca2e737fefcd5024c2515ff22e0..7f12e7df4885a3d442cb6a74dfc47c66d9a3b4d6 100644
--- a/api/formula_runtime.py
+++ b/api/formula_runtime.py
@@ -83,6 +83,8 @@ FN_ARITY = {
"ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
"FILTER": (2, 3), "SORT": (1, 4),
+ # S67-T24: the reference workbooks' `IF(ISNUMBER(SEARCH(...)))` and `ISFORMULA(...)`.
+ "SEARCH": (2, 3), "SEARCHB": (2, 3), "ISFORMULA": (1, 1),
}
# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
@@ -457,6 +459,15 @@ FN_CATALOGUE = {
"description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
"ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
"description": "Returns TRUE when a value is text.", "contexts": _BOTH},
+ "ISFORMULA": {"category": "Information", "signature": "ISFORMULA(reference)",
+ "description": "Returns TRUE when the cell it names holds a formula.",
+ "contexts": _SHEET},
+ "SEARCH": {"category": "Text", "signature": "SEARCH(find_text, within_text, [start_num])",
+ "description": "Returns the position of one text value inside another, ignoring "
+ "case; ? matches one character and * any run.", "contexts": _BOTH},
+ "SEARCHB": {"category": "Text", "signature": "SEARCHB(find_text, within_text, [start_num])",
+ "description": "Counts like SEARCH; each character is one byte in this workbook.",
+ "contexts": _BOTH},
"ISERROR": {"category": "Information", "signature": "ISERROR(value)",
"description": "Returns TRUE when a formula would refuse to produce a value.",
"contexts": _BOTH},
@@ -625,6 +636,9 @@ PARITY_FIXTURES = {
"EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
"ISNUMBER": ("ISNUMBER(1)", "TRUE"),
"ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
+ "SEARCH": ("SEARCH(\"B\", \"abc\")", 2),
+ "SEARCHB": ("SEARCHB(\"c?\", \"abcd\")", 3),
+ "ISFORMULA": ("ISFORMULA(B2)", "FALSE"),
"ISERROR": ("ISERROR(1 / 0)", "TRUE"),
}
@@ -917,7 +931,7 @@ RANGE_ARGS = {
#: ⭐ THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
#: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
-ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,)}
+ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,), "ISFORMULA": (0,)}
def _is_range_position(fn, index):
@@ -1792,6 +1806,11 @@ def _eval_value(node, row, today, diagnostics=None):
return _array_binary(a, b, node["op"], diagnostics)
if isinstance(row, _SheetInputs):
def comparison(value):
+ # HOTFIX 2026-09-28: AN EMPTY CELL COMPARES AS EMPTY TEXT, exactly as the browser's
+ # `formulaEval.ts` does ("value" in cell ? cell.value : ""). As a number it made
+ # `=IF(E13="",E12,E13)` FALSE here and TRUE in the browser, so Daily Sales showed a
+ # date the server refused and every binding filtered on row 7 read #ERROR!.
+ if value is None: return (1, "")
if isinstance(value, bool): return (2, value)
if isinstance(value, str): return (1, value.casefold())
return (0, _excel_number(value or 0))
@@ -1858,6 +1877,17 @@ def _eval_value(node, row, today, diagnostics=None):
for code in local:
_diagnose(diagnostics, code)
return "" if first is None and isinstance(row, _SheetInputs) and not local else first
+ if fn in ("ISNUMBER", "ISTEXT") and len(node.get("args") or ()) == 1:
+ # ⭐ S67-T24: AN IS-FUNCTION NEVER PASSES AN ERROR ON. Excel answers FALSE for
+ # `ISNUMBER(#VALUE!)`, which is what makes `IF(ISNUMBER(SEARCH(x, y)), ...)` (2,512 cells
+ # in the Inventory System's Catalog) take its FALSE branch instead of failing whole.
+ local = []
+ try:
+ value = _eval(node["args"][0], row, today, local)
+ except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error,
+ FormulaError, _ReferenceFault):
+ return False
+ return False if local else _call(fn, [value], today, diagnostics)
if fn == "ISERROR":
# The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
local = []
@@ -1888,6 +1918,11 @@ def _eval_value(node, row, today, diagnostics=None):
_diagnose(diagnostics, "bad_value")
return None
args.append(here)
+ if fn == "ISFORMULA":
+ # ⭐ S67-T24: ASKED OF THE CELL, NOT OF ITS VALUE. Only a Sheet document knows which
+ # cells hold formulas (`SheetCalculation.specs`); a grid row has no formula cells.
+ spec = row.engine.specs.get(args[0]) if isinstance(row, _SheetInputs) and args[0] else None
+ return bool(isinstance(spec, dict) and spec.get("formula"))
return _call(fn, args, today, diagnostics)
# ⭐ RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
# `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
@@ -1923,6 +1958,27 @@ def _eval_value(node, row, today, diagnostics=None):
raise FormulaError("formula AST node is not supported")
+def _display_text(value):
+ """A calculated value as the grid shows it: Excel's TRUE / FALSE for a boolean (S67-T24; a
+ Python `str(True)` painted "True"), blank for nothing, else the value's own text."""
+ if value is None:
+ return ""
+ if isinstance(value, bool):
+ return "TRUE" if value else "FALSE"
+ return str(value)
+
+
+def _search_pattern(text):
+ """SEARCH's wildcard text as a regex: `?` one character, `*` any run, `~x` a literal x."""
+ out, index = [], 0
+ while index < len(text):
+ ch = text[index]
+ if ch == "~" and index + 1 < len(text) and text[index + 1] in "?*~":
+ out.append(re.escape(text[index + 1])); index += 2; continue
+ out.append(".*?" if ch == "*" else "." if ch == "?" else re.escape(ch)); index += 1
+ return "".join(out)
+
+
def _addr_arg(node, diagnostics):
"""The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
if isinstance(node, dict) and node.get("t") == "a1":
@@ -2319,6 +2375,19 @@ def _call(fn, a, today, diagnostics=None):
if fn == "FIND":
start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
return pos+1 if pos >= 0 else None
+ if fn in ("SEARCH", "SEARCHB"):
+ # Excel's SEARCH: case-insensitive, `?` one character, `*` any run, `~` escapes the next;
+ # absent text or a start outside the text is #VALUE!. SEARCHB counts bytes only in a
+ # double-byte locale, so in this one it is SEARCH.
+ within = _text(a[1])
+ start = _strict_num(a[2], diagnostics) if len(a) > 2 else 1
+ if start is None or not 1 <= int(start) <= len(within) + 1:
+ _diagnose(diagnostics, "bad_value"); return None
+ found = re.compile(_search_pattern(_text(a[0])), re.IGNORECASE | re.DOTALL).search(
+ within, int(start) - 1)
+ if not found:
+ _diagnose(diagnostics, "bad_value"); return None
+ return found.start() + 1
if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
if fn == "REGEX_EXTRACT": return m.group(0) if m else None
@@ -3722,7 +3791,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
if excel: cell["error"]["excelCode"] = excel
else:
- cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"]))
+ cell.update(state="fresh", display=_display_text(cell["value"]))
cell.pop("refusal", None)
cell.pop("error", None)
cell.pop("spill", None)
@@ -3765,7 +3834,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
base = engine.specs.get(child, {})
style = {**(owner.get("style") or {}), **(base.get("style") or {})}
record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
- "display": "" if engine.values.get(child) is None else str(engine.values[child]),
+ "display": _display_text(engine.values.get(child)),
"spillParent": origin, "style": style,
"formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
**({"base": base} if base else {})}}}
diff --git a/api/main.py b/api/main.py
index 4bc409a04a6922346510f13cc3b2a8b56a06e6f8..6eb60bb1df7f5c3690273f0435da5bc8c34318a8 100644
--- a/api/main.py
+++ b/api/main.py
@@ -92,6 +92,7 @@ from core import store # noqa: E402
# 409: a view write whose caller was looking at an older revision. See its handler below.
from core import view_audit # noqa: E402
from deps import Session, module_gate # noqa: E402
+from deps import require_session as deps_require_session # noqa: E402 (S67-T20 freshness door)
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
@@ -351,6 +352,21 @@ app.include_router(routes_uploads.router)
# control is that the enumeration must NAME every router in this file.**
app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated
app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory
+
+
+@app.get("/api/v1/connectors/odoo/freshness")
+def odoo_sync_freshness(session: Session = Depends(deps_require_session)):
+ """⭐⭐ S67-T20 / C4 — IS THIS WORKSPACE'S ODOO DATA STALE, AND DID THE LAST SYNC FAIL.
+
+ `{stale, lastGoodSyncAt, schedule, staleAfterSeconds, lastRunAt, phase, frozen, failure}`
+ from `odoo_relational.sync_freshness`. The Connectors card and Connectors > Odoo word it.
+ ⚠ A SIBLING OF `GET /connectors/odoo/status`, not a field on it, because that route is another
+ lane's file this wave; the integrator may fold these fields into it (COMBINE-AMEND, S67-T20).
+ Same administrator door as that route.
+ """
+ routes_connectors._odoo_admin(session)
+ import odoo_relational as _rel_fresh
+ return _rel_fresh.sync_freshness(session.runtime)
# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so
# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
@@ -527,6 +543,45 @@ def _assert_contract_floor():
"Run aios-web/verify_fields_contract.py.")
+def _assert_store_guards(cadence=None):
+ """Refuse an unbounded store or a cadence below the deployment's own floor."""
+ from core import store_pg
+
+ budget = store_pg.byte_budget_status()
+ if not budget["armed"]:
+ raise RuntimeError("byte budget disarmed: " + "; ".join(budget["reasons"]))
+
+ # Read the deployment independently of the resolver: a broken resolver must
+ # not be able to supply both sides of its own comparison.
+ raw = os.environ.get("AIOS_STORE_RESYNC_SECONDS")
+ deployed = None
+ if raw is not None:
+ try:
+ deployed = int(raw.strip())
+ except (TypeError, ValueError):
+ raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
+ "positive integer") from None
+ if deployed <= 0:
+ raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
+ "positive integer")
+ if cadence is None:
+ cadence = _resolve_resync_cadence()
+ resolved = cadence.get("resolvedSeconds")
+ if type(resolved) is not int or resolved <= 0:
+ raise RuntimeError("resync guard disarmed: resolved cadence must be a positive integer")
+ if deployed is not None and resolved < deployed:
+ raise RuntimeError(f"resync guard disarmed: resolved cadence {resolved}s is below "
+ f"deployed floor {deployed}s")
+ deployment = f"{deployed}s deployed" if deployed is not None else "not deployed"
+ return {"ok": True, "byte_budget": budget,
+ "resync": {"ok": True, "resolved_seconds": resolved, "deployed_seconds": deployed},
+ "message": ("Byte budget armed: yes. "
+ f"Table {budget['max_table_bytes'] / 1048576:g} MB; "
+ f"document {budget['max_document_bytes'] / 1048576:g} MB; "
+ f"store ceiling {budget['document_ceiling_bytes'] / 1048576:g} MB. "
+ f"Resync: {resolved}s resolved vs {deployment}.")}
+
+
def _startup_notes():
"""Say the two things an operator must know, once, at import — never in a response body."""
if aios_session.EPHEMERAL_SECRET:
@@ -535,10 +590,6 @@ def _startup_notes():
"Set it in production.")
-_assert_contract_floor()
-_startup_notes()
-
-
# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
# page cannot live behind a password prompt, so the shell's own assets must be public — which they
@@ -746,6 +797,7 @@ def _pull_amazon(why):
try:
from harness import amazon_store as _amazon
from harness import runtime as _runtime
+ import routes_odoo_tables as _rot
# The existing boot/resync projection targets tenant #0. Keep credential, mirror and
# projection on that same tenant; never fall back to an operator's environment credential.
tenant_key = "royal-imports"
@@ -758,7 +810,19 @@ def _pull_amazon(why):
# bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
# ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
# the markers live in the container's mirror, so every restart began again from orders.
- if _amazon.needs_first_load(tenant_key, rt=rt):
+ first_load_needed = _amazon.needs_first_load(tenant_key, rt=rt)
+ # D-3495: `needs_first_load` (and `sync`, below) call `restore_mirror` as their very
+ # first step, which writes the Amazon mirror tables directly on the shared datastore
+ # instance. Nothing else republished the read-through registry until the Odoo relational
+ # rebuild's `refresh_state` ran, minutes later (`_rebuild_odoo_relational`, further down
+ # this boot) — so a grid read landing in between (staging, 2026-09-29: `ut_amzn_orders`
+ # 409 for ~30 min after "Amazon mirror restored") answered stale or absent. Invalidate
+ # right here, cheaply: this only clears the registry and its built flag, so the next read
+ # (or `_sources()` below) does a fresh per-table `count(*)` probe against the mirror that
+ # was just restored — never the slow `plan`/`apply_plan` write `refresh_state` does.
+ _rot.invalidate_sources()
+ _rot._sources()
+ if first_load_needed:
from core import connector_sync as _connector_sync
receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
print(f"[aios-api] amazon sync ({why}): first load "
@@ -766,6 +830,8 @@ def _pull_amazon(why):
return True
rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
+ _rot.invalidate_sources()
+ _rot._sources()
for problem in rep.get("problems") or []:
print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
@@ -1184,7 +1250,7 @@ def _sweep_automation_schemas(why):
def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
- tenant=None, cadence_seconds=None):
+ tenant=None, cadence_seconds=None, table_keys=None):
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
could have asked" are different diagnoses.
@@ -1207,13 +1273,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
from core import user_tables as _ut
default_cid = routes_connectors._odoo_connection_slot(_rt, None)
connection_ids = [default_cid]
- try:
- import core.keychain as _keychain
- connection_ids.extend(str(row.get("connection_id") or row.get("id"))
- for row in _keychain.list_entries(_rt)
- if row.get("type") == "odoo")
- except Exception: # noqa: BLE001
- pass
+ if table_keys is None:
+ try:
+ import core.keychain as _keychain
+ connection_ids.extend(str(row.get("connection_id") or row.get("id"))
+ for row in _keychain.list_entries(_rt)
+ if row.get("type") == "odoo")
+ except Exception: # noqa: BLE001
+ pass
connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
def _health(cid, stage, state, **kwargs):
@@ -1240,10 +1307,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
# because a gate that counts RAW occurrences of a call would count the one in this
# comment too. That gate happens to strip comments first; the next one may not.
# See `odoo_relational.refresh` and its `all_connections` arm.
- counts = _rel.refresh(_rt, tenant_key, all_connections=True)
- raw_models, schema_errors = _rel.authoritative_schema_snapshot()
- _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
- errors=schema_errors, st=_rt)
+ # Only the designated default owns this mirror. A selection must never be lent to
+ # another connection; the full recovery path still refreshes every connection.
+ counts = _rel.refresh(_rt, tenant_key, all_connections=table_keys is None,
+ **({"table_keys": table_keys} if table_keys is not None else {}))
+ if table_keys is None:
+ raw_models, schema_errors = _rel.authoritative_schema_snapshot()
+ _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
+ errors=schema_errors, st=_rt)
rebuild_errors = []
# ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
#
@@ -1319,14 +1390,25 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
cid = str(cid or default_cid)
report = report if isinstance(report, dict) else {}
failed = report.get("failed")
+ # S67-T19: a read-through `limits` notice is the design working, not a failed
+ # database; health reads only the entries that withheld rows (`_rel.health_view`).
database_failures, unmatched_failures = (
- routes_connectors._odoo_result_database_failures(_rt, cid, report))
+ routes_connectors._odoo_result_database_failures(
+ _rt, cid, _rel.health_view(report)))
failure = (routes_connectors._odoo_failure(failed, "materialization")
if failed else None)
- if not failure and (database_failures or unmatched_failures):
+ # S67-T19: count drift within tolerance is recorded per database (below) and does not
+ # fail the connection; a real failure or a stale mirror's gap still does.
+ if not failure and (_rel.blocking_failures(database_failures, report)
+ or unmatched_failures):
failure = routes_connectors._odoo_failure(
"one or more databases failed", "materialization")
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
+ for key, delta in sorted(tables.items()):
+ if isinstance(delta, dict):
+ print(f"[aios-api] odoo relational table rebuilt ({why}): {key} "
+ + str({name: delta.get(name, 0)
+ for name in ("added", "updated", "removed")}))
database_receipts = report.get("databases")
database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
canonical_receipts = report.get("canonicalDatabases")
@@ -1395,6 +1477,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
_ut.record_odoo_sync_status(
phase="error" if safe_errors else "ok", errors=safe_errors,
st=_rt, connection_id=cid)
+ # S67-T20 / C4: the tenant-wide freshness stamp follows the DEFAULT connection, the
+ # one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
+ if cid == str(default_cid):
+ try:
+ _rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at)
+ except Exception: # noqa: BLE001
+ pass
meta_state = _rebuild_meta_relational(
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
if isinstance(meta_state_out, dict):
@@ -1424,26 +1513,96 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
_ut.record_odoo_sync_status(
phase="error", errors=[f"{failure['code']}: {failure['message']}"],
st=_rt, connection_id=cid)
+ # S67-T20 / C4: a failed rebuild is a sync that did NOT end well.
+ import odoo_relational as _rel_failed
+ _rel_failed.record_sync_outcome(_rt, ok=False)
except Exception:
pass
return False
-# W39-T31 / A63. `sync_all` already knows whether Odoo supplied any change, but this loop used to
-# rebuild every relational table (and its large `user_tables` document) after every poll anyway.
-# A nonempty all-live zero report is the only evidence strong enough to skip that Odoo half. An
-# absent report is not zero: it means manual mode, paused source, or a failed/unknown sync, all of
-# which force the pre-existing rebuild path. `delete_report=None` is one of the ordinary passes
-# between delete reconciliations; it is unknown, not falsely claimed clean.
+# D-780: fetched rows include an idempotent cursor overlap. Only mirror content compared with
+# the last successful apply proves equality; scalar backfills need not move write_date.
_RESYNC_REBUILD_MAX_AGE_SECONDS = 24 * 60 * 60
-def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
+# Dependencies of odoo_relational.plan/read_* and the two read-through table specs. Customers
+# and products are compiled databases, no longer refreshable ut_odoo_* tables. Unknown entities
+# (including their sources) deliberately select full recovery, never an empty selection.
+_RESYNC_ENTITY_TABLE_KEYS = {
+ "account_move": ("ut_odoo_invoices", "ut_odoo_bills", "ut_odoo_vendors"),
+ "sale_order": ("ut_odoo_orders",),
+ "res_partner": ("ut_odoo_invoices", "ut_odoo_orders", "ut_odoo_agents", "ut_odoo_vendors"),
+ "account_invoice_line_agent": ("ut_odoo_invoices", "ut_odoo_agents"),
+ "account_account": ("ut_odoo_accounts",),
+ "sale_order_line": ("ut_odoo_order_lines",),
+ "account_move_line": ("ut_odoo_gl_lines",),
+}
+
+
+def _resync_source_fingerprints(sync_report, cur=None):
+ """Count and hash every column, without fetching rows out of the local mirror.
+
+ Use the sync writer's cursor factory (also used for status/validation), never a second
+ DuckDB instance or a tenant-store read. A missing optional table has an explicit fingerprint;
+ a failed read returns unknown and forces recovery. A lent fixture cursor is never closed.
+ """
+ owned = cur is None
+ try:
+ if owned:
+ from harness import datastore as _ds
+ cur = _ds.connect()
+ cur.execute("BEGIN TRANSACTION")
+ available = {row[0] for row in cur.execute(
+ "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main'"
+ ).fetchall()}
+ entities = set(_RESYNC_ENTITY_TABLE_KEYS) | set(sync_report or {})
+ fingerprints = {}
+ for entity in sorted(entities):
+ if entity not in available:
+ fingerprints[entity] = None
+ continue
+ quoted = '"' + entity.replace('"', '""') + '"'
+ fingerprints[entity] = tuple(cur.execute(
+ f"SELECT count(*), coalesce(bit_xor(hash(t)), 0) FROM {quoted} AS t"
+ ).fetchone())
+ if owned:
+ cur.execute("COMMIT")
+ return fingerprints
+ except Exception as exc: # noqa: BLE001
+ print(f"[aios-api] odoo source fingerprint unconfirmed ({type(exc).__name__})")
+ return {}
+ finally:
+ if owned and cur is not None:
+ cur.close()
+
+
+def _resync_changed_entities(previous_fingerprints, current_fingerprints):
+ """The union catches disappearing entities as well as additions and scalar changes."""
+ previous = previous_fingerprints or {}
+ current = current_fingerprints or {}
+ return sorted(entity for entity in set(previous) | set(current)
+ if entity not in previous or entity not in current
+ or previous[entity] != current[entity])
+
+
+def _resync_table_keys(changed_entities):
+ """None means full recovery; an unmapped entity can never suppress an apply."""
+ if not changed_entities or any(entity not in _RESYNC_ENTITY_TABLE_KEYS
+ for entity in changed_entities):
+ return None
+ return sorted({key for entity in changed_entities for key in _RESYNC_ENTITY_TABLE_KEYS[entity]})
+
+
+def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now,
+ previous_fingerprints=None, current_fingerprints=None):
"""Return a fail-safe Odoo-rebuild reason, or ``None`` for a proven no-op.
The policy stays pure so it can be tested without Odoo, DuckDB, or PostgreSQL. Hard deletes
are noticed by the existing every-fourth-pass reconciliation; an intervening pass does not
- pretend that it checked them, preserving the old visibility bound.
+ pretend that it checked them, preserving the old visibility bound. The four-argument legacy
+ caller retains its conservative fetched-row signal; the resident loop always supplies a
+ fingerprint map (an empty map means a failed read, never equality).
"""
if not isinstance(sync_report, dict) or not sync_report:
return "sync-unconfirmed"
@@ -1451,7 +1610,7 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
if not isinstance(report, dict) or report.get("phase") != "live":
return "sync-unconfirmed"
try:
- if int(report.get("pulled") or 0) > 0:
+ if int(report.get("pulled") or 0) > 0 and current_fingerprints is None:
return "source-change"
except (TypeError, ValueError):
return "sync-unconfirmed"
@@ -1463,8 +1622,20 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
return "delete-change"
except (TypeError, ValueError):
return "sync-unconfirmed"
+ if current_fingerprints is not None:
+ if not isinstance(current_fingerprints, dict) or not current_fingerprints:
+ return "fingerprint-unconfirmed"
+ if not isinstance(previous_fingerprints, dict) or not previous_fingerprints:
+ return "no-fingerprint"
if last_success_at is None or now - last_success_at >= _RESYNC_REBUILD_MAX_AGE_SECONDS:
return "daily-recovery"
+ if current_fingerprints is not None:
+ # Kept inline so the pure decision can still be AST-extracted on its own by old gates.
+ changed = sorted(entity for entity in set(previous_fingerprints) | set(current_fingerprints)
+ if entity not in previous_fingerprints or entity not in current_fingerprints
+ or previous_fingerprints[entity] != current_fingerprints[entity])
+ if changed:
+ return "source-change: " + ",".join(changed)
return None
@@ -1619,6 +1790,14 @@ def _resolve_resync_cadence():
"manual": manual, "disagreement": disagreement}
+# All import-time refusals run before any boot or resync thread can start.
+_assert_contract_floor()
+STORE_GUARDS = _assert_store_guards()
+from core import neon_usage as _neon_usage # noqa: E402
+_neon_usage.set_guards(STORE_GUARDS)
+_startup_notes()
+
+
def _resync_mirror_pass(manual, passes):
"""ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
@@ -1729,6 +1908,7 @@ def _store_resync_loop():
from core import connector_sync
passes = 0
last_relational_success_at = None
+ last_source_fingerprints = None
last_meta_fingerprint = None
last_meta_applied_at = None
# ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
@@ -1844,6 +2024,7 @@ def _store_resync_loop():
if _odoo_requested is None:
_odoo_requested = _run_sync_requests()
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
+ source_fingerprints = _resync_source_fingerprints(sync_report)
RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
RESYNC_STATE["lastPassOk"] = _err is None
RESYNC_STATE["lastError"] = _err
@@ -1896,14 +2077,19 @@ def _store_resync_loop():
_meta_previous = last_meta_fingerprint if _meta_reason is None else None
_amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
_reason = _resync_rebuild_reason(sync_report, delete_report,
- last_relational_success_at, _meta_now)
+ last_relational_success_at, _meta_now,
+ last_source_fingerprints, source_fingerprints)
# The scheduled pass may now be quiet because Sync now just advanced this mirror.
# Its zero-change report cannot suppress projection of the explicit Odoo pull.
if _odoo_requested:
_reason = "sync now"
_meta_state = {}
if _reason is not None:
- print(f"[aios-api] odoo relational rebuild required ({_reason})")
+ _table_keys = (_resync_table_keys(_resync_changed_entities(
+ last_source_fingerprints, source_fingerprints))
+ if _reason.startswith("source-change:") else None)
+ print(f"[aios-api] odoo relational rebuild required ({_reason}); "
+ f"table_keys={_table_keys if _table_keys is not None else 'all'}")
# Test/recovery harnesses replace the rebuild seam with its historical three-argument
# callable. Bind tenant/cadence whenever the active implementation accepts them,
# without turning an internal TypeError into a second materialization attempt.
@@ -1913,14 +2099,20 @@ def _store_resync_loop():
p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
_rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
if _rebuild_variadic or "tenant" in _rebuild_params else {})
+ if _rebuild_variadic or "table_keys" in _rebuild_params:
+ _rebuild_scope["table_keys"] = _table_keys
if _rebuild_odoo_relational(
"resync", _meta_previous, _meta_state, **_rebuild_scope):
- last_relational_success_at = _t.time()
+ last_source_fingerprints = source_fingerprints or None
+ # Partial successes cannot postpone the mandatory full daily recovery forever.
+ if _table_keys is None:
+ last_relational_success_at = _t.time()
else:
# A prior successful projection no longer proves the changed mirror was applied.
# Keep recovery due on the next cadence even when that sync has no new rows.
last_relational_success_at = None
else:
+ print("[aios-api] odoo relational rebuild skipped (unchanged)")
# A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
# whether this cadence may skip both the full-document write and relation pass.
try:
@@ -2113,10 +2305,9 @@ if os.environ.get("AIOS_PREWARM") == "1":
#
# ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
# is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
-# SETS it. TWO STRUCTURAL CAUSES REMAIN OPEN AS REGISTER ROWS, and neither is a hotfix:
-# * `_resync_rebuild_reason` treats `pulled > 0` as a change, but `pulled` counts rows FETCHED,
-# not rows CHANGED — live rebuilt on passes its own report scored `added/updated/removed: 0`.
-# Narrowing it is what would make an hourly cadence nearly free.
+# SETS it. D-780 now compares local content fingerprints with the last successful apply and
+# logs unchanged skips; changed mapped entities select only their dependent table keys.
+# The other structural cost remains outside that skip policy:
# * the whole-document store write itself. A partial-JSONB write is a wave.
# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
if os.environ.get("AIOS_STORE_RESYNC") == "1":
diff --git a/api/odoo_relational.py b/api/odoo_relational.py
index 5cce979f40292f783c2a2da6c572faec3476cb8a..de4fc7159627e489e0a2c628da3cd6822d6ab9af 100644
--- a/api/odoo_relational.py
+++ b/api/odoo_relational.py
@@ -60,6 +60,7 @@ RI_SLUGS = ("", "royal-imports")
INVOICES_KEY = "ut_odoo_invoices"
CUSTOMERS_KEY = "ut_odoo_customers"
ORDERS_KEY = "ut_odoo_orders"
+SHIPMENTS_KEY = "ut_odoo_shipments"
PRODUCTS_KEY = "ut_odoo_products"
#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
@@ -169,6 +170,34 @@ def _refreshed_field():
"default": False, "description": "When this row was last reconciled against Odoo."}
+def shipment_fields():
+ """Outgoing pickings, with elapsed delivery time measured at actual completion."""
+ return [_preset({"source": "overlay", **field}) for field in (
+ {"key": "picking_no", "type": "text", "label": "Shipment", "pinned": True,
+ "default": True},
+ {"key": "odoo_id", "type": "int", "label": "Odoo ID", "default": False,
+ "description": "The `stock.picking` id. Also this row's id."},
+ {"key": "origin", "type": "text", "label": "Source document", "default": True},
+ {"key": "order_no", "type": "text", "label": "Order", "default": True},
+ {"key": "order_id", "type": "int", "label": "Odoo order id", "default": False},
+ {"key": "order_link", "type": "link", "label": "Order record", "default": True,
+ "link": {"on": "odoo_id", "from": "order_id", "table": ORDERS_KEY}},
+ {"key": "customer", "type": "text", "label": "Customer", "default": True},
+ {"key": "partner_id", "type": "int", "label": "Odoo partner id", "default": False},
+ {"key": "customer_link", "type": "link", "label": "Customer record", "default": False,
+ "link": {"on": "pid", "from": "partner_id", "table": "customer_data"}},
+ {"key": "state", "type": "select", "label": "State", "default": True,
+ "options": ["draft", "waiting", "confirmed", "assigned", "done", "cancel"]},
+ {"key": "scheduled_date", "type": "date", "label": "Scheduled date", "default": True},
+ {"key": "date_done", "type": "date", "label": "Ship date", "default": True},
+ {"key": "order_date", "type": "date", "label": "Order date", "default": True},
+ {"key": "delivery_days", "type": "float", "agg": "avg", "label": "Delivery days",
+ "default": True, "description": "Elapsed days from the sale order date to the actual "
+ "shipment completion. Blank when either date is missing."},
+ _refreshed_field(),
+ )]
+
+
def agent_fields():
"""One row per SALES AGENT, keyed on the `res.partner` id.
@@ -1854,6 +1883,9 @@ ODOO_TABLE_MODELS = {
ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
}
+# Shipments deliberately stay outside this map and TABLES: its consumers route mapped models
+# through the DuckDB refresh and suppress their discovered-model reads. There is no picking
+# mirror. The live shipment writer below stamps its model directly on the stored connector.
def authoritative_schema_snapshot():
@@ -2589,6 +2621,7 @@ def _aios_grid():
#: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids.
_REGISTRY_COL_TYPE = {
"int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE",
+ "float": "DOUBLE",
"measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP",
}
@@ -2610,7 +2643,7 @@ def _registry_col_value(value, kind):
try:
if kind == "int":
return int(value)
- if kind in ("currency", "number", "pct", "measure"):
+ if kind in ("currency", "number", "pct", "measure", "float"):
return float(value)
if kind == "checkbox":
return bool(value)
@@ -2623,7 +2656,7 @@ def _registry_col_value(value, kind):
return str(value)
-def registry_grid_spec(cur, table_key, fields, rows):
+def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False):
"""A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway
DuckDB TEMP TABLE on THIS cursor (W52-T11).
@@ -2637,7 +2670,8 @@ def registry_grid_spec(cur, table_key, fields, rows):
request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped
read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`.
- ⭐ THE JOIN KEY IS ALWAYS `pid`, NEVER A DECLARED FIELD. Both grains carry it as a raw row
+ The default identity is `pid`; stored shipments explicitly use `odoo_id` instead.
+ Both registry grains carry `pid` as a raw row
datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id;
`product_data.pid` is `crc32(default_code)` — see `_related_source_value`'s own note), so it
is forced into the materialised columns here even when the caller's field list never
@@ -2645,10 +2679,10 @@ def registry_grid_spec(cur, table_key, fields, rows):
"""
keys = list(dict.fromkeys(
str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key")))
- if "pid" not in keys:
- keys.insert(0, "pid")
+ if id_key not in keys:
+ keys.insert(0, id_key)
by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)}
- by_key.setdefault("pid", {"type": "int"})
+ by_key.setdefault(id_key, {"type": "int"})
table = f"_registry_grid_{table_key}"
cols_sql = ", ".join(
@@ -2658,9 +2692,15 @@ def registry_grid_spec(cur, table_key, fields, rows):
values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
for r in (rows or []) if isinstance(r, dict)]
if values:
- placeholders = ", ".join(["?"] * len(keys))
- cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
- return {"table": table, "id": '"pid"', "cols": {k: (f'"{k}"', str) for k in keys}}
+ if bulk:
+ # Column arrays avoid 40,000 Python/DuckDB execute round trips for stored shipments.
+ selects = ", ".join(
+ f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
+ cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
+ else:
+ placeholders = ", ".join(["?"] * len(keys))
+ cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
+ return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
def _declared_from_spec(fields):
@@ -2766,6 +2806,7 @@ TABLE_SUBJECTS = {
# spec row tried to bring it back.
INVOICES_KEY: "odoo:account.move.customer",
ORDERS_KEY: "odoo:sale.order",
+ SHIPMENTS_KEY: "odoo:stock.picking.outgoing",
AGENTS_KEY: "odoo:res.partner.agent",
ACCOUNTS_KEY: "odoo:account.account",
VENDORS_KEY: "odoo:res.partner.vendor",
@@ -3150,7 +3191,8 @@ DEFAULT_CONNECTION = "default"
#: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a
#: document that still holds one resolves to its connection rather than reading as a stranger's
#: table — `apply_plan` has to know whose copy it is popping.
-BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES] + list(RETIRED_KEYS))
+BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES]
+ + list(RETIRED_KEYS) + [SHIPMENTS_KEY])
#: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated
#: key is a key that can collide with another connection's. The longest base key is
@@ -3577,6 +3619,112 @@ def sync_seconds(rt):
return max(int(secs), SYNC_FLOOR_SECONDS)
+#: ⭐⭐ S67-T20 / C4 — how far past its cadence a sync may run before its data is called stale.
+#: Two hours of grace on top of the interval: a daily sync is stale after 26 hours (C4's number),
+#: long enough that one slow pass does not raise a warning and short enough that a dead sync is
+#: named on the next day, not three weeks later as staging's was.
+STALE_GRACE_SECONDS = 2 * 3600
+
+
+def _utc_now():
+ return _dt.datetime.now(_dt.timezone.utc)
+
+
+def _parse_iso(value):
+ """An ISO timestamp as an aware UTC datetime, or None when it is not one."""
+ if not value:
+ return None
+ try:
+ at = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ return at if at.tzinfo else at.replace(tzinfo=_dt.timezone.utc)
+
+
+def record_sync_outcome(rt, ok, at=None):
+ """Stamp this tenant's last sync, and its last GOOD sync when `ok`, on the connector's config.
+
+ ⛔ ON THE CONFIG DOCUMENT, NOT ON `odoo_sync_status`, because that document is REPLACED whole
+ on every phase write (`core.user_tables.record_odoo_sync_status`), so a stamp kept there would
+ be erased by the next `syncing`. Every writer of this config document updates it in place.
+ """
+ stamp = at or _utc_now().isoformat()
+
+ def _up(cur):
+ cur = dict(cur) if isinstance(cur, dict) else {}
+ cur["lastSyncAt"] = stamp
+ if ok:
+ cur["lastGoodSyncAt"] = stamp
+ return cur
+
+ rt.update(CONFIG_KEY, _up, flush="sync")
+ return stamp
+
+
+def _effective_sync_seconds(preset):
+ """The interval a sync can actually keep: the tenant's preset, floored, and never faster than
+ the deployment's own resync floor (`AIOS_STORE_RESYNC_SECONDS`), which `main.py` applies with
+ `max()`. None for `manual`: a person who asked not to be synced has no schedule to fall
+ behind."""
+ secs = SYNC_PRESETS.get(preset)
+ if secs is None:
+ return None
+ secs = max(int(secs), SYNC_FLOOR_SECONDS)
+ import os
+ try:
+ env = int(str(os.environ.get("AIOS_STORE_RESYNC_SECONDS") or "").strip() or 0)
+ except ValueError:
+ env = 0
+ return max(secs, env)
+
+
+def sync_freshness(rt, now=None):
+ """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
+ fail. PURE over the store reads.
+
+ `stale` is true when the last good sync is older than the effective cadence plus
+ `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
+ (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
+ the last run's own reason, its code prefix removed, when that run ended in error.
+ """
+ now = now or _utc_now()
+ try:
+ raw = rt.get(CONFIG_KEY) if rt is not None else None
+ except Exception: # noqa: BLE001
+ raw = None
+ raw = raw if isinstance(raw, dict) else {}
+ cfg = read_config(rt)
+ try:
+ status = rt.get(_ut().ODOO_SYNC_STATUS_KEY) if rt is not None else None
+ except Exception: # noqa: BLE001
+ status = None
+ status = status if isinstance(status, dict) else {}
+ phase = str(status.get("phase") or "unknown")
+ last_good = raw.get("lastGoodSyncAt") or None
+ # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
+ if not last_good and phase == "ok":
+ last_good = status.get("lastRun") or None
+ every = _effective_sync_seconds(cfg["syncEvery"])
+ allowed = None if every is None else every + STALE_GRACE_SECONDS
+ stale = False
+ if allowed is not None and not cfg["frozen"]:
+ good_at = _parse_iso(last_good)
+ stale = good_at is None or (now - good_at).total_seconds() > allowed
+ failure = None
+ errors = status.get("errors") if isinstance(status.get("errors"), list) else []
+ if phase == "error" and errors:
+ text = str(errors[0])
+ code, _sep, message = text.partition(": ")
+ if not _sep:
+ code, message = "sync_failed", text
+ failure = {"code": code.strip()[:80], "message": message.strip()[:300],
+ "at": status.get("lastRun") or None}
+ return {"stale": bool(stale), "lastGoodSyncAt": last_good,
+ "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
+ "lastRunAt": status.get("lastRun") or None, "phase": phase,
+ "frozen": cfg["frozen"], "failure": failure}
+
+
def frozen(rt):
"""Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential."""
@@ -3697,15 +3845,9 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
continue
built[bucket] = reader(cur, excluded)
problems = []
-
- if rt is not None:
- needed = [k for k in keys.values() if k not in existing]
- if needed and len(existing) + len(needed) > ut.MAX_TABLES:
- problems.append(
- f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and "
- f"needs {len(needed)} more ({', '.join(needed)}) for connection "
- f"{connection_id or 'default'}; refusing rather than creating "
- f"part of a linked set")
+ # ⭐⭐ D-3284 (owner ruling 2026-09-27): connector databases take NO `MAX_TABLES` slot
+ # (`user_tables.counts_toward_table_cap`), so the cap can no longer refuse, or half-spawn, a
+ # linked set. The refusal that stood here counted every database, people's and connectors'.
for bucket, _base, _label, _fields in tables:
key = keys[bucket]
@@ -3751,6 +3893,221 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
return built
+def _shipment_datetime(value):
+ """Odoo's naive datetimes are UTC; preserve seconds before deriving date cells."""
+ if not value:
+ return None
+ parsed = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ return (parsed.replace(tzinfo=_dt.timezone.utc) if parsed.tzinfo is None
+ else parsed.astimezone(_dt.timezone.utc))
+
+
+def _shipment_relation_id(value):
+ if isinstance(value, (list, tuple)):
+ value = value[0] if value else None
+ return int(value) if value else None
+
+
+def read_shipments_live(client):
+ """All outgoing pickings in id pages, then their sale orders in bounded read batches.
+
+ Only guarded read methods are used. The guarded OdooClient has no ``read`` convenience
+ method, so sale-order batches use search_read with an explicit id domain as well.
+ """
+ pickings, last_id = [], 0
+ fields = ["name", "origin", "sale_id", "partner_id", "state", "scheduled_date", "date_done"]
+ while True:
+ domain = [("picking_type_code", "=", "outgoing")]
+ if last_id:
+ domain.append(("id", ">", last_id))
+ page = client.search_read("stock.picking", domain, fields,
+ limit=MODEL_READ_PAGE, order="id")
+ if not page:
+ break
+ for picking in page:
+ rid = int(picking["id"])
+ if rid <= last_id:
+ raise Refused("stock.picking: source ids did not advance; no shipments written")
+ last_id = rid
+ pickings.append(picking)
+ # Continue until an empty page, even if the server returned less than requested.
+
+ ids = sorted({oid for picking in pickings
+ if (oid := _shipment_relation_id(picking.get("sale_id"))) is not None})
+ orders = {}
+ for start in range(0, len(ids), MODEL_READ_PAGE):
+ chunk = ids[start:start + MODEL_READ_PAGE]
+ batch = client.search_read("sale.order", [("id", "in", chunk)],
+ ["name", "date_order"], limit=len(chunk), order="id")
+ if len(batch) != len(chunk) or {int(row["id"]) for row in batch} != set(chunk):
+ raise Refused("sale.order: referenced shipment orders were not all readable; "
+ "the prior database was preserved. Check Odoo read access and retry.")
+ orders.update((int(row["id"]), row) for row in batch)
+
+ rows, stamp = {}, _iso_today()
+ for picking in pickings:
+ rid = int(picking["id"])
+ oid = _shipment_relation_id(picking.get("sale_id"))
+ partner = picking.get("partner_id")
+ pid = _shipment_relation_id(partner)
+ order = orders.get(oid) or {}
+ ordered = _shipment_datetime(order.get("date_order"))
+ done = _shipment_datetime(picking.get("date_done"))
+ scheduled = _shipment_datetime(picking.get("scheduled_date"))
+ rows[str(rid)] = {
+ "picking_no": picking.get("name") or "", "odoo_id": rid,
+ "origin": picking.get("origin") or "", "order_no": order.get("name") or "",
+ "order_id": oid, **({"order_link": str(oid)} if oid is not None else {}),
+ "customer": (partner[1] or "")
+ if isinstance(partner, (list, tuple)) and len(partner) > 1 else "",
+ "partner_id": pid, **({"customer_link": str(pid)} if pid is not None else {}),
+ "state": picking.get("state") or "",
+ "scheduled_date": scheduled.date().isoformat() if scheduled else "",
+ "date_done": done.date().isoformat() if done else "",
+ "order_date": ordered.date().isoformat() if ordered else "",
+ "delivery_days": (done - ordered).total_seconds() / 86400
+ if done and ordered else None,
+ "refreshed": stamp,
+ }
+ return rows
+
+
+def reconcile_shipments(rows, client, year):
+ """Independent source count for completed outgoing shipments in one UTC calendar year."""
+ year = int(year)
+ start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
+ values = rows.values() if isinstance(rows, dict) else rows
+ count = sum(1 for row in values if row.get("state") == "done"
+ and start <= (row.get("date_done") or "") < end)
+ source_count = client.search_count("stock.picking", [
+ ("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
+ ("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
+ return {"year": year, "rowCount": count, "sourceCount": source_count,
+ "ok": type(source_count) is int and source_count == count}
+
+
+def _shipment_problems(rows, existing, key):
+ ut = _ut()
+ problems = []
+ stored = len((existing.get(key) or {}).get("rows") or {})
+ if stored and len(rows) < stored * MAX_SHRINK:
+ problems.append(f"{key}: read returned {len(rows)} rows against {stored} stored; "
+ "preserved the prior database. Check Odoo read access and retry sync.")
+ # D-3284: no MAX_TABLES check here; a connector database takes no slot.
+ size = ut.rows_bytes(rows)
+ if size > ut.MAX_TABLE_BYTES:
+ problems.append(f"{key}: {size} bytes exceeds MAX_TABLE_BYTES={ut.MAX_TABLE_BYTES}; "
+ "the prior database was preserved. Use read-through storage before retrying.")
+ return problems
+
+
+def plan_shipments(rt, connection_id=None, client=None, today=None):
+ """Prepare shipments without writing a store; injected clients keep gates hermetic."""
+ if client is None:
+ source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
+ if source is None:
+ raise Refused("the selected Odoo connection is unavailable; no shipments read")
+ from core.odoo import get_odoo
+ with source._bound():
+ return plan_shipments(rt, connection_id, get_odoo(), today)
+ _cid, namespace = connection_namespace(rt, connection_id)
+ rows = read_shipments_live(client)
+ for row in rows.values():
+ row["refreshed"] = today or _iso_today()
+ if namespace:
+ row.pop("customer_link", None) # customer_data belongs to the default connection
+ key = table_key_for(SHIPMENTS_KEY, namespace)
+ result = {"rows": rows, "problems": _shipment_problems(
+ rows, rt.get(_ut().STORE_KEY) or {}, key)}
+ if namespace:
+ result["connection_id"] = namespace
+ return result
+
+
+def apply_shipments(rt, planned, username="automation", today=None):
+ """Publish via the relational writer, refusing partial reads before touching stored rows."""
+ from copy import deepcopy
+ from core import connector_sync
+ if planned.get("problems"):
+ raise Refused("; ".join(planned["problems"]))
+ ut, stamp = _ut(), today or _iso_today()
+ namespace = planned.get("connection_id")
+ key = table_key_for(SHIPMENTS_KEY, namespace)
+ rows = {rid: {**row, "refreshed": stamp} for rid, row in planned["rows"].items()}
+ problems = _shipment_problems(rows, rt.get(ut.STORE_KEY) or {}, key)
+ if problems:
+ raise Refused("; ".join(problems))
+ written = {}
+
+ def apply(current):
+ current = current if isinstance(current, dict) else {}
+ # Recheck the current document under the store's update lock, including concurrent growth.
+ problems = _shipment_problems(rows, current, key)
+ if problems:
+ raise Refused("; ".join(problems))
+ current = dict(current)
+ if key in current:
+ current[key] = deepcopy(current[key])
+ previous = (current.get(key) or {}).get("rows") or {}
+ link_only_changes = sum(
+ 1 for rid, row in rows.items() if rid in previous
+ and any(link in previous[rid] and link not in row
+ for link in ("order_link", "customer_link"))
+ and all(str(previous[rid].get(k, "")) == str(v)
+ for k, v in row.items() if k != "refreshed"))
+ new = key not in current
+ written[key] = _ensure_table_inplace(
+ current, key, table_label_for("Odoo shipments", connection_label(rt, namespace)),
+ namespaced_fields(shipment_fields(), namespace),
+ [{**row, "_id": rid} for rid, row in rows.items()], username, stamp)
+ if new:
+ current[key].pop("created", None) # match the existing staging definition
+ for rid, row in rows.items():
+ for link in ("order_link", "customer_link"):
+ if link not in row:
+ current[key]["rows"][rid].pop(link, None)
+ written[key]["updated"] += link_only_changes
+ current[key]["connector"] = {
+ **connector_sync.database_identity(
+ "odoo", connection_for_table_key(key, rt), key, "stock.picking"),
+ "key": "odoo", "readOnly": True}
+ # Existing user cells also count toward the unchanged byte budget.
+ problems = _shipment_problems(current[key]["rows"], current, key)
+ if problems:
+ raise Refused("; ".join(problems))
+ return current
+
+ rt.update(ut.STORE_KEY, apply, flush="sync")
+ ut.register_connected(key)
+ return written
+
+
+def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
+ username="automation", client=None):
+ """Reconcile before publication, then independently count the rows read back from storage."""
+ if client is None:
+ source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
+ if source is None:
+ raise Refused("the selected Odoo connection is unavailable; no shipments read")
+ from core.odoo import get_odoo
+ with source._bound():
+ return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
+ username=username, client=get_odoo())
+ stamp = today or _iso_today()
+ year = int(year) if year is not None else int(stamp[:4])
+ planned = plan_shipments(rt, connection_id, client, stamp)
+ reconciled = reconcile_shipments(planned["rows"], client, year)
+ if not reconciled["ok"]:
+ raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
+ f"({reconciled['rowCount']} planned, {reconciled['sourceCount']} source); "
+ "nothing written")
+ applied = apply_shipments(rt, planned, username, stamp)
+ key = table_key_for(SHIPMENTS_KEY, planned.get("connection_id"))
+ stored = ((rt.get(_ut().STORE_KEY, fresh=True) or {}).get(key) or {}).get("rows") or {}
+ return {"planned": planned, "reconcile_planned": reconciled, "apply": applied,
+ "stored_rows": stored, "reconcile_stored": reconcile_shipments(stored, client, year)}
+
+
def apply_plan(rt, built, username="automation", today=None, report=None,
connection_id=None, retire=True):
"""Create-or-merge every table in `built` and its rows. Idempotent by construction.
@@ -3864,10 +4221,8 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
subject = table_subject_for(key)
table = cur.get(key)
if table is None:
- if len(cur) >= ut.MAX_TABLES:
- # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a
- # successful refresh over a table that does not exist.
- raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created")
+ # D-3284 (owner ruling 2026-09-27): every table this writer creates is a connector
+ # database (Odoo, Meta, Amazon), and those take no MAX_TABLES slot, so no cap refusal here.
# ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE.
#
# ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This
@@ -4195,11 +4550,8 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
f"{choice['model']}: too large to store, served nowhere until read-through"
for choice in oversized]}
keys = [str(choice["key"]) for choice in selected]
- missing = [key for key in keys if key not in existing]
- if missing and len(existing) + len(missing) > ut.MAX_TABLES:
- raise Refused(f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} databases and "
- f"needs {len(missing)} more enabled Odoo models; nothing was created")
- source = (rt.odoo_source() if source_connection_id is None
+ # D-3284: enabled Odoo models are connector databases and take no MAX_TABLES slot.
+ source =(rt.odoo_source() if source_connection_id is None
else rt.odoo_source(source_connection_id))
if source is None:
raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
@@ -4312,6 +4664,93 @@ def available(rt=None, tenant_key=None, connection_id=None):
return False
+#: ⭐ S67-T19 — the `limits` effects that REPORT a serving choice rather than withheld rows.
+#: `read_through` says a database is served THROUGH the mirror with no ceiling at all
+#: (`core.user_tables.limit_report`), which is the design working, not a failure.
+INFORMATIONAL_LIMIT_EFFECTS = frozenset({"read_through"})
+
+
+def failing_limits(limits):
+ """Only the `limits` entries that describe rows a pass did NOT serve, PURE.
+
+ ⛔⛔ STAGING'S `materialization_failed` WAS THIS, NAMED 2026-09-28 FROM ITS OWN RECEIPTS. The
+ rebuild handed its whole `limits` map to the health classifier, which reads EVERY entry as a
+ materialization failure, so the two read-through grains (`ut_odoo_order_lines`,
+ `ut_odoo_gl_lines`, "no ceiling applies to them") failed the connection on every single sync
+ and `odoo_sync_status` could never read `ok`. The report keeps them; only health drops them.
+ """
+ if not isinstance(limits, dict):
+ return limits
+ return {key: entry for key, entry in limits.items()
+ if not (isinstance(entry, dict)
+ and entry.get("effect") in INFORMATIONAL_LIMIT_EFFECTS)}
+
+
+#: ⭐ S67-T19 — the reconciliation DRIFT a good sync may carry: Odoo keeps taking invoices while a
+#: rebuild runs, and a hard delete leaves a mirror row until `reconcile_deletes` (every fourth
+#: pass). Measured on a mirror pulled 25 minutes before its rebuild: invoices 32,980 vs 32,978,
+#: orders 34,059 vs 34,058, agents 17 vs 18, GL lines 1,024,102 vs 1,024,178, customers 3,715 vs
+#: 3,716. A stale mirror is nothing like that: staging's unpulled seed was 626 orders and 732
+#: invoices short. The receipt keeps its exact counts either way; this only decides whether the
+#: CONNECTION's sync counts as good.
+RECONCILE_DRIFT_ROWS = 10
+RECONCILE_DRIFT_RATIO = 0.002
+
+
+def reconciliation_drift(receipt):
+ """`(difference, tolerance)` for a count-reconciliation receipt, or None when it has no counts."""
+ if not isinstance(receipt, dict):
+ return None
+ source = receipt.get("sourceCount")
+ if isinstance(receipt.get("unexplainedCount"), int):
+ diff = int(receipt["unexplainedCount"])
+ else:
+ mirror = receipt.get("mirrorCount")
+ if not isinstance(source, int) or not isinstance(mirror, int):
+ return None
+ diff = abs(source - mirror)
+ base = source if isinstance(source, int) and source > 0 else 0
+ return diff, max(RECONCILE_DRIFT_ROWS, int(base * RECONCILE_DRIFT_RATIO))
+
+
+def blocking_failures(database_failures, report):
+ """The database failures that make a sync NOT good, PURE.
+
+ A reconciliation failure whose count drift is within `reconciliation_drift`'s tolerance is
+ recorded against its database and does not fail the connection; every other failure does.
+ """
+ receipts = {}
+ for name in ("databases", "canonicalDatabases"):
+ group = report.get(name) if isinstance(report, dict) else None
+ if isinstance(group, dict):
+ receipts.update(group)
+ out = {}
+ for key, failure in (database_failures or {}).items():
+ if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
+ drift = reconciliation_drift(receipts.get(key))
+ if drift is not None and drift[0] <= drift[1]:
+ continue
+ out[key] = failure
+ return out
+
+
+def health_view(report):
+ """The refresh report as the sync-health classifier must read it, PURE.
+
+ Identical to `report` except that `limits` carries only `failing_limits`; the returned report
+ itself (the log line, the Sync report) still names every read-through database.
+ """
+ if not isinstance(report, dict) or not isinstance(report.get("limits"), dict):
+ return report
+ view = dict(report)
+ remaining = failing_limits(report["limits"])
+ if remaining:
+ view["limits"] = remaining
+ else:
+ view.pop("limits", None)
+ return view
+
+
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
today=None, retire=True, on_apply=None, source_client=None):
"""Refresh each enabled database independently; retain failed databases unchanged.
@@ -4598,6 +5037,16 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
return out
+#: ⭐ S67-T19 — the store errors a connection refresh retries ONCE. Staging's rebuild at
+#: 2026-09-28 16:03Z failed whole with `ProtocolViolation` (SQLSTATE 08P01), which is how the
+#: DigitalOcean pool (PgBouncer) reports its own timeouts while every lane container shares it;
+#: nothing was written and the next scheduled pass was a day away. `refresh` is idempotent, so a
+#: second attempt after a pause is safe; a refusal or any other exception is never retried.
+TRANSIENT_STORE_ERRORS = frozenset({"ProtocolViolation", "OperationalError", "PoolTimeout",
+ "ConnectionTimeout", "AdminShutdown"})
+REFRESH_RETRY_SECONDS = 20
+
+
def refresh_all(rt, tenant, username="automation", cur=None, today=None):
"""Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
@@ -4634,12 +5083,20 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
primary = None
for entry_cid in order:
named = None if (entry_cid is None or entry_cid == default) else entry_cid
- try:
- one = refresh(rt, tenant, username=username,
- cur=(cur if named is None else None),
- today=today, connection_id=named)
- except Exception as exc: # noqa: BLE001
- one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
+ for attempt in (1, 2):
+ try:
+ one = refresh(rt, tenant, username=username,
+ cur=(cur if named is None else None),
+ today=today, connection_id=named)
+ break
+ except Exception as exc: # noqa: BLE001
+ # S67-T19: one retry for a TRANSIENT store error only; every other failure is
+ # recorded exactly as before.
+ if attempt == 1 and type(exc).__name__ in TRANSIENT_STORE_ERRORS:
+ _time.sleep(REFRESH_RETRY_SECONDS)
+ continue
+ one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
+ break
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
if named is None and primary is None:
primary = one
diff --git a/api/routes_admin.py b/api/routes_admin.py
index 5083129c25661dcbb08c9e7d1737dab6b34dd5ca..453813d2b6c58c18894eef2309b21d9b5043574b 100644
--- a/api/routes_admin.py
+++ b/api/routes_admin.py
@@ -605,7 +605,8 @@ def _perm_modules(session, surfaces=False):
skeys = set()
if surfaces:
# ⭐⭐ OWNER RULING 2026-08-18 — the Assistant and Agents surfaces are
- # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and
+ # governable ACCESS toggles in the ACCOUNT editor, alongside R3/C6's AI apps (MCP)
+ # gateway capability. `surfaces` is caller-explicit and
# defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor
# (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE
# (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle
@@ -869,11 +870,11 @@ def _module_fields(key, session=None):
import aios_grid
key = str(key or "")
if key in {s["key"] for s in perms.governable_surface_modules()}:
- # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest
+ # ⭐ A SURFACE (Assistant, Agents, AI apps) has NO field schema, and `[]` is the honest
# answer rather than the wave-26 empty-option trap: the editor's schemaless rendering
# is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any
# filter or hiddenFields validated against this empty vocabulary. An unknown key
- # still RAISES below — this arm admits exactly the derived surface list.
+ # still RAISES below: this arm admits the page surfaces and R3/C6's gateway capability.
return []
if key.startswith("ut_"):
if session is None:
@@ -2337,7 +2338,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
# save of ANY unrelated change would write the explicit deny for real. `nav_may_open`
# is the evaluator the nav and the route gate actually ask, legacy fallback included,
# so the box shows what the account can reach and an untouched save stays a no-op.
- _default = (perm_scope.nav_may_open(users._public(uname, rec), k)
+ # AI apps also uses this evaluator: absence grants even for a restricted legacy list.
+ _default = (perm_scope.nav_may_open(users._public(uname, rec), k, st=session.runtime)
if any(m.get("surface") and m["key"] == k for m in modules) else
perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
# ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
diff --git a/api/routes_auth.py b/api/routes_auth.py
index 3dbe9f68d9ac2ea259312ece79176ef596b11255..9965f0978ebcfd3a93766f74e767be9478ce44c8 100644
--- a/api/routes_auth.py
+++ b/api/routes_auth.py
@@ -69,6 +69,17 @@ def _throttled(key, now=None):
raise _auth_store_down("_throttled", exc)
+def _verify(uname, pw):
+ """`users.verify`, with an UNREADABLE registry answered 503 before `_note_failure` runs.
+
+ HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct
+ password got 401 `invalid_credentials` and was charged to the lockout budget."""
+ try:
+ return users.verify(uname, pw) if (uname and pw) else None
+ except users.RegistryUnavailable as exc:
+ raise _auth_store_down("verify", exc)
+
+
def _note_failure(key, now=None):
if key is None:
return 0
@@ -196,7 +207,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
raise err(429, "too_many_attempts",
f"too many failed attempts. Try again in {wait} seconds")
if not user:
- user = users.verify(uname, pw) if (uname and pw) else None
+ user = _verify(uname, pw)
if not user:
# CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
# record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
@@ -356,7 +367,7 @@ def _mfa_credentials(uname, password):
print(f"[aios-api] {exc}")
raise err(429, "too_many_attempts",
f"too many failed attempts. Try again in {wait} seconds")
- user = users.verify(uname, password) if (uname and password) else None
+ user = _verify(uname, password)
if not user:
try:
users._hash(password or "x", "00" * 16)
@@ -620,7 +631,7 @@ def change_password(body: dict = Body(default=None)):
print(f"[aios-api] {exc}")
raise err(429, "too_many_attempts",
f"too many failed attempts. Try again in {wait} seconds")
- user = users.verify(uname, pw) if (uname and pw) else None
+ user = _verify(uname, pw)
if not user:
try:
users._hash(pw or "x", "00" * 16)
diff --git a/api/routes_connected_tables.py b/api/routes_connected_tables.py
index f51584efca4a275be7c3adc439d61dbbe3bf21bc..6a549ff2e413955b45d7f323358568d770bbf70e 100644
--- a/api/routes_connected_tables.py
+++ b/api/routes_connected_tables.py
@@ -30,7 +30,7 @@ behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be
(`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
`verify_meta` both enumerates `main.app.routes` for this path AND calls it.
"""
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Body, Depends, Query
from deps import Session, err, require_session
@@ -45,6 +45,8 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
# Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
definition = _defn_or_refuse(session, table_key, defs_only=True)
+ from routes_odoo_tables import require_amazon_mirror
+ require_amazon_mirror(table_key, session, definition)
identity = user_tables.connector_database_identity(
table_key, rt=session.runtime, definition=definition)
if identity is None:
@@ -54,6 +56,7 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
@router.get("/connected-tables/{table_key}/rows")
+@router.post("/connected-tables/{table_key}/rows")
def connected_table_rows(table_key: str,
offset: int = Query(default=0, ge=0),
limit: int = Query(default=0),
@@ -64,7 +67,8 @@ def connected_table_rows(table_key: str,
related: str = Query(default=None),
groupBy: str = Query(default=None),
groupKey: str = Query(default=None),
- session: Session = Depends(require_session)):
+ session: Session = Depends(require_session),
+ body: dict = Body(default=None)):
"""ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
limits, identity, recordsMutable}`, whichever connector the rows come from.
@@ -92,4 +96,4 @@ def connected_table_rows(table_key: str,
return routes_odoo_tables.odoo_table_rows(
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
- session=session)
+ session=session, body=body)
diff --git a/api/routes_connectors.py b/api/routes_connectors.py
index 83148bd4972f378385095a41f75d61b3280777dc..832f240d6239efc43f1fb7502106b1035ffb5213 100644
--- a/api/routes_connectors.py
+++ b/api/routes_connectors.py
@@ -21,14 +21,17 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
from __future__ import annotations
import os
+import copy
import ssl
import time
import threading
import xmlrpc.client
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
+from collections import OrderedDict
+from typing import Annotated, Literal
-from fastapi import APIRouter, Body, Depends
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
import oauth_connect
from deps import Session, err, require_session, running_main
@@ -83,6 +86,75 @@ from routes_monid import router as monid_router
router = APIRouter(prefix="/api/v1")
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
+_ODOO_DETAIL_CATALOG_CACHE = OrderedDict()
+_ODOO_DETAIL_CATALOG_LOCK = threading.Lock()
+_ODOO_DETAIL_CATALOG_CACHE_SIZE = 16
+
+
+class _ConnectorDetailReads:
+ """Request-local copies only; credentials, grants and enablements never outlive a read.
+
+ Keep projections separate from whole documents and honour fresh reads. HF projections still
+ download the whole document on a cold store; removing that requires a store-layer change.
+ """
+
+ def __init__(self, runtime):
+ self.runtime = runtime
+ self.documents = {}
+ self.projections = {}
+
+ def get(self, name, fresh=False):
+ if fresh:
+ return self.runtime.get(name, fresh=True)
+ if name not in self.documents:
+ self.documents[name] = self.runtime.get(name)
+ return copy.deepcopy(self.documents[name])
+
+ def get_projection(self, name, drop=()):
+ key = (name, tuple(sorted(drop)))
+ if key not in self.projections:
+ self.projections[key] = self.runtime.get_projection(name, drop=drop)
+ return copy.deepcopy(self.projections[key])
+
+ def __getattr__(self, name):
+ return getattr(self.runtime, name)
+
+
+class _DatabasePage:
+ """Select before hydrating Odoo fields; the total always describes the unsliced list."""
+
+ def __init__(self, offset=0, limit=50, database_key=None):
+ self.offset, self.limit, self.database_key = offset, limit, database_key
+ self.total = 0
+ self.creates = []
+
+ def select(self, rows, *, overview_rows=None):
+ self.total = len(rows)
+ self.creates = [str(row.get("label") or "")
+ for row in (rows if overview_rows is None else overview_rows)
+ if row.get("enabled") is True and row.get("label")]
+ if self.database_key is not None:
+ return [row for row in rows if row.get("key") == self.database_key]
+ return rows[self.offset:self.offset + self.limit]
+
+ def dto(self):
+ next_offset = self.offset + self.limit
+ return {"offset": self.offset, "limit": self.limit, "total": self.total,
+ "next": next_offset if next_offset < self.total else None}
+
+
+def _detail_page(databases_offset, databases_limit, fields):
+ if fields not in ("summary", "full"):
+ raise err(422, "invalid_database_fields", "Fields must be summary or full.")
+ if databases_offset is None and databases_limit is None and fields == "full":
+ return None
+ offset = 0 if databases_offset is None else databases_offset
+ limit = 50 if databases_limit is None else databases_limit
+ if (type(offset) is not int or offset < 0 or type(limit) is not int or limit < 1):
+ raise err(422, "invalid_database_page", "Use a nonnegative offset and a positive limit.")
+ return _DatabasePage(offset, limit)
+
+
SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
SCRAPER_RECIPE_MAX_ROWS = 30
SCRAPER_RUN_HISTORY_LIMIT = 20
@@ -254,9 +326,11 @@ TOKEN_CONNECTORS = [
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
"it, then paste client_id, client_secret, refresh_token, seller_id and "
"marketplace_id.",
- "note": "The credential is verified live against your marketplace participations before "
- "it is stored. Bulk reads use the Reports API, so a backfill is one document "
- "rather than a page loop."},
+ # S66-T27: storing is offline (`keychain._test_amazon_entry` probes only on an explicit
+ # Test), so this no longer claims the key was verified before it was stored.
+ "note": "Choose Test in the Keychain tab to verify the credential live against your "
+ "marketplace participations. Bulk reads use the Reports API, so a backfill is one "
+ "document rather than a page loop."},
]
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
@@ -374,6 +448,11 @@ _register_probers()
# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace,
# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
+#: The directory states that mean "this workspace holds this connection": connected, stored but
+#: asking to be reconnected, or stored past its rotation deadline (`ConnectorsPage`'s `held`).
+_GRAPH_HELD_STATES = frozenset(("connected", "reconnect", "expired"))
+
+
def _graph_refusal(connection_key, reason, status="refused"):
return {"connectionKey": str(connection_key or ""), "status": status,
"reason": str(reason or ""), "tables": [], "relationships": []}
@@ -526,7 +605,12 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False):
graphs = {}
connected = set()
for connection_key, row in row_by_key.items():
- if row.get("state") == "connected":
+ # ⭐ S67-T31 / D-718 — A HELD CONNECTION HAS A GRAPH. Staging's Amazon card reads
+ # `reconnect` (its stored key has no passing test on record) while 38 `ut_amzn_*` tables
+ # are materialised, and the old `== "connected"` test refused the graph as "Connect this
+ # provider" over a connection that is plainly there. The nodes are still only tables that
+ # exist and that this reader may open, so admitting the held states draws no template.
+ if row.get("state") in _GRAPH_HELD_STATES:
connected.add(connection_key)
graphs[connection_key] = _graph_refusal(
connection_key,
@@ -906,9 +990,21 @@ def directory(rt, uname, is_admin=False, incomplete=None):
else "keychain_personal"),
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
# key is real and useful, and it is not the same as the data being on screen.
- "note": "Storing the key connects the account; reading its data lands in a later "
- "release.",
+ # ⭐ S66-T27 / S67-T31 — EXCEPT AMAZON, whose data DOES land (38 `ut_amzn_*`
+ # tables on staging), so the generic sentence told an administrator the opposite
+ # of the truth. Amazon carries its own authored note.
+ # S67 lane D: so does Meta Ads, whose authored note this line used to overwrite
+ # with the generic one; every card now shows its own (Stripe's and Shopify's own
+ # note IS the generic sentence, so theirs is unchanged).
+ "note": t["note"],
})
+ # S66-T27 — WHAT `reconnect` MEANS, said on the card. On Amazon it is "a key is stored
+ # but no passing Test is on record for it" (`keychain.entry_verified`), measured on
+ # staging's admin view with the data still syncing. The bare word read as a broken
+ # connection with no way to tell what to do.
+ if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
+ rows[-1]["needs"] = ("Amazon access for the stored key is not confirmed yet. "
+ "Choose Test in the Keychain tab, or replace the key.")
contribute("token", _tokens,
"the token-connector registry could not be composed",
@@ -2150,19 +2246,21 @@ def _schema_catalog(session):
return _schema_catalog_rt(session.runtime)
-def _odoo_raw_models(rt, snapshot=None, connection_id=None):
+def _odoo_raw_models(rt, snapshot=None, connection_id=None, *, discovered=None):
"""Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
- if snapshot is None:
- import core.user_tables as user_tables
- snapshot = user_tables.odoo_schema_snapshot(rt)
- legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {}
- try:
- import odoo_relational
- discovered = odoo_relational.readable_model_catalog(rt, connection_id)
- except Exception: # noqa: BLE001
- discovered = {}
+ if discovered is None:
+ try:
+ import odoo_relational
+ discovered = odoo_relational.readable_model_catalog(rt, connection_id)
+ except Exception: # noqa: BLE001
+ discovered = {}
if not discovered.get("capturedAt"):
- return legacy
+ if connection_id is not None:
+ return {}
+ if snapshot is None:
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(rt)
+ return dict((snapshot or {}).get("models") or {})
# A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
# than adding snapshot-only models that the current account never proved it may read.
return {str(item.get("model") or ""): dict(item.get("fields") or {})
@@ -2170,7 +2268,7 @@ def _odoo_raw_models(rt, snapshot=None, connection_id=None):
if str(item.get("model") or "")}
-def _schema_catalog_rt(rt, database_key=None, connection_id=None):
+def _schema_catalog_rt(rt, database_key=None, connection_id=None, *, raw_models=None):
"""The same catalogue, over a RUNTIME rather than a session.
⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
@@ -2179,10 +2277,9 @@ def _schema_catalog_rt(rt, database_key=None, connection_id=None):
already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
signature so every existing caller is untouched.
"""
- import core.user_tables as user_tables
enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
- snapshot = user_tables.odoo_schema_snapshot(rt)
- raw_models = _odoo_raw_models(rt, snapshot, connection_id)
+ if raw_models is None:
+ raw_models = _odoo_raw_models(rt, connection_id=connection_id)
if raw_models:
rows = []
for model, raw_fields in raw_models.items():
@@ -3089,34 +3186,72 @@ def _detail_keychain(rt, uname, is_admin, row):
"connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
-def _odoo_catalog_by_model(rt, connection_id=None):
+def _odoo_catalog_by_model(rt, connection_id=None, *, models=None):
"""`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
`excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
the catalogue starts filtering instead of needing somebody to remember.
"""
- raw_models = _odoo_raw_models(rt, connection_id=connection_id)
+ import odoo_relational
+ discovered = odoo_relational.readable_model_catalog(rt, connection_id)
+ marker = ("discovered", discovered.get("version"), discovered.get("capturedAt"),
+ discovered.get("complete"))
+ snapshot = None
+ if not discovered.get("capturedAt"):
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(rt) if connection_id is None else {}
+ marker = ("snapshot", snapshot.get("version"), snapshot.get("capturedAt"),
+ snapshot.get("complete"))
+ owner = rt.runtime if isinstance(rt, _ConnectorDetailReads) else rt
+ cache_key = (id(owner), str(connection_id or ""), marker)
+ wanted = None if models is None else set(models)
+ # Cache only normalized metadata, never permissions or field switch values. Keep the owner
+ # alive with the entry so Python cannot reuse a runtime id for another tenant's catalogue.
+ with _ODOO_DETAIL_CATALOG_LOCK:
+ entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key) if marker[2] else None
+ cached = dict(entry[1]) if entry else {}
+ if entry:
+ _ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
+ raw_models = _odoo_raw_models(rt, snapshot, connection_id, discovered=discovered)
+ if wanted is not None:
+ raw_models = {model: fields for model, fields in raw_models.items() if model in wanted}
+ missing = {model: fields for model, fields in raw_models.items() if model not in cached}
offered = {}
- for row in _schema_catalog_rt(rt, connection_id=connection_id):
+ for row in (_schema_catalog_rt(rt, connection_id=connection_id, raw_models=missing)
+ if missing else ()):
offered.setdefault(str(row.get("model") or ""), []).append(row)
- out = {}
- for model, raw_fields in raw_models.items():
+ built = {}
+ for model, raw_fields in missing.items():
rows = offered.get(str(model), [])
offered_raw = {str(row.get("rawName") or "") for row in rows}
- out[str(model)] = (
+ built[str(model)] = (
[{"key": str(row.get("key") or ""),
"label": str(row.get("label") or ""),
"type": str(row.get("type") or "text"),
"source_field": str(row.get("rawName") or ""),
- "enabled": bool(row.get("enabled"))}
+ "enabled": False}
for row in rows],
[{"source_field": str(name),
"reason": "This source field was captured from Odoo but the field catalogue did not "
"offer it."}
for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
)
- return out
+ if marker[2]:
+ with _ODOO_DETAIL_CATALOG_LOCK:
+ entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key)
+ merged = {**(entry[1] if entry else {}), **built}
+ _ODOO_DETAIL_CATALOG_CACHE[cache_key] = (owner, merged)
+ _ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
+ while len(_ODOO_DETAIL_CATALOG_CACHE) > _ODOO_DETAIL_CATALOG_CACHE_SIZE:
+ _ODOO_DETAIL_CATALOG_CACHE.popitem(last=False)
+ cached.update(built)
+ enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
+ return {model: ([{**field, "enabled": connection_id is None
+ and enabled.get(field["key"]) is True}
+ for field in cached[model][0]],
+ [dict(item) for item in cached[model][1]])
+ for model in raw_models}
def _spec_field(item, name):
@@ -3495,7 +3630,8 @@ def _scraper_databases(rt, is_admin, uname):
return _named_databases(rt, is_admin, uname, rows)
-def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
+def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None, *,
+ page=None, fields="full"):
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
⛔ ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
@@ -3523,19 +3659,34 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
is walled, per row, by `canRename`.
"""
+ def select(rows):
+ return page.select(rows) if page is not None else rows
+
+ def disclose(rows):
+ if fields == "summary":
+ return [{**row, "fields": [], "excluded": [], "fieldsDeferred": True}
+ for row in rows]
+ return rows
+
if str(connector_key) == "scraper":
- return _scraper_databases(rt, is_admin, uname)
+ return disclose(select(_scraper_databases(rt, is_admin, uname)))
if not connector_directory.is_introspectable(connector_key):
return []
spec = _connector_spec_tables(connector_key)
if str(connector_key) == "amazon_sp":
- return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))
+ return disclose(select(_named_databases(
+ rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))))
try:
import odoo_relational
namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
if connection_id else None)
choices = odoo_relational.grid_choices(rt, connection_id)
models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
+ if page is not None:
+ choices = page.select(choices, overview_rows=schema_overlay.merge_database_labels([
+ {**choice, "label": str((spec.get(choice.get("baseKey") or choice["key"])
+ or {}).get("label") or choice.get("label") or choice["key"])}
+ for choice in choices], _database_name_overrides(rt)))
# This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
# catalogue was warm, so modal detail must never turn a missing cache into inline work.
census = odoo_relational.population_census(rt, namespace)
@@ -3547,9 +3698,11 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
census_at = str(census.get("capturedAt") or "")
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
catalog = {}
- if is_admin:
+ if is_admin and fields == "full" and choices:
try:
- catalog = _odoo_catalog_by_model(rt, namespace)
+ catalog = _odoo_catalog_by_model(rt, namespace, models={
+ str(choice.get("model") or models.get(choice.get("key")) or "")
+ for choice in choices})
except Exception: # noqa: BLE001
catalog = {}
enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
@@ -3576,9 +3729,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
key = str(choice.get("key") or "")
resolved_connection_id = resolve_connection_id(key)
model = str(choice.get("model") or models.get(key) or "")
- fields, excluded = catalog.get(model, ([], []))
+ field_rows, excluded = catalog.get(model, ([], []))
scoped_fields = []
- for field in fields:
+ for field in field_rows:
field_key = _odoo_field_key(rt, key, model, field["source_field"],
connection_id=resolved_connection_id)
scoped_fields.append({**field, "key": field_key,
@@ -3612,7 +3765,7 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
"fields": scoped_fields,
"excluded": [dict(item) for item in excluded],
"sharesFieldsWith": []})
- return _named_databases(rt, is_admin, uname, out)
+ return disclose(_named_databases(rt, is_admin, uname, out))
def _amazon_report_name(table):
@@ -3662,7 +3815,17 @@ def _amazon_databases(rt, is_admin, spec=None):
enabled = enablement.get("databases") or {}
cat = amazon_schema.catalogue()
tables = cat.get("tables") or {}
- states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
+ # ⭐ S67-T31 — A STATUS THE GUARD REFUSES COSTS ITS OWN ROW, NEVER THE MODAL. `sync_state`
+ # trims an over-long cause itself; a credential shape still refuses, and the refusal carries
+ # the guarded rows that passed, so only the refused tables read "Sync status is unavailable
+ # for this database." (a missing status is exactly what `_amazon_database_sync_health`
+ # degrades). Any other failure reading status degrades every row, never the whole page.
+ try:
+ states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
+ except cx.AmazonStatusSafetyError as refused:
+ states = dict(getattr(refused, "safe_status", None) or {})
+ except Exception: # noqa: BLE001
+ states = {}
# Which OTHER declared tables are filled from the SAME report document. Derived from the
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
@@ -3833,12 +3996,14 @@ def _detail_introspection(rt, is_admin, row):
captured = ""
census = {}
try:
- import core.user_tables as user_tables
- snapshot = user_tables.odoo_schema_snapshot(rt)
import odoo_relational
discovered = odoo_relational.readable_model_catalog(rt)
census = odoo_relational.population_census(rt)
has_discovery = bool(discovered.get("capturedAt"))
+ snapshot = {}
+ if not has_discovery:
+ import core.user_tables as user_tables
+ snapshot = user_tables.odoo_schema_snapshot(rt)
complete = (discovered.get("complete") is True if has_discovery
else snapshot.get("complete") is True)
captured = str((discovered.get("capturedAt") if has_discovery
@@ -4130,13 +4295,16 @@ def _scraper_database_run(rt, uname, is_admin, database_key):
return {"key": key, "state": state, "counts": counts, "receipt": receipt}
-def connector_detail(rt, user, uname, is_admin, connector_key):
+def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offset=None,
+ databases_limit=None, fields="full"):
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
Pure over the session for the same reason `directory` is: a gate can drive it without a
request, so the modal's payload is provable without a browser.
"""
budget = _ConnectorDetailBudget()
+ page = _detail_page(databases_offset, databases_limit, fields)
+ rt = _ConnectorDetailReads(rt)
rows = directory(rt, uname, bool(is_admin))
budget.check()
key = str(connector_key or "")
@@ -4144,15 +4312,23 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
if str(candidate.get("key") or "") == key), None)
if row is None:
return None
- graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal(
- key, "No actual relationship graph was returned for this connection. A provider template "
- "is not shown.")
+ # Summary pages defer the independent graph as well, otherwise an all-table field graph
+ # would recreate the unbounded payload beside the paged database list.
+ graph = (None if fields == "summary" else
+ relationship_graphs([row], user, rt, uname, bool(is_admin)).get(key)
+ or _graph_refusal(key, "No actual relationship graph was returned for this connection. "
+ "A provider template is not shown."))
budget.check()
# ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
# question about a PERSON and a table, not about an administrator flag alone.
- databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""))
+ databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""),
+ page=page, fields=fields)
+ if key == "amazon_sp":
+ databases = _screen_amazon_database_rows(databases)
budget.check()
overview = _detail_overview(row, databases, graph)
+ if page is not None and page.total:
+ overview["creates"] = page.creates
budget.check()
keychain = _detail_keychain(rt, uname, bool(is_admin), row)
budget.check()
@@ -4163,6 +4339,9 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
return _safe_connector_status(key, {"overview": overview,
"keychain": keychain,
"databases": databases,
+ **({"databasesPage": page.dto()} if page is not None else {}),
+ **({"relationshipsPath": f"/connectors/{key}/relationships"}
+ if fields == "summary" else {}),
"syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
if is_admin and key in connector_sync.providers() else None),
"materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
@@ -4182,6 +4361,36 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
"relationships": graph})
+def _screen_amazon_database_rows(rows):
+ """S67-T31: each Amazon database row answers the status guard ON ITS OWN.
+
+ `_safe_connector_status` still guards the whole detail below, and it used to be the first and
+ only judge, so one oversized scalar in one row aborted every row, the Overview and the
+ Keychain with a bare 500. Here an over-long string is trimmed, and a row that still fails (a
+ credential shape anywhere in it) keeps only its key, its shipped name and its switch, with
+ "Sync status is unavailable for this database." as its status. The refused value is dropped,
+ never echoed.
+ """
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns the guard
+ listed = list(rows or [])
+ safe, _refused = cx.screen_status_rows(dict(enumerate(listed)))
+ out = []
+ for index, row in enumerate(listed):
+ if index in safe:
+ out.append(safe[index])
+ continue
+ key = str((row or {}).get("key") or "")
+ enabled = (row or {}).get("enabled") is True
+ label = LABELS_AMAZON.get(key) or key
+ out.append({"key": key, "label": label, "defaultLabel": label, "native": "",
+ "enabled": enabled,
+ **({"sync": None} if "sync" in (row or {}) else {}),
+ "syncHealth": _amazon_database_sync_health(None, enabled),
+ "fields": [], "excluded": [], "sharesFieldsWith": [],
+ **({"fieldsDeferred": True} if (row or {}).get("fieldsDeferred") else {})})
+ return out
+
+
def _set_amazon_database_enablement(database_key, body, session):
"""The testable core of Amazon's tenant-scoped administrator enablement door."""
if not session.admin:
@@ -4253,22 +4462,100 @@ def scraper_database_run(database_key: str, session: Session = Depends(require_s
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
# collide with this three-segment path at all.
@router.get("/connectors/{connector_key}/detail")
-def connectors_detail(connector_key: str, session: Session = Depends(require_session)):
+def connectors_detail(connector_key: str, session: Session = Depends(require_session),
+ databases_offset: Annotated[int | None, Query(ge=0)] = None,
+ databases_limit: Annotated[int | None, Query(ge=1)] = None,
+ fields: Literal["summary", "full"] = "full"):
"""C4 / R26: everything the connector modal shows, for THIS reader, in one read.
Session-gated exactly like the directory it is opened from: a member may look at what the
workspace is connected to. What a member does NOT get is the tenant-wide credential door
(`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
"""
- detail = connector_detail(
- session.runtime, session.user, session.uname,
- bool(session.admin), connector_key)
+ try:
+ detail = connector_detail(
+ session.runtime, session.user, session.uname,
+ bool(session.admin), connector_key, databases_offset=databases_offset,
+ databases_limit=databases_limit, fields=fields)
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_detail_failed",
+ "The connector details could not be read.")
if detail is None:
raise err(404, "unknown_connector",
"this workspace has no connector with that key")
return detail
+def _plain_read_failure(exc, code, sentence):
+ """S67-T31: an unexpected failure on a connector read answers the app's JSON error shape.
+
+ The owner's modal read "The server answered 500." because a guard refusal escaped as a bare
+ `Internal Server Error` with no body. Every read the modal makes now answers
+ `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
+ and no exception text (which can echo a payload) reaches the wire.
+ """
+ import logging # noqa: PLC0415
+ logging.getLogger(__name__).exception("connector read failed: %s", code)
+ name = type(exc).__name__
+ if name in ("StoreUnavailable", "PoolTimeout"):
+ return err(503, "store_unavailable",
+ "The workspace store did not answer in time, so nothing could be read.")
+ try:
+ import connectors_amazon as cx # noqa: PLC0415
+ if isinstance(exc, cx.AmazonStatusSafetyError):
+ return err(500, "connector_status_withheld",
+ "This connector's status could not be shown safely, so it was withheld.")
+ except Exception: # noqa: BLE001
+ pass
+ return err(500, code, sentence)
+
+
+def _detail_row(rt, uname, is_admin, connector_key):
+ row = next((row for row in directory(rt, uname, is_admin)
+ if row.get("key") == connector_key), None)
+ if row is None:
+ raise err(404, "unknown_connector", "this workspace has no connector with that key")
+ return row
+
+
+@router.get("/connectors/{connector_key}/databases/{database_key}/fields")
+def connector_database_fields(connector_key: str, database_key: str,
+ session: Session = Depends(require_session)):
+ """The same field disclosure and database access as detail, for one database only."""
+ rt = _ConnectorDetailReads(session.runtime)
+ try:
+ _detail_row(rt, session.uname, bool(session.admin), connector_key)
+ rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname,
+ page=_DatabasePage(database_key=database_key))
+ if not rows:
+ raise err(404, "unknown_database",
+ "This connector has no visible database with that key.")
+ return _safe_connector_status(connector_key, {
+ "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]})
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_fields_failed",
+ "The fields of this database could not be read.")
+
+
+@router.get("/connectors/{connector_key}/relationships")
+def connector_relationships(connector_key: str, session: Session = Depends(require_session)):
+ """Load the permitted graph when the summary client's Relationships tab opens."""
+ rt = _ConnectorDetailReads(session.runtime)
+ try:
+ row = _detail_row(rt, session.uname, bool(session.admin), connector_key)
+ return _safe_connector_status(connector_key, relationship_graphs(
+ [row], session.user, rt, session.uname, bool(session.admin))[connector_key])
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_relationships_failed",
+ "The relationship graph could not be read.")
+
+
class _ReplacementConflict(Exception):
"""The addressed keychain row changed between replacement preparation and its store update."""
diff --git a/api/routes_external.py b/api/routes_external.py
index 172fdd6123fb33e8026f291a90264f9c95009eb3..f479438ef786347f27b90e520fba1c7084bdaefa 100644
--- a/api/routes_external.py
+++ b/api/routes_external.py
@@ -19,7 +19,7 @@ from starlette.concurrency import run_in_threadpool
import deps
import aios_session
from deps import Session, err, require_session
-from core import audit, external_access, external_oauth, external_tools, sheets
+from core import audit, external_access, external_oauth, external_tools, perm_scope, sheets
from harness import runtime
@@ -621,14 +621,16 @@ def require_external_session(
def require_external_analyst(
session: Session = Depends(require_external_session),
) -> Session:
- """Apply the same Analyst grant wall used by the in-app Assistant and Query."""
- session.require("analyst")
+ """Check the separate AI apps grant on every request, including existing bearer tokens."""
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
+ if reason:
+ raise err(403, "forbidden", reason)
return session
def _can_use_analyst(session: Session) -> bool:
try:
- session.require("analyst")
+ require_external_analyst(session)
except HTTPException as exc:
if exc.status_code == 403:
return False
@@ -726,8 +728,12 @@ def _resolve_external_scope(
if include_schema:
if row["answerable"] or strict_schema:
try:
+ # ⭐ S67-T34 (D-2630): THE FIELD CONTRACT, NOT THE ROWS. Describing a
+ # database used to read (and count) every row first, so the largest
+ # connected table refused its own schema. "schema" runs every wall and
+ # reads no row; sources that are not mirror-served ignore it.
scoped = deps.assistant_read_scope(
- session, database, fields=None, filters=None,
+ session, database, fields=None, filters=None, mode="schema",
)
except HTTPException as exc:
# HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
@@ -999,7 +1005,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
f" '{surface_url}/mcp'",
"```",
"",
- ("Sheet tools require analyst and sheets grants. Geometry patches carry the revision "
+ ("Sheet tools require AI apps (MCP) and Sheet access. Geometry patches carry the revision "
"read by the server; pass rev to also protect the revision you read. Cell writes "
"are previewed before confirmation. A partially refused patch changes no cells."),
"",
@@ -1043,7 +1049,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
@router.post("/keys", status_code=201)
def create_key(body=Body(default=None), session: Session = Depends(require_session)):
"""Mint a credential for the signed-in human. Bearer auth is not accepted here."""
- session.require("analyst")
+ require_external_analyst(session)
label = body.get("label") if isinstance(body, dict) else None
access = body.get("access") if isinstance(body, dict) else None
databases = body.get("databases") if isinstance(body, dict) else None
@@ -1182,7 +1188,7 @@ def oauth_metadata(request: Request):
@router.post("/oauth/clients", status_code=201)
def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
"""Register one user-owned public client; PKCE replaces a shared client secret."""
- session.require("analyst")
+ require_external_analyst(session)
body = body if isinstance(body, dict) else {}
try:
client = external_oauth.register_client(
@@ -1470,16 +1476,18 @@ def oauth_authorize(
if redirect_uri not in meta["r"]:
return _refusal_page("The app sent a callback it did not register. Start the "
"connection again from the app.")
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
+ if reason:
+ return _refusal_page(reason, 403)
try:
- session.require("analyst")
_require_oauth_resource(request, resource)
if response_type != "code":
raise external_oauth.ExternalOAuthError("oauth_response_type_invalid")
external_oauth._normalise_state(state)
external_oauth._normalise_challenge(code_challenge, code_challenge_method)
except HTTPException:
- return _refusal_page("Your Loopable role cannot connect AI apps. Ask an "
- "administrator.", 403)
+ return _refusal_page("The app requested an unavailable resource. Start the "
+ "connection again from the app.", 403)
except external_access.ExternalAccessError:
return _refusal_page("The app sent an incomplete sign-in request. Start the "
"connection again from the app.")
@@ -1490,7 +1498,9 @@ def oauth_authorize(
return _consent_page(request, session, meta, fields)
if session is None:
raise err(401, "no_session", "sign in to continue")
- session.require("analyst")
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
+ if reason:
+ return _refusal_page(reason, 403)
try:
audience = _require_oauth_resource(request, resource)
result = external_oauth.authorize(
@@ -1532,11 +1542,9 @@ async def oauth_consent(request: Request, session: Session = Depends(require_ses
databases = chosen
else:
databases = external_access.ALL_DATABASES
- try:
- session.require("analyst")
- except HTTPException:
- return _refusal_page("Your Loopable role cannot connect AI apps. Ask an administrator.",
- 403)
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
+ if reason:
+ return _refusal_page(reason, 403)
try:
audience = _require_oauth_resource(request, fields.get("resource"))
client = external_oauth.adopt_dynamic_client(
@@ -1719,7 +1727,7 @@ def external_setup(request: Request):
@router.get("/skill.md")
def portable_skill(request: Request, session: Session = Depends(require_session)):
"""Download current governed external read and build instructions."""
- session.require("analyst")
+ require_external_analyst(session)
document = _portable_skill_document(session, str(request.base_url))
return Response(
content=document,
@@ -1750,7 +1758,7 @@ def whoami(session: Session = Depends(require_external_analyst)):
@router.get("/scope/databases")
def scope_databases(session: Session = Depends(require_session)):
"""List the databases this signed-in maker could put inside a new credential's scope."""
- session.require("analyst")
+ require_external_analyst(session)
return {"databases": [
{key: row[key] for key in ("database", "label", "permitted", "answerable")}
for row in _resolve_external_scope(session)
@@ -1817,8 +1825,11 @@ def _query_database(session: Session, database: str, payload: dict):
# This is the same permissioned record reader the in-app Assistant uses. It resolves the
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
+ # ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
+ # its aggregates run in SQL over every permitted row and its rows are read only as needed.
scoped = deps.assistant_read_scope(
- session, database, fields=None, filters=payload.get("filters"))
+ session, database, fields=None, filters=payload.get("filters"), mode="schema")
+ mirror = scoped.get("source_kind") == "mirror_user_table"
visible_fields = list(scoped.get("fields") or [])
visible_keys = {
field.get("key") for field in visible_fields
@@ -1847,6 +1858,104 @@ def _query_database(session: Session, database: str, payload: dict):
sorts.append({"field": rule["field"], "direction": rule["direction"]})
aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
+ server_aggregates = None
+ paged_total = None
+ if mirror:
+ # ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
+ # are computed by the mirror in SQL over exactly the rows this caller may read, never
+ # over a fetched slice, and never by pulling a million rows into this process.
+ if aggregates:
+ summary = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"),
+ mode=("aggregate", aggregates))
+ server_aggregates = summary.get("server_aggregates")
+ import assistant_mirror
+ # The summary already counted the permitted population, so a table past the one-pass
+ # ceiling goes straight to SQL paging instead of pulling a first 5,000-row page only to
+ # refuse it.
+ known = (summary.get("total_rows") if aggregates else None)
+ try:
+ if known is not None and int(known) > assistant_mirror.ASSISTANT_READ_MAX:
+ raise err(409, "assistant_source_too_large",
+ f"this database matches {int(known):,} rows for that question, "
+ f"past the {assistant_mirror.ASSISTANT_READ_MAX:,} rows one answer "
+ f"holds in memory")
+ scoped = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"))
+ except HTTPException as exc:
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
+ if info.get("code") != "assistant_source_too_large":
+ raise
+ # ⭐ RULE 1: PAST THE ONE-PASS CEILING THE ROWS ARE PAGED IN SQL, NOT REFUSED. Only
+ # this page reaches the process. A sort cannot be honoured at this size without
+ # holding every row, so it is refused by name with the fix, never silently dropped.
+ if sorts:
+ raise err(409, "query_sort_too_large",
+ f"{info.get('message') or 'This database is too large to sort here.'} "
+ f"Sorting is refused at this size because it would need every row in "
+ f"memory. Ask without sorts to page it in record order, or narrow it "
+ f"with a filter and sort again.") from exc
+ paged_total = True
+ if paged_total:
+ offset = (0 if payload.get("cursor") is None else None)
+ page_digest = _json_digest({
+ "database": database, "fields": selected_keys, "filters": payload.get("filters"),
+ "sorts": [], "aggregates": aggregates, "limit": limit, "paged": True,
+ })
+ first = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"), mode=("page", 0, 1))
+ snap = _json_digest({
+ "source_version": first.get("source_version"),
+ "visible_fields": [field["key"] for field in visible_fields
+ if isinstance(field, dict) and field.get("key") in visible_keys],
+ "total_rows": first.get("total_rows"),
+ })
+ if offset is None:
+ offset = _decode_query_cursor(session, payload.get("cursor"), page_digest, snap)
+ total_rows = int(first.get("total_rows") or 0)
+ if offset > total_rows:
+ raise err(409, "query_cursor_stale",
+ "The permitted result set changed. Start again without a cursor.")
+ page = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"),
+ mode=("page", offset, limit))
+ page_rows = list(page.get("records") or [])
+ for row in page_rows:
+ _assert_finite_query_value({"pid": row.get("pid"),
+ **{key: row.get(key) for key in selected_keys}})
+ records = [{"pid": copy.deepcopy(row.get("pid")),
+ **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
+ for row in page_rows]
+ next_offset = offset + limit
+ next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset)
+ if next_offset < total_rows else None)
+ result = {
+ "database": page.get("database"), "label": page.get("label"),
+ "source_kind": page.get("source_kind"),
+ "source_version": copy.deepcopy(page.get("source_version")),
+ "retrieved_at": page.get("retrieved_at"), "permission_scope_applied": True,
+ "fields": [copy.deepcopy(field) for field in visible_fields
+ if field.get("key") in selected_keys],
+ "filters": copy.deepcopy(page.get("filters")), "sorts": [],
+ "records": records, "aggregates": copy.deepcopy(server_aggregates or []),
+ "page": {"limit": limit, "returned": len(records), "offset": offset,
+ "has_more": next_cursor is not None, "next_cursor": next_cursor},
+ "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
+ "max_aggregates": _QUERY_MAX_AGGREGATES,
+ "max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
+ "max_filter_nodes": _QUERY_MAX_FILTER_NODES,
+ "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
+ }
+ _assert_finite_query_value(result)
+ audit.record("external.query", session.tenant, session.uname, {
+ "surface": "ai_api",
+ "authentication": session.claims.get("external_auth_type", "api_key"),
+ "field_count": len(selected_keys), "filter_applied": page.get("filters") is not None,
+ "sort_count": 0, "aggregate_count": len(aggregates), "page_size": limit,
+ "returned_count": len(records), "cursor_used": offset > 0,
+ })
+ return result
+
rows = list(scoped.get("records") or [])
inspected_keys = (set(selected_keys)
| {rule["field"] for rule in sorts}
@@ -1897,7 +2006,8 @@ def _query_database(session: Session, database: str, payload: dict):
"filters": copy.deepcopy(scoped.get("filters")),
"sorts": copy.deepcopy(sorts),
"records": records,
- "aggregates": _aggregate_records(rows, aggregates),
+ "aggregates": (copy.deepcopy(server_aggregates) if server_aggregates is not None
+ else _aggregate_records(rows, aggregates)),
"page": {"limit": limit, "returned": len(records), "offset": offset,
"has_more": next_cursor is not None, "next_cursor": next_cursor},
"caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
@@ -2730,7 +2840,7 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
def _strong_confirmation_required(name, args):
if name == "database_object":
return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
- if name in {"view_delete", "app_delete"}:
+ if name in {"view_delete", "app_delete", "sheet_delete_tab"}:
return True
if name == "sheet_write_cells":
return any(isinstance(cell, dict) and cell.get("kind") == "empty"
@@ -2913,7 +3023,7 @@ def _confirm_change(session, args):
"Enter the exact stronger confirmation phrase shown in the preview.")
_confirmation_already_used(session, payload)
- # Freshly resolve the account, Analyst grant, credential revocation/label, credential scope,
+ # Freshly resolve the account, AI apps grant, credential revocation/label, credential scope,
# database walls and writer semantics before claiming the nonce or touching customer state.
fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
@@ -3100,7 +3210,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
return protocol_error(rid, -32602, "MCP params must be an object.")
name = params.get("name")
try:
- # Same Analyst wall as the read doors, inside the dispatcher so a grant refusal
+ # Same AI apps wall as the read doors, inside the dispatcher so a grant refusal
# remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
require_external_analyst(session)
if method == "initialize":
@@ -3163,7 +3273,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
@router.post("/mcp")
async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
- """Remote MCP without a browser session, with the read doors' maker and Analyst wall."""
+ """Remote MCP without a browser session, with the read doors' maker and AI apps wall."""
try:
req = await request.json()
except (ValueError, UnicodeError):
@@ -3177,6 +3287,7 @@ async def external_mcp(request: Request, session: Session = Depends(require_exte
# A new write in the registry without a matching literal raises at import, before serving.
_WRITE_TOOLS = frozenset({
"sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
+ "sheet_delete_tab",
"sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
"sheet_add_rule", "sheet_set_chart", "sheet_structure",
"sheet_save_template",
diff --git a/api/routes_keychain.py b/api/routes_keychain.py
index c557a66edd40d8f0c54876d788498bf0ed548c13..9dec177d26dd5e8de290927c43748b0a83fca64f 100644
--- a/api/routes_keychain.py
+++ b/api/routes_keychain.py
@@ -782,23 +782,63 @@ def _credential_revocation(row, rt):
def _revocation_residue(entry_type):
- """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
-
- Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
- deletion stays operator-only because it is permanent, exact-tenant work outside the product
- store transaction.
- """
+ """Fallback cleanup instructions when automatic retention could not be scheduled."""
if entry_type != "amazon_sp":
return []
return [{
"where": "The tenant DuckDB mirror",
"why": "Connection removal clears product databases but not the raw Amazon mirror.",
- "fix": "After every Amazon credential is revoked, an operator must run "
+ "fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run "
"platform/harness/amazon_store.py --purge --tenant "
"--confirm-tenant and retain the residue-free receipt.",
}]
+def _amazon_lifecycle_warning(session, action):
+ """Access already changed. Warn and audit a lifecycle failure without turning it into 5xx."""
+ rt = session.runtime
+ removing = action == "revocation"
+ warning = ("The Amazon credential was removed, but its automatic data deletion could not be scheduled. "
+ "Contact an administrator to arrange deletion within 30 days."
+ if removing else
+ "The Amazon credential was saved, but its pending data deletion could not be cancelled. "
+ "Contact an administrator before the scheduled deletion date.")
+ kind = "credential.revoked" if removing else "credential.added"
+ try:
+ from core import audit
+ detail = {"connector": "amazon_sp",
+ "deletion_trigger_recorded" if removing else "deletion_cancellation_recorded": False}
+ if audit.record(kind, rt.key, session.uname, detail) is None:
+ raise RuntimeError("lifecycle audit could not be saved")
+ except Exception:
+ try:
+ from harness import amazon_store as az
+ az._append_ledger(rt.key, rt, "audit_failures",
+ {"tenant": rt.key, "kind": kind, "checked_at": az._control_stamp()})
+ except Exception:
+ pass
+ warning += " The audit event could not be saved."
+ print(f"Amazon lifecycle warning: {warning}")
+ return {"warning": warning}
+
+
+def _amazon_deletion_receipt(session, *, product_purge_succeeded=True):
+ try:
+ if not product_purge_succeeded:
+ raise RuntimeError("product cleanup is incomplete")
+ from harness import amazon_store as az
+ rt = session.runtime
+ az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname)
+ plan = az.scheduled_purge(rt.key, rt)
+ return {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"],
+ "residue": [{"where": "Amazon data in this workspace",
+ "why": az.deletion_schedule_message(plan),
+ "fix": "Add the Amazon credential again before the scheduled date to cancel deletion."}]}
+ except Exception:
+ return {**_amazon_lifecycle_warning(session, "revocation"),
+ "residue": _revocation_residue("amazon_sp")}
+
+
def _write_revocation_warning(rt, entry_id, warning=None):
"""Prune expired consent on every write; failed persistence requires explicit confirmation."""
now = int(time.time())
@@ -852,6 +892,7 @@ def list_keychain(session: Session = Depends(require_session)):
@router.post("/admin/keychain", status_code=201)
def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)):
kc = _kc()
+ lifecycle = {}
body = body or {}
if not session.runtime.available():
raise err(503, "store_unavailable", "the tenant store is unavailable")
@@ -921,6 +962,11 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
except Exception: # noqa: BLE001
pass
elif etype == "amazon_sp":
+ try:
+ from harness import amazon_store as az
+ az.cancel_deletion_trigger(session.runtime.key, rt=session.runtime, principal=session.uname)
+ except Exception:
+ lifecycle = _amazon_lifecycle_warning(session, "cancellation")
# D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
# remains the mirror's single writer. A queue failure must not undo a stored credential.
try:
@@ -939,7 +985,7 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
raise err(503, "store_unavailable",
"the key was not saved; its sharing setting could not be stored, so "
"nothing was kept. Try again.")
- return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}}
+ return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
@router.put("/admin/keychain/{entry_id}")
@@ -1089,14 +1135,16 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
raise err(503, "store_unavailable", "The delete did not land. Try again.")
if not purges:
_write_revocation_warning(rt, entry_id)
+ lifecycle = (_amazon_deletion_receipt(session, product_purge_succeeded=footprint["rowsKnown"])
+ if row["type"] == "amazon_sp" else {})
if footprint["rowsKnown"]:
- return {"ok": True}
+ return {"ok": True, **lifecycle}
# Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
# that incomplete cleanup explicit instead of returning an invented "nothing to purge".
return {"ok": True, "registry": registered, "footprint": footprint,
"purged": {"tables": [], "rows": 0, "rowsKnown": False,
"refused": [], "failed": []},
- "residue": _revocation_residue(row["type"])}
+ "residue": _revocation_residue(row["type"]), **lifecycle}
from core import user_tables as ut
try:
result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
@@ -1105,11 +1153,14 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
"failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
_write_revocation_warning(rt, entry_id)
+ lifecycle = (_amazon_deletion_receipt(
+ session, product_purge_succeeded=not result["failed"] and not result["refused"])
+ if row["type"] == "amazon_sp" else {})
return {"ok": True,
"purged": {"tables": result["purged"], "rows": result["rows"],
"rowsKnown": result["rowsKnown"], "refused": result["refused"],
"failed": result["failed"]},
- "residue": _revocation_residue(row["type"])}
+ "residue": _revocation_residue(row["type"]), **lifecycle}
@router.post("/admin/keychain/{entry_id}/test")
diff --git a/api/routes_module_index.py b/api/routes_module_index.py
index 5618005eceb358f2e1e20dbf0b58140afd87e7f3..b9d246e393da257a09298d657a1e2c06566172dc 100644
--- a/api/routes_module_index.py
+++ b/api/routes_module_index.py
@@ -1,12 +1,15 @@
-"""W61-T20 / C6: personal App and Agents folder trees in the tenant store."""
+"""Personal App, Agents and Data folder trees in the tenant store."""
import copy
from fastapi import APIRouter, Body, Depends, HTTPException
from core.sheets import SheetFolderRefusal, validate_folder_tree
from deps import Session, err, module_gate, require_session
+import application_registry
import routes_assets
import routes_automation
+from routes_applications import _may_open
+from routes_nav import _placeable_top_keys
import voice_control_plane as voice
router = APIRouter(prefix="/api/v1/module-index")
@@ -17,16 +20,22 @@ def _sentence(value):
return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
+# D-3282 (standing rule 1): a limit the server cannot lift is reported with its cause and the fix.
+_REFUSAL_SENTENCES = {
+ "too_many_folders": "Folders hold at most 200 per rail. Delete or merge a folder, then try again.",
+}
+
+
def _refuse(exc):
status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
"module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
- return err(status, exc.code, _sentence(exc.message))
+ return err(status, exc.code, _sentence(_REFUSAL_SENTENCES.get(exc.code, exc.message)))
def _access(module: str, session: Session = Depends(require_session)):
- if module not in {"app", "agents"}:
+ if module not in {"app", "agents", "data"}:
raise _refuse(SheetFolderRefusal(
- "module_index_unknown", f'Folder index "{module}" is not supported; use app or agents.'))
+ "module_index_unknown", f'Folder index "{module}" is not supported; use app, agents or data.'))
# App has the same session-only door as GET /applications; its rows have their own wall.
if module == "agents":
try:
@@ -47,11 +56,28 @@ def _owner(session, user):
def _readable_ids(module, session):
+ if module == "data":
+ keys, enumerated = _placeable_top_keys(session)
+ if not enumerated:
+ raise err(503, "store_unavailable", "Databases could not be loaded. Reload folders to retry.")
+ return keys
# The projection is hygiene; the per-item list door is the wall. The client renders
# only listed items, so an unverifiable placement renders no tile. Use local voice
# records, never audience-option assembly, just to verify placement ids.
if module == "app":
- return {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
+ # S67-T38 (O24): the App home also renders the seeded Finance projections, so the wall
+ # admits their ids through the SAME grant filter /nav/applications uses. Never ungated:
+ # a Finance card the session cannot open is not placeable.
+ stored = {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
+ finance = application_registry.finance_application_records(
+ may_open=lambda permission: _may_open(session, permission))
+ # S67-T41 owner correction: a Content Studio App is bound to a USER table, which the
+ # topic-only `/applications` index refuses, so its tile arrives from /nav/applications.
+ # Its id is admitted through the registry's own `may_read` (the same read that lists it).
+ studios = {row["id"] for row in application_registry.list_apps(
+ session.runtime, session.uname, session.admin)
+ if row.get("templateKind") == application_registry.CONTENT_STUDIO_TEMPLATE_KIND}
+ return stored | {row["id"] for row in finance} | studios
# ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
# *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
# it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
@@ -94,11 +120,41 @@ def _available(session):
raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
+def _carry_data_index(module, owner, session):
+ """Copy the legacy bucket once, inside the index transaction; never clean or write nav_prefs.
+
+ Presence, including an empty bucket, is authoritative. Revision one makes an unread rev-zero
+ PUT stale even when it is the first request after upgrade (W61-T19's wholesale-erase guard).
+ """
+ if module != "data":
+ return
+ doc = session.runtime.get("module_index_data") or {}
+ if owner in (doc.get("users") or {}):
+ return
+ legacy = (session.runtime.get("nav_prefs") or {}).get(owner) or {}
+
+ def carry(current):
+ state = copy.deepcopy(current) if isinstance(current, dict) else {}
+ users = state.setdefault("users", {})
+ if owner not in users:
+ users[owner] = {
+ "folders": [{**folder, "parent": None} for folder in legacy.get("folders", [])],
+ "itemFolders": copy.deepcopy(legacy.get("placement", {})),
+ "rev": 1,
+ "carriedFrom": "nav_prefs",
+ }
+ state["schema"] = 1
+ return state
+
+ session.runtime.update("module_index_data", carry, flush="sync")
+
+
@router.get("/{module}")
def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
try:
owner = _owner(session, user)
_available(session)
+ _carry_data_index(module, owner, session)
doc = session.runtime.get("module_index_" + module) or {}
bucket = (doc.get("users") or {}).get(owner) or {}
ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
@@ -117,6 +173,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
raise SheetFolderRefusal("module_index_user_forbidden",
f'The folder tree for "{owner}" can only be changed by its owner.')
_available(session)
+ _carry_data_index(module, owner, session)
raw = body or {}
expect_rev = raw.get("rev")
if type(expect_rev) is not int or expect_rev < 0:
@@ -141,6 +198,8 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
raise SheetFolderRefusal("index_revision_stale",
"Folders changed somewhere else. Reload folders and try again.")
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
+ if "carriedFrom" in (users.get(owner) or {}):
+ saved["carriedFrom"] = users[owner]["carriedFrom"]
if collapsed is not None:
saved["collapsedGroups"] = collapsed
elif "collapsedGroups" in (users.get(owner) or {}):
diff --git a/api/routes_nav.py b/api/routes_nav.py
index 6613020e5468ad25e844cf4a1c348fea72ae0f2a..f5495fb916fbdbd4d8e2a70361b3b73d635e07c8 100644
--- a/api/routes_nav.py
+++ b/api/routes_nav.py
@@ -528,7 +528,7 @@ def _granted_view_ids(session):
#:
#: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
#: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
-#: "Loopable" group, and the branch below puts user tables in "Custom". A connection someone named
+#: "Loopable" group; the legacy "Custom" group is reserved too. A connection someone named
#: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
#: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
_RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
@@ -704,7 +704,7 @@ def nav(session: Session = Depends(require_session)):
continue
pages.append({"key": e["key"], "label": e["label"],
"source": e.get("source") or "Blank",
- "sourceGroup": e.get("sourceGroup") or "Custom", "chrome": "main"})
+ "sourceGroup": e.get("sourceGroup") or "Loopable", "chrome": "main"})
except Exception:
# ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT.
#
@@ -757,7 +757,7 @@ def nav(session: Session = Depends(require_session)):
if _folder:
page["sourceGroup"] = _folder
elif page.get("user_table") or source in {"", "Blank", "Custom"}:
- page.setdefault("sourceGroup", "Custom")
+ page.setdefault("sourceGroup", "Loopable")
# ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
#
# W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py
index b6a96f94af25e558df0c2e79ef91ad9ad06a841e..8a0457587d279e52a80df39d2b5be4b84911b74e 100644
--- a/api/routes_odoo_tables.py
+++ b/api/routes_odoo_tables.py
@@ -23,6 +23,7 @@ import hashlib
import hmac
import json
import math
+import threading
import time
from contextlib import contextmanager
@@ -66,7 +67,7 @@ def _rt():
# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
-def _mirror_cur(session):
+def _mirror_cur(session, table_key=None):
"""A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
Returns `(cursor, mismatch_sentence)` — exactly one of the two is None, so a caller cannot
@@ -88,11 +89,32 @@ def _mirror_cur(session):
"""
rtm = _rt()
try:
- return rtm.mirror_cursor(session.runtime), None
+ return _table_cursor(session.runtime, table_key), None
except rtm.DatastoreMismatch as e:
return None, str(e)
+def _table_cursor(rt, table_key=None):
+ """THE one place a mirror cursor is chosen FOR A TABLE: tenant assertion first, then the gate.
+
+ Amazon sync is independent of Odoo's `ready()` phases, so a `ut_amzn_*` grid asserts the same
+ tenant and then opens the loader's local mirror without that Odoo gate; every other table keeps
+ `mirror_cursor`, which refuses until the Odoo backfill is live (partial totals are worse than a
+ retry).
+ ⛔ HOTFIX 2026-09-27: this rule lived only in `_mirror_cur`, so the window route served Amazon
+ rows while `whole_pool` (every `GET /tables//rows` and the workspace envelope of a
+ read-through grid) still answered `503 store_not_ready` on a staging mirror with 0 of 8 Odoo
+ sources ready. Every door now asks here, so the rule cannot hold in one door and not another.
+ `rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents.
+ """
+ from harness import datastore
+ if str(table_key or "").startswith("ut_amzn_"):
+ if rt is not None:
+ rt.assert_datastore_matches()
+ return datastore.ro_cursor()
+ return _rt().mirror_cursor(rt) if rt is not None else datastore.ro_con()
+
+
@router.post("/odoo-tables/refresh")
def refresh_odoo_tables(session: Session = Depends(require_session)):
"""Rebuild all four locked Odoo databases from the tenant's mirror — customers, products,
@@ -178,10 +200,15 @@ def odoo_tables_status(brief: bool = Query(False),
rel = _rel()
if brief:
sources = _sources()
- return {"ok": True, "tables": {
- key: {"readThrough": key in sources}
- for _bucket, key, _label, _fields in rel.TABLES
- }}
+ tables = {key: {"readThrough": key in sources}
+ for _bucket, key, _label, _fields in rel.TABLES}
+ # S66-T43 (lead, observed 2026-09-25): the grid asks THIS answer which tables read
+ # through, and it listed Odoo's roster only, so an Amazon read-through table opened on the
+ # ordinary door and painted its empty stub. Every Amazon key the registered provider
+ # serves is read-through; none is listed when this container's mirror holds no Amazon rows.
+ tables.update({key: {"readThrough": True} for key in sources
+ if str(key).startswith("ut_amzn_") and key not in tables})
+ return {"ok": True, "tables": tables}
import core.user_tables as user_tables
# The detailed operator view genuinely needs rows to report counts/freshness, but it needs
@@ -596,6 +623,28 @@ GRID_SOURCES = {}
_SOURCE_PROVIDERS = []
_SOURCE_ANALYSES = {}
+#: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
+#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
+#: keys to the shared dict as it went, and returned the dict to any caller that found it
+#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
+#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
+#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
+#: this lock and publishes it in one `dict.update`; a reader waits on the lock until
+#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
+#: Amazon fallback) happens only after a build has been published, so it can never stand in
+#: for one either.
+_SOURCES_LOCK = threading.RLock()
+_SOURCES_BUILT = False
+_SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
+
+
+def invalidate_sources():
+ """Forget measured bindings after a connector publishes a new mirror schema."""
+ global _SOURCES_BUILT
+ with _SOURCES_LOCK:
+ _SOURCES_BUILT = False
+ GRID_SOURCES.clear()
+
def register_source_provider(fn):
"""Add a `() -> {table_key: spec}` builder to the connected-grid registry.
@@ -641,20 +690,60 @@ def _sources():
⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
importing that module at file-import time would drag the Odoo layer into every process that
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
+
+ ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
+ lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
+ and so does one that holds only admissions made since the last invalidation.
"""
- if GRID_SOURCES:
+ global _SOURCES_BUILT, _SOURCES_BUILDER
+ if _SOURCES_BUILT and GRID_SOURCES:
return GRID_SOURCES
- for build in list(_SOURCE_PROVIDERS):
+ with _SOURCES_LOCK:
+ if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
+ return GRID_SOURCES
+ _SOURCES_BUILDER = threading.get_ident()
try:
- extra = build() or {}
- except Exception: # noqa: BLE001
- # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must
- # not go dark because a newer connector's module is unhappy — and vice versa.
- continue
- if extra:
+ built = {}
+ for build in list(_SOURCE_PROVIDERS):
+ try:
+ extra = build() or {}
+ except Exception: # noqa: BLE001
+ # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
+ # grids must not go dark because a newer connector's module is unhappy, and
+ # vice versa.
+ continue
+ if extra:
+ built.update(extra)
+ _ut().register_connected(*extra)
+ # A key admitted since the last invalidation was measured at least as recently as
+ # this build, so it stands; everything else comes from the build. ONE update, so a
+ # lock-free reader of the dict sees the old entry or the new one, never a gap.
+ GRID_SOURCES.update({**built, **GRID_SOURCES})
+ _SOURCES_BUILT = True
+ finally:
+ _SOURCES_BUILDER = None
+ return GRID_SOURCES
+
+
+def _learn_amazon_sources():
+ """Publish the Amazon provider's keys into a registry that was built without any of them.
+
+ HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot
+ BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates
+ it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through
+ Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more
+ than one window. A registry that already knows one Amazon key is not re-asked.
+ """
+ _sources()
+ if any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
+ return
+ extra = _amazon_sources()
+ if not extra:
+ return
+ with _SOURCES_LOCK:
+ if not any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
GRID_SOURCES.update(extra)
_ut().register_connected(*extra)
- return GRID_SOURCES
def _odoo_sources():
@@ -1014,7 +1103,67 @@ def _odoo_spawned():
#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
+def _amazon_sources():
+ try:
+ import amazon_relational
+ return amazon_relational.grid_sources()
+ except Exception:
+ import logging
+ logging.getLogger(__name__).exception("Amazon grid sources could not be loaded")
+ return {}
+
+
+def require_amazon_mirror(table_key, session, definition=None, st=None):
+ """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
+
+ The provider contributes no keys on an Odoo-only mirror. A persisted read-through
+ definition still must refuse there instead of falling through to its empty stored rows.
+ S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
+ window route stops paying a second definitions projection for this one check.
+ """
+ if not str(table_key).startswith("ut_amzn_"):
+ return
+ import amazon_relational as amazon
+ from routes_tables import _defn_or_refuse
+
+ if table_key not in amazon.SOURCE:
+ return
+ if definition is None:
+ definition = _defn_or_refuse(session, table_key, st=st, defs_only=True,
+ scope_applied=True)
+ try:
+ cur, mismatch = _mirror_cur(session, table_key)
+ except RuntimeError as exc:
+ raise err(503, "store_not_ready", str(exc))
+ if mismatch:
+ raise err(409, "cross_tenant_store", mismatch)
+ try:
+ spec = _sources().get(table_key)
+ if (spec is not None or definition.get("readThrough")
+ or table_key in amazon.READ_THROUGH):
+ # Check current table presence even if a binding was cached before a restart.
+ current = amazon.grid_source(cur, table_key)
+ if current.get("missing"):
+ raise err(503, "amazon_mirror_not_loaded", current["missing"])
+ # A dynamically oversized snapshot may have a durable stub before this process
+ # has measured it. Admit its binding without changing another source's registry.
+ # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
+ # key. Admitted into an empty or Amazon-less registry, this one key used to make
+ # `_sources()` skip its build, so the grid resolved while its link targets
+ # (orders, read-through) were dispatched as stored tables: a first-page 500.
+ if spec is None or spec.get("missing"):
+ _learn_amazon_sources()
+ with _SOURCES_LOCK:
+ _sources()
+ held = GRID_SOURCES.get(table_key)
+ if held is None or held.get("missing"):
+ GRID_SOURCES[table_key] = current
+ finally:
+ cur.close()
+
+
register_source_provider(_odoo_sources)
+register_source_provider(_amazon_sources)
register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
@@ -1243,8 +1392,18 @@ def _source_for(cur, table_key):
if module_cache_source(key) is not None:
return None
spec = _sources().get(key)
+ if not spec and key.startswith("ut_amzn_"):
+ # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
+ # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
+ # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
+ # read-through Amazon grid answered "no read-through binding". Ask the Amazon provider
+ # alone, once it can answer; a registry that already knows one Amazon key is not re-asked.
+ _learn_amazon_sources()
+ spec = GRID_SOURCES.get(key)
if not spec:
return None
+ if spec.get("missing"):
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
from harness import datastore
for opt in (spec.get("optional") or ()):
wants = tuple(opt.get("absent") or ())
@@ -1271,6 +1430,10 @@ def _source_for(cur, table_key):
for alias, tname in aliases.items():
cols = datastore.columns_of(tname, cur=cur)
if not cols:
+ if key.startswith("ut_amzn_"):
+ import amazon_relational
+ raise err(503, "amazon_mirror_not_loaded",
+ amazon_relational.missing_source(key)["missing"])
return None # the mirror has no such table on this deployment
have_by_alias[alias] = cols
excluded = ""
@@ -1624,11 +1787,133 @@ def _related_source_value(session, entry, record):
def _materialised_table(table_key):
- """Storage dispatch, independent of the browser's fallible status roster."""
- return module_cache_source(table_key) is not None or table_key not in _sources()
+ """Storage dispatch, independent of the browser's fallible status roster.
+
+ ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
+ (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
+ the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
+ """
+ if module_cache_source(table_key) is not None:
+ return True
+ if table_key in _sources():
+ return False
+ if str(table_key).startswith("ut_amzn_"):
+ _learn_amazon_sources()
+ return table_key not in GRID_SOURCES
+ return True
+
+
+#: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
+#: ROWS, each held against the durable `user_tables` revision it was read at. A grouped Amazon page
+#: used to re-read the definitions projection once for its own wall and once more for EVERY link
+#: target, plus the stored listings entry, on every page; on staging each of those reads is one
+#: large JSONB projection (measured 33 to 186 s a page under load, 2026-09-28). Every `put` and
+#: `update` bumps the revision column in the same statement, so an unchanged token proves the
+#: bytes unchanged; the token is read BEFORE the value, so a write racing the read can only cache
+#: newer bytes under the older token (a miss next time, never a stale hit). ⚠ Nothing cached here
+#: is a permission: the wall (`may_read`, `apply_row_scope`, the compiled predicate) is evaluated
+#: per request against the session's own user, only the bytes it evaluates over are reused.
+_REV_MEMO: dict = {}
+_REV_MEMO_MAX = 64
+# Route handlers run on a thread pool: the lock makes the evict (`next(iter(...))`) and the
+# insert one step, so a concurrent insert can never raise "dictionary changed size".
+_REV_MEMO_LOCK = threading.Lock()
+#: "No token was handed in, read one." Distinct from None, which means "unreadable, hold nothing".
+#: It is the `...` literal (a singleton) so a signature can spell its default `...` and still
+#: define when `verify_grid_ux.py` execs `_relation_rows` on its own, without this module name.
+_UNREAD = ...
+
+
+def _user_tables_token(rt):
+ """The durable change token of this tenant's `user_tables` bucket, or None when unreadable."""
+ try:
+ return rt.revision(_ut().STORE_KEY)["token"]
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
+ """`read()`, reused while this tenant's `user_tables` revision is unchanged.
+
+ ⛔ `clone` runs on the way in AND on every hit, so a caller that edits what it was handed
+ cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
+ answer never is (it is the shape a failed read takes, and caching it would make one outage
+ outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
+ `token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
+ trip, not one per memo. It was read before any value this call reads, which is the only
+ ordering the staleness argument above needs.
+ """
+ if token is _UNREAD:
+ token = _user_tables_token(rt)
+ tenant = getattr(rt, "key", None)
+ slot = (kind, tenant, name)
+ if token is not None and tenant:
+ with _REV_MEMO_LOCK:
+ hit = _REV_MEMO.get(slot)
+ if hit is not None and hit[0] == token:
+ return clone(hit[1])
+ value = read()
+ if token is not None and tenant and keep(value):
+ held = (token, clone(value))
+ with _REV_MEMO_LOCK:
+ if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
+ _REV_MEMO.pop(next(iter(_REV_MEMO)), None)
+ _REV_MEMO[slot] = held
+ return value
+
+
+def _page_defs(session):
+ """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's.
+
+ Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked
+ against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link
+ target does not pay a second revision round trip.
+ The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
+ `user_tables` revision instead of once per question. Kept only when every definition really is
+ a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
+ 28 MB document on a failed read and that must never be held, let alone copied per request.
+ ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
+ lends lazily live for this one request and are never carried to the next.
+ """
+ import copy
+ import core.store as store
+
+ ut = _ut()
+ rt = session.runtime
+ token = _user_tables_token(rt)
+
+ def keep(v):
+ tables = [d for d in (v or {}).values() if isinstance(d, dict)]
+ return bool(tables) and all(store.is_projected(d) for d in tables)
+ defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt),
+ keep=keep, clone=copy.deepcopy, token=token)
+ return ut.lend(rt, **{ut.STORE_KEY: defs}), token
-def _traversal_defn_or_refuse(session, table_key):
+
+def _stored_link_rows(session, table, token=_UNREAD):
+ """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
+
+ ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
+ cache, an app reader) or a read-through binding answers from somewhere this token does not
+ watch, so those read exactly as before. The wall is NOT applied here; the caller applies
+ `apply_row_scope` for its own user on every request. Holding it across users is sound only
+ because `perm_scope._read_user_table` never consults `user` for a `ut_*` key: every
+ user-dependent step (hidden fields, row wall, BU scope) sits above it, in the caller.
+ """
+ import core.perm_scope as perm_scope
+
+ rt = session.runtime
+ if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
+ or module_cache_source(table) is not None or not _materialised_table(table)):
+ return perm_scope._read(table, session.user, rt)
+ return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
+ keep=lambda v: bool(v[1]),
+ clone=lambda v: ([dict(f) for f in v[0]], [dict(r) for r in v[1]]),
+ token=token)
+
+
+def _traversal_defn_or_refuse(session, table_key, st=None):
"""R50 / C4: admit an intermediate after the caller has established both end grants.
The session may already open both ends of a declared two-hop link. Its intermediate
@@ -1636,7 +1921,7 @@ def _traversal_defn_or_refuse(session, table_key):
read still applies the row wall, hidden-field closure, BU scope and mirror guard.
"""
ut = _ut()
- st = ut.lend_defs(session.runtime)
+ st = st if st is not None else ut.lend_defs(session.runtime)
defn = ut.get(table_key, st=st)
if not defn:
raise err(404, "unknown_table", "that database does not exist")
@@ -1644,21 +1929,26 @@ def _traversal_defn_or_refuse(session, table_key):
def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
- traversal_of=None):
+ traversal_of=None, st=None, rev=...): # `...` IS `_UNREAD`
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
Join columns may be hidden on the wire. Only identities leave the resolver; the native
payload builders still apply their complete hidden-field closure before returning cells.
With traversal_of, the caller must already have established both ends of the declared
edge named by that key. Only join values leave this intermediate read, never a cell.
+ S67-T28: `st` is a lent DEFINITIONS snapshot the caller already holds (`_page_defs`); the wall
+ is asked of it instead of re-reading every definition per target. A stored target's rows come
+ through `_stored_link_rows`, reused per `user_tables` revision; its wall still runs here.
+ `rev` is the revision token `_page_defs` already read for this request; absent, the stored
+ target is read directly, as before S67-T28.
"""
import core.perm_scope as perm_scope
import core.relations as relations
columns = set(columns) | {relations.IDENTITY_COL}
if not _materialised_table(table):
- read = (_compiled_read(table, session, traversal_intermediate=True)
- if traversal_of is not None else _compiled_read(table, session))
+ read = (_compiled_read(table, session, traversal_intermediate=True, st=st)
+ if traversal_of is not None else _compiled_read(table, session, st=st))
spec = read["spec"]
bindings = dict(spec["cols"], pid=(spec["id"], int))
if columns - set(bindings):
@@ -1681,11 +1971,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
# This is an internal join, never a cell response. Apply the same permanent wall on the
# complete contract before taking any join value, including a join on a hidden column.
- if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime):
+ if traversal_of is None and not perm_scope.may_read(
+ session.user, table, st=st if st is not None else session.runtime):
raise err(403, "forbidden_table", "This account cannot open the linked database.")
if module_cache_source(table) is not None:
session.require(table)
- fields, rows = perm_scope._read(table, session.user, session.runtime)
+ # The revision memo serves a caller that handed this request's token (the page's link fills,
+ # S67-T28); any other caller reads exactly as before, which is also what lets
+ # `verify_grid_ux.py` exec this function alone with a stubbed `perm_scope`.
+ fields, rows = (perm_scope._read(table, session.user, session.runtime) if rev is ...
+ else _stored_link_rows(session, table, token=rev))
rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
if where:
raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
@@ -1730,6 +2025,11 @@ def _relation_leg_ids(session, source, source_ids, entry):
# identity. Treating a missing/hidden/blank source value as an empty match would
# turn "the join input is unavailable" into the business claim "no related rows".
if source_col != relations.IDENTITY_COL and not values:
+ # A visible Amazon row with an empty business key has no related records.
+ # A missing/inaccessible source row must retain the ordinary refusal.
+ if (source.startswith("ut_amzn_") and target.startswith("ut_amzn_")
+ and source_rows and all(source_col in row for row in source_rows)):
+ return set()
raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
if (manual and not reverse) or (inverse and reverse):
values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
@@ -1794,6 +2094,10 @@ def _relation_target_ids(session, target, related):
if not _may_open(session, leg["target"]):
raise err(403, "forbidden_table",
f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
+ if not ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
+ # An earlier, authorised Amazon hop matched nothing. There is no next
+ # source record to read, and that is an empty relation, not a missing key.
+ continue
ids = _relation_leg_ids(session, table, ids, leg)
return ids
@@ -2464,7 +2768,8 @@ def _may_see_view(session, vid, store):
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
- search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False):
+ search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
+ st=None):
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
@@ -2524,17 +2829,25 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
+ # S67-T28: `st` lends a definitions snapshot the caller already holds (`_page_defs`), the
+ # same `st=` `_defn_or_refuse` has always accepted. None keeps its own `lend_defs` read.
if traversal_intermediate:
- defn = _traversal_defn_or_refuse(session, table_key)
+ defn = _traversal_defn_or_refuse(session, table_key, st=st)
else:
- defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
+ defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
fields = None
for _bucket, key, _label, mk_fields in rel.TABLES:
if key == table_key:
fields = mk_fields()
break
if fields is None:
- raise err(404, "not_connected", "that database is not a connected Odoo grid")
+ if table_key in _sources():
+ fields = defn.get("fields") or []
+ if not fields:
+ raise err(409, "schema_not_loaded",
+ "This connected database has no saved fields yet. Sync its connector first.")
+ else:
+ raise err(404, "not_connected", "that database is not a connected grid")
# ⭐⭐ W31-T45 / D-169 — THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
# That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
@@ -2549,7 +2862,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
# turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
# try, not inside it.
try:
- cur, mismatch = _mirror_cur(session)
+ cur, mismatch = (_mirror_cur(session, table_key) if table_key.startswith("ut_amzn_")
+ else _mirror_cur(session))
except RuntimeError as e:
# ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
# mismatch subclass, which is what makes this broad clause safe to keep here.
@@ -2787,7 +3101,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
@contextmanager
-def _materialised_read(table_key, session, query, related):
+def _materialised_read(table_key, session, query, related, ids=None):
"""One compiler for native rows and their aggregate sibling, over the walled payload.
The temporary connection contains only this request's already stripped, pinned rows and is
@@ -2806,6 +3120,10 @@ def _materialised_read(table_key, session, query, related):
else:
from routes_tables import table_rows
payload = table_rows(table_key, session=session, related=related)
+ if ids is not None:
+ wanted = set(ids)
+ payload = {**payload, "rows": [row for row in payload["rows"]
+ if int(row["pid"]) in wanted]}
fields = payload["fields"]
with duckdb.connect(":memory:") as cur:
cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
@@ -2882,13 +3200,38 @@ def _group_pin(cols, group_by, group_key):
return f"({gexpr}) = ?", [str(key)]
+def _record_ids(body):
+ """An explicit empty set stays empty. Never truncate a drill's population."""
+ if "ids" not in body:
+ return None
+ ids = body["ids"]
+ if not isinstance(ids, list) or any(type(pid) is not int for pid in ids):
+ raise err(400, "bad_record_ids", "ids must be a list of integer record IDs")
+ if any(pid < -(2 ** 63) or pid >= 2 ** 63 for pid in ids):
+ raise err(400, "bad_record_ids", "Record IDs must fit a signed 64-bit integer")
+ return list(dict.fromkeys(ids))
+
+
+def _restrict_row_ids(read, ids):
+ """Intersect the walled read with one bound array, independent of the ID count."""
+ if ids is None:
+ return read
+ pin = f'{read["spec"]["id"]} IN (SELECT unnest(?::BIGINT[]))'
+ narrowed = dict(read)
+ for where_key, params_key in (("where", "params"), ("base_where", "wall_params")):
+ where = read[where_key]
+ narrowed[where_key] = f"({where}) AND {pin}" if where else pin
+ narrowed[params_key] = list(read[params_key]) + [ids]
+ return narrowed
+
+
def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
- groupBy=None, groupKey=None):
+ groupBy=None, groupKey=None, ids=None):
from harness import datastore
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
"sorts": _json_arg(sorts, "sorts"), "search": search})
- with _materialised_read(table_key, session, query, related) as read:
+ with _materialised_read(table_key, session, query, related, ids=ids) as read:
spec = read["spec"]
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
where, params = read["where"], list(read["params"])
@@ -2912,6 +3255,7 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
@router.get("/odoo-tables/{table_key}/rows")
+@router.post("/odoo-tables/{table_key}/rows")
def odoo_table_rows(table_key: str,
offset: int = Query(default=0, ge=0),
limit: int = Query(default=0),
@@ -2922,7 +3266,8 @@ def odoo_table_rows(table_key: str,
related: str = Query(default=None),
groupBy: str = Query(default=None),
groupKey: str = Query(default=None),
- session: Session = Depends(require_session)):
+ session: Session = Depends(require_session),
+ body: dict = Body(default=None)):
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
`{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` —
@@ -2940,19 +3285,36 @@ def odoo_table_rows(table_key: str,
argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
window to the rows linked to that ONE record, ahead of `filters`, and `total` /
`totalUnfiltered` count inside the pin rather than around it.
- ⛔ `routes_connected_tables.connected_table_rows` - the source-neutral ALIAS onto this
- function - forwards its arguments BY NAME and does not forward this one, so a pinned read must
- name THIS route. That file is in no lane's fence this wave; the one-line forward is booked in
- W51-T05's handoff. Degrading to an UNPINNED read is the behaviour that shipped rather than a
- new wrong answer, which is the one direction this gap is allowed to be wrong in.
+ S66-T48: POST is the same window with one more narrowing, a JSON body `{"ids": [...]}`: a
+ cell's record set, which can run to thousands of ids and would overflow a URL. Every other
+ part of the question stays in the query string, spelled exactly as GET spells it (one
+ vocabulary, one parser). The set narrows the rows AND their counts before LIMIT/OFFSET, so
+ `total` and the pager describe the cell's records. The connected-tables alias forwards the
+ body.
"""
import aios_grid
import core.perm_scope as perm_scope
from harness import datastore
+ # A direct Python caller omits the Body sentinel, and a GET has no body: both read unpinned.
+ ids = None
+ if isinstance(body, dict):
+ # A window part sent in the body would be IGNORED, and an ignored offset or filter is a
+ # wrong page that looks right. Refuse it by name instead.
+ if set(body) - {"ids"}:
+ raise err(400, "bad_window", "Only ids travel in the request body. Send offset, "
+ "limit, filters, sorts and search in the query string.")
+ ids = _record_ids(body)
+
+ # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
+ # asks it instead of reading its own definitions projection (one store round trip fewer on
+ # every Amazon page). Not reordered past it: the check may register this grid's binding,
+ # which is what `_materialised_table` below then reads.
+ page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
+ require_amazon_mirror(table_key, session, st=page[0] if page else None)
if _materialised_table(table_key):
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
- sorts, search, related, groupBy, groupKey)
+ sorts, search, related, groupBy, groupKey, ids=ids)
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
@@ -2960,8 +3322,12 @@ def odoo_table_rows(table_key: str,
# is a page of* - and that is provable only while there is one of it. What stays here is what
# a WINDOW alone needs: the total order, the projection, the overlay merge and the two
# sentences a page earns (`window_clamped`, the deep-page walk).
+ # S67-T28 (D-3480 point 1): ONE definitions snapshot answers this page's own wall and every
+ # link target's below, re-read only when `user_tables` has moved (`_page_defs`).
+ page_st, page_rev = page if page is not None else _page_defs(session)
read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
- sorts=sorts, search=search, related=related)
+ sorts=sorts, search=search, related=related, st=page_st)
+ read = _restrict_row_ids(read, ids)
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
cols, limits, frm = read["cols"], read["limits"], read["frm"]
base_where, wall_params = read["base_where"], read["wall_params"]
@@ -3010,6 +3376,40 @@ def odoo_table_rows(table_key: str,
rows_src = _pool_from_window(spec, win,
stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
session.runtime))
+ if table_key.startswith("ut_amzn_"):
+ import amazon_relational
+ from routes_grid import _may_open
+
+ # ⛔⛔ HOTFIX 2026-09-28: ONE READ PER STORED TARGET PER REQUEST. FBA customer returns
+ # links listings twice (`seller_sku`, `asin1`); each edge read the whole stored table, so
+ # every page paid two tenant-document reads (~3 s each on staging, /rows 9.5 s against
+ # /aggregates' 1 s) and a burst of group-page clicks starved the store pool until sign-in
+ # failed. A read-through target (orders, 559k rows) keeps its `match=` SQL read per edge:
+ # reading it whole to filter here would be far worse.
+ _opened, _stored = {}, {}
+ _stored_cols = {}
+ for _f, _target, _frm, _on in amazon_relational.link_edges(fields):
+ _stored_cols.setdefault(_target, set()).add(_on)
+
+ def read_amazon_targets(target, column, values):
+ if target not in _opened:
+ _opened[target] = _may_open(session, target, st=page_st)
+ if not _opened[target]:
+ return []
+ if not _materialised_table(target):
+ return _relation_rows(session, target, [column], match=(column, values),
+ st=page_st, rev=page_rev)
+ if target not in _stored:
+ _stored[target] = _relation_rows(
+ session, target, sorted(_stored_cols.get(target, set()) | {column}),
+ st=page_st, rev=page_rev)
+ wanted = {str(value) for value in values}
+ # The same string comparison `_relation_rows`' own `match=` applies to a stored table.
+ return [row for row in _stored[target] if str(row.get(column)) in wanted]
+
+ # Fields are the current reader's contract; target rows cross the same wall
+ # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
+ amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets)
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
# ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
@@ -3025,11 +3425,14 @@ def odoo_table_rows(table_key: str,
# the wall and the predicate, so the argument it demands is the list we just fetched.
# ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
# answered "may this session open this surface".
- shared_defs = _so().fields(table_key, st=session.runtime)
+ # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request
+ # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
+ # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
+ shared_defs = _so().fields(table_key, st=page_st)
if shared_defs:
fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
- st=session.runtime).items():
+ st=page_st).items():
overlays.setdefault(pid, {}).update(cells)
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
@@ -3335,6 +3738,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
# hand: `filters` a list, `filterConj` in and/or, `search` text under 1,000 characters. A
# second spelling of "what a predicate on the wire looks like" is how two of them come apart.
query = _selection_query(payload)
+ ids = _record_ids(payload)
aggs = payload.get("aggs")
if aggs is None:
aggs = []
@@ -3348,7 +3752,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
raise err(400, "bad_selection", "groupBy must be a column id")
if _materialised_table(table_key):
- with _materialised_read(table_key, session, query, payload.get("related")) as read:
+ with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
read["params"], read["frm"], aggs, group_by=group_by,
limits=read["limits"])
@@ -3359,9 +3763,12 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
# who assumed both were bodies would have wired the rows and left the summaries answering over
# the entire database under a tab that names one record. `_compiled_read` accepts either shape,
# so the two doors cannot pin differently without this argument being deleted.
+ # S67-T28: the same page snapshot `/rows` takes, so opening a grouped view (its group counts
+ # come from here) no longer re-reads every definition while `user_tables` is unchanged.
read = _compiled_read(table_key, session, tree=query["filters"],
filterConj=query["filterConj"], search=query["search"],
- related=payload.get("related"))
+ related=payload.get("related"), st=_page_defs(session)[0])
+ read = _restrict_row_ids(read, ids)
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
read["params"], read["frm"], aggs, group_by=group_by,
limits=read["limits"])
@@ -3487,7 +3894,7 @@ def population(table_key, cur=None, rt=None):
from harness import datastore
if rt is not None:
rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
- cur = cur if cur is not None else datastore.ro_con()
+ cur = cur if cur is not None else _table_cursor(rt, table_key)
spec = _source_for(cur, table_key)
if spec is None:
return None
@@ -3520,9 +3927,9 @@ def whole_pool(table_key, cur=None, rt=None):
"""
from harness import datastore
if rt is not None:
- cur = cur if cur is not None else _rt().mirror_cursor(rt)
+ cur = cur if cur is not None else _table_cursor(rt, table_key)
rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
- cur = cur if cur is not None else datastore.ro_con()
+ cur = cur if cur is not None else _table_cursor(None, table_key)
spec = _source_for(cur, table_key)
if spec is None:
raise TooBigToMaterialise(
diff --git a/api/routes_platform_admin.py b/api/routes_platform_admin.py
index 731414254b55d7548864c76b0a3ab13c67396389..91bbdd4d6770ed92d9a360b3170b8d5c1b6e3e25 100644
--- a/api/routes_platform_admin.py
+++ b/api/routes_platform_admin.py
@@ -37,16 +37,60 @@ import subprocess
import sys
import time
from pathlib import Path
+from contextlib import asynccontextmanager
from fastapi import APIRouter, Body, Depends, Header, Request
import core.platform_admin as platform_admin
+import core.neon_usage as neon_usage
+import core.do_usage as do_usage
import core.store as store
import operator_control
import voice_control_plane as voice
from deps import Session, err, require_session, users
-router = APIRouter(prefix="/api/v1/platform-admin")
+@asynccontextmanager
+async def _neon_lifespan(app):
+ _start_neon_sampler()
+ _start_do_sampler()
+ try:
+ yield
+ finally:
+ neon_usage.stop_sampler()
+ do_usage.stop_sampler()
+
+
+router = APIRouter(prefix="/api/v1/platform-admin", lifespan=_neon_lifespan)
+
+
+def _start_neon_sampler():
+ # These callbacks are invoked only by the sampler after its management API
+ # awake check. In particular, the card's GET never resolves a store runtime.
+ def load():
+ from harness import runtime
+ return runtime.get_runtime("loopable").get(neon_usage.STORE_KEY)
+
+ def persist(document):
+ from harness import runtime
+ runtime.get_runtime("loopable").update(neon_usage.STORE_KEY, lambda _: document, flush="sync")
+
+ neon_usage.start_sampler(load=load, persist=persist)
+
+
+def _start_do_sampler():
+ def load():
+ from harness import runtime
+ return runtime.get_runtime("loopable").get(do_usage.STORE_KEY)
+
+ def persist(document):
+ from harness import runtime
+ runtime.get_runtime("loopable").update(do_usage.STORE_KEY, lambda _: document, flush="sync")
+
+ do_usage.start_sampler(load=load, persist=persist)
+
+
+# FastAPI 0.139 carries a router lifespan into include_router; legacy startup
+# event registration does not fire there. Defining it does no work at import.
def _now_iso():
@@ -736,6 +780,18 @@ def platform_automations(tenant: str = "", session: Session = Depends(padmin_gat
"resident": resident}}
+@router.get("/neon")
+def platform_neon(session: Session = Depends(padmin_gate)):
+ """Project billing totals plus the sampler's bounded, in-memory history."""
+ return neon_usage.report()
+
+
+@router.get("/do")
+def platform_do(session: Session = Depends(padmin_gate)):
+ """DigitalOcean management-plane billing, with no database read on this path."""
+ return do_usage.report()
+
+
@router.get("/aws")
def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
"""AWS automation usage, verbatim, or an honest note explaining its absence."""
diff --git a/api/routes_query.py b/api/routes_query.py
index 75dc5d74852681e27f9d9a58907514ea35331e28..6fbdca0cccecb5c31f8ffc0ed772942181cb5ee3 100644
--- a/api/routes_query.py
+++ b/api/routes_query.py
@@ -410,6 +410,7 @@ def _spec_schema(field_keys):
"filters": {"type": "array", "items": {"type": "object", "properties": {
"colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)},
"value": {"type": "string"},
+ "values": {"type": "array", "items": {"type": "string"}},
"rhs": {"type": "object", "properties": {
"kind": {"type": "string", "enum": ["field"]}, "colId": col,
}, "required": ["kind", "colId"]},
@@ -897,6 +898,10 @@ def _validate(spec, fields):
for sent, kept in zip(raw_filters, filters):
if sent.get("rhs") and not kept.get("rhs"):
return None, "that column cannot be compared against another column", "rhs_dropped"
+ # D-658: an empty multi-value leaf survives cleaning but narrows no records.
+ if sent.get("op") in _grid().MULTI_VALUE_OPS and (
+ not sent.get("values") or not kept.get("values")):
+ return None, "that filter needs at least one non-blank value", "values_missing"
aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {}
op = str(aggregation.get("op") or "").lower()
diff --git a/api/routes_sheet.py b/api/routes_sheet.py
index 19a6515f406aceeb13a338f05f7f5cbbc1afdae8..f41ef72efbcfdc678d846d355be869c02f1daae4 100644
--- a/api/routes_sheet.py
+++ b/api/routes_sheet.py
@@ -35,6 +35,7 @@ while its surface is a 404.
(standing rule 2), and each names both the cause and the subject.
"""
+import contextlib
import hashlib
import json
import os
@@ -115,14 +116,67 @@ def _mirror(session: Session):
# ⚠ `TABLES` ITSELF IS UNTOUCHED — it has other consumers (`declared_schema_fields`,
# `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
- if fields is None and not is_registry_grid:
- raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
+ is_shipments = table_key == rel.SHIPMENTS_KEY
+ # ⭐⭐ S66-T43 — THE THIRD ROSTER IS THE READ-THROUGH REGISTRY ITSELF. Lane G made every
+ # windowed Amazon table a `user_tables` stub served from the mirror through a provider in
+ # `routes_odoo_tables._sources()`, the same registry the Data grid's rows door reads. A key
+ # found there is exactly as connected as one in `TABLES`, and its FIELDS are the stub's own
+ # (`defn["fields"]`), which is where the rows door takes them from too.
+ # ⚠ `require_amazon_mirror` runs FIRST for an Amazon key: it admits a stub on a cold
+ # registry and refuses 503 with cause and fix when the mirror has not loaded the table,
+ # instead of this door calling a real Amazon table "not connected".
+ # ⚠ A STORED Amazon snapshot (under `MAX_ROWS`, e.g. listings) is served from the mirror
+ # too: its stored ids ARE the mirror's (`amazon_relational._read` stores `grid_source`'s
+ # id), and a relation from one Amazon table to another must resolve on ONE cursor
+ # (`_binding_related_query` refuses `rcur is not cur`), so both paths agree.
+ connector_defn = None
+ if fields is None and not is_registry_grid and not is_shipments:
+ connector_defn = _connector_definition(session, table_key)
+ if connector_defn is None:
+ raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
+ fields = list(connector_defn.get("fields") or [])
+ if not fields:
+ raise err(409, "schema_not_loaded",
+ f"{table_key} has no saved fields yet. Sync its connector first.")
try:
- cur, mismatch = rot._mirror_cur(session)
+ # The Amazon mirror opens without Odoo's ready() gate, exactly as the rows door does.
+ if str(table_key).startswith("ut_amzn_"):
+ # ONE Amazon cursor per request: `datastore.ro_cursor()` opens a new one per call,
+ # and a related condition joins two Amazon tables on the same cursor.
+ cur, mismatch = getattr(session, "_sheet_amzn_cursor", None) or (None, None)
+ if cur is None:
+ cur, mismatch = rot._mirror_cur(session, table_key)
+ if cur is not None:
+ session._sheet_amzn_cursor = (cur, None)
+ else:
+ cur, mismatch = rot._mirror_cur(session)
except RuntimeError as exc:
raise err(503, "store_not_ready", str(exc))
if mismatch:
raise err(409, "cross_tenant_store", mismatch)
+ if is_shipments:
+ from core import user_tables, perm_scope
+ from modules import sheet as sheet_mod
+ memo = getattr(session, "_sheet_stored_mirrors", None)
+ if memo is None:
+ memo = session._sheet_stored_mirrors = {}
+ if table_key not in memo:
+ stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
+ # The store keeps records as {row id: row}. Iterating that dict as a list yields only
+ # the ids, which materialises an EMPTY table and answers every cell blank.
+ rows = [row for row in (stored.values() if isinstance(stored, dict) else stored)
+ if isinstance(row, dict)]
+ if not rows:
+ raise err(409, "sheet_database_empty",
+ f"{table_key} has no stored shipment rows. Sync shipments before binding this cell.")
+ fields = rel.shipment_fields()
+ hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
+ fields = [field for field in fields if field["key"] not in hidden]
+ spec_fields = sheet_mod.mirror_fields(fields)
+ spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows,
+ id_key="odoo_id", bulk=True)
+ memo[table_key] = (cur, {**spec, "fields": fields}, rot._filter_columns(spec, spec_fields))
+ return memo[table_key]
if is_registry_grid:
# ⛔⛔ ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
# resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
@@ -165,6 +219,8 @@ def _mirror(session: Session):
memo[slot] = (None if want is None else set(want), made)
return made
spec = rot._source_for(cur, table_key)
+ if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
+ spec = _amazon_stored_spec(cur, table_key)
if spec is None:
raise err(409, "not_read_through",
f"{table_key} has no read-through binding on this deployment, so a cell cannot "
@@ -179,6 +235,49 @@ def _mirror(session: Session):
return _for
+def _connector_definition(session: Session, table_key: str):
+ """The stored stub of a read-through CONNECTOR grid (S66-T43), or None when it is not one.
+
+ Only a key the read-through registry serves qualifies (`routes_odoo_tables._sources()`), or an
+ Amazon table the connector declares (`amazon_relational.SOURCE`, stored or read-through), so an
+ ordinary typed database still answers `not_connected`. Memoised on the Session for the reason
+ `_mirror` memoises: `resolve_bindings` asks once per CELL and the Session dies with the request.
+ ⚠ `scope_applied=True` because every Sheet read compiles the row wall into its own SQL.
+ """
+ import routes_odoo_tables as rot
+ import routes_tables
+ memo = getattr(session, "_sheet_connector_defs", None)
+ if memo is None:
+ memo = session._sheet_connector_defs = {}
+ if table_key in memo:
+ return memo[table_key]
+ key = str(table_key or "")
+ if not key.startswith("ut_"):
+ return None
+ defn = None
+ if key.startswith("ut_amzn_"):
+ import amazon_relational
+ if key not in amazon_relational.SOURCE:
+ return None
+ defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
+ rot.require_amazon_mirror(key, session, defn)
+ elif key not in rot._sources():
+ return None
+ if defn is None:
+ defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
+ memo[table_key] = defn
+ return defn
+
+
+def _amazon_stored_spec(cur, table_key):
+ """The mirror binding for a STORED Amazon snapshot, or the 503 that names the missing load."""
+ import amazon_relational
+ spec = amazon_relational.grid_source(cur, table_key)
+ if spec.get("missing"):
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
+ return spec
+
+
def _registry_assembly(session: Session, table_key: str):
"""The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
@@ -254,157 +353,144 @@ def _registry_assembly(session: Session, table_key: str):
"shown rather than shown from an older set of columns. Try again in a moment.")
-def _linked_customer_bridges(database):
- """Only declared reverse customer edges with an unconditional, served relation."""
- from core.relations import BRIDGES
- return [bridge for bridge in BRIDGES if bridge.get("target") == database
- and bridge.get("source") == "customer_data" and bridge.get("served")
- and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")]
+def _related_declarations(session, database):
+ """The same permission-scoped catalogue for reads, rewrites and the picker.
+ HOTFIX 2026-09-29: answered once per request inside `_sheet_request_scope`. Every
+ `related:` leaf used to re-read `user_tables.all_defs()` and re-scope every definition, once
+ per BINDING, so a Refresh of Daily Sales (~470 cells filtered on the customer's pricelist) read
+ the whole catalogue ~470 times (SDLC 67 S67-T21 finding).
+ """
+ memo = getattr(session, "_sheet_read_memo", None)
+ slot = ("related", database)
+ if isinstance(memo, dict) and slot in memo:
+ return memo[slot]
+ from core import perm_scope, relations, user_tables
+ from routes_grid import _source_fields
-def _linked_customer_fields(session, bridge):
+ fields = _source_fields(session, database)
+ defs = {}
+ for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
+ if perm_scope.nav_may_open(session.user, key, st=session.runtime):
+ declared = definition.get("fields") or []
+ hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
+ defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
+ entries = relations.related(database, fields=fields, defs=defs)
+ # S66-T13 / C2: `relations.bridge_entries` lists a bridge only from its SOURCE, so
+ # `ut_odoo_orders` has no edge back to `customer_data` although every one-release
+ # `linked:bridge_customer_orders:` leaf stands on exactly that edge. Read from the
+ # target the columns swap, as `reverse_links` swaps them. A bridge carrying `where` is
+ # skipped as `linked:` always skipped it, because that predicate narrows this side's rows.
+ seen = {relations.edge_identity(entry) for entry in entries}
+ for bridge in relations.BRIDGES:
+ if bridge.get("target") != database or bridge.get("where"):
+ continue
+ entry = {"key": bridge["key"], "label": bridge.get("source_label") or bridge["source"],
+ "via": {"direction": "bridge", "key": bridge["key"],
+ "on": bridge["source_col"], "from": bridge["target_col"]},
+ "target": bridge["source"], "served": bool(bridge.get("served"))}
+ if relations.edge_identity(entry) not in seen:
+ seen.add(relations.edge_identity(entry))
+ entries.append(entry)
+ if isinstance(memo, dict):
+ memo[slot] = (fields, defs, entries)
+ return fields, defs, entries
+
+
+def _legacy_linked_keys(database, entry):
+ """The one-release `linked:` bridge keys an offered relationship replaces."""
+ from core import relations
+ via = entry.get("via") or {}
+ return [bridge["key"] for bridge in relations.BRIDGES
+ if bridge.get("target") == database and bridge.get("source") == entry.get("target")
+ and bridge.get("target_col") == via.get("from")
+ and bridge.get("source_col") == via.get("on") and not bridge.get("where")]
+
+
+def _related_condition_fields(session, database):
from core import perm_scope
- _require_database(session, bridge["source"])
- g = _registry_assembly(session, bridge["source"])
- fields = list(g.get("fields") or [])
- hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime)
- # Only values the server can read per record. A browser formula or linked rollup needs
- # another evaluator; offering it here would make a missing value look like a blank value.
- fields = [f for f in fields if f.get("key") not in hidden
- and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"}
- and not f.get("measure")]
- return g, fields
+ _require_database(session, database)
+ fields = list(_registry_assembly(session, database).get("fields") or [])
+ hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
+ return [field for field in fields if field.get("key") not in hidden
+ and field.get("type") not in {"formula", "link", "lookup", "rollup", "metric", "measure"}
+ and not field.get("measure")]
-def _linked_relation_column(session, bridge, cur, spec, columns):
- from core import perm_scope
+def _rewrite_related_filters(session, database, nodes):
+ """Read the one-release linked spelling as an ordinary related condition."""
+ from core import relations
from modules import sheet as sheet_mod
- key = str(bridge.get("target_col") or "")
- subject = bridge["key"] + ":" + (key or "(missing column)")
- hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [],
- st=session.runtime)
- if key in hidden:
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "the relation column is hidden from this account.")
- # The catalogue owns expressions such as sol.order_partner_id and aml.partner_id.
- # The declaration supplies the key, and is a fallback only when the catalogue has none.
- if (columns.get(key) or {}).get("sql"):
- return key
- if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "the link does not declare a readable relation column.")
- source = spec.get("from_sql") or spec.get("table")
+ if not sheet_mod.linked_filter_ids(nodes):
+ return nodes
+ _fields, _defs, entries = _related_declarations(session, database)
+ bridges = {b["key"]: b for b in relations.BRIDGES if b.get("target") == database}
+
+ def resolve(leaf):
+ bridge_key, field_key = sheet_mod.linked_field_parts(leaf["colId"])
+ bridge = bridges.get(bridge_key)
+ entry = next((entry for entry in entries if bridge
+ and entry["target"] == bridge["source"]
+ and entry["via"].get("from") == bridge.get("target_col")
+ and entry["via"].get("on") == bridge.get("source_col")
+ and not entry.get("where")), None)
+ if entry is None:
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", leaf["colId"],
+ "its declared relationship is unavailable.")
+ return {**leaf, "colId": f'related:{entry["key"]}:{field_key}'}
+
try:
- cur.execute(f'SELECT {key} FROM {source} LIMIT 0')
- except Exception:
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "the declared relation column is unavailable on this database.")
- columns[key] = {"sql": key, "type": "int", "aggregate": False}
- return key
+ return sheet_mod.rewrite_linked_filters(nodes, resolve)
+ except sheet_mod.SheetRefusal as exc:
+ raise err(409, exc.code, exc.message)
@router.get("/sheets/filter-fields/{database}")
def linked_filter_fields(database: str, session: Session = Depends(require_session)):
- """The linked Filter offer, through the same field wall as the related grid."""
- from modules import sheet as sheet_mod
+ """Offer ordinary conditions grouped by their readable related database."""
+ from core import perm_scope, relations
database = _require_database(session, database)
- bridges = _linked_customer_bridges(database)
- if not bridges:
- return {"fields": []}
- cur, spec, columns = _mirror(session)(database)
- columns = dict(columns)
- offered = []
- for bridge in bridges:
+ _fields, defs, entries = _related_declarations(session, database)
+ offered, databases, seen = [], [], set()
+ for candidate in entries:
+ if not candidate.get("served") or not perm_scope.nav_may_open(
+ session.user, candidate["target"], st=session.runtime):
+ continue
+ identity = relations.edge_identity(candidate)
+ if identity in seen:
+ continue
try:
- _linked_relation_column(session, bridge, cur, spec, columns)
- _g, fields = _linked_customer_fields(session, bridge)
+ entry = _binding_related_entry(session, database, candidate["key"])
+ fields = _related_condition_fields(session, entry["target"])
except HTTPException as exc:
- if exc.status_code == 403:
+ code = ((exc.detail or {}).get("error") or {}).get("code") if isinstance(exc.detail, dict) else None
+ if exc.status_code == 403 or code in {
+ "relation_source_unresolvable", "relation_target_unavailable", "unknown_relation"}:
continue
raise
- except sheet_mod.SheetRefusal:
+ if not entry["via"].get("from") or not entry["via"].get("on"):
continue
- link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}'
+ seen.add(identity)
+ target = entry["target"]
+ from core import registry
+ from routes_nav import _read_nav_meta
+ topic = registry.BY_KEY.get(target) or {}
+ label = (_read_nav_meta(session.runtime).get(target, {}).get("name")
+ or defs.get(target, {}).get("label")
+ or topic.get("label") or target)
+ metadata = {"via": entry["key"], "database": target, "databaseLabel": label,
+ "relationLabel": entry.get("label") or entry["key"]}
+ databases.append(metadata)
+ # The panel maps a saved one-release `linked:` leaf onto its related key with these.
+ legacy = _legacy_linked_keys(database, entry)
for field in fields:
- # Copy the existing field vocabulary, including options. No customer values or
- # hidden definitions travel on the picker wire.
- offered.append({**field,
- "key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}',
- "label": f'{link_label}: {field.get("label") or field["key"]}'})
- return {"fields": offered}
-
-
-def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None):
- """Read linked values in Python, then give the relation-blind compiler flat predicates."""
- from modules import sheet as sheet_mod
- if not sheet_mod.linked_filter_ids(nodes):
- return nodes
- import aios_grid
- from core import perm_scope, shared_overlay, view_templates
- from harness import filter_eval, filter_sql
-
- bridges = {b["key"]: b for b in _linked_customer_bridges(database)}
- related = {}
-
- def resolve(leaf):
- subject = leaf["colId"]
- bridge_key, field_key = sheet_mod.linked_field_parts(subject)
- bridge = bridges.get(bridge_key)
- if bridge is None:
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "this database no longer has that available customer link.")
- try:
- relation_column = _linked_relation_column(session, bridge, cur, spec, columns)
- if bridge_key not in related:
- g, fields = _linked_customer_fields(session, bridge)
- # Use the rows door's wall, including its shared-field enrichment. The pool's
- # identity is pid; it must never be confused with the orders' partner_id.
- rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"],
- aios_grid.FIELDS, st=session.runtime)
- related[bridge_key] = (g, fields, rows)
- g, fields, source_rows = related[bridge_key]
- except HTTPException as exc:
- if exc.status_code != 403:
- raise
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "this account cannot read the linked customer database.")
- field = next((f for f in fields if f.get("key") == field_key), None)
- if field is None:
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "the field is hidden, removed, or has no readable per-record value.")
- local = {**leaf, "colId": field_key}
- # Validate before evaluating even an empty pool, so malformed/stale leaves refuse
- # rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above.
- probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}}
- try:
- compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today)
- if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None
- or field_key not in compiled.columns_used):
- raise ValueError("unusable linked condition")
- except (ValueError, sheet_mod.SheetRefusal):
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "its value or operator cannot be applied to this field.")
- storage_key = view_templates.workspace_key(bridge["source"])
- try:
- definitions = shared_overlay.fields(storage_key, st=session.runtime)
- rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"),
- derived=g.get("derived"))
- if field_key in definitions:
- cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime)
- rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")}
- for row in rows]
- ctx = filter_eval.EvalCtx(today=today)
- ids = [row[bridge["source_col"]] for row in rows
- if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)]
- return sheet_mod.linked_id_predicate(relation_column, ids)
- except Exception:
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
- "the linked customer values could not be read. Retry after the data is available.")
-
- try:
- return sheet_mod.rewrite_linked_filters(nodes, resolve)
- except sheet_mod.SheetRefusal as exc:
- raise err(409, exc.code, exc.message)
+ field_label = field.get("label") or field["key"]
+ offered.append({**field, "key": f'related:{entry["key"]}:{field["key"]}',
+ "label": f'{label} via {metadata["relationLabel"]}: {field_label}',
+ "related": {**metadata, "field": field["key"], "fieldLabel": field_label,
+ "legacyKeys": [f"linked:{key}:{field['key']}"
+ for key in legacy]}})
+ return {"fields": offered, "databases": databases}
def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
@@ -448,6 +534,19 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
derived=derived)
+ if table_key == "customer_data" and (needed is None or "agent_id" in needed):
+ from core import perm_scope
+ agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
+ hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime)
+ if "agent_id" not in hidden and "agent" not in hidden:
+ # Join only to already-scoped customer rows. pid is the actual res.partner id.
+ have = rel.columns(cur, "res_partner")
+ if "agent_id" in have:
+ agents = dict(cur.execute("SELECT id, agent_id FROM res_partner").fetchall())
+ rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows]
+ fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
+ elif needed is not None and "agent_id" in needed:
+ raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
# The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
# `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
kept = (fields if needed is None
@@ -815,6 +914,9 @@ def _refuse(exc):
return HTTPException(status_code=409, detail={"error": body})
if exc.code in _FORBIDDEN_CODES:
return err(403, exc.code, exc.message)
+ if exc.code == "workbook_unavailable":
+ # The store did not answer; the workbook is not gone. 503 tells the client to retry.
+ return err(503, exc.code, exc.message)
status = 404 if exc.code in _MISSING_CODES else 400
return err(status, exc.code, exc.message)
@@ -1022,17 +1124,43 @@ def _has_cell_inputs(binding):
related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
nodes = [*nodes, *related_nodes]
# Linked values and related-table grants can change independently of the orders' cache key.
- if sheet_mod.linked_filter_ids(nodes):
+ if (sheet_mod.linked_filter_ids(nodes) or sheet_mod.related_filter_ids(nodes)
+ or related is not None):
return True
return any(sheet_mod.filter_cell_nodes(nodes))
-def _refresh_context(book, binding, memo=None):
- """Authored inputs and relative dates invalidate refresh, but never an open's snapshot."""
+def _refresh_context(book, binding, memo=None, *, tab=None, session=None):
+ """Authored inputs and relative dates invalidate refresh, but never an open's snapshot.
+
+ HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
+ update"): with its `tab` and a request scope, a binding's inputs are the cells it actually
+ READS, transitively (`_input_closure`), not the whole workbook. The whole-workbook fingerprint
+ meant one edit anywhere invalidated every one of Daily Sales' ~470 bindings, so every Refresh
+ after any edit re-resolved all of them. The tab list rides the fingerprint, so a rename or a
+ reorder still invalidates everything. Without a scope the old whole-workbook reading stands.
+ """
from core.sheet_binding import fingerprint
from datetime import date
payload = {}
- if _has_cell_inputs(binding):
+ graph = None
+ if session is not None and tab is not None and _has_cell_inputs(binding):
+ import formula_runtime as fr
+ graph = _read_graph(session, book, fr, None)
+ if graph is not None:
+ import formula_runtime as fr
+ tabs = book.get("sheets") or []
+ home = _resolve_home_tab(book, tabs, graph["names"], tab)
+ engine = _read_engine(graph, book, fr, None)[0]
+ closure = _input_closure(graph, engine, fr, _qualified_refs(binding, home, fr, graph))
+ authored = {}
+ for key in sorted(closure):
+ cell = graph["cells"].get(key)
+ authored[key] = None if not isinstance(cell, dict) else {
+ k: cell.get(k) for k in ("kind", "text", "formula", "binding", "value")
+ if k != "value" or cell.get("kind") in ("literal", "empty")}
+ payload["inputs"] = fingerprint([[[t.get("id"), t.get("name")] for t in tabs], authored])
+ elif _has_cell_inputs(binding):
if memo is None:
memo = {}
if "inputs" not in memo:
@@ -1143,8 +1271,11 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
# and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
# F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
# no client engine to cover for it, so the blank is what it would have shown.
+ # HOTFIX 2026-09-29: a `refused` formula is a COMPUTED answer (an error is a result), so it no
+ # longer forces a whole-workbook recompute on every tab read. Only a formula with no answer at
+ # all (`stale`, or no value stored) does. One `#DIV/0!` anywhere used to cost every read ~1 s.
stale_formula = any(
- cell.get("kind") == "formula" and (cell.get("state") != "fresh" or "value" not in cell)
+ cell.get("kind") == "formula" and (cell.get("state") == "stale" or "value" not in cell)
for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
if changed_values or stale_formula:
from modules import sheet as sheet_mod
@@ -1760,10 +1891,17 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
A formula sitting on a tab nobody has open still shows its last answer until that tab is
opened or the workbook is refreshed, exactly as a stale database-bound cell does.
"""
+ with _sheet_request_scope(session):
+ return _recomputed_tab_cells_scoped(session, workbook_id, tab_ref, changes=changes)
+
+
+def _recomputed_tab_cells_scoped(session: Session, workbook_id: str, tab_ref: str, *, changes=None):
from core import sheets
from modules import sheet as sheet_mod
import formula_runtime as fr
+ from time import perf_counter
+ started = perf_counter()
try:
book = sheets.get_workbook(_store(session), workbook_id)
found = sheets.find_tab(book, tab_ref)
@@ -1777,17 +1915,43 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
(found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
_recompute_formulas(book, fr, sheet_mod,
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
- # Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
+ # HOTFIX 2026-09-29 (owner: "we should only update or refresh cells that are affected by an
+ # operation"): a commit re-resolves ONLY the database cells whose inputs read what it changed.
+ # This loop used to resolve EVERY bound cell with a cell input on EVERY commit, whatever was
+ # edited: ~470 SQL resolves (each rebuilding the whole formula graph) per keystroke on Daily
+ # Sales. `_dependent_bound_keys` walks the formula graph forward from the edited cells.
+ graph = _read_graph(session, book, fr, None)
+ targets, mode = None, "all"
+ if edits is not None and graph is not None:
+ try:
+ targets = _dependent_bound_keys(book, graph, fr, set(edits))
+ # A bound cell written by this commit resolves too, as it always did.
+ targets |= {key for key in edits if key in graph["bound"]}
+ mode = "dependents"
+ except Exception as exc: # noqa: BLE001
+ # Unprovable dependents fall back to the old whole-workbook pass: slow, never wrong.
+ try:
+ import harness.telemetry as _tel
+ _tel.error("api:sheet-dependents", exc)
+ except Exception: # noqa: BLE001
+ pass
+ targets, mode = None, "all"
ending_cells = set()
refreshed_entries = []
context_memo = {}
- for source in book.get("sheets") or []:
+ view_memo = {}
+ names = _tab_scope_names(book.get("sheets") or [])
+ for source, name in zip(book.get("sheets") or [], names):
for addr, cell in (source.get("cells") or {}).items():
binding = cell.get("binding") or {}
- if cell.get("kind") != "bound" or not _has_cell_inputs(binding):
+ if cell.get("kind") != "bound" or _is_view_binding(cell) or not _has_cell_inputs(binding):
+ continue
+ key = fr.a1_key(addr, name)
+ if targets is not None and key not in targets:
continue
try:
- result = _resolve_cell_binding(session, binding, book, source.get("id"))
+ result = _resolve_cell_binding(session, binding, book, source.get("id"),
+ view_memo=view_memo)
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
cell["refreshed_at"] = _now_iso()
if result.get("limits"):
@@ -1795,6 +1959,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
else:
cell.pop("limits", None)
cell.pop("refusal", None)
+ if graph is not None:
+ graph["bound_values"][key] = result
except HTTPException as exc:
info = ((exc.detail or {}).get("error") or {})
cell.update(_binding_refusal(
@@ -1804,7 +1970,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
ending_cells.add((source.get("id"), addr))
refreshed_entries.append((str(source.get("id")) + "!" + addr,
_binding_scope_key(session, str(binding.get("database") or ""), binding),
- {"tokens": {}, "context": _refresh_context(book, binding, context_memo),
+ {"tokens": {}, "context": _refresh_context(book, binding, context_memo,
+ tab=source.get("id"), session=session),
"cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
if k in cell}}))
if ending_cells:
@@ -1813,6 +1980,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
{c.get("binding", {}).get("database") for t in book.get("sheets", [])
for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
_recompute_formulas(book, fr, sheet_mod)
+ stats = _read_stats(session) or {}
+ print(f"[sheet] commit wb={workbook_id} tab={tab_ref} edited={len(edits or {})} mode={mode} "
+ f"bound_resolved={len(ending_cells)} engines={stats.get('engines', 0)} "
+ f"input_hits={stats.get('input_hits', 0)} "
+ f"ms={round((perf_counter() - started) * 1000.0)}", flush=True)
found = sheets.find_tab(book, tab_ref)
out = {}
for addr, cell in (found.get("cells") or {}).items():
@@ -1986,12 +2158,25 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
@router.post("/sheets/{workbook_id}/refresh")
-def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
+def refresh_sheet(workbook_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
"""C2/R4: Refresh resolves only databases whose token or binding inputs changed.
Viewers refresh their own server cache. Only editors save the workbook document.
One token is read per distinct database on the unchanged path, regardless of cell count.
+
+ HOTFIX 2026-09-29: an optional body `{"cells": ["!", ...]}` refreshes ONLY
+ those bound cells (the open-time repair of the few cells this reader has no answer for, and
+ the cells a row or column edit left stale). Every other cell keeps its stored answer.
"""
+ wanted = (body or {}).get("cells") if isinstance(body, dict) else None
+ only = ({str(item) for item in wanted if isinstance(item, str) and "!" in item}
+ if isinstance(wanted, list) else None)
+ with _sheet_request_scope(session):
+ return _refresh_sheet_scoped(workbook_id, session, only=only)
+
+
+def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None):
from copy import deepcopy
from core import sheets, sheet_binding
from modules import sheet as sheet_mod
@@ -2019,10 +2204,12 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
def input_databases(binding, tab_ref):
nonlocal input_graph
- nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
- related_nodes, _ = _fe_parts((binding.get("related") or {}).get("filter"))
+ binding = dict(binding)
+ if "related" in binding:
+ binding = sheet_mod.clean_binding(binding)
+ nodes, _ = _binding_filter_nodes(session, binding["database"], binding, memo=view_memo)
refs = [str((node.get(key) or {}).get("ref") or "")
- for node in sheet_mod.filter_cell_nodes([*nodes, *related_nodes])
+ for node in sheet_mod.filter_cell_nodes(nodes)
for key in ("valueRef", "value2Ref") if key in node]
period = binding.get("period") or {}
if period.get("ref"):
@@ -2032,15 +2219,25 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
tabs = book.get("sheets") or []
names = _tab_scope_names(tabs)
if input_graph is None:
- documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
- for addr, cell in (tab.get("cells") or {}).items()}
- engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
- for key, cell in documents.items()}, sheets=names,
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
- input_graph = documents, engine
+ shared = _read_graph(session, book, fr, None)
+ if shared is not None:
+ # The request's one engine (bound cells pending): its edges are the same edges.
+ input_graph = shared["cells"], _read_engine(shared, book, fr, None)[0]
+ else:
+ documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
+ for addr, cell in (tab.get("cells") or {}).items()}
+ engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
+ for key, cell in documents.items()}, sheets=names,
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
+ input_graph = documents, engine
documents, engine = input_graph
home = _resolve_home_tab(book, tabs, names, tab_ref)
- pending, visited, deps = [_asked_key(ref, home, fr) for ref in refs], set(), set()
+ shared = _read_graph(session, book, fr, None)
+ pending = ([key for ref in refs for key in
+ (_formula_slot_keys(shared, fr, ref, home) if ref.startswith("=")
+ else [_asked_key(ref, home, fr)])] if shared is not None
+ else [_asked_key(ref, home, fr) for ref in refs if not ref.startswith("=")])
+ visited, deps = set(), set()
while pending:
key = pending.pop()
if key in visited:
@@ -2048,7 +2245,14 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
visited.add(key)
cell = documents.get(key) or {}
if cell.get("kind") == "bound":
- deps.add(str((cell.get("binding") or {}).get("database") or ""))
+ source = dict(cell.get("binding") or {})
+ if "related" in source:
+ source = sheet_mod.clean_binding(source)
+ source_db = str(source.get("database") or "")
+ deps.add(source_db)
+ source_nodes, _ = _binding_filter_nodes(session, source_db, source, memo=view_memo)
+ deps.update(_binding_related_entry(session, source_db, via)["target"]
+ for via in sheet_mod.related_vias(source_nodes))
pending.extend(engine.consumed.get(key, ()))
return deps
@@ -2069,16 +2273,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
def dependencies(db, binding, tab_ref):
deps = {db}
- if binding.get("related"):
- key = (db, binding["related"].get("via"))
+ binding = dict(binding)
+ if "related" in binding:
+ binding = sheet_mod.clean_binding(binding)
+ nodes, _ = _binding_filter_nodes(session, db, binding, memo=view_memo)
+ for via in sheet_mod.related_vias(nodes):
+ key = (db, via)
if key not in related_memo:
- related_memo[key] = _binding_related_entry(session, db, binding)
+ related_memo[key] = _binding_related_entry(session, db, via)
deps.add(related_memo[key]["target"])
- nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
- linked = sheet_mod.linked_filter_ids(nodes)
- if linked:
- names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
- deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
deps.update(input_databases(binding, tab_ref))
return deps
@@ -2098,11 +2301,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
if (hit and all(sources.values()) and hit.get("tokens") == sources
- and hit.get("context") == _refresh_context(book, binding, context_memo)
+ and hit.get("context") == _refresh_context(book, binding, context_memo,
+ tab=tab_ref, session=session)
and hit["cell"].get("state") == "fresh"):
return hit["cell"]
return None
+ started = perf_counter()
+ resolved_count = 0
+ scope_names = {id(tab): name for tab, name in
+ zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))}
for tab in book.get("sheets") or []:
for addr, cell in (tab.get("cells") or {}).items():
if cell.get("kind") != "bound" or _is_view_binding(cell):
@@ -2110,8 +2318,11 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
binding = cell.get("binding") or {}
db = str(binding.get("database") or "")
cell_key = str(tab.get("id")) + "!" + addr
+ if only is not None and cell_key not in only:
+ continue
scope_key = _binding_scope_key(session, db, binding)
- context = _refresh_context(book, binding, context_memo)
+ context = _refresh_context(book, binding, context_memo, tab=tab.get("id"),
+ session=session)
hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
source_tokens = {}
try:
@@ -2131,12 +2342,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
status.setdefault(db, "skipped")
continue
status[db] = "resolved"
+ resolved_count += 1
previous = getattr(session, "_sheet_refresh_input", None)
session._sheet_refresh_input = cached_input
try:
out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
finally:
session._sheet_refresh_input = previous
+ shared = _read_graph(session, book, fr, None)
+ if shared is not None:
+ shared["bound_values"][fr.a1_key(addr, scope_names.get(id(tab), ""))] = out
limits = list(out.get("limits") or [])
if not all(source_tokens.values()):
limits.append({"code": "change_token_unavailable",
@@ -2147,8 +2362,10 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
"refreshed_at": stamp}
if limits:
snapshot["limits"] = limits
- except HTTPException as exc:
- info = ((exc.detail or {}).get("error") or {})
+ except (HTTPException, sheet_mod.SheetRefusal) as exc:
+ info = ({"code": exc.code, "message": exc.message}
+ if isinstance(exc, sheet_mod.SheetRefusal)
+ else ((exc.detail or {}).get("error") or {}))
snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
info.get("message", "This cell could not be refreshed."))
status[db] = "refused"
@@ -2179,9 +2396,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
tab_ref, addr = cell_key.split("!", 1)
_snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
+ resolve_ms = round((perf_counter() - started) * 1000.0)
started = perf_counter()
_recompute_formulas(book, fr, sheet_mod)
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
+ stats = _read_stats(session) or {}
+ print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
+ f"bound_resolved={resolved_count} engines={stats.get('engines', 0)} "
+ f"input_hits={stats.get('input_hits', 0)} resolve_ms={resolve_ms} "
+ f"recalc_ms={recalc_ms}", flush=True)
if editable:
try:
saved = sheets.put_workbook(store, workbook_id,
@@ -2196,6 +2419,13 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
saved["refreshedFor"] = str(session.uname or "")
if entries:
sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
+ if only is not None:
+ # ⛔ C5: a targeted answer carries ONLY the cells it resolved for this caller. The rest of
+ # the document holds the owner's stored answers, which a scoped reader must never adopt.
+ return {"workbook": {key: saved.get(key) for key in ("id", "name", "rev", "refreshed_at")},
+ "cells": {cell_key: entry["cell"] for cell_key, _, entry in entries},
+ "recalc_ms": recalc_ms,
+ "binding_refresh": {"databases": status, "token_reads": token_reads}}
return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
"binding_refresh": {"databases": status, "token_reads": token_reads}}
@@ -2341,6 +2571,10 @@ _METRIC_COLUMN_CREATE_LOCK = threading.RLock()
def _forget_sheet_database(session, database):
"""A field write invalidates both request-local schema and materialised-column memos."""
getattr(session, "_sheet_assemblies", {}).pop(database, None)
+ memo = getattr(session, "_sheet_read_memo", None)
+ if isinstance(memo, dict):
+ for slot in [slot for slot in memo if isinstance(slot, tuple) and slot[0] == "related"]:
+ memo.pop(slot, None)
mirrors = getattr(session, "_sheet_mirrors", {})
for slot in list(mirrors):
if slot[0] == database:
@@ -2467,49 +2701,361 @@ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()
return sheet_mod.period_window(period, today, cell_date=date)
+@contextlib.contextmanager
+def _sheet_request_scope(session):
+ """HOTFIX 2026-09-29: one memo for one Sheet request, dropped when the request ends.
+
+ ⭐ The Session is built fresh per request (`deps.require_session`), but gates reuse one
+ SimpleNamespace across calls, so the memo is OPT-IN, installed here and removed on exit.
+ Inside it the workbook's authored cells cannot change: every door that opens a scope reads one
+ document and resolves against it, so a cell read or a bound input answered once is the answer
+ for the rest of the request. ⛔ Never open a scope around code that edits cells between two
+ reads of the same document object (the Daily Sales repair projects in rounds and does not).
+ """
+ previous = getattr(session, "_sheet_read_memo", None)
+ memo = {"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
+ try:
+ session._sheet_read_memo = memo
+ except Exception: # noqa: BLE001
+ yield None
+ return
+ try:
+ yield memo
+ finally:
+ session._sheet_read_memo = previous
+
+
+def _request_scoped(route):
+ """Run a route inside `_sheet_request_scope`, keeping its signature for FastAPI and its body in
+ place for the source gates that read it."""
+ import functools
+ import inspect as _inspect
+ signature = _inspect.signature(route)
+
+ @functools.wraps(route)
+ def scoped(*args, **kwargs):
+ session = signature.bind_partial(*args, **kwargs).arguments.get("session")
+ if session is None:
+ return route(*args, **kwargs)
+ with _sheet_request_scope(session):
+ return route(*args, **kwargs)
+ return scoped
+
+
+def _read_stats(session):
+ memo = getattr(session, "_sheet_read_memo", None)
+ return memo.get("stats") if isinstance(memo, dict) else None
+
+
+def _read_graph(session, book, fr, today):
+ """This request's cell map and formula graph for `book`, or None outside a request scope."""
+ memo = getattr(session, "_sheet_read_memo", None)
+ if not isinstance(memo, dict) or not isinstance(book, dict):
+ return None
+ slot = ("graph", id(book), str(today or ""))
+ graph = memo.get(slot)
+ if graph is None or graph["book"] is not book:
+ graph = memo[slot] = _new_read_graph(book, fr, memo.get("stats"))
+ return graph
+
+
+def _new_read_graph(book, fr, stats=None):
+ tabs = book.get("sheets") or []
+ names = _tab_scope_names(tabs)
+ cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
+ for addr, cell in (source.get("cells") or {}).items() if isinstance(cell, dict)}
+ return {"book": book, "names": names, "cells": cells,
+ "bound": {k: c for k, c in cells.items() if c.get("kind") == "bound"},
+ "engine": None, "attempted": set(), "failures": {}, "bound_values": {},
+ "formulas": {}, "stats": stats}
+
+
+#: Where a filter formula is evaluated: a private one-formula engine, never a workbook address.
+_FORMULA_SLOT_ADDR = "XFD1048576"
+
+
+def _formula_slot(graph, fr, formula, home):
+ """A filter formula, compiled once per request: (slot key, cells it reads, ranges it reads).
+
+ HOTFIX 2026-09-29 (owner: "make formula native inside the Cell database Filter so it can point
+ to an existing date but we can use something like a date function refer to a cell"). A filter
+ value slot holds a cell reference (`F7`) OR a formula (`=DATE($E$24,1,1)`), so a monthly total
+ needs no helper cell holding its first and last day. `cells` is None when it does not compile.
+ """
+ cache = graph.setdefault("formulas", {})
+ slot = (str(home).upper(), formula)
+ if slot not in cache:
+ vkey = fr.a1_key(_FORMULA_SLOT_ADDR, home)
+ book = graph["book"]
+ try:
+ probe = fr.SheetCalculation({vkey: {"formula": formula, "sheet": home}},
+ sheets=graph["names"], named_ranges=book.get("namedRanges"),
+ date_system=book.get("dateSystem", "1900"))
+ compiled = probe.compiled.get(vkey)
+ except Exception: # noqa: BLE001
+ compiled = None
+ cache[slot] = (vkey, set(compiled["cells"]) if compiled else None,
+ list(compiled["ranges"]) if compiled else [])
+ return cache[slot]
+
+
+def _formula_slot_keys(graph, fr, formula, home):
+ """Every workbook cell a filter formula reads (range members included), or [] if unreadable."""
+ _vkey, reads, ranges = _formula_slot(graph, fr, formula, home)
+ keys = set(reads or ())
+ if ranges:
+ engine = _read_engine(graph, graph["book"], fr, None)[0]
+ for bounds in ranges:
+ keys.update(engine.positions.within(bounds))
+ return sorted(keys)
+
+
+def _read_binding_formula(session, formula, *, book, tab, today, seen, code, as_date):
+ """A filter / ending value computed by a formula over this workbook's cells, never stored."""
+ from modules import sheet as sheet_mod
+ import formula_runtime as fr
+
+ def refuse(reason):
+ exc = sheet_mod.SheetRefusal(code, formula, reason)
+ exc.message = exc.message.replace("from cell =", "from formula =", 1)
+ exc.args = (exc.message,)
+ raise exc
+
+ graph = _read_graph(session, book, fr, today) or _new_read_graph(book, fr)
+ home = _resolve_home_tab(book, book.get("sheets") or [], graph["names"], tab)
+ vkey, reads, _ranges = _formula_slot(graph, fr, formula, home)
+ if reads is None:
+ refuse("the formula could not be read.")
+ specs = {vkey: {"formula": formula, "sheet": home}}
+ reasons = {}
+ for key in _formula_slot_keys(graph, fr, formula, home):
+ if key == vkey:
+ continue
+ if key in seen:
+ refuse("the reference is circular.")
+ cell = graph["cells"].get(key)
+ if not isinstance(cell, dict) or cell.get("kind") == "empty":
+ if key in reads:
+ # A cell the formula names directly must hold something: a deleted input refuses
+ # by name, as a cell reference does, never a year-zero date. The browser twin is
+ # `sheetState.readFormulaSlot`. A blank range member stays blank.
+ refuse(f"cell {key.rpartition('!')[2]} is blank.")
+ continue
+ try:
+ specs[key] = {"value": _read_binding_cell(session, key, book=book, tab=tab, today=today,
+ seen=seen, code=code),
+ **({"style": cell["style"]} if isinstance(cell.get("style"), dict) else {})}
+ except sheet_mod.SheetRefusal as exc:
+ specs[key] = {"refused": exc.code}
+ reasons[key] = exc.message
+ probe = fr.SheetCalculation(specs, sheets=graph["names"], today=today,
+ named_ranges=book.get("namedRanges"),
+ date_system=book.get("dateSystem", "1900"))
+ if probe.errors.get(vkey):
+ error_code = probe.errors[vkey]
+ refuse(next(iter(reasons.values()), None) or fr._sheet_error_message(error_code))
+ value = probe.values.get(vkey)
+ if value is None or isinstance(value, str) and not value.strip():
+ refuse("its result is blank.")
+ if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
+ and sheet_mod.date_formatted_cell({"style": {"format": probe.formats.get(vkey)}})):
+ from harness import windows as W
+ day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
+ if not day:
+ refuse("its value is not a real date.")
+ return day
+ return value
+
+
+def _read_engine(graph, book, fr, today):
+ """ONE formula engine per request, database inputs pending until a read needs them."""
+ if graph["engine"] is None:
+ specs = {k: ({"refused": "ending_input_pending"} if k in graph["bound"]
+ else fr._document_spec(c, k.rpartition("!")[0]))
+ for k, c in graph["cells"].items()}
+ graph["engine"] = fr.SheetCalculation(
+ specs, sheets=graph["names"], today=today,
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
+ stats = graph.get("stats")
+ if stats is not None:
+ stats["engines"] += 1
+ return graph["engine"], graph["attempted"], graph["failures"]
+
+
+def _bound_input(session, key, cell, *, book, today, seen, graph):
+ """A bound cell read as an INPUT (a filter value, an ending date, a formula's operand).
+
+ Refresh's certified cache first, then this request's memo, then one real resolve. A refusal
+ is memoized as the refusal: the same binding over the same document and the same database
+ cannot answer differently twice in one request.
+ """
+ lookup = getattr(session, "_sheet_refresh_input", None)
+ cached = lookup(key) if lookup is not None else None
+ if cached is not None:
+ return cached
+ memo = graph["bound_values"] if graph is not None else None
+ if memo is not None and key in memo:
+ stats = _read_stats(session)
+ if stats is not None:
+ stats["input_hits"] += 1
+ hit = memo[key]
+ if isinstance(hit, HTTPException):
+ raise hit
+ return hit
+ try:
+ result = _resolve_binding(session, cell.get("binding"), today=today, book=book,
+ tab=key.rpartition("!")[0], ending_seen=seen)
+ except HTTPException as exc:
+ if memo is not None:
+ memo[key] = exc
+ raise
+ if memo is not None:
+ memo[key] = result
+ return result
+
+
+def _binding_input_refs(binding):
+ """Every cell reference a binding reads: filter values at any depth and a period's ref."""
+ from modules import sheet as sheet_mod
+ if not isinstance(binding, dict):
+ return []
+ nodes = []
+ for part in (binding.get("scope"), binding.get("related")):
+ if isinstance(part, dict):
+ found, _ = _fe_parts(part.get("filter"))
+ nodes.extend(found)
+ refs = [str((node.get(slot) or {}).get("ref") or "")
+ for node in sheet_mod.filter_cell_nodes(nodes)
+ for slot in ("valueRef", "value2Ref") if isinstance(node.get(slot), dict)]
+ period = binding.get("period")
+ if isinstance(period, dict) and period.get("ref"):
+ refs.append(str(period["ref"]))
+ return [ref for ref in refs if ref.strip()]
+
+
+def _qualified_refs(binding, home, fr, graph=None):
+ out = []
+ for ref in _binding_input_refs(binding):
+ if ref.startswith("="):
+ if graph is not None:
+ out.extend(_formula_slot_keys(graph, fr, ref, home))
+ continue
+ try:
+ out.append(_asked_key(ref, home, fr))
+ except Exception: # noqa: BLE001
+ continue
+ return out
+
+
+def _dependent_bound_keys(book, graph, fr, edited):
+ """HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
+ update, this includes the cell database"): the bound cells an edit can change, and no other.
+
+ Walks FORWARD from the edited keys over the formula graph's own reverse edges (the same three
+ `SheetCalculation.update` walks: static reads, dynamic reads, range consumers) plus one more
+ edge the formula graph cannot see: a bound cell's filter value / period reference. So editing
+ `F6` reaches `F7 = G6-1`, and every binding filtering on `F7`, and every formula over those.
+ ⛔ STRUCTURE, NEVER A VALUE DIFF: the stored values are last computed, not current, so an
+ edit reverted to its old value would look unchanged and leave a binding on the edited answer.
+ """
+ from collections import deque
+ engine = _read_engine(graph, book, fr, None)[0]
+ readers = {}
+ for key, cell in graph["bound"].items():
+ if _is_view_binding(cell):
+ continue
+ for ref in _qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph):
+ readers.setdefault(ref, set()).add(key)
+ seen, queue = set(edited), deque(edited)
+ while queue:
+ key = queue.popleft()
+ owners = (set(engine.read_by.get(key, ())) | set(engine.consumed_by.get(key, ()))
+ | set(engine.ranges.at(key, engine.stats)) | readers.get(key, set()))
+ for owner in owners:
+ if owner not in seen:
+ seen.add(owner)
+ queue.append(owner)
+ return {key for key in seen if key in graph["bound"] and key not in edited}
+
+
+def _input_closure(graph, engine, fr, roots):
+ """Every cell a set of references reads, transitively, through formulas and bound inputs."""
+ seen, pending = set(), list(roots)
+ while pending:
+ key = pending.pop()
+ if key in seen:
+ continue
+ seen.add(key)
+ cell = graph["cells"].get(key)
+ if isinstance(cell, dict) and cell.get("kind") == "bound":
+ pending.extend(_qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph))
+ continue
+ pending.extend(engine.reads.get(key, ()))
+ pending.extend(engine.consumed.get(key, ()))
+ compiled = engine.compiled.get(key)
+ for bounds in (compiled or {}).get("ranges") or ():
+ pending.extend(engine.positions.within(bounds))
+ return seen
+
+
def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
- code="ending_cell_unreadable"):
- """Shared cell read for Ending and Filter, including scoped bound and formula inputs."""
+ code="ending_cell_unreadable", as_date=False):
+ """Shared cell read for Ending and Filter, including scoped bound and formula inputs.
+
+ S67-T10: with ``as_date`` a date-formatted number (typed, or a formula's result such as
+ 46293) is returned as its ISO day, so a Filter condition compares it as that date for every
+ operator; a plain number with no date format is returned unchanged.
+ """
from modules import sheet as sheet_mod
import formula_runtime as fr
def refuse(reason):
raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
+ if str(ref or "").strip().startswith("=") and book is not None:
+ return _read_binding_formula(session, str(ref).strip(), book=book, tab=tab, today=today,
+ seen=seen, code=code, as_date=as_date)
if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
refuse("its reference is invalid.")
if book is None:
refuse("the source workbook is unavailable.")
tabs = book.get("sheets") or []
- names = _tab_scope_names(tabs)
+ # HOTFIX 2026-09-29 (owner: "MAKE IT FAST"): one request reads ONE graph. Before this, every
+ # filter reference of every bound cell rebuilt the whole workbook's cell map and, for a formula
+ # reference, a whole-workbook SheetCalculation (about 0.6 s each): Daily Sales has ~470 bound
+ # cells reading `$E$10` and `F7`, so one commit or one Refresh paid ~900 engine builds.
+ graph = _read_graph(session, book, fr, today)
+ names = graph["names"] if graph is not None else _tab_scope_names(tabs)
home = _resolve_home_tab(book, tabs, names, tab)
key = _asked_key(ref, home, fr)
if key in seen:
refuse("the reference is circular.")
- cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
- for addr, cell in (source.get("cells") or {}).items()}
+ cells = graph["cells"] if graph is not None else {
+ fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
+ for addr, cell in (source.get("cells") or {}).items()}
cell = cells.get(key)
if not isinstance(cell, dict) or cell.get("kind") == "empty":
refuse("the cell or its sheet is missing or blank.")
if cell.get("kind") == "bound":
try:
- lookup = getattr(session, "_sheet_refresh_input", None)
- cached = lookup(key) if lookup is not None else None
- value = (cached["value"] if cached is not None else _resolve_binding(
- session, cell.get("binding"), today=today, book=book,
- tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
+ value = _bound_input(session, key, cell, book=book, today=today,
+ seen=(*seen, key), graph=graph)["value"]
except HTTPException as exc:
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
refuse(reason)
elif cell.get("kind") == "formula":
# The existing formula graph discovers actual inputs, including dynamic references.
# A database input starts refused, so a cached value can never bypass this reader's wall.
- bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"}
- specs = {k: ({"refused": "ending_input_pending"} if k in bound
- else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
- engine = fr.SheetCalculation(specs, sheets=names, today=today,
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
- attempted, failures = set(), {}
+ bound = graph["bound"] if graph is not None else {
+ k: c for k, c in cells.items() if c.get("kind") == "bound"}
+ if graph is not None:
+ engine, attempted, failures = _read_engine(graph, book, fr, today)
+ else:
+ specs = {k: ({"refused": "ending_input_pending"} if k in bound
+ else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
+ engine = fr.SheetCalculation(specs, sheets=names, today=today,
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
+ attempted, failures = set(), {}
while True:
dependencies, pending = set(), [key]
while pending:
@@ -2525,11 +3071,8 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
for dependency in needed:
attempted.add(dependency)
try:
- lookup = getattr(session, "_sheet_refresh_input", None)
- result = lookup(dependency) if lookup is not None else None
- if result is None:
- result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
- book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
+ result = _bound_input(session, dependency, bound[dependency], book=book,
+ today=today, seen=(*seen, key, dependency), graph=graph)
changes[dependency] = {"value": result["value"]}
except HTTPException as exc:
info = ((exc.detail or {}).get("error") or {})
@@ -2551,6 +3094,13 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
if cell.get("state") == "stale":
refuse("the cell needs to be refreshed.")
value = cell.get("value", cell.get("text"))
+ if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
+ and sheet_mod.date_formatted_cell(cell)):
+ from harness import windows as W
+ day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
+ if not day:
+ refuse("its value is not a real date.")
+ return day
return value
@@ -2558,7 +3108,8 @@ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen
from modules import sheet as sheet_mod
try:
return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
- session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"))
+ session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable",
+ as_date=True))
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
@@ -2602,25 +3153,14 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
return _resolve_binding(session, binding, **kwargs)
-def _binding_related_entry(session, database, binding):
+def _binding_related_entry(session, database, name):
"""Resolve one authored edge key against the permission-scoped declarations."""
- if "related" not in binding:
- return None
- from core import perm_scope, relations, user_tables
- from routes_grid import _source_fields
-
- fields = _source_fields(session, database)
- defs = {}
- for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
- if perm_scope.nav_may_open(session.user, key, st=session.runtime):
- declared = definition.get("fields") or []
- hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
- defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
- entries = relations.related(database, fields=fields, defs=defs)
- name = binding["related"]["via"]
+ from core import perm_scope, relations
+ fields, defs, entries = _related_declarations(session, database)
matches = [entry for entry in entries if entry["key"] == name]
via = matches[0]["via"] if len(matches) == 1 else {"key": name}
- entry = relations.find(database, via, fields=fields, defs=defs)
+ # `pick` over the SCOPED list, which also carries the target-side bridge entries above.
+ entry = relations.pick(entries, via)
if entry is None:
candidates = [item["key"] for item in relations.via_candidates(entries, via)
if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
@@ -2640,23 +3180,20 @@ def _binding_related_entry(session, database, binding):
declaration = next((f for f in defs.get(target, {}).get("fields", [])
if f.get("key") == entry["via"].get("field")), {})
link = declaration.get("link") or {}
- if link.get("through") or link.get("selectionMode") == "manual":
+ if (link.get("through") or link.get("selectionMode") == "manual"
+ or not entry["via"].get("from") or not entry["via"].get("on")):
raise err(409, "relation_source_unresolvable",
f"Relationship '{name}' does not declare a direct column join. "
"Choose a relationship with one readable column on each database.")
return entry
-def _binding_related_query(session, binding, entry, cur, spec, columns, *, window=None,
- today=None, book=None, tab=None, ending_seen=()):
+def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None):
"""Materialise the far side on the near cursor, then hand metadata to the compiler."""
- if entry is None:
- return None
from core import perm_scope, relations
from modules import sheet as sheet_mod
- database, target = binding["database"], entry["target"]
- name = binding["related"]["via"]
+ target, name = entry["target"], entry["key"]
local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
def map_join(key, table, source_spec, cols, side):
@@ -2682,11 +3219,7 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
if perm_scope.row_scope_applies(session.user, target) else None)
wall_nodes, wall_conj = _fe_parts(wall)
- scope_nodes, scope_conj = _fe_parts(binding["related"]["filter"])
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
- today=today, seen=ending_seen)
- needed = (sheet_mod.filter_columns_named(scope_nodes)
- | sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
+ needed = set(fields) | sheet_mod.filter_columns_named(wall_nodes) | {remote_key}
try:
rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
except HTTPException as exc:
@@ -2703,13 +3236,69 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
st=session.runtime)
rcols = {key: value for key, value in rcols.items() if key not in hidden}
+ declared = {field["key"]: field for field in rspec.get("fields") or []}
+
+ def unknown_field(key):
+ return err(409, "related_field_unknown",
+ f"Field '{key}' on relationship '{name}' is hidden, removed, or has no "
+ "readable per-record value. Choose a readable field on this relationship.")
+
+ for key in sorted(fields):
+ field = declared.get(key) or {}
+ if (key in hidden
+ or field.get("type") in {"formula", "link", "lookup", "rollup", "metric", "measure"}
+ or field.get("measure")):
+ raise unknown_field(key)
map_join(remote_key, target, rspec, rcols, "target")
- depends = sheet_mod.filter_columns_named(scope_nodes)
+ for key in sorted(fields):
+ expr = (rcols.get(key) or {}).get("sql")
+ if not expr or str(expr).strip().upper() == "NULL":
+ raise unknown_field(key)
return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
"spec": rspec, "columns": rcols, "where": entry.get("where"),
- "scope_nodes": scope_nodes, "scope_conj": scope_conj,
"wall_nodes": wall_nodes, "wall_conj": wall_conj,
- "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in depends]}
+ "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]}
+
+
+def _related_leaf_bags(session, database, nodes, needed):
+ """Prepare joins before the local mirror, then materialise their bags on its cursor."""
+ from modules import sheet as sheet_mod
+ entries, fields = {}, {}
+ try:
+ for key in sorted(sheet_mod.related_filter_ids(nodes)):
+ via, field = sheet_mod.related_field_parts(key)
+ if via not in entries:
+ entries[via] = _binding_related_entry(session, database, via)
+ needed.add(entries[via]["via"].get("from"))
+ fields.setdefault(via, set()).add(field)
+ needed.difference_update(sheet_mod.related_filter_ids(nodes))
+ except sheet_mod.SheetRefusal as exc:
+ raise err(409, exc.code, exc.message)
+
+ def materialise(cur, spec, columns, *, window=None):
+ return {via: _binding_related_query(session, database, entry, cur, spec, columns,
+ fields=fields[via], window=window)
+ for via, entry in entries.items()}
+ return materialise
+
+
+def _binding_filter_nodes(session, database, binding, *, memo=None):
+ """A saved view and an authored extra scope are always intersected."""
+ scope = binding.get("scope") or {}
+ nodes, conj = _fe_parts(scope.get("filter"))
+ from modules import sheet as sheet_mod
+ if sheet_mod.linked_filter_ids(nodes):
+ nodes = _rewrite_related_filters(session, database, nodes)
+ binding["scope"] = {**scope, "filter": {"conj": conj, "nodes": nodes}}
+ if scope.get("view_id"):
+ view_nodes, view_conj = _view_filter(session, database, scope["view_id"],
+ scope.get("view_name", ""), memo=memo)
+ if nodes:
+ nodes, conj = [{"conj": view_conj, "children": view_nodes},
+ {"conj": conj, "children": nodes}], "and"
+ else:
+ nodes, conj = view_nodes, view_conj
+ return _rewrite_related_filters(session, database, nodes), conj
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
@@ -2760,12 +3349,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
if perm_scope.row_scope_applies(session.user, database) else None)
wall_nodes, wall_conj = _fe_parts(wall_tree)
- scope = binding["scope"]
- if scope.get("view_id"):
- scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
- scope.get("view_name", ""), memo=view_memo)
- else:
- scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
+ scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
today=today, seen=ending_seen)
needed = (sheet_mod.filter_columns_named(scope_nodes)
@@ -2773,12 +3357,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
if binding["aggregate"].get("field"):
needed.add(str(binding["aggregate"]["field"]))
- related_entry = _binding_related_entry(session, database, binding)
- if related_entry:
- needed.add(related_entry["via"].get("from"))
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
cols = dict(cols)
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
# ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
# real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
@@ -2817,9 +3398,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
# spec's column expressions; applying it a second time would double-count the exclusion and
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
try:
- related = _binding_related_query(
- session, binding, related_entry, cur, spec, cols, window=window, today=today,
- book=book, tab=tab, ending_seen=ending_seen)
+ related = related_bags(cur, spec, cols, window=window)
sql, params, limits = sheet_mod.build_aggregate_query(
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
@@ -2922,6 +3501,11 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
binding editor show a number before anyone commits the cell. R10 may create a missing Metric
field through the database's ordinary field writer; reusing a column needs no field write.
"""
+ with _sheet_request_scope(session):
+ return _preview_cell_scoped(workbook_id, addr, body, session)
+
+
+def _preview_cell_scoped(workbook_id: str, addr: str, body, session: Session):
raw = dict(body or {})
binding = raw.get("binding") or {}
book = None
@@ -3029,6 +3613,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
⚠ `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
its next PATCH would carry a revision the server has already moved past.
"""
+ with _sheet_request_scope(session):
+ return _compute_cell_scoped(workbook_id, addr, body, session)
+
+
+def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
from core import sheets
try:
@@ -3162,9 +3751,10 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
if perm_scope.row_scope_applies(session.user, database) else None)
wall_nodes, wall_conj = _fe_parts(wall_tree)
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
+ scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
- if not key.startswith(sheet_mod.LINKED_FILTER_PREFIX) and key not in visible:
+ if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
return JSONResponse(status_code=400, content={
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
today = date.today()
@@ -3173,10 +3763,13 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
window = _ending_window(session, binding, book=book, today=today)
needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
| sheet_mod.filter_columns_named(wall_nodes))
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
cols = dict(cols)
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
+ related = related_bags(cur, spec, cols, window=window)
for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
+ if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
+ continue
if key in hidden or not (cols.get(key) or {}).get("sql"):
return JSONResponse(status_code=400, content={
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
@@ -3186,7 +3779,8 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
raise ValueError("unapplied row wall")
count_sql, params, limits = sheet_mod.build_aggregate_query(
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
+ related=related)
if limits:
raise ValueError("unapplied view filter")
except sheet_mod.SheetRefusal as exc:
@@ -3227,8 +3821,20 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
@router.get("/sheets/{workbook_id}/cells/{addr}/drill")
-def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
- """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
+@_request_scoped
+def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
+ session: Session = Depends(require_session)):
+ """`{database, view_id?, filter?, record_ids, count, id_limit}` for `LinkGridModal` (R6, R13, R17).
+
+ ⭐⭐ S66-T48 — `tab` (an id or a name, as `GET /sheets/{id}/tabs/{tab}` accepts) NAMES THE TAB
+ THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a
+ Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old
+ first-tab rule still answers, so an older client keeps working.
+
+ ⭐⭐ S66-T48 — NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching
+ id; the popup pages them and sends the set in a POST body. Only a database whose records a
+ person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so
+ the popup says "showing X of N" rather than presenting X as the whole.
⛔ A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
@@ -3249,10 +3855,18 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
key = str(addr or "").replace("$", "").upper()
cell = None
- for tab in book.get("sheets") or ():
- if key in (tab.get("cells") or {}):
- cell = tab["cells"][key]
- break
+ home = None
+ if str(tab or "").strip():
+ try:
+ home = sheets.find_tab(book, str(tab).strip())
+ except sheets.SheetRefusal as exc:
+ raise _refuse(exc)
+ cell = (home.get("cells") or {}).get(key)
+ else:
+ for candidate in book.get("sheets") or ():
+ if key in (candidate.get("cells") or {}):
+ home, cell = candidate, candidate["cells"][key]
+ break
if not isinstance(cell, dict):
raise err(404, "cell_missing", f"Cell {key} is not filled in.")
if cell.get("kind") == "formula":
@@ -3282,11 +3896,7 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
wall_nodes, wall_conj = _fe_parts(wall_tree)
scope = binding["scope"]
- if scope.get("view_id"):
- scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
- scope.get("view_name", ""))
- else:
- scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
+ scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding)
# ⚠ THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
# reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
@@ -3297,25 +3907,20 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
# window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
# the column set instead, where the time actually is.
try:
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id"))
- window = _ending_window(session, binding, book=book, tab=tab.get("id"))
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=home.get("id"))
+ window = _ending_window(session, binding, book=book, tab=home.get("id"))
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
- related_entry = _binding_related_entry(session, database, binding)
needed = (sheet_mod.filter_columns_named(scope_nodes)
| sheet_mod.filter_columns_named(wall_nodes))
- if related_entry:
- needed.add(related_entry["via"].get("from"))
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
cur, spec, cols = _mirror(session)(
database, metric_window=window,
needed=needed)
cols = dict(cols)
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
try:
- related = _binding_related_query(
- session, binding, related_entry, cur, spec, cols, window=window,
- book=book, tab=tab.get("id"))
+ related = related_bags(cur, spec, cols, window=window)
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
@@ -3330,10 +3935,21 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
"connection. Edit the scope or ask an administrator to update the restriction.")
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
+ # ⭐⭐ S66-T48 — THE CAP IS DECIDED BY THE DATA'S SOURCE, NOT BY THE POPUP. A connector's records
+ # (`records_mutable` False) are listed in full: the popup renders one page and the id set
+ # travels in a POST body, which were the two reasons the old 500 id cap gave. A typed database
+ # keeps `MAX_ROWS`, asked only when the count passes it (the predicate reads the store), and
+ # the limit is RETURNED so the popup states it instead of enforcing it silently.
+ from core import user_tables
+ id_limit = None
+ if (int(total or 0) > user_tables.MAX_ROWS
+ and user_tables.records_mutable(database, st=session.runtime)):
+ id_limit = user_tables.MAX_ROWS
+ ids_sql = f"{ids_sql} LIMIT {int(id_limit)}"
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
- "limits": limits}
+ "id_limit": id_limit, "limits": limits}
if scope.get("view_id"):
out["view_id"] = scope["view_id"]
else:
@@ -3442,6 +4058,11 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
database work, and a batch's cost is dominated by the number of distinct (database, period)
pairs in it rather than by the number of cells.
"""
+ with _sheet_request_scope(session):
+ return _resolve_bindings_scoped(workbook_id, body, session)
+
+
+def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
from core import sheets
import formula_runtime as fr
diff --git a/api/routes_tables.py b/api/routes_tables.py
index b73d44dee2b8b64a9a5cff72f2697fd7ad6fd985..2bca3420b1cbeb35e574537ae1afd69d10a4c304 100644
--- a/api/routes_tables.py
+++ b/api/routes_tables.py
@@ -2116,7 +2116,7 @@ def create_table(body: dict = Body(default=None),
if not key:
raise err(400, "refused",
f"could not create it. The name may be empty or this tenant already has "
- f"{ut.MAX_TABLES} databases")
+ f"{ut.MAX_TABLES} databases of its own (connected-source databases do not count)")
return {"key": key}
diff --git a/api/routes_templates.py b/api/routes_templates.py
index 278d43922770fa5c6186a809c4c3a9a5ac93d090..0c7ada55c0c120ee79cbf30734cade841497e0a4 100644
--- a/api/routes_templates.py
+++ b/api/routes_templates.py
@@ -80,8 +80,8 @@ def list_templates(table: str = "", session: Session = Depends(require_session),
`missing_columns`, which is the same function the POST below re-runs).
"""
if module:
- if module not in ("sheet", "data", "app"):
- raise err(400, "unknown_module", "Choose Sheet, Data or App.")
+ if module not in ("sheet", "data", "app", "agents"):
+ raise err(400, "unknown_module", "Choose Sheet, Data, App or Agents.")
return _module_templates(session)
fields, source = _target_or_refuse(session, table)
vt = _templates()
@@ -137,6 +137,16 @@ def apply_template(key: str, body: dict = Body(default=None),
import core.table_store as table_store
ops = table_store.make(ws_key, st=session.runtime)
applied = []
+ # S67-T39 (R12, D-2233): a template View bound to an App that no longer exists instantiates as
+ # a plain view of its database; the live App ids are read once, and an unreadable registry
+ # keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
+ try:
+ import application_registry
+ raw_registry = session.runtime.get(application_registry._KEY)
+ live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
+ and isinstance(raw_registry.get("applications"), dict) else None)
+ except Exception:
+ live_apps = None
for view in tpl.get('views') or ():
# ⛔ A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
# — mutating one would write this caller's username into the template every other tenant
@@ -145,6 +155,10 @@ def apply_template(key: str, body: dict = Body(default=None),
# side.
payload = dict(view)
payload['config'] = dict(view.get('config') or {})
+ dead_ref = payload['config'].get('application')
+ if (live_apps is not None and isinstance(dead_ref, dict)
+ and str(dead_ref.get('id') or '') not in live_apps):
+ payload['config'].pop('application')
payload['createdBy'] = session.uname
saved = ops.save_view(session.uname, payload)
applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
@@ -265,8 +279,68 @@ def _data_template_database(template, definitions):
return ""
+def _agent_templates():
+ """Fresh starter definitions; saving one never arms its schedule."""
+ import os
+
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").strip().rstrip("/")
+
+ def report(instruction, route):
+ return {"id": "act_report", "kind": "ai_agent", "config": {
+ "url": base + "/#/" + route if base else "",
+ "instruction": instruction,
+ "field": "", "timeoutMs": 20000, "maxSteps": 6}}
+
+ def starter(key, name, description, cron, databases, actions):
+ return {"key": key, "name": name, "description": description, "databases": databases,
+ "definition": {"name": name, "kind": "plain", "trigger": {"key": "schedule"},
+ "schedule": {"cron": cron, "enabled": False},
+ "flow": {"actions": actions}}}
+
+ return [
+ starter("odoo_sync_check", "Odoo sync check", "Check Odoo database freshness each day.",
+ "0 8 * * *", ["ut_odoo_orders"], [
+ {"id": "act_find", "kind": "find_records", "config": {
+ "table": "ut_odoo_orders", "cond": None, "limit": 25}},
+ report("Read the Odoo connection's database sync statuses in this workspace. "
+ "Report tables whose last successful sync is older than 24 hours or failed. "
+ "Name each table and the observed timestamp or error. If freshness cannot "
+ "be read, report it as unknown. Only read; do not sync, change settings, "
+ "edit records or send messages.", "connectors")]),
+ starter("ar_follow_up_digest", "AR follow-up digest", "Draft a weekly digest of overdue invoices.",
+ "0 9 * * 1", ["ut_odoo_invoices"], [
+ # The condition vocabulary orders numbers only. Evaluate due dates in the
+ # report at run time, never against a date frozen when the template was saved.
+ {"id": "act_find", "kind": "find_records", "config": {
+ "table": "ut_odoo_invoices",
+ "cond": {"field": "residual", "op": ">", "value": 0}, "limit": 100}},
+ report("Read open invoices in this workspace's Odoo invoices database. Compare "
+ "due_date with today's date and include only overdue invoices with residual "
+ "above zero. Draft a follow-up digest grouped by customer, with invoice, "
+ "due date, open amount and suggested next action. State any missing data "
+ "or incomplete coverage. Only read; do not edit records or send messages.",
+ "ut_odoo_invoices")]),
+ starter("data_health_watcher", "Data-health watcher", "Check for stale or empty databases each day.",
+ "0 7 * * *", [], [
+ report("Read the databases visible in this workspace. Flag empty databases and "
+ "those whose last successful refresh is older than 24 hours. Report each "
+ "database name, observed row count and refresh time when available. "
+ "Treat missing freshness information as unknown, never as healthy. "
+ "Only read; do not edit records, change settings or send messages.", "data")]),
+ ]
+
+
+def _content_studio():
+ """The Content Studio App's own module, or None when it is absent (rule 13: delete-safe)."""
+ try:
+ import content_studio
+ except ImportError:
+ return None
+ return content_studio
+
+
def _module_templates(session):
- from core import sheet_templates, user_tables
+ from core import registry, sheet_templates, user_tables
from routes_connectors import _entry_types
from routes_keychain import env_odoo_available
from routes_nav import application_templates
@@ -281,9 +355,13 @@ def _module_templates(session):
saved = _saved_sheet_templates(session.runtime)
data_templates = [(row, _data_template_database(row, definitions))
for row in _templates().TEMPLATES]
+ # S67-T36 (R25): an archived Agents module offers no agent template; the create route stays.
+ agent_templates = (_agent_templates() if _may_open(session, "automation")
+ and not registry.is_archived("automation") else [])
databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
databases.update(db for row in saved for db in row["databases"])
databases.update(database for _row, database in data_templates if database)
+ databases.update(db for row in agent_templates for db in row["databases"])
database_states = _database_states(session, definitions, databases)
requirement = lambda database, connector="": _requirement(
database, definitions, have, database_states, connector)
@@ -305,9 +383,25 @@ def _module_templates(session):
app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
creation_mode="post")
for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
- return {"sections": [{"module": "sheet", "templates": sheet},
- {"module": "data", "templates": data},
- {"module": "app", "templates": app}]}
+ sections = [{"module": "sheet", "templates": sheet},
+ {"module": "data", "templates": data},
+ {"module": "app", "templates": app}]
+ # S67-T40 (R18): every tenant's App catalogue offers Content Studio, which creates its own
+ # databases. Guarded (rule 13): without the module the catalogue simply omits the row.
+ # ⚠ Appended AFTER `sections` (it is the App section's own list, so the order stays Finance,
+ # then Studio): `verify_shell_rail.py` scopes its S65-T10 needles to the span from the Finance
+ # offer to `sections = [`, and its finance-offer-wizard control mutates the FIRST
+ # `creation_mode="post"` in this file. A Studio row inside that span, or above it, masks it.
+ studio = _content_studio()
+ if studio is not None:
+ app.append(_catalog_row(studio.TEMPLATE_KEY, studio.LABEL, studio.DESCRIPTION, [],
+ creation_mode="post"))
+ if agent_templates:
+ sections.append({"module": "agents", "templates": [
+ _catalog_row(row["key"], row["name"], row["description"],
+ [requirement(db, "odoo") for db in row["databases"]], creation_mode="post")
+ for row in agent_templates]})
+ return {"sections": sections}
@router.get("/sheet-templates")
@@ -412,6 +506,9 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
from urllib.parse import quote
body = body if isinstance(body, dict) else {}
+ if module == "agents":
+ # A direct call to create_automation does not run its FastAPI dependency.
+ session.require("automation")
if module == "app" and key.startswith("finance:"):
import application_registry
from modules.finance_apps import finance_application
@@ -447,6 +544,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
raise err(404, "unknown_template", "That template does not exist.")
if not template["creatable"]:
raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
+ studio = _content_studio() if module == "app" else None
+ if studio is not None and key == studio.TEMPLATE_KEY:
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was created.")
+ try:
+ return studio.create(session, body.get("name") or "")
+ except studio.ContentStudioError as exc:
+ raise err(409, "content_studio_refused", str(exc))
+ except PermissionError as exc:
+ raise err(403, "forbidden", str(exc))
if module == "sheet":
database = template["requires"][0]["database"] if template["requires"] else ""
result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
@@ -455,9 +562,28 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
database = template["requires"][0]["database"]
result = apply_template(key, {"table": database}, session)
return {"address": "#/" + quote(database, safe=""), **result}
+ if module == "agents":
+ from routes_automation import create_automation
+
+ definition = next(row["definition"] for row in _agent_templates() if row["key"] == key)
+ result = create_automation(definition, session=session)
+ return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result}
# The existing wizard still owns its POST-then-PATCH share sequence. API clients may
# dispatch creation here; the existing App writer retains schema and permission walls.
result = create_nav_application({**body, "templateKind": key}, session=session)
app = result["application"]
return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
**result}
+
+# S67-T41: Content Studio's own routes (`/api/v1/content-studio/*`) ride this router because the
+# App wires no mount of its own into main.py; GUARDED, so deleting content_studio.py drops the
+# routes and nothing else (rule 13).
+# ⚠ An inline guarded import, NOT a call to `_content_studio()`: this tail belongs to the
+# dispatcher block, which `verify_home.py`'s W61-T23 declaration-order control moves ABOVE that
+# helper's definition, where a call is a NameError that crashes the whole gate.
+try:
+ import content_studio as _studio_module
+except ImportError:
+ _studio_module = None
+if _studio_module is not None and getattr(_studio_module, "router", None) is not None:
+ router.include_router(_studio_module.router)
diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py
index 45aa4249d8573e27187f5bd0800b37185f73d06c..f10d98ad9eefb00ab18fbb5260b58734fa134230 100644
--- a/platform/core/connector_spec.py
+++ b/platform/core/connector_spec.py
@@ -278,7 +278,7 @@ AMAZON_TABLES = (
# ⚠ THE MONEY-TO-ORDER BRIDGE — what turns "an order happened" into "this is what we were
# actually paid for it". The V2 settlement report carries the order id per line.
links=(
- ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
+ ("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
),
),
@@ -320,6 +320,309 @@ AMAZON_TABLES = (
# about this product", and a link would be misleading.
links=(),
),
+ ConnectorTableSpec(
+ key=_AMZN + "economics", native="analytics_economics_2024_03_15",
+ source="amzn_economics", label="Amazon economics", pinned="msku",
+ identity=("start_date", "msku", "marketplace_id"),
+ links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("child_asin", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "transactions", native="finances/2024-06-19/transactions",
+ source="amzn_transactions", label="Amazon financial transactions", pinned="transaction_id",
+ identity=("transaction_id", "item_position"),
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
+ ("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA",
+ source="amzn_fba_fee_estimates", label="Amazon FBA fee estimates", pinned="sku",
+ identity=("_snapshot_date", "sku", "amazon_store"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT",
+ source="amzn_referral_fees", label="Amazon referral fee preview", pinned="seller_sku",
+ identity=("_snapshot_date", "seller_sku"),
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA",
+ source="amzn_storage_fees", label="Amazon storage fees", pinned="asin",
+ identity=("month", "asin", "fnsku", "fulfillment_center"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
+ source="amzn_search_query_performance", label="Amazon search query performance",
+ pinned="search_query_data_search_query",
+ identity=("start_date", "end_date", "asin", "search_query_data_search_query"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
+ source="amzn_long_term_storage_fees", label="Amazon long term storage fees", pinned="sku",
+ identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA",
+ source="amzn_storage_overage_fees", label="Amazon storage overage fees", pinned="storage_type",
+ identity=("_period_start", "storage_type", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA",
+ source="amzn_reimbursements", label="Amazon reimbursements", pinned="reimbursement_id",
+ identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
+ source="amzn_fba_customer_returns", label="Amazon FBA customer returns", pinned="order_id",
+ identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
+ source="amzn_replacements", label="Amazon replacements", pinned="replacement_amazon_order_id",
+ identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
+ source="amzn_fba_shipped_sales", label="Amazon FBA shipped sales", pinned="amazon_order_id",
+ identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
+ source="amzn_fba_shipment_promotions", label="Amazon FBA shipment promotions", pinned="amazon_order_id",
+ identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"),
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA",
+ source="amzn_inventory_ledger_detail", label="Amazon inventory ledger detail", pinned="msku",
+ identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
+ links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA",
+ source="amzn_inventory_planning", label="Amazon inventory planning", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
+ source="amzn_manage_inventory", label="Amazon manage inventory", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA",
+ source="amzn_reserved_inventory", label="Amazon reserved inventory", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
+ source="amzn_restock_recommendations", label="Amazon restock recommendations", pinned="merchant_sku",
+ identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
+ links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA",
+ source="amzn_stranded_inventory", label="Amazon stranded inventory", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
+ source="amzn_inbound_noncompliance", label="Amazon inbound noncompliance", pinned="shipment_id",
+ identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
+ source="amzn_inventory_expiration_dates", label="Amazon inventory expiration dates", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA",
+ source="amzn_recommended_removals", label="Amazon recommended removals", pinned="sku",
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
+ source="amzn_removal_orders", label="Amazon removal orders", pinned="order_id",
+ identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
+ source="amzn_removal_shipments", label="Amazon removal shipments", pinned="order_id",
+ identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA",
+ source="amzn_subscribe_save_forecast", label="Amazon Subscribe and Save forecast", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA",
+ source="amzn_subscribe_save_performance", label="Amazon Subscribe and Save performance", pinned="sku",
+ identity=("_period_start", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY",
+ source="amzn_remote_fulfillment_eligibility", label="Amazon remote fulfillment eligibility", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(), # Measured header has no product key; do not declare a guessed join.
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
+ source="amzn_search_catalog_performance", label="Amazon search catalog performance", pinned="asin",
+ identity=("_period_start", "asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
+ source="amzn_market_basket", label="Amazon market basket", pinned="asin",
+ identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
+ source="amzn_repeat_purchase", label="Amazon repeat purchase", pinned="asin",
+ identity=("_period_start", "asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT",
+ source="amzn_promotion_performance", label="Amazon promotion performance", pinned="promotion_id",
+ identity=("_period_start", "promotion_id", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT",
+ source="amzn_coupon_performance", label="Amazon coupon performance", pinned="coupon_id",
+ identity=("_period_start", "coupon_id", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT",
+ source="amzn_account_health", label="Amazon account health", pinned="marketplace_id",
+ identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA",
+ source="amzn_inactive_listings", label="Amazon inactive listings", pinned="seller_sku",
+ identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin1", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA",
+ source="amzn_cancelled_listings", label="Amazon cancelled listings", pinned="seller_sku",
+ identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
+ ("asin1", _AMZN + "listings", "asin1", "many_to_one")),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT",
+ source="amzn_suppressed_listings", label="Amazon suppressed listings", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA",
+ source="amzn_listing_defects", label="Amazon listing defects", pinned="sku",
+ identity=("_snapshot_date", "sku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
+ source="amzn_sold_listings", label="Amazon sold listings", pinned="sku",
+ identity=("_snapshot_date", "sku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
+ source="amzn_mfn_returns", label="Amazon MFN returns", pinned="order_id",
+ identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"),
+ links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
+ source="amzn_b2b_recommended", label="Amazon B2B product opportunities", pinned="asin",
+ identity=("_snapshot_date", "asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
+ source="amzn_b2b_not_yet_on_amazon", label="Amazon B2B products not yet on Amazon", pinned="ean",
+ identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES",
+ source="amzn_geo_opportunities", label="Amazon geo opportunities", pinned="asin",
+ identity=("_snapshot_date", "asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION",
+ source="amzn_out_of_stock_mfn", label="Amazon out of stock MFN listings", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD",
+ source="amzn_b2b_quotes", label="Amazon B2B quantity discounts", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT",
+ source="amzn_fee_discounts", label="Amazon fee discounts", pinned="sku",
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
+ links=(), # The measured empty report supplies metadata only.
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "order_metrics", native="sales/v1/orderMetrics",
+ source="amzn_order_metrics", label="Amazon order metrics", pinned="interval",
+ identity=("_period_start", "interval", "_report_row_number"),
+ links=(),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries",
+ source="amzn_fba_inventory", label="Amazon FBA inventory summaries", pinned="seller_sku",
+ identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}",
+ source="amzn_listing_items", label="Amazon listing status and issues", pinned="sku",
+ identity=("_snapshot_date", "sku", "_report_row_number"),
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
+ ),
+ ConnectorTableSpec(
+ key=_AMZN + "catalog_items", native="catalog/2022-04-01/items",
+ source="amzn_catalog_items", label="Amazon catalog attributes", pinned="asin",
+ identity=("_snapshot_date", "asin", "_report_row_number"),
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
+ ),
)
diff --git a/platform/core/daily_sales_repair.py b/platform/core/daily_sales_repair.py
index 3618c8e2046729f16cd8fc8905dddb46e0d3f290..c069974c44d057c66e32b6bf98bd7fd7d8fdaa56 100644
--- a/platform/core/daily_sales_repair.py
+++ b/platform/core/daily_sales_repair.py
@@ -40,6 +40,39 @@ _SUMMARY_RATIO_CASCADES = {
"R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
}
+# Exact authored formulas at the staging snapshot; an owner edit is never an oracle.
+# Q64 deliberately keeps its formula but participates in the recalculation/write plan.
+_FORMULA_REPAIRS = {
+ ("Summary (SR)", "A1", '=CONCATENATE(Print!C6," Sales Report")'):
+ ('=CONCATENATE(C6," Sales Report")', "resolved", "C6 holds this tab's sales representative."),
+ ("Summary (SR)", "W8", "=Print!C6"):
+ ("=C6", "resolved", "C6 holds this tab's sales representative."),
+ ("Summary (SR)", "Q39", '=_xlfn.XLOOKUP(Print!$C$6,Print!C14:C30,Print!E14:E30,"")'):
+ ('=""', "deliberate blank", "The Print target table is absent from this workbook."),
+ ("Summary (SR)", "O43", "=Q39-O41"):
+ ('=IFERROR(Q39-O41,"")', "deliberate blank", "The target in Q39 is unavailable."),
+ ("Summary (SR)", "R64", '=COUNTIF(Uncollectible!E7:E1000,"*")'):
+ ('=""', "deliberate blank", "The Uncollectible tab is absent from this workbook."),
+ ("Summary (SR)", "R65", '=COUNTIF(Closed!E7:E1000,"*")'):
+ ('=""', "deliberate blank", "The Closed tab is absent from this workbook."),
+ ("Summary (SR)", "Q64", "=R64"):
+ ("=R64", "deliberate blank", "The unchanged reference follows the deliberate blank in R64."),
+ ("Summary (SR)", "F71", "=(F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
+ ('=IFERROR((F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
+ "deliberate blank", "The ratio is blank while its denominator is unavailable."),
+ ("Summary (SR)", "F72", "=(F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
+ ('=IFERROR((F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
+ "deliberate blank", "The ratio is blank while its denominator is unavailable."),
+ ("Summary (SR)", "F73", "=F71-F72"):
+ ('=IFERROR(F71-F72,"")', "deliberate blank", "The difference is blank while either ratio is unavailable."),
+ ("Summary (SR)", "Q63", "=R63"):
+ ('=IF(ISBLANK(R63),"",R63)', "deliberate blank", "An empty R63 has no figure to report."),
+ ("Summary (SR)", "R68", "=R67-Q67"):
+ ('=IFERROR(R67-Q67,"")', "deliberate blank", "The difference is blank while R67 is unavailable."),
+ (SUMMARY_TAB, "F52", "=G51-F51"):
+ ('=IFERROR(G51-F51,"")', "deliberate blank", "The difference is blank while F51 is unavailable."),
+}
+
def operation_for(workbook_id):
"""Bind a repair operation to the workbook the caller names."""
@@ -141,10 +174,27 @@ def _validate_binding(name, binding, *, window_kind=None):
# Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
# both referenced date bounds keeps a missing period from becoming an unbounded sum.
nodes = tree.get("nodes") if isinstance(tree, dict) else None
- has_filter = (isinstance(nodes, list) and bool(nodes) and tree.get("conj") == "and"
- and all(isinstance(n, dict) and n.get("colId") and n.get("op")
- and ("value" in n or isinstance(n.get("valueRef"), dict))
- for n in nodes))
+ def conjunction_leaves(items):
+ if not isinstance(items, list) or not items:
+ return None
+ leaves = []
+ for node in items:
+ if not isinstance(node, dict):
+ return None
+ if "children" in node:
+ children = conjunction_leaves(node["children"])
+ if node.get("conj", "and") != "and" or children is None:
+ return None
+ leaves.extend(children)
+ elif node.get("colId") and node.get("op") and (
+ "value" in node or isinstance(node.get("valueRef"), dict)):
+ leaves.append(node)
+ else:
+ return None
+ return leaves
+
+ leaves = conjunction_leaves(nodes)
+ has_filter = leaves is not None and tree.get("conj") == "and"
if not has_view and not has_filter:
raise RepairRefusal(
"repair_binding_scope_required",
@@ -158,7 +208,7 @@ def _validate_binding(name, binding, *, window_kind=None):
period = binding.get("period")
if period is None and has_filter and not has_view:
bounds = {}
- for node in nodes:
+ for node in leaves:
slot = node.get("valueRef")
if slot is not None:
if not isinstance(slot, dict) or set(slot) != {"ref"}:
@@ -281,6 +331,70 @@ def check_summary_formula_errors(book):
"negative_control": {"planted": len(expected), "found": len(found)}}
+def _recalculated_formula_cycles(book, formula_check):
+ checked = copy.deepcopy(book)
+ try:
+ result = formula_check(checked)
+ except Exception as exc:
+ raise RepairRefusal("repair_cycle_check_failed",
+ "The formula engine could not complete the cycle check.") from exc
+ if isinstance(result, dict) and isinstance(result.get("book"), dict):
+ checked = result["book"]
+ # The engine gives all members of a cyclic component the same origin. Preserve
+ # authored tab names in the report; engine lookup keys are upper case.
+ groups = {}
+ for tab in checked.get("sheets") or ():
+ for address, cell in (tab.get("cells") or {}).items():
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
+ continue
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
+ if not isinstance(error, dict) or error.get("code") != "circular_reference":
+ continue
+ label = str(tab.get("name") or "") + "!" + address
+ origin = str(error.get("origin") or label).upper()
+ groups.setdefault(origin, set()).add(label)
+ return sorted(sorted(addresses) for addresses in groups.values())
+
+
+def formula_cycles(book, formula_check):
+ """Recalculate a copy and report the engine's circular references by component.
+
+ The injected checker is the same API-side engine used by the repair. Every call
+ first proves that both a two-cell cycle and a self-loop are detectable, without
+ trusting stored errors or changing the supplied workbook.
+ """
+ if not callable(formula_check):
+ raise RepairRefusal("repair_cycle_check_required",
+ "The cycle check requires a formula engine.")
+ probe = {"sheets": [{"id": "cycle-probe", "name": "Cycle probe", "cells": {
+ address: {"kind": "formula", "state": "stale", "formula": formula}
+ for address, formula in (("A1", "=B1"), ("B1", "=A1"), ("C1", "=C1"))
+ }}]}
+ expected = [["Cycle probe!A1", "Cycle probe!B1"], ["Cycle probe!C1"]]
+ found = _recalculated_formula_cycles(probe, formula_check)
+ if found != expected:
+ raise RepairRefusal("repair_cycle_census_blind",
+ "The cycle check failed to find its planted formula cycles.")
+ cycles = _recalculated_formula_cycles(book, formula_check)
+ return {"count": len(cycles), "cycles": cycles,
+ "negative_control": {"planted": len(expected), "found": len(found), "cycles": found}}
+
+
+def _formula_cycle_comparison(before, after, formula_check):
+ """Diagnostic evidence only: keep execute's existing acceptance/refusal contract.
+
+ Legacy callers may omit the engine. A missing or blind checker is explicitly
+ reported as a named refusal, never represented as a successful zero census.
+ """
+ def report(book):
+ try:
+ return formula_cycles(book, formula_check)
+ except RepairRefusal as exc:
+ return {"refusal": {"code": exc.code, "message": exc.message}}
+
+ return {"before": report(before), "after": report(after)}
+
+
def _formula_error_comparison(before, after, edits, reported_errors=()):
"""Reject new errors anywhere and every residual error in explicitly planned cells.
@@ -331,6 +445,21 @@ def _cell_map(book):
for address, cell in (tab.get("cells") or {}).items()}
+def _same_unplanned_error(previous, recalculated):
+ """True when recalculation leaves an erroring formula with the same formula and error code."""
+ def code(cell):
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
+ return error.get("code") if isinstance(error, dict) else None
+
+ return (isinstance(previous, dict) and isinstance(recalculated, dict)
+ and previous.get("kind") == recalculated.get("kind") == "formula"
+ and isinstance(previous.get("formula"), str)
+ and previous.get("formula") == recalculated.get("formula")
+ and code(previous) is not None and code(previous) == code(recalculated)
+ and {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous}
+ == {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated})
+
+
def _checked_projection(projected, edits, formula_check):
"""Make formula output part of the plan, never an unpersisted preflight side effect."""
if not callable(formula_check):
@@ -351,6 +480,10 @@ def _checked_projection(projected, edits, formula_check):
if before.get(slot) == after.get(slot):
continue
row = planned.get(slot)
+ if row is None and _same_unplanned_error(before.get(slot), after.get(slot)):
+ # An unrelated formula that was an error and recalculates to the SAME error is
+ # neither repaired nor broken by this plan, so it stays exactly as stored.
+ continue
if row is None or row["kind"] == "recalculated":
previous, recalculated = before.get(slot), after.get(slot)
# Recalculation may refresh a healthy formula's result, but may not repair
@@ -365,7 +498,8 @@ def _checked_projection(projected, edits, formula_check):
!= {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
raise RepairRefusal(
"repair_unplanned_formula_change",
- "Formula validation would change a cell outside the supplied source diff.")
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
if row is None:
sheet = sheet_names[slot[0]]
@@ -382,7 +516,8 @@ def _checked_projection(projected, edits, formula_check):
or row["after"].get("kind") != "formula":
raise RepairRefusal(
"repair_unplanned_formula_change",
- "Formula validation would change a cell outside the supplied source diff.")
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
recalculated = after.get(slot)
if not isinstance(recalculated, dict) \
or (row["kind"] == "source"
@@ -441,10 +576,10 @@ def _summary_date(value):
"Every summary date must be a real ISO date from the sheet or mirror.") from exc
-def _resolved(binding, book, resolve_binding):
+def _resolved(binding, book, resolve_binding, *, tab_name=SUMMARY_TAB):
if not callable(resolve_binding):
raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
- result = resolve_binding(copy.deepcopy(binding), book, _tab(book, SUMMARY_TAB))
+ result = resolve_binding(copy.deepcopy(binding), book, _tab(book, tab_name))
if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
if result.get("limits"):
@@ -599,9 +734,9 @@ def _build_summary_plan(current, request, resolve_binding, formula_check):
"scope": {"filter": {"conj": "and", "nodes": [
{"colId": "invoice_date", "op": "within", "dateWindow": {"kind": kind},
"valueRef": {"ref": end_ref}},
- {"colId": "move_type", "op": "eq", "value": "out_invoice"}]}},
- "related": {"via": "customer_link", "filter": {"conj": "and", "nodes": [
- {"colId": "pricelist", "op": "eq", "valueRef": {"ref": scope_ref}}]}}}
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"},
+ {"colId": "related:customer_link:pricelist", "op": "eq",
+ "valueRef": {"ref": scope_ref}}]}}}
_validate_binding(col + str(row_number), binding)
value = _resolved(binding, projected, resolve_binding)
if type(value) not in (int, float) or not math.isfinite(value):
@@ -685,6 +820,919 @@ def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, form
return checks
+def _binding_ref_slots(value):
+ """Yield both operand slots at any depth without changing other binding metadata."""
+ if isinstance(value, dict):
+ for key, child in value.items():
+ if key in {"valueRef", "value2Ref"} and isinstance(child, dict):
+ yield child
+ yield from _binding_ref_slots(child)
+ elif isinstance(value, list):
+ for child in value:
+ yield from _binding_ref_slots(child)
+
+
+def _has_legacy_related(value):
+ def linked(node):
+ if isinstance(node, dict):
+ key = node.get("colId")
+ return ((isinstance(key, str) and key.startswith("linked:"))
+ or any(linked(child) for child in node.values()))
+ return isinstance(node, list) and any(linked(child) for child in node)
+
+ return isinstance(value, dict) and ("related" in value or linked(value))
+
+
+def _remap_cell(book, tab_name, ref):
+ """Read an explicitly supplied A1 ref, relative to the migrated cell's tab."""
+ match = re.fullmatch(r"(?:(?:'((?:[^']|'')+)'|([^'!]+))!)?(\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6})",
+ ref) if isinstance(ref, str) else None
+ if not match:
+ raise RepairRefusal("repair_ref_remap_invalid", "Invalid remap reference " + str(ref) + ".")
+ name = (match[1].replace("''", "'") if match[1] else match[2]) or tab_name
+ try:
+ return (_tab(book, name).get("cells") or {}).get(_addr(match[3])) or {}
+ except RepairRefusal as exc:
+ raise RepairRefusal("repair_ref_remap_invalid",
+ "Remap reference " + ref + " cannot be read.") from exc
+
+
+def _build_related_migration_plan(current, request, resolve_binding, formula_check=None):
+ """Migrate by content, proving explicit moved helpers and every stored answer first."""
+ from core.sheet_binding import BindingMetadataRefusal, rewrite_legacy_related
+
+ migration = request["related_migration"]
+ if not isinstance(migration, dict) or not isinstance(migration.get("ref_remap", {}), dict):
+ raise RepairRefusal("repair_ref_remap_invalid", "Supply a reference remap object for the migration.")
+ if any(key in request for key in ("summary_block", "source_cells", "binding_cells")):
+ raise RepairRefusal("repair_migration_plan_conflict", "Run the related migration as its own repair.")
+ remap = migration.get("ref_remap", {})
+ edits = []
+ for tab in current.get("sheets") or ():
+ for address, before in (tab.get("cells") or {}).items():
+ if not isinstance(before, dict) or before.get("kind") != "bound" \
+ or not isinstance(before.get("binding"), dict) \
+ or not _has_legacy_related(before["binding"]):
+ continue
+ try:
+ binding = rewrite_legacy_related(before["binding"], linked=True)
+ except BindingMetadataRefusal as exc:
+ raise RepairRefusal(exc.code, exc.message) from exc
+ if _has_legacy_related(binding):
+ raise RepairRefusal("related_invalid", "A nested legacy relationship cannot be migrated.")
+ edits.append({"kind": "binding", "name": tab["name"] + "!" + address,
+ "sheet": tab["name"], "address": address, "before": copy.deepcopy(before),
+ "after": {**copy.deepcopy(before), "binding": binding}})
+ # A retry with the same remap must be a no-op, not an unused-remap refusal.
+ if not edits:
+ return [], []
+ if not str(request.get("refreshed_at") or "").strip():
+ raise RepairRefusal("repair_timestamp_required", "The migration needs an explicit refresh timestamp.")
+ edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
+ recalculated = []
+
+ def current_after_recalculation(tab_name, ref, stored):
+ # A structure change leaves a helper formula `stale` until the next recalculation, so the
+ # formula engine, not the stored flag, decides whether the stored text is still its answer.
+ if not callable(formula_check):
+ return False
+ if not recalculated:
+ checked = formula_check(copy.deepcopy(current))
+ recalculated.append(checked.get("book") if isinstance(checked, dict) else None)
+ if not isinstance(recalculated[0], dict):
+ return False
+ again = _remap_cell(recalculated[0], tab_name, ref)
+ return (again.get("kind") == "formula" and again.get("state") == "fresh"
+ and not again.get("error") and not again.get("refusal")
+ and again.get("value") == stored.get("value"))
+
+ used = set()
+ for row in edits:
+ for slot in _binding_ref_slots(row["after"]["binding"]):
+ ref = slot.get("ref")
+ if not isinstance(ref, str) or ref not in remap:
+ continue
+ target = remap[ref]
+ source_cell = _remap_cell(current, row["sheet"], ref)
+ target_cell = _remap_cell(current, row["sheet"], target)
+ source_empty = (source_cell.get("kind", "empty") in {"empty", "literal"}
+ and source_cell.get("value") in (None, "")
+ and not any(source_cell.get(key) for key in ("formula", "binding", "text")))
+ current_text = target_cell.get("state") == "fresh" or (
+ target_cell.get("state") == "stale"
+ and current_after_recalculation(row["sheet"], target, target_cell))
+ if not source_empty or target_cell.get("kind") != "formula" \
+ or not str(target_cell.get("formula") or "").startswith("=") \
+ or not current_text \
+ or target_cell.get("error") or target_cell.get("refusal") \
+ or not isinstance(target_cell.get("value"), str) or not target_cell["value"].strip():
+ raise RepairRefusal(
+ "repair_ref_remap_invalid",
+ "Remap " + ref + " to " + str(target) + " on " + row["sheet"]
+ + " requires an empty source and a current formula with a non-empty string value.")
+ used.add(ref)
+ slot["ref"] = target
+ unused = set(remap) - used
+ if unused:
+ raise RepairRefusal("repair_ref_remap_unused",
+ "No migrated binding uses remap reference " + ", ".join(sorted(map(str, unused))) + ".")
+
+ projected = _project(current, edits)
+ migrated = []
+ for row in edits:
+ value = _resolved(row["after"]["binding"], projected, resolve_binding, tab_name=row["sheet"])
+ before_value = row["before"].get("value")
+ numeric = type(value) in (int, float) and type(before_value) in (int, float)
+ equal = (math.isfinite(value) and math.isfinite(before_value)
+ and math.isclose(value, before_value, rel_tol=1e-9, abs_tol=1e-6)) if numeric else (
+ type(value) is type(before_value) and value == before_value)
+ if not equal:
+ raise RepairRefusal("repair_migration_value_changed",
+ row["name"] + " would change from " + repr(before_value)
+ + " to " + repr(value) + "; no migration was applied.")
+ desired = _preserve_visual(row["before"], {
+ **row["after"], "kind": "bound", "state": "fresh", "value": value,
+ "display": str(value), "refreshed_at": request["refreshed_at"]})
+ cleaned = sheets.clean_cell(desired, row["address"])
+ if cleaned != desired:
+ raise RepairRefusal("repair_migration_metadata_changed",
+ row["name"] + " cannot be migrated without changing its saved metadata.")
+ row["after"] = cleaned
+ _tab(projected, row["sheet"])["cells"][row["address"]] = copy.deepcopy(cleaned)
+ migrated.append({"sheet": row["sheet"], "address": row["address"],
+ "before_value": before_value, "after_value": value})
+ return edits, migrated
+
+
+def _formula_repair_census(book):
+ errors = formula_error_census(book)
+ return {name: {"count": sum(row["sheet"] == name for row in errors),
+ "errors": [row for row in errors if row["sheet"] == name]}
+ for name in (SUMMARY_TAB, "Summary (SR)")}
+
+
+def _check_formula_repair_errors(book):
+ """All codes count, including blank, missing tabs and circular references."""
+ for name, address, _before in _FORMULA_REPAIRS:
+ cell = (_tab(book, name).get("cells") or {}).get(address) or {}
+ if cell.get("kind") != "formula" or cell.get("state") != "fresh" \
+ or cell.get("error") or cell.get("refusal") or cell.get("value") is None:
+ raise RepairRefusal("repair_error_cell_unresolved",
+ name + "!" + address + " did not recalculate to a non-error value.")
+ census = _formula_repair_census(book)
+ remaining = [row for tab in census.values() for row in tab["errors"]]
+ if remaining:
+ raise RepairRefusal("repair_formula_errors_remain",
+ "Formula errors remain: " + json.dumps(remaining, sort_keys=True) + ".")
+ return census
+
+
+def _build_formula_repair_plan(current, request, formula_check):
+ """Restore the two stored marker masters and the exact, enumerated error formulas."""
+ from openpyxl.formula.translate import Translator
+
+ if not isinstance(request["formula_repair"], dict) or any(
+ key in request for key in ("related_migration", "summary_block", "source_cells", "binding_cells")):
+ raise RepairRefusal("repair_formula_plan_conflict", "Run the formula repair as its own repair.")
+ if not callable(formula_check):
+ raise RepairRefusal("repair_formula_check_required", "The formula repair requires a formula engine.")
+ edits, evidence = [], []
+
+ def append(name, address, formula):
+ before = copy.deepcopy((_tab(current, name).get("cells") or {}).get(address))
+ desired = None
+ if formula is not None:
+ desired = {**copy.deepcopy(before or {}), "kind": "formula",
+ "formula": formula, "state": "stale", "value": None}
+ for key in ("error", "refusal", "display"):
+ desired.pop(key, None)
+ after = _preserve_visual(before, desired)
+ if after is not None:
+ after = sheets.clean_cell(after, address)
+ # Even unchanged formula text must be planned: Q64's old error is repaired by R64.
+ if before != after:
+ edits.append({"kind": "derived", "name": name + "!" + address,
+ "sheet": name, "address": address, "before": before, "after": after})
+
+ def empty(cell):
+ return (cell.get("kind", "empty") in {"empty", "literal"}
+ and cell.get("value") in (None, "")
+ and not any(cell.get(key) for key in ("formula", "binding", "text")))
+
+ cached_markers = []
+ for name, marker_row, last_copy in ((SUMMARY_TAB, 4, 6), ("Summary (SR)", 5, 7)):
+ cells = _tab(current, name).get("cells") or {}
+ source, origin = "E" + str(marker_row), "F" + str(marker_row)
+ stored = cells.get(source) or {}
+ master = stored.get("formula") if stored.get("kind") == "formula" else None
+ retry = empty(stored)
+ if retry:
+ # A completed repair retains the verbatim master in F, after clearing E.
+ held = cells.get(origin) or {}
+ master = held.get("formula") if held.get("kind") == "formula" else None
+ if not isinstance(master, str) or not master.startswith('=IF(' + source + '="",'):
+ raise RepairRefusal("repair_marker_master_missing",
+ name + "!" + source + " must hold its self-referencing marker master.")
+ translated = {origin: master}
+ translated.update({chr(col) + str(marker_row): Translator(master, origin=origin).translate_formula(
+ chr(col) + str(marker_row)) for col in range(ord("G"), ord("Q") + 1)})
+ if retry and (not all(empty(cells.get("E" + str(row)) or {})
+ for row in range(3, last_copy + 1)) or not all(
+ (cells.get(address) or {}).get("kind") == "formula"
+ and cells[address].get("formula") == formula for address, formula in translated.items())):
+ raise RepairRefusal("repair_marker_master_missing",
+ name + "!" + source + " is missing and the restored marker row is incomplete.")
+ for address, formula in translated.items():
+ before = cells.get(address) or {}
+ exact_formula = before.get("kind") == "formula" and before.get("formula") == formula
+ # Excel's cached month-end markers (H4 2026-06-30, M4 2026-07-31, Q4 2026-08-31 on
+ # staging) are typed ISO dates; one is accepted only if the restored formula
+ # recalculates to exactly it (checked after projection, below), never by address.
+ cached_date = (before.get("kind") == "literal" and isinstance(before.get("value"), str)
+ and re.fullmatch(r"\d{4}-\d{2}-\d{2}", before["value"]) is not None)
+ if not (empty(before) or cached_date or exact_formula):
+ raise RepairRefusal("repair_marker_row_occupied",
+ name + "!" + address + " contains an owner value outside the marker repair.")
+ if cached_date:
+ cached_markers.append((name, address, before["value"]))
+ append(name, address, formula)
+ for row in range(3, last_copy + 1):
+ append(name, "E" + str(row), None)
+
+ for (name, address, expected), (formula, resolution, reason) in _FORMULA_REPAIRS.items():
+ before = (_tab(current, name).get("cells") or {}).get(address) or {}
+ if before.get("kind") != "formula" or before.get("formula") not in (expected, formula):
+ raise RepairRefusal("repair_error_cell_changed",
+ name + "!" + address + " no longer holds the expected error formula or its repair.")
+ evidence.append({"tab": name, "address": address, "before": before["formula"],
+ "after": formula, "resolution": resolution, "reason": reason})
+ append(name, address, formula)
+
+ def checked(book):
+ result = formula_check(book)
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
+ raise RepairRefusal("repair_formula_check_required",
+ "The formula repair requires a recalculated workbook from its engine.")
+ # Name unresolved targets before the generic unplanned-change guard can mask them.
+ _check_formula_repair_errors(result["book"])
+ return result
+
+ projected, _errors = _checked_projection(_project(current, edits), edits, checked)
+ for name, address, cached in cached_markers:
+ got = ((_tab(projected, name).get("cells") or {}).get(address) or {}).get("value")
+ if got != cached:
+ raise RepairRefusal("repair_marker_row_occupied",
+ name + "!" + address + " holds " + cached + " but its restored marker "
+ "recalculates to " + json.dumps(got) + ", so it is an owner value.")
+ cycles = formula_cycles(projected, formula_check)
+ if cycles["count"]:
+ raise RepairRefusal("repair_formula_errors_remain",
+ "Formula cycles remain: " + json.dumps(cycles["cycles"]) + ".")
+ return [row for row in edits if row["before"] != row["after"]], evidence
+
+
+def _daily_sales_formula_checker(formula_check):
+ """Keep number-format inference from repainting this content-only repair."""
+ if not callable(formula_check):
+ raise RepairRefusal("repair_formula_check_required", "Daily Sales requires a formula engine.")
+
+ def checked(book):
+ before = _cell_map(book)
+ result = formula_check(book)
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
+ raise RepairRefusal("repair_formula_check_required",
+ "Daily Sales requires a recalculated workbook from its engine.")
+ for tab in result["book"].get("sheets") or ():
+ for address, cell in (tab.get("cells") or {}).items():
+ prior = before.get((str(tab.get("name") or "").casefold(), address)) or {}
+ inferred = (cell.get("formulaMeta") or {}).get("inferredFormat")
+ if cell.get("kind") != "formula" or not isinstance(inferred, dict):
+ continue
+ style, old_style = cell.get("style") or {}, prior.get("style") or {}
+ if inferred.get("format") == style.get("format") and {
+ k: v for k, v in style.items() if k != "format"} == {
+ k: v for k, v in old_style.items() if k != "format"}:
+ if "style" in prior:
+ cell["style"] = copy.deepcopy(prior["style"])
+ else:
+ cell.pop("style", None)
+ meta = cell.get("formulaMeta") or {}
+ old_inferred = (prior.get("formulaMeta") or {}).get("inferredFormat")
+ if old_inferred:
+ meta["inferredFormat"] = copy.deepcopy(old_inferred)
+ else:
+ meta.pop("inferredFormat", None)
+ if not meta:
+ cell.pop("formulaMeta", None)
+ return result
+ return checked
+
+
+def _daily_sales_invariants(book, formula_check):
+ census = _formula_repair_census(book)
+ if any(row["count"] for row in census.values()):
+ raise RepairRefusal("repair_formula_errors_remain",
+ "Daily Sales formula errors remain: " + json.dumps(census, sort_keys=True))
+ for name in (SUMMARY_TAB, "Summary (SR)"):
+ for address, cell in (_tab(book, name).get("cells") or {}).items():
+ if cell.get("kind") == "formula" and cell.get("state") != "fresh":
+ raise RepairRefusal("repair_formula_check_required", name + "!" + address
+ + " has not been recalculated.")
+ cycles = formula_cycles(book, formula_check)
+ if cycles["count"]:
+ raise RepairRefusal("repair_formula_errors_remain", "Daily Sales formula cycles remain.")
+ return census
+
+
+def _build_daily_sales_bindings_plan(current, request, resolve_binding, formula_check):
+ """R18-R21: locate the authored report by labels and resolve every proposed binding."""
+ from calendar import month_name, monthrange
+ from openpyxl.formula.translate import Translator
+ from openpyxl.utils.cell import column_index_from_string, get_column_letter
+
+ if not isinstance(request["daily_sales_bindings"], dict) or any(key in request for key in (
+ "formula_repair", "related_migration", "summary_block", "source_cells", "binding_cells")):
+ raise RepairRefusal("repair_daily_sales_plan_conflict", "Run Daily Sales bindings as their own repair.")
+ if not callable(formula_check) or not callable(resolve_binding):
+ raise RepairRefusal("repair_binding_resolver_required",
+ "Daily Sales bindings require the resolver and formula engine.")
+ if not str(request.get("refreshed_at") or "").strip():
+ raise RepairRefusal("repair_timestamp_required", "Daily Sales bindings require a refresh timestamp.")
+ tab = _tab(current, SUMMARY_TAB)
+ _tab(current, "Summary (SR)")
+ cells = tab.get("cells") or {}
+
+ def label(text, column="D", lo=0, hi=sheets.DOCUMENT_MAX_ROWS + 1):
+ found = [address for address, cell in cells.items()
+ if re.fullmatch(column + r"[1-9][0-9]*", address)
+ and lo < int(address[len(column):]) < hi
+ and str(cell.get("value") or "").strip() == text]
+ if len(found) != 1:
+ raise RepairRefusal("repair_label_missing", SUMMARY_TAB + " label '" + text
+ + "' must occur exactly once in its report block.")
+ return found[0]
+
+ def row(text, **kwargs):
+ return int(re.search(r"\d+$", label(text, **kwargs))[0])
+
+ ending = row("End. Date")
+ monthly = row("Monthly metrics")
+ allocated = row("Allocated client summary")
+ frequency = row("Average order frequency", lo=allocated)
+ orders = row("Number of Orders", lo=frequency)
+ aov = row("Avg. Order Value", lo=orders)
+ delivery = row("Average Delivery Time", lo=aov)
+ sales = row("Total Sales", hi=monthly)
+ blocks = {}
+ for name, start, end in (("sales", sales, monthly), ("frequency", frequency, orders),
+ ("orders", orders, aov), ("aov", aov, delivery),
+ ("delivery", delivery, sheets.DOCUMENT_MAX_ROWS + 1)):
+ blocks[name] = [row(text, lo=start, hi=end) for text in YTD_NAMES]
+ allocated_row = row("No. of allocated client", lo=allocated, hi=frequency)
+ purchasing_row = row("No. of purchasing client", lo=allocated, hi=frequency)
+ previous_row = row("Previous value", lo=allocated, hi=frequency)
+ percent_row = row("% of purchasing client", lo=allocated, hi=frequency)
+ helpers = [address for address, cell in cells.items() if cell.get("kind") == "formula"
+ and str(cell.get("formula") or "").startswith('=IF($D$9="RI","Royal 1 (USD)"')]
+ if len(helpers) != 1:
+ raise RepairRefusal("repair_label_missing", "Daily Sales BU helper formula must occur exactly once.")
+ helper = helpers[0]
+ columns = []
+ for address, cell in cells.items():
+ match = re.fullmatch(r"([A-Z]+)([1-9][0-9]*)", address)
+ if match and int(match[2]) == ending and column_index_from_string(match[1]) >= 6:
+ try:
+ _summary_date(cell.get("value"))
+ except RepairRefusal:
+ continue
+ columns.append(match[1])
+ columns.sort(key=column_index_from_string)
+ if tuple(columns) != SUMMARY_COLUMNS:
+ raise RepairRefusal("repair_label_missing", "End. Date requires one date for each F through R column.")
+ edits, observed = [], []
+
+ def cache_or_empty(cell):
+ return (cell.get("kind", "empty") in {"literal", "empty"}
+ and (cell.get("value") in (None, "") or type(cell.get("value")) in (int, float))
+ and not any(cell.get(key) for key in ("formula", "binding", "text")))
+
+ def append(address, desired, kind):
+ before = copy.deepcopy(cells.get(address))
+ after = sheets.clean_cell(_preserve_visual(before, desired), address)
+ edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
+ "sheet": SUMMARY_TAB, "address": address, "before": before, "after": after})
+
+ def formula_fill(origin, targets):
+ master = (cells.get(origin) or {}).get("formula")
+ if not isinstance(master, str) or not master.startswith("="):
+ raise RepairRefusal("repair_formula_master_missing", SUMMARY_TAB + "!" + origin
+ + " must hold its authored formula master.")
+ for address in targets:
+ text = Translator(master, origin=origin).translate_formula(address)
+ before = cells.get(address) or {}
+ if not (cache_or_empty(before) or before.get("kind") == "formula"
+ and before.get("formula") == text):
+ raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
+ + " contains an owner value outside this formula fill.")
+ append(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
+
+ def bind(address, database, fn, nodes, field=None):
+ aggregate = {"fn": fn, **({"field": field} if field else {})}
+ binding = sheets.clean_cell({"kind": "bound", "binding": {
+ "database": database, "aggregate": aggregate,
+ "scope": {"filter": {"conj": "and", "nodes": nodes}}}}, address)["binding"]
+ before = cells.get(address) or {}
+ if not (cache_or_empty(before) or before.get("kind") == "bound" and before.get("binding") == binding):
+ raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
+ + " contains an owner value outside this binding plan.")
+ try:
+ result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(current), copy.deepcopy(tab))
+ except Exception as exc:
+ detail = getattr(exc, "detail", None)
+ info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
+ code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
+ raise RepairRefusal(code, SUMMARY_TAB + "!" + address + " binding resolution refused.") from exc
+ if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
+ or result.get("error") or result.get("state") not in (None, "fresh"):
+ raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
+ + " did not resolve a fresh bound value.")
+ if result.get("limits"):
+ raise RepairRefusal("repair_binding_limits", SUMMARY_TAB + "!" + address
+ + " has unapplied or limited binding conditions.")
+ value = result["value"]
+ if not (value is None and fn == "avg") and (
+ type(value) not in (int, float) or not math.isfinite(value)):
+ raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
+ + " must resolve a finite number.")
+ prior = before.get("value")
+ if before.get("kind") == "bound" and before.get("binding") == binding \
+ and type(prior) in (int, float) and type(value) in (int, float) \
+ and math.isclose(prior, value, rel_tol=1e-9, abs_tol=1e-9):
+ # A float SUM/AVG can differ in its last bits between runs (parallel summation
+ # order: 168809.64999999997 then 168809.65). It is the same number, so the stored
+ # one is kept and a rerun changes nothing.
+ value = prior
+ desired = {"kind": "bound", "state": "fresh", "binding": binding, "value": value,
+ "display": "" if value is None else str(value), "refreshed_at": request["refreshed_at"]}
+ if all(before.get(key) == desired[key] for key in ("kind", "state", "binding", "value", "display")) \
+ and not any(before.get(key) for key in ("limits", "refusal", "error")):
+ desired["refreshed_at"] = before.get("refreshed_at", request["refreshed_at"])
+ append(address, desired, "binding")
+ observed.append({"address": address, "binding": binding, "value": value})
+
+ def bu(via=""):
+ return {"colId": ("related:" + via + ":" if via else "") + "pricelist",
+ "op": "eq", "valueRef": {"ref": helper}}
+
+ def window(field, col, kind):
+ return {"colId": field, "op": "within", "valueRef": {"ref": col + str(ending)},
+ "dateWindow": {"kind": kind}}
+
+ for half, label_column, months in (("H1", "D", range(1, 7)), ("H2", "I", range(7, 13))):
+ header = label(half, column=label_column, lo=monthly, hi=allocated)
+ header_row = int(re.search(r"\d+$", header)[0])
+ base_col = column_index_from_string(label_column)
+ month_rows = [row(month_name[month], column=label_column, lo=header_row, hi=allocated)
+ for month in months]
+ for offset in (1, 2):
+ value_col = get_column_letter(base_col + offset)
+ year = (cells.get(value_col + str(header_row)) or {}).get("value")
+ numeric_year = (type(year) in (int, float) and math.isfinite(year) and int(year) == year)
+ if not (numeric_year or isinstance(year, str) and year.isdigit()) \
+ or not 1900 <= int(year) <= 9999:
+ raise RepairRefusal("repair_label_missing", half + " requires a calendar year at "
+ + value_col + str(header_row) + ".")
+ year = int(year)
+ for month, month_row in zip(months, month_rows):
+ bind(value_col + str(month_row), "ut_odoo_invoices", "sum", [
+ {"colId": "invoice_date", "op": "between", "value": f"{year:04d}-{month:02d}-01",
+ "value2": f"{year:04d}-{month:02d}-{monthrange(year, month)[1]:02d}"},
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"}, bu("customer_link")],
+ "amount_untaxed")
+ growth = get_column_letter(base_col + 3)
+ formula_fill(growth + str(month_rows[0]), [growth + str(r) for r in month_rows])
+
+ for col in columns:
+ bind(col + str(allocated_row), "customer_data", "count", [bu(), {"conj": "and", "children": [
+ {"colId": "related:invoices:agent_id", "op": "eq", "rhs": {"colId": "agent_id"}},
+ {"colId": "related:invoices:invoice_date", "op": "lte", "valueRef": {"ref": col + str(ending)}},
+ {"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
+ for i, kind in enumerate(YTD_NAMES.values()):
+ bind(col + str((purchasing_row, previous_row)[i]), "customer_data", "count", [bu(),
+ {"conj": "and", "children": [window("related:invoices:invoice_date", col, kind),
+ {"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
+ bind(col + str(blocks["orders"][i]), "ut_odoo_orders", "count", [
+ window("order_date", col, kind), {"colId": "state", "op": "isAnyOf", "values": ["sale", "done"]},
+ bu("bridge_customer_orders")])
+ bind(col + str(blocks["delivery"][i]), "ut_odoo_shipments", "avg", [
+ {"colId": "state", "op": "eq", "value": "done"}, window("date_done", col, kind),
+ bu("customer_link")], "delivery_days")
+ for target_row in [percent_row, *blocks["frequency"], *blocks["aov"]]:
+ formula_fill("F" + str(target_row), [col + str(target_row) for col in columns])
+ projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
+ _daily_sales_invariants(projected, formula_check)
+ return [edit for edit in edits if edit["before"] != edit["after"]], {
+ "helper": helper, "ending_row": ending, "observed": observed}
+
+
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ S67-T21 / R34 — EVERY METRIC COMPUTED; ONLY THE OWNER'S OWN INPUTS STAY TYPED
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# The owner's weekly scorecard read pasted numbers: the weekly block and the middle of a dozen
+# rows were values copied once and never recomputed. This mode applies an explicit, fully named
+# cell plan (each cell a formula, a Cell Database binding or a blank), resolved and recalculated
+# BEFORE a byte moves, and REFUSES unless the projected workbook keeps no hardcoded number but the
+# owner's inputs. The plan itself is built by the operator from the Excel reference; this module is
+# what makes it safe to apply: owner inputs and labels are untouchable, every binding resolves to
+# a fresh value with no limits, the formula census stays empty, and the backup lands first.
+
+SR_TAB = "Summary (SR)"
+
+#: R34 (owner, 2026-09-28), verbatim cells. Summary (All)'s six are the owner's inputs; Summary
+#: (SR)'s are the only numbers the Excel file itself types on that tab (its targets and goals).
+OWNER_INPUTS = {
+ SUMMARY_TAB: frozenset({"Q25", "T20", "T37", "T38", "T51", "T55"}),
+ SR_TAB: frozenset({"T24", "T25", "T34", "T67", "T71", "T105", "T112"}),
+}
+_PLANNED_KINDS = ("formula", "bound", "empty")
+#: The tab-level visual keys a plan may DECLARE (O42: match the Excel file's freeze). Anything else
+#: visual still refuses, exactly as every content-only repair above does.
+_PLANNED_TAB_VISUAL_KEYS = ("freeze",)
+
+
+def hardcoded_numbers(book):
+ """`{tab name: {address: number}}` for every literal numeric cell, PURE. R34's census."""
+ out = {}
+ for tab in book.get("sheets") or ():
+ found = {}
+ for address, cell in (tab.get("cells") or {}).items():
+ value = (cell or {}).get("value")
+ if (cell or {}).get("kind") == "literal" and type(value) in (int, float):
+ found[address] = value
+ out[str(tab.get("name") or "")] = found
+ return out
+
+
+def _bound_value(value, fn):
+ """A resolved binding's value as the document stores it, or a refusal.
+
+ ⭐ A `min`/`max` over a date column answers a DATE ("Latest data" is `max(invoice_date)`), which
+ is stored as its ISO text, the shape every formula date in this workbook already has.
+ """
+ import datetime as _dtm
+ if isinstance(value, (_dtm.date, _dtm.datetime)):
+ if fn not in ("min", "max"):
+ raise RepairRefusal("repair_binding_unreadable", "Only min or max may answer a date.")
+ return value.isoformat()[:10]
+ if isinstance(value, str) and fn in ("min", "max") and re.fullmatch(r"\d{4}-\d{2}-\d{2}.*", value):
+ return value[:10]
+ if value is None and fn in ("avg", "min", "max"):
+ return None
+ if type(value) not in (int, float) or not math.isfinite(value):
+ raise RepairRefusal("repair_binding_unreadable", "A binding must resolve a finite number.")
+ return value
+
+
+def _visual_without(book, planned_notes, keys=("note",)):
+ """`visual_metadata` with each planned cell's declared visual `keys` removed (a note for
+ T21, a sparkline for T23), so a declared change is the only visual change a plan may make."""
+ shaped = copy.deepcopy(book)
+ for tab in shaped.get("sheets") or ():
+ name = str(tab.get("name") or "").casefold()
+ for address, cell in (tab.get("cells") or {}).items():
+ if (name, address) in planned_notes and isinstance(cell, dict):
+ for key in keys:
+ cell.pop(key, None)
+ return visual_metadata(shaped)
+
+
+def _build_computed_metrics_plan(current, request, resolve_binding, formula_check):
+ """R34: validate and resolve an explicit per-cell plan; refuse unless it leaves no hardcode."""
+ spec = request["computed_metrics"]
+ if not isinstance(spec, dict) or any(key in request for key in (
+ "formula_repair", "related_migration", "summary_block", "source_cells",
+ "binding_cells", "daily_sales_bindings")):
+ raise RepairRefusal("repair_metrics_plan_conflict",
+ "Run the computed metrics plan as its own repair.")
+ if not callable(formula_check) or not callable(resolve_binding):
+ raise RepairRefusal("repair_binding_resolver_required",
+ "Computed metrics require the resolver and the formula engine.")
+ refreshed_at = str(request.get("refreshed_at") or "").strip()
+ if not refreshed_at:
+ raise RepairRefusal("repair_timestamp_required", "Computed metrics require a refresh timestamp.")
+ rows = spec.get("cells")
+ if not isinstance(rows, list) or not rows:
+ raise RepairRefusal("repair_metrics_plan_empty", "The computed metrics plan names no cell.")
+ edits, observed, seen, planned_notes = [], [], set(), set()
+ # ⭐ A binding may read a cell this same plan writes (a month's helper start/end dates, a
+ # year header), so bindings resolve against the workbook WITH the planned formulas and blanks
+ # applied and recalculated, never against the stale current one.
+ staged = []
+ for row in rows:
+ want = row.get("cell") if isinstance(row, dict) else None
+ if isinstance(want, dict) and want.get("kind") in ("formula", "empty"):
+ tab = _tab(current, str(row.get("sheet") or ""))
+ address = _addr(row.get("address"))
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ cell = ({"kind": "formula", "state": "stale", "formula": str(want.get("formula") or "")}
+ if want["kind"] == "formula" else {"kind": "empty", "state": "typed"})
+ staged.append({"sheet": str(tab.get("name")), "address": address, "before": before,
+ "after": sheets.clean_cell(_preserve_visual(before, cell) or cell, address)})
+ def recalculated(extra):
+ projected = _project(current, staged + extra)
+ checked = formula_check(copy.deepcopy(projected))
+ return checked["book"] if isinstance(checked, dict) and isinstance(
+ checked.get("book"), dict) else projected
+
+ def resolve_row(row, book):
+ sheet = str(_tab(current, str(row.get("sheet") or "")).get("name"))
+ address = _addr(row.get("address"))
+ binding = sheets.clean_cell({"kind": "bound", "binding": row["cell"].get("binding")},
+ address)["binding"]
+ try:
+ result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(book),
+ copy.deepcopy(_tab(book, sheet)))
+ except Exception as exc: # noqa: BLE001
+ detail = getattr(exc, "detail", None)
+ info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
+ code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
+ return None, RepairRefusal(code, sheet + "!" + address + " binding resolution refused.")
+ return (binding, result), None
+
+ # ⭐ ROUNDS, because a binding may read a date another binding produces through formulas
+ # (Summary (SR)'s week axis runs E12 `max(invoice_date)` -> E15 -> R8 -> F8 -> F31's window).
+ # Each round resolves what it can read, the next projection carries those values, and only a
+ # cell whose filter input was not yet readable waits; no progress in a round refuses by name.
+ resolved, pending = {}, [row for row in rows if isinstance(row, dict)
+ and isinstance(row.get("cell"), dict)
+ and row["cell"].get("kind") == "bound"]
+ for _round in range(6):
+ if not pending:
+ break
+ extra = []
+ for (sheet, address), (binding, result) in resolved.items():
+ value = result.get("value") if isinstance(result, dict) else None
+ try:
+ value = _bound_value(value, (binding.get("aggregate") or {}).get("fn"))
+ except RepairRefusal:
+ value = None
+ extra.append({"sheet": sheet, "address": address, "before": None,
+ "after": sheets.clean_cell({"kind": "bound", "state": "fresh",
+ "binding": binding, "value": value,
+ "display": "" if value is None else str(value)},
+ address)})
+ book = recalculated(extra)
+ waiting, first_error = [], None
+ for row in pending:
+ got, error = resolve_row(row, book)
+ if got is not None:
+ resolved[(str(_tab(current, str(row["sheet"])).get("name")),
+ _addr(row.get("address")))] = got
+ elif error.code == "filter_cell_unreadable":
+ waiting.append(row)
+ first_error = first_error or error
+ else:
+ raise error
+ if len(waiting) == len(pending):
+ raise first_error
+ pending = waiting
+ if pending:
+ raise RepairRefusal("filter_cell_unreadable", "A binding's filter cell never became readable.")
+ for row in rows:
+ if not isinstance(row, dict):
+ raise RepairRefusal("repair_metrics_cell_invalid", "Every planned cell must be an object.")
+ sheet_name = str(row.get("sheet") or "").strip()
+ address = _addr(row.get("address"))
+ tab = _tab(current, sheet_name)
+ sheet_name = str(tab.get("name") or sheet_name)
+ slot = (sheet_name.casefold(), address)
+ if slot in seen:
+ raise RepairRefusal("repair_edit_duplicate", "A planned cell may be named only once.")
+ seen.add(slot)
+ if address in OWNER_INPUTS.get(sheet_name, ()):
+ raise RepairRefusal("repair_owner_input", sheet_name + "!" + address
+ + " is one of the owner's typed inputs and stays typed.")
+ cells = tab.get("cells") or {}
+ before = copy.deepcopy(cells.get(address))
+ prior_value = (before or {}).get("value")
+ want = row.get("cell")
+ if not isinstance(want, dict) or want.get("kind") not in (*_PLANNED_KINDS, "note_only"):
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
+ + " must be planned as a formula, a binding, a blank or a note.")
+ kind = want["kind"]
+ # A typed ISO date is a value (Summary (SR)'s frozen E13/E15), never a label.
+ if (before or {}).get("kind") == "literal" and isinstance(prior_value, str) \
+ and prior_value.strip() and kind != "note_only" \
+ and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", prior_value.strip()):
+ raise RepairRefusal("repair_label_protected", sheet_name + "!" + address
+ + " holds a label, which a metrics plan never replaces.")
+ if kind == "note_only":
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
+ elif kind == "formula":
+ text = str(want.get("formula") or "")
+ if not text.startswith("="):
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
+ + " needs a formula that starts with =.")
+ desired = _preserve_visual(before, {"kind": "formula", "state": "stale", "formula": text})
+ elif kind == "empty":
+ desired = _preserve_visual(before, {"kind": "empty", "state": "typed"}) \
+ or {"kind": "empty", "state": "typed"}
+ else:
+ binding, result = resolved[(sheet_name, address)]
+ fn = (binding.get("aggregate") or {}).get("fn")
+ if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
+ or result.get("error") or result.get("state") not in (None, "fresh"):
+ raise RepairRefusal("repair_binding_unreadable", sheet_name + "!" + address
+ + " did not resolve a fresh bound value.")
+ if result.get("limits"):
+ raise RepairRefusal("repair_binding_limits", sheet_name + "!" + address
+ + " has unapplied or limited binding conditions.")
+ value = _bound_value(result["value"], fn)
+ if (before or {}).get("kind") == "bound" and before.get("binding") == binding \
+ and type(prior_value) in (int, float) and type(value) in (int, float) \
+ and math.isclose(prior_value, value, rel_tol=1e-9, abs_tol=1e-9):
+ value = prior_value # the same number; a rerun changes nothing
+ desired = _preserve_visual(before, {
+ "kind": "bound", "state": "fresh", "binding": binding, "value": value,
+ "display": "" if value is None else str(value), "refreshed_at": refreshed_at})
+ observed.append({"sheet": sheet_name, "address": address, "value": value})
+ if "note" in row:
+ note = str(row.get("note") or "").strip()
+ if note:
+ desired["note"] = note
+ else:
+ desired.pop("note", None)
+ planned_notes.add(slot)
+ after = sheets.clean_cell(desired, address)
+ # A planned formula is `derived`: the recalculation below is what fills its value, and
+ # `_checked_projection` accepts a recalculated planned cell only under that kind.
+ edit_kind = {"formula": "derived", "bound": "binding"}.get(kind, "metric")
+ edits.append({"kind": edit_kind, "name": sheet_name + "!" + address, "sheet": sheet_name,
+ "address": address, "before": before, "after": after})
+ tab_visual = spec.get("tab_visual") or {}
+ if not isinstance(tab_visual, dict):
+ raise RepairRefusal("repair_metrics_visual_invalid", "Tab visual changes must be an object.")
+ for sheet_name, keys in tab_visual.items():
+ _tab(current, sheet_name)
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_PLANNED_TAB_VISUAL_KEYS):
+ raise RepairRefusal("repair_metrics_visual_invalid",
+ "A metrics plan may declare only a tab's freeze.")
+ projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
+ _daily_sales_invariants(projected, formula_check)
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
+ for name, found in hardcoded_numbers(projected).items()
+ if name in OWNER_INPUTS}
+ if any(remaining.values()):
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers would remain: "
+ + json.dumps(remaining, sort_keys=True))
+ return [edit for edit in edits if edit["before"] != edit["after"]], {
+ "observed": observed, "planned_notes": sorted(f"{s}!{a}" for s, a in planned_notes),
+ "tab_visual": copy.deepcopy(tab_visual),
+ "hardcoded_before": hardcoded_numbers(current), "hardcoded_after": hardcoded_numbers(projected)}
+
+
+def _apply_tab_visual(st, workbook_id, tab_visual, *, actor="", actor_name=""):
+ """Write the DECLARED tab-level visual keys through the ordinary document door, then prove
+ nothing else moved. Returns the readback snapshot."""
+ doc = sheets.workbook_snapshot(st, workbook_id)
+ before_cells = {str(t.get("name")): copy.deepcopy(t.get("cells") or {}) for t in doc["sheets"]}
+ before_visual = visual_metadata(doc)
+ for sheet_name, keys in tab_visual.items():
+ tab = _tab(doc, sheet_name)
+ for key, value in keys.items():
+ tab[key] = copy.deepcopy(value)
+ sheets.put_workbook(st, workbook_id, doc, expect_rev=int(doc.get("rev") or 0),
+ actor=actor, actor_name=actor_name, lossless=True)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ for sheet_name, keys in tab_visual.items():
+ tab = _tab(readback, sheet_name)
+ for key, value in keys.items():
+ if _canonical(tab.get(key)) != _canonical(value):
+ raise RepairRefusal("repair_visual_readback", sheet_name + " " + key
+ + " did not read back as declared.")
+ if {str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} != before_cells:
+ raise RepairRefusal("repair_visual_readback", "A declared visual change moved cell content.")
+ expected = copy.deepcopy(before_visual)
+ for item in expected:
+ for key, value in (tab_visual.get(item.get("name")) or {}).items():
+ item[key] = copy.deepcopy(value)
+ if visual_metadata(readback) != _canonical(expected):
+ raise RepairRefusal("repair_visual_readback", "An undeclared visual property changed.")
+ return readback
+
+
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ S67-T22 / S67-T23 — DECLARED VISUAL REPAIRS: Excel's rule colours and its 3M sparklines
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# Every repair above REFUSES a visual change, on purpose. These two tickets ARE visual changes, so
+# they get their own mode that names exactly what may move: a tab's `rules` (T22, O31) and a cell's
+# `sparkline` (T23, O32). Cell content, every other visual key and every other cell must read back
+# unchanged; the backup lands before anything is written.
+
+_VISUAL_REPAIR_TAB_KEYS = ("rules",)
+
+
+def _build_visual_repair_plan(current, request):
+ spec = request["visual_repair"]
+ if not isinstance(spec, dict) or any(key in request for key in (
+ "formula_repair", "related_migration", "summary_block", "source_cells",
+ "binding_cells", "daily_sales_bindings", "computed_metrics")):
+ raise RepairRefusal("repair_visual_plan_conflict", "Run the visual repair as its own repair.")
+ tab_visual = copy.deepcopy(spec.get("tab_visual") or {})
+ for sheet_name, keys in tab_visual.items():
+ _tab(current, sheet_name)
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_VISUAL_REPAIR_TAB_KEYS):
+ raise RepairRefusal("repair_visual_invalid", "A visual repair may declare only a tab's rules.")
+ keys["rules"] = sheets._clean_rules(keys["rules"], lossless=True)
+ edits, seen = [], set()
+ for row in spec.get("cells") or ():
+ if not isinstance(row, dict) or "sparkline" not in row:
+ raise RepairRefusal("repair_visual_invalid", "A visual repair cell names only its sparkline.")
+ tab = _tab(current, str(row.get("sheet") or ""))
+ sheet_name, address = str(tab.get("name")), _addr(row.get("address"))
+ if (sheet_name.casefold(), address) in seen:
+ raise RepairRefusal("repair_edit_duplicate", "A repaired cell may be named only once.")
+ seen.add((sheet_name.casefold(), address))
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
+ if row["sparkline"] is None:
+ desired.pop("sparkline", None)
+ else:
+ clean = sheets._clean_sparkline(row["sparkline"])
+ if clean is None:
+ raise RepairRefusal("repair_visual_invalid", sheet_name + "!" + address
+ + " names a sparkline that cannot be drawn.")
+ desired["sparkline"] = clean
+ after = sheets.clean_cell(desired, address)
+ content = ("kind", "state", "value", "display", "formula", "binding", "text")
+ if any((before or {}).get(key) != after.get(key) for key in content if key != "state") \
+ and not (before is None and after.get("kind") == "empty"):
+ raise RepairRefusal("repair_visual_content_changed", sheet_name + "!" + address
+ + " would change content, which a visual repair never does.")
+ edits.append({"kind": "visual", "name": sheet_name + "!" + address, "sheet": sheet_name,
+ "address": address, "before": before, "after": after})
+ return [e for e in edits if e["before"] != e["after"]], {
+ "tab_visual": tab_visual, "sparkline_cells": sorted(e["name"] for e in edits)}
+
+def _execute_visual_repair(st, backup_store, request, current, revision, digest, workbook_id,
+ *, actor="", actor_name=""):
+ """Back up, then write ONLY the declared sparklines (exact cell edits) and tab rules."""
+ edits, evidence = _build_visual_repair_plan(current, request)
+ if not edits and not evidence["tab_visual"]:
+ return {"workbook_id": workbook_id, "revision_before": revision, "revision_after": revision,
+ "applied": 0, "noop": True, "visual_repair": evidence}
+ planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
+ projected = _project(current, edits)
+ if _visual_without(projected, planned, ("sparkline",)) != \
+ _visual_without(current, planned, ("sparkline",)):
+ raise RepairRefusal("repair_visual_metadata_changed",
+ "The projected repair changes a visual property it did not declare.")
+ exact = [{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
+ for row in edits]
+ diff_digest = sheets.workbook_digest({"edits": exact, "tab_visual": evidence["tab_visual"]})
+ backup_key = _write_backup(backup_store, current, digest, diff_digest, workbook_id)
+ revision_after = revision
+ if exact:
+ sheets.apply_exact_cell_edits(st, workbook_id, exact, revision, digest,
+ actor=actor, actor_name=actor_name)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ if int(readback.get("rev") or 0) != revision + 1:
+ raise RepairRefusal("repair_readback_revision",
+ "The repaired workbook revision did not read back as expected.")
+ for row in edits:
+ actual = (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"])
+ if actual != sheets.clean_cell(row["after"], row["address"]):
+ raise RepairRefusal("repair_readback_cell",
+ row["sheet"] + "!" + row["address"] + " did not read back exactly.")
+ revision_after = int(readback["rev"])
+ if evidence["tab_visual"]:
+ revision_after = int(_apply_tab_visual(st, workbook_id, evidence["tab_visual"],
+ actor=actor, actor_name=actor_name)["rev"])
+ return {"workbook_id": workbook_id, "revision_before": revision,
+ "revision_after": revision_after, "applied": len(exact), "backup_key": backup_key,
+ "backup_digest": digest, "diff_digest": diff_digest, "visual_repair": evidence}
+
+
+def restore_backup(st, backup_store, backup_key, *, workbook_id, actor="", actor_name=""):
+ """NC for every mode here: put a backup's workbook back and prove it round-trips."""
+ record = backup_store.get(backup_key)
+ if not isinstance(record, dict) or record.get("workbook_id") != workbook_id:
+ raise RepairRefusal("repair_backup_missing", "That backup does not exist for this workbook.")
+ saved = copy.deepcopy(record["workbook"])
+ current = sheets.workbook_snapshot(st, workbook_id)
+ sheets.put_workbook(st, workbook_id, saved, expect_rev=int(current.get("rev") or 0),
+ actor=actor, actor_name=actor_name, lossless=True)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ same = ({str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]}
+ == {str(t.get("name")): t.get("cells") or {} for t in saved["sheets"]}
+ and visual_metadata(readback) == visual_metadata(saved))
+ if not same:
+ raise RepairRefusal("repair_restore_unverified", "The restored workbook did not round-trip.")
+ return readback
+
+
def _build_plan(current, request, resolve_binding):
if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
raise RepairRefusal(
@@ -877,15 +1925,62 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
raise RepairRefusal("repair_revision_mismatch",
"The named workbook is not at the expected repair revision.")
digest = sheets.workbook_digest(current)
+ if "visual_repair" in request:
+ return _execute_visual_repair(st, backup_store, request, current, revision, digest,
+ target_workbook_id, actor=actor, actor_name=actor_name)
summary_evidence = None
summary_errors_before = None
- if "summary_block" in request:
+ migrated = None
+ formula_evidence = None
+ daily_evidence = None
+ metrics_evidence = None
+ if "computed_metrics" in request:
+ formula_check = _daily_sales_formula_checker(formula_check)
+ edits, metrics_evidence = _build_computed_metrics_plan(
+ current, request, resolve_binding, formula_check)
+ if not edits and not metrics_evidence["tab_visual"]:
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
+ "revision_after": revision, "applied": 0, "noop": True,
+ "computed_metrics": metrics_evidence}
+ elif "daily_sales_bindings" in request:
+ formula_check = _daily_sales_formula_checker(formula_check)
+ edits, daily_evidence = _build_daily_sales_bindings_plan(
+ current, request, resolve_binding, formula_check)
+ if not edits:
+ census = _daily_sales_invariants(current, formula_check)
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
+ "revision_after": revision, "applied": 0, "noop": True,
+ "daily_sales_bindings": daily_evidence, "formula_errors": [],
+ "formula_error_census": {"before": census, "after": copy.deepcopy(census)},
+ "formula_cycles": _formula_cycle_comparison(current, current, formula_check),
+ "visual_metadata_preserved": True}
+ elif "formula_repair" in request:
+ edits, formula_evidence = _build_formula_repair_plan(current, request, formula_check)
+ if not edits:
+ census = _formula_repair_census(current)
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
+ "revision_after": revision, "applied": 0, "noop": True,
+ "formula_repair": formula_evidence, "formula_errors": formula_error_census(current),
+ "formula_error_census": {"before": census, "after": copy.deepcopy(census)},
+ "formula_cycles": _formula_cycle_comparison(current, current, formula_check),
+ "visual_metadata_preserved": True}
+ elif "related_migration" in request:
+ edits, migrated = _build_related_migration_plan(
+ current, request, resolve_binding, formula_check)
+ if not edits:
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
+ "revision_after": revision, "applied": 0, "migrated": [], "noop": True}
+ elif "summary_block" in request:
summary_errors_before = check_summary_formula_errors(current)
edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
else:
edits = _build_plan(current, request, resolve_binding)
projected = _project(current, edits)
projected, projected_errors = _checked_projection(projected, edits, formula_check)
+ if formula_evidence is not None:
+ _check_formula_repair_errors(projected)
+ if daily_evidence is not None:
+ _daily_sales_invariants(projected, formula_check)
comparison = _formula_error_comparison(current, projected, edits, projected_errors)
summary_errors_after = (check_summary_formula_errors(projected)
if summary_evidence is not None else None)
@@ -896,7 +1991,11 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
"The projected repair introduces a formula error or leaves one in its repair scope, "
"so no repair was applied.")
visual_before = visual_metadata(current)
- if visual_metadata(projected) != visual_before:
+ planned_notes = set()
+ if metrics_evidence is not None:
+ planned_notes = {tuple(item.split("!", 1)) for item in metrics_evidence["planned_notes"]}
+ planned_notes = {(sheet.casefold(), address) for sheet, address in planned_notes}
+ if _visual_without(projected, planned_notes) != _visual_without(current, planned_notes):
raise RepairRefusal("repair_visual_metadata_changed",
"The projected repair changes Daily Sales visual metadata.")
@@ -928,9 +2027,13 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
or _cell_map(checked_readback) != _cell_map(readback):
raise RepairRefusal("repair_readback_formula_state",
"The repaired workbook did not read back calculation-current.")
- if visual_metadata(readback) != visual_before:
+ if _visual_without(readback, planned_notes) != _visual_without(current, planned_notes):
raise RepairRefusal("repair_readback_visual_metadata",
"Daily Sales visual metadata changed during repair.")
+ if formula_evidence is not None:
+ _check_formula_repair_errors(readback)
+ if daily_evidence is not None:
+ _daily_sales_invariants(readback, formula_check)
if summary_evidence is not None:
tab = _tab(readback, SUMMARY_TAB)
if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
@@ -963,8 +2066,36 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
"bindings": bindings,
"formula_errors": formula_error_census(readback),
"formula_error_comparison": readback_comparison,
+ "formula_cycles": _formula_cycle_comparison(current, readback, formula_check),
"visual_metadata_preserved": True,
}
if summary_evidence is not None:
result["summary_block"] = summary_evidence
+ if formula_evidence is not None:
+ result["formula_repair"] = formula_evidence
+ result["formula_error_census"] = {
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
+ if migrated is not None:
+ result["migrated"] = [
+ {**row, "after_value": _tab(readback, row["sheet"])["cells"][row["address"]]["value"]}
+ for row in migrated]
+ if daily_evidence is not None:
+ result["daily_sales_bindings"] = daily_evidence
+ result["formula_error_census"] = {
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
+ if metrics_evidence is not None:
+ _daily_sales_invariants(readback, formula_check)
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
+ for name, found in hardcoded_numbers(readback).items() if name in OWNER_INPUTS}
+ if any(remaining.values()):
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers remain after the write: "
+ + json.dumps(remaining, sort_keys=True))
+ if metrics_evidence["tab_visual"]:
+ visual_readback = _apply_tab_visual(st, target_workbook_id, metrics_evidence["tab_visual"],
+ actor=actor, actor_name=actor_name)
+ result["revision_after"] = visual_readback["rev"]
+ metrics_evidence["hardcoded_readback"] = hardcoded_numbers(readback)
+ result["computed_metrics"] = metrics_evidence
+ result["formula_error_census"] = {
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
return result
diff --git a/platform/core/data_binding.py b/platform/core/data_binding.py
index b5a47464eb95ae33276255b603bf4ad412fd66f3..ad3cb149883173fd5259578c8806f2b5b7c5119f 100644
--- a/platform/core/data_binding.py
+++ b/platform/core/data_binding.py
@@ -167,6 +167,69 @@ def local_override():
return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1'
+#: W-219. Inherited by every child a gate process starts, so a subprocess a gate spawns is a gate too.
+_GATE_PROCESS_VAR = 'AIOS_GATE_PROCESS'
+
+#: The directories the product gate battery lives in (`aios-web/api`, `aios-web/web`, `aios-web`,
+#: `platform`). `ops/verify_*.py` are OPERATOR tools, some of which read or probe a live store on
+#: purpose, so they are not gates here.
+_GATE_DIRS = frozenset({'api', 'web', 'aios-web', 'platform'})
+
+
+def _argv_gate():
+ """The `verify_*.py` gate script this process was started as, or `''`."""
+ import sys
+ argv0 = str((getattr(sys, 'argv', None) or [''])[0] or '')
+ name = os.path.basename(argv0).lower()
+ if not (name.startswith('verify_') and name.endswith('.py')):
+ return ''
+ parent = os.path.basename(os.path.dirname(os.path.abspath(argv0))).lower()
+ return name if parent in _GATE_DIRS else ''
+
+
+def gate_process():
+ """The gate this process is, or `''`. W-219: a gate writes FIXTURE stores only, never a real one.
+
+ ⛔⛔ WHY THIS LIVES AT THE STORE DOOR AND NOT IN EACH GATE. A lane container carries the exact
+ isolated staging posture `store_pg.write_refusal` is written to ALLOW (`STORE_BACKEND=pg`, the
+ staging `DATABASE_URL`, `AIOS_DEPLOYMENT_ID=fsanyoto/loopable`, `AIOS_SANDBOX_TENANTS=
+ royal-imports`), and `_fake_store.installed` swaps module functions a bound `PgStore` never
+ calls. So `verify_meta.py` run in SDLC 61 wrote staging's `t_royal_imports` through the real
+ app, and a pin at the top of each gate (W-219, W-240) only covers the gates somebody pinned:
+ `verify_amazon` and `verify_odoo_relational` carry none. Asking the PROCESS covers every gate,
+ including one written tomorrow.
+
+ Two ways to be one: started as `verify_*.py` from a battery directory (every battery row and
+ every lane runs `python /verify_.py`), or `AIOS_GATE_PROCESS=1` inherited from one.
+ ⛔ There is no way OUT through the environment, because a lane container exports whatever it
+ was handed: the only off switch is an in-process patch of this function, which is what a gate
+ testing the PRODUCT rules does.
+ """
+ name = _argv_gate()
+ if name:
+ return name
+ return 'a process started by a gate' if _env(_GATE_PROCESS_VAR) == '1' else ''
+
+
+if _argv_gate():
+ os.environ[_GATE_PROCESS_VAR] = '1' # W-219: a child this gate spawns is a gate too
+
+
+def gate_write_refusal(repo):
+ """`None` unless this is a gate process writing a PRODUCTION store, else why not. W-219.
+
+ Consulted by `check_write` BEFORE `write_refusal`, so neither `AIOS_ALLOW_PRODUCTION_DATA=1`
+ on a laptop nor a production identity opens a production store to a gate.
+ """
+ who = gate_process()
+ rid = str(repo or '').strip()
+ if not who or rid not in PRODUCTION_STORES:
+ return None
+ return (f'this process is a verification gate ({who}), and a gate never writes {rid}, which '
+ f'holds real customer data. A gate writes fixture stores only: empty HF_TOKEN and '
+ f'point OS_DATA_REPO at a fixture store (W-219).')
+
+
def default_store():
"""What `OS_DATA_REPO` resolves to when it is unset.
@@ -221,7 +284,7 @@ def write_refusal(repo):
def check_write(repo, operation='write'):
"""Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
- reason = write_refusal(repo)
+ reason = gate_write_refusal(repo) or write_refusal(repo)
if reason:
raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
diff --git a/platform/core/do_usage.py b/platform/core/do_usage.py
new file mode 100644
index 0000000000000000000000000000000000000000..598b0e1698b8bd51d419cfa034e591a3f4cbf351
--- /dev/null
+++ b/platform/core/do_usage.py
@@ -0,0 +1,583 @@
+"""DigitalOcean billing observations, never SQL or a database wake-up probe.
+
+The card uses GETs and memory only. The optional sampler receives storage
+callbacks from the host and shares Neon's management-plane store-awake guard.
+"""
+from __future__ import annotations
+
+import copy
+import json
+import math
+import os
+import re
+import threading
+from datetime import datetime, timezone
+from urllib.error import HTTPError, URLError
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+from core.neon_usage import send_telegram, store_is_awake
+
+
+API_BASE = "https://api.digitalocean.com/v2"
+HTTP_TIMEOUT = 5
+DEFAULT_CLUSTER_ID = "0c4049b4-8541-43a7-ab6e-a69c34a8c604"
+STORE_KEY = "platform_do_usage_samples"
+MAX_SAMPLES = 240
+FIXED_PRICES = {"db-s-1vcpu-1gb": {"usd": 15.15, "source": "R28 fixed monthly price per node"}}
+CLUSTER_SOURCE = "DigitalOcean databases API"
+BALANCE_SOURCE = "DigitalOcean balance API (daily snapshot, account-wide)"
+BILLING_SOURCE = "DigitalOcean billing history API"
+
+
+class UsageReadError(Exception):
+ """Fixed, credential-free cause safe to show to an operator."""
+
+
+class _NoRedirect(HTTPRedirectHandler):
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ return None
+
+
+def _get(path, token):
+ try:
+ request = Request(API_BASE + path, headers={"Authorization": "Bearer " + token}, method="GET")
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
+ payload = json.loads(response.read(1_000_001))
+ if not isinstance(payload, dict):
+ raise ValueError()
+ return payload
+ except HTTPError as exc:
+ raise UsageReadError(f"DigitalOcean API returned HTTP {exc.code}.") from None
+ except TimeoutError:
+ raise UsageReadError("DigitalOcean API timed out.") from None
+ except URLError as exc:
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
+ raise UsageReadError(f"DigitalOcean API {cause}.") from None
+ except Exception:
+ # Do not return the URL, response body or exception text: any can contain credentials.
+ raise UsageReadError("DigitalOcean API returned an unreadable response.") from None
+
+
+def _now():
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _stamp(value):
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ return parsed.astimezone(timezone.utc) if parsed.tzinfo else None
+ except (ValueError, TypeError):
+ return None
+
+
+def _text(value):
+ if not isinstance(value, str) or not value.strip():
+ raise ValueError()
+ return value.replace("\u2014", "; ").replace("\u2013", " to ")[:512]
+
+
+def _money(value):
+ if isinstance(value, bool) or value is None:
+ raise ValueError()
+ number = float(value)
+ if not math.isfinite(number):
+ raise ValueError()
+ return number
+
+
+def read_account():
+ """Independently read each API block; allowlist fields to exclude connection secrets."""
+ read_at = _now()
+ cluster = {"available": False, "cause": None, "source": CLUSTER_SOURCE, "read_at": read_at,
+ **dict.fromkeys(("id", "name", "size", "num_nodes", "region", "status", "version", "storage_size_mib"))}
+ balance = {"available": False, "cause": None, "source": BALANCE_SOURCE, "read_at": read_at,
+ **dict.fromkeys(("month_to_date_usage", "account_balance", "month_to_date_balance", "generated_at"))}
+ billing = {"available": False, "cause": None, "source": BILLING_SOURCE, "read_at": read_at}
+ price = {"monthly_usd": None, "source": None, "read_at": read_at, "cause": None}
+ result = {"available": False, "cause": None, "read_at": read_at, "cluster": cluster,
+ "balance": balance, "billing": billing, "billing_history": [], "fixed_price": price}
+ token = os.environ.get("AIOS_DO_USAGE_TOKEN") or os.environ.get("DIGITALOCEAN_TOKEN", "")
+ if not token:
+ cause = "No DigitalOcean token is configured on this deployment, so usage cannot be read."
+ result["cause"] = price["cause"] = cause
+ for block in (cluster, balance, billing):
+ block["cause"] = cause
+ return result
+ selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
+
+ try:
+ if not re.fullmatch(r"[a-zA-Z0-9-]{1,128}", selected):
+ raise UsageReadError("AIOS_DO_CLUSTER_ID is invalid; the cluster could not be read.")
+ database = _get("/databases/" + selected, token).get("database")
+ if not isinstance(database, dict) or database.get("id") != selected:
+ raise ValueError()
+ fields = {key: _text(database.get(key)) for key in ("id", "name", "size", "region", "status", "version")}
+ nodes = database.get("num_nodes")
+ if type(nodes) is not int or nodes < 1:
+ raise ValueError()
+ storage = database.get("storage_size_mib")
+ fields.update(num_nodes=nodes, storage_size_mib=storage if type(storage) is int and storage >= 0 else None)
+ cluster.update(fields, available=True, read_at=_now())
+ quoted = FIXED_PRICES.get(fields["size"])
+ if quoted:
+ price.update(monthly_usd=round(quoted["usd"] * nodes, 2), source=quoted["source"], read_at=cluster["read_at"])
+ else:
+ price["cause"] = f"Fixed price UNKNOWN for size {fields['size']}; no fixed price is configured."
+ except UsageReadError as exc:
+ cluster["cause"] = str(exc)
+ except Exception:
+ cluster["cause"] = "DigitalOcean cluster response is missing valid cluster details."
+ if not cluster["available"]:
+ price["cause"] = cluster["cause"]
+
+ try:
+ payload = _get("/customers/my/balance", token)
+ amounts = {key: _money(payload.get(key)) for key in
+ ("month_to_date_usage", "account_balance", "month_to_date_balance")}
+ generated_at = payload.get("generated_at")
+ if amounts["month_to_date_usage"] < 0 or not _stamp(generated_at):
+ raise ValueError()
+ balance.update(amounts, generated_at=generated_at, available=True, read_at=_now())
+ except UsageReadError as exc:
+ balance["cause"] = str(exc)
+ except Exception:
+ balance["cause"] = "DigitalOcean balance response is missing valid amounts or its snapshot time."
+
+ try:
+ payload = _get("/customers/my/billing_history", token)
+ rows = payload.get("billing_history")
+ # A new account's empty history arrives as the bare list envelope {"links": {}, "meta": {}}
+ # with the key omitted (read 2026-09-25): an empty history, not an unreadable one.
+ if rows is None and isinstance(payload, dict) and "meta" in payload:
+ rows = []
+ if not isinstance(rows, list):
+ raise ValueError()
+ history = []
+ for row in rows:
+ date = row.get("date")
+ if not _stamp(date):
+ raise ValueError()
+ history.append({"date": date, "description": _text(row.get("description")),
+ "amount": _money(row.get("amount")), "type": _text(row.get("type"))})
+ # Report a bounded page honestly; never follow an API-provided URL with a bearer token.
+ pages = (payload.get("links") or {}).get("pages") or {}
+ billing.update(available=True, read_at=_now(), has_more=bool(pages.get("next")))
+ result["billing_history"] = history
+ except UsageReadError as exc:
+ billing["cause"] = str(exc)
+ except Exception:
+ billing["cause"] = "DigitalOcean billing history response could not be read."
+ result["available"] = any(block["available"] for block in (cluster, balance, billing))
+ if not result["available"]:
+ result["cause"] = " ".join(f"{name}: {block['cause']}" for name, block in
+ (("Cluster", cluster), ("Balance", balance), ("Billing history", billing)))
+ return result
+
+
+def evaluate_alarm(account):
+ """Account-wide MTD against the sum of the configured cluster's fixed node prices."""
+ balance, price = account["balance"], account["fixed_price"]
+ if not balance["available"]:
+ return None, "Alert evaluation deferred. " + balance["cause"]
+ expected = price["monthly_usd"]
+ if expected is None:
+ return None, "Alert evaluation deferred. " + price["cause"]
+ override = os.environ.get("AIOS_DO_ALARM_USD")
+ try:
+ threshold = expected if override is None else _money(override)
+ except (ValueError, TypeError, OverflowError):
+ return None, "AIOS_DO_ALARM_USD is invalid; alert evaluation is deferred."
+ if balance["month_to_date_usage"] <= threshold:
+ return None, None
+ test = override is not None
+ cluster = account["cluster"]
+ message = ("TEST " if test else "") + (
+ f"DigitalOcean month-to-date spend USD {balance['month_to_date_usage']:.2f}, "
+ f"as of {balance['generated_at']} ({balance['source']}; read {balance['read_at']}). "
+ f"Expected fixed price USD {expected:.2f} a month ({price['source']}; read {price['read_at']}). "
+ f"Cluster {cluster['name']} ({cluster['size']}, {cluster['num_nodes']} "
+ f"node{'' if cluster['num_nodes'] == 1 else 's'}, {cluster['region']}).")
+ if test:
+ message += f" TEST threshold USD {threshold:.2f} (AIOS_DO_ALARM_USD; read {account['read_at']})."
+ return {"level": "red", "message": message, "test": test, "threshold": threshold,
+ "period": _stamp(balance["generated_at"]).strftime("%Y-%m")}, None
+
+
+_lock = threading.RLock()
+_sample_lock = threading.Lock()
+_thread = None
+_stop = threading.Event()
+_samples = []
+_alerted = {}
+_loaded = False
+_cluster_id = None
+_status = {"every_s": 0, "last_at": None, "persisted": None,
+ "cause": "Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset or zero."}
+
+
+def snapshot():
+ with _lock:
+ return copy.deepcopy(_samples), dict(_status), dict(_alerted)
+
+
+def report():
+ """The route performs management GETs only; it never loads storage or sends alerts."""
+ account = read_account()
+ samples, status, alerted = snapshot()
+ alarm, cause = evaluate_alarm(account)
+ return {**account, "samples": samples, "sampler": status, "alarm": alarm,
+ "alarm_cause": cause, "alerted": alerted}
+
+
+def _clean_samples(samples):
+ clean = {}
+ for row in samples if isinstance(samples, list) else []:
+ try:
+ if not _stamp(row.get("at")) or not _stamp(row.get("generated_at")):
+ continue
+ usage = _money(row.get("month_to_date_usage"))
+ if usage < 0:
+ continue
+ clean[row["at"]] = {"at": row["at"], "generated_at": row["generated_at"],
+ "month_to_date_usage": usage, "source": BALANCE_SOURCE}
+ except (AttributeError, TypeError, ValueError, OverflowError):
+ continue
+ return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
+
+
+def _clean_alerted(alerted):
+ if not isinstance(alerted, dict):
+ raise ValueError()
+ for month, at in alerted.items():
+ if not isinstance(month, str) or not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", month) or not _stamp(at):
+ raise ValueError()
+ return {key: alerted[key] for key in sorted(alerted)[-2:]}
+
+
+def _sample_once(load, persist, *, sender=None):
+ # Serialize manual and scheduled ticks as well as delivery marker changes.
+ with _sample_lock:
+ _sample_tick(load, persist, sender=sender)
+
+
+def _sample_tick(load, persist, *, sender=None):
+ global _loaded, _samples, _alerted, _cluster_id
+ account = read_account()
+ balance = account["balance"]
+ alarm, alarm_cause = evaluate_alarm(account)
+ if not balance["available"]:
+ with _lock:
+ _status["cause"] = alarm_cause
+ return
+ selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
+ with _lock:
+ if _cluster_id != selected:
+ _cluster_id, _loaded, _samples, _alerted = selected, False, [], {}
+ _status["persisted"] = None
+ _samples = _clean_samples([*_samples, {"at": balance["read_at"],
+ "generated_at": balance["generated_at"], "month_to_date_usage": balance["month_to_date_usage"]}])
+ _status["last_at"] = balance["read_at"]
+ awake, cause = store_is_awake()
+ if not awake:
+ with _lock:
+ _status["cause"] = cause
+ return
+ try:
+ if not _loaded:
+ saved = load()
+ saved = {} if saved is None else saved
+ matching = saved.get("cluster_id") == selected
+ history = _clean_samples(saved.get("samples", [])) if matching else []
+ markers = _clean_alerted(saved.get("alerted", {})) if matching else {}
+ with _lock:
+ _samples = _clean_samples([*history, *_samples])
+ _alerted, _loaded = markers, True
+ except Exception:
+ with _lock:
+ _status["cause"] = "Sample history and alert markers could not be loaded; alerts deferred."
+ return
+ sent = False
+ try:
+ def save():
+ awake, cause = store_is_awake()
+ if not awake:
+ raise UsageReadError(cause)
+ samples, _, markers = snapshot()
+ persist({"cluster_id": selected, "samples": samples, "alerted": markers})
+ with _lock:
+ _status.update(persisted=_now(), cause=alarm_cause)
+
+ # Flush any successful delivery marker before considering another send.
+ save()
+ if not alarm or alarm["period"] in _alerted:
+ return
+ try:
+ ok, delivery = (sender or send_telegram)(alarm["message"])
+ except Exception:
+ ok, delivery = False, "Telegram alert could not be sent."
+ if not ok:
+ with _lock:
+ _status["cause"] = delivery
+ return
+ sent = True
+ with _lock:
+ _alerted = _clean_alerted({**_alerted, alarm["period"]: _now()})
+ save()
+ except Exception:
+ with _lock:
+ _status["cause"] = (
+ "Telegram alert sent; its marker remains pending in memory because storage is unavailable."
+ if sent else "Sample history could not be stored; alerts deferred until storage is available.")
+
+
+def start_sampler(*, load=None, persist=None, sender=None):
+ """Off by default; at most one sampler per process, with a one-minute floor."""
+ global _thread
+ try:
+ every_s = int(os.environ.get("AIOS_DO_SAMPLE_SECONDS", "0"))
+ except ValueError:
+ every_s = 0
+ if every_s <= 0 or load is None or persist is None:
+ with _lock:
+ _status.update(every_s=0, cause="Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset, invalid or zero, or storage callbacks are unavailable.")
+ return False
+ every_s = max(60, every_s)
+ with _lock:
+ if _thread is not None and _thread.is_alive():
+ return True
+ _status.update(every_s=every_s, cause="Waiting for the first scheduled sample.")
+ _stop.clear()
+
+ def run():
+ while not _stop.is_set():
+ try:
+ _sample_once(load, persist, sender=sender)
+ except Exception:
+ with _lock:
+ _status["cause"] = "The scheduled sample could not be completed; it will retry."
+ if _stop.wait(every_s):
+ break
+
+ _thread = threading.Thread(target=run, name="do-usage-sampler", daemon=True)
+ _thread.start()
+ return True
+
+
+def stop_sampler():
+ _stop.set()
+
+
+def _selftest():
+ """Run without credentials, sockets, runtimes, files or a real Telegram sender."""
+ import sys
+ from unittest.mock import patch
+ from core import neon_usage
+
+ module = sys.modules[__name__]
+ at = "2026-09-25T12:00:00Z"
+ snapshot_at = "2026-09-25T00:00:00Z"
+ database = {"id": DEFAULT_CLUSTER_ID, "name": "fixture-cluster", "size": "db-s-1vcpu-1gb",
+ "num_nodes": 1, "region": "nyc3", "status": "online", "version": "18",
+ "storage_size_mib": 10240, "connection": {"password": "fixture-secret"}}
+ responses = {
+ "/databases/" + DEFAULT_CLUSTER_ID: {"database": database},
+ "/customers/my/balance": {"month_to_date_usage": "20.00", "account_balance": "-1.00",
+ "month_to_date_balance": "19.00", "generated_at": snapshot_at},
+ "/customers/my/billing_history": {"billing_history": []},
+ }
+ calls, sends, documents = [], [], []
+
+ def fake_get(path, token):
+ calls.append(path)
+ return copy.deepcopy(responses[path])
+
+ def fake_sender(message):
+ sends.append(message)
+ return True, len(sends)
+
+ def persist(document):
+ documents.append(copy.deepcopy(document))
+
+ def reset():
+ global _loaded, _samples, _alerted, _cluster_id
+ _loaded, _samples, _alerted, _cluster_id = False, [], {}, None
+ sends.clear()
+ documents.clear()
+
+ # Double protection: all requests are fake, including if a test forgets sender=.
+ with patch.dict(os.environ, {}, clear=True), patch.object(module, "_get", side_effect=fake_get), \
+ patch.object(module, "send_telegram", side_effect=fake_sender), \
+ patch.object(neon_usage, "send_telegram", side_effect=fake_sender), \
+ patch.object(module, "_now", return_value=at):
+ missing = read_account()
+ assert not missing["available"] and missing["cause"] and not calls
+ assert missing["balance"]["month_to_date_usage"] is None
+ assert missing["fixed_price"]["monthly_usd"] is None
+ assert not start_sampler(load=lambda: {}, persist=persist, sender=fake_sender)
+ assert "disabled" in report()["sampler"]["cause"]
+ os.environ["DIGITALOCEAN_TOKEN"] = "fixture-fallback"
+ account = read_account()
+ assert account["fixed_price"]["monthly_usd"] == 15.15
+ assert account["balance"]["account_balance"] == -1
+ assert account["billing"]["available"] and account["billing_history"] == []
+ # The real empty history omits the key: {"links": {}, "meta": {}} is empty, not unreadable.
+ responses["/customers/my/billing_history"] = {"links": {}, "meta": {}}
+ enveloped = read_account()
+ assert enveloped["billing"]["available"] and enveloped["billing_history"] == []
+ responses["/customers/my/billing_history"] = {"links": {}}
+ assert not read_account()["billing"]["available"]
+ responses["/customers/my/billing_history"] = {"billing_history": []}
+ assert "fixture-secret" not in json.dumps(account)
+ assert account["balance"]["source"] == BALANCE_SOURCE and account["balance"]["generated_at"] == snapshot_at
+ os.environ["AIOS_DO_USAGE_TOKEN"] = "fixture-preferred"
+ def preferred_get(path, token):
+ assert token == "fixture-preferred"
+ return fake_get(path, token)
+ with patch.object(module, "_get", side_effect=preferred_get):
+ assert read_account()["available"]
+
+ for failed_path, failed_block in (("/databases/" + DEFAULT_CLUSTER_ID, "cluster"),
+ ("/customers/my/balance", "balance"),
+ ("/customers/my/billing_history", "billing")):
+ def fail_one(path, token):
+ if path == failed_path:
+ raise UsageReadError("DigitalOcean API returned HTTP 403.")
+ return fake_get(path, token)
+ with patch.object(module, "_get", side_effect=fail_one):
+ partial = read_account()
+ assert partial["available"] and not partial[failed_block]["available"]
+ assert "403" in partial[failed_block]["cause"]
+ assert all(partial[key]["available"] for key in ("cluster", "balance", "billing") if key != failed_block)
+ if failed_block == "balance":
+ assert partial["balance"]["month_to_date_usage"] is None
+ assert evaluate_alarm(partial)[0] is None and evaluate_alarm(partial)[1]
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ assert not sends and "403" in snapshot()[1]["cause"]
+
+ original_balance = copy.deepcopy(responses["/customers/my/balance"])
+ for invalid in (None, True, "NaN", "Infinity", "unreadable"):
+ responses["/customers/my/balance"]["month_to_date_usage"] = invalid
+ malformed = read_account()
+ assert not malformed["balance"]["available"] and malformed["balance"]["cause"]
+ assert malformed["balance"]["month_to_date_usage"] is None
+ assert evaluate_alarm(malformed)[0] is None
+ responses["/customers/my/balance"] = original_balance
+
+ database["size"] = "unknown-size"
+ unknown = read_account()
+ assert unknown["fixed_price"]["monthly_usd"] is None
+ assert "UNKNOWN" in unknown["fixed_price"]["cause"] and "unknown-size" in unknown["fixed_price"]["cause"]
+ assert evaluate_alarm(unknown)[0] is None
+ database["size"] = "db-s-1vcpu-1gb"
+ database["num_nodes"] = 2
+ assert read_account()["fixed_price"]["monthly_usd"] == 30.30
+ database["num_nodes"] = 1
+
+ reset()
+ os.environ["AIOS_DO_ALARM_USD"] = "1"
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ _sample_once(lambda: documents[-1], persist, sender=fake_sender)
+ assert len(sends) == 1 and sends[0].startswith("TEST ")
+ assert all(word in sends[0] for word in ("20.00", "15.15", snapshot_at, "fixture-cluster", "R28"))
+ saved = copy.deepcopy(documents[-1])
+ reset()
+ _sample_once(lambda: saved, persist, sender=fake_sender)
+ assert not sends and "2026-09" in documents[-1]["alerted"]
+ responses["/customers/my/balance"]["generated_at"] = "2026-10-01T00:00:00Z"
+ _sample_once(lambda: saved, persist, sender=fake_sender)
+ assert len(sends) == 1 # A new billing month permits one new delivery.
+ responses["/customers/my/balance"]["generated_at"] = snapshot_at
+
+ reset()
+ del os.environ["AIOS_DO_ALARM_USD"]
+ responses["/customers/my/balance"]["month_to_date_usage"] = "15.15"
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ assert not sends
+ responses["/customers/my/balance"]["month_to_date_usage"] = "0.00"
+ zero = report()
+ assert zero["balance"]["month_to_date_usage"] == 0 and not zero["alarm"]
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ assert not sends
+ os.environ["AIOS_DO_ALARM_USD"] = "-1"
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ assert len(sends) == 1 and sends[0].startswith("TEST ")
+ os.environ["AIOS_DO_ALARM_USD"] = "NaN"
+ assert report()["alarm"] is None and "invalid" in report()["alarm_cause"]
+
+ reset()
+ os.environ["AIOS_DO_ALARM_USD"] = "-1"
+ os.environ["STORE_BACKEND"] = "pg"
+ os.environ["DATABASE_URL"] = "postgresql://fixture@ep-fixture.neon.tech/app"
+ with patch.object(neon_usage, "read_project", return_value={"available": True, "id": "fixture"}), \
+ patch.object(neon_usage, "_get", return_value={"endpoints": [
+ {"id": "ep-fixture", "current_state": "idle"}]}):
+ def forbidden(*args):
+ raise AssertionError("Idle compute must never call storage")
+ _sample_once(forbidden, forbidden, sender=fake_sender)
+ assert not sends and "idle" in snapshot()[1]["cause"]
+ assert report()["balance"]["available"]
+ del os.environ["STORE_BACKEND"]
+ del os.environ["DATABASE_URL"]
+ with patch.object(neon_usage, "read_project", side_effect=AssertionError("Unexpected Neon read")):
+ assert store_is_awake(backend="pg", database_url="postgresql://fixture@fixture.db.ondigitalocean.com/app")[0]
+ assert not store_is_awake(backend="pg", database_url="")[0]
+ assert not store_is_awake({"available": False, "cause": "offline"}, backend="pg",
+ database_url="postgresql://fixture@ep-fixture.neon.tech/app")[0]
+ reset()
+ def failed_load():
+ raise RuntimeError("fixture-secret")
+ _sample_once(failed_load, persist, sender=fake_sender)
+ assert not sends and "loaded" in snapshot()[1]["cause"]
+ reset()
+ def failed_persist(document):
+ raise RuntimeError("fixture-secret")
+ _sample_once(lambda: {}, failed_persist, sender=fake_sender)
+ assert not sends and "stored" in snapshot()[1]["cause"]
+ assert "fixture-secret" not in json.dumps(report())
+ reset()
+ def fail_after_delivery(document):
+ if sends:
+ raise RuntimeError("fixture-secret")
+ persist(document)
+ _sample_once(lambda: {}, fail_after_delivery, sender=fake_sender)
+ assert len(sends) == 1 and "pending in memory" in snapshot()[1]["cause"]
+ _sample_once(lambda: {}, persist, sender=fake_sender)
+ assert len(sends) == 1 and "2026-09" in documents[-1]["alerted"]
+ rows = [{"at": f"2026-09-{1 + n // 24:02}T{n % 24:02}:00:00Z", "generated_at": snapshot_at,
+ "month_to_date_usage": n} for n in range(MAX_SAMPLES + 1)]
+ assert len(_clean_samples(rows)) == MAX_SAMPLES
+
+ # Exercise the actual HTTP boundary with a fake opener, including credential-bearing errors.
+ class Response:
+ def __enter__(self):
+ return self
+ def __exit__(self, *args):
+ pass
+ def read(self, limit):
+ return b'{"ok": true}'
+
+ class Opener:
+ def open(self, request, *, timeout):
+ assert request.get_method() == "GET" and timeout == 5
+ return Response()
+
+ with patch.object(module, "build_opener", return_value=Opener()) as opener:
+ assert _get("/fixture", "fixture-token") == {"ok": True}
+ assert isinstance(opener.call_args.args[0], _NoRedirect)
+ assert opener.call_args.args[0].redirect_request(None, None, 302, None, None, None) is None
+ for error in (HTTPError("fixture-private-url", 403, "fixture-token", None, None),
+ URLError("fixture-private-url"), TimeoutError("fixture-token"), ValueError("fixture-token")):
+ with patch.object(module, "build_opener", side_effect=error):
+ try:
+ _get("/fixture", "fixture-token")
+ raise AssertionError("The HTTP error must fail closed")
+ except UsageReadError as exc:
+ assert "fixture-" not in str(exc)
+ print("PASS DigitalOcean usage selftest: isolated API failures, pricing, snapshot provenance, monthly alerts, restart markers, idle-store guard and safe HTTP errors; fake senders only.")
+
+
+if __name__ == "__main__":
+ import sys
+ if sys.argv[1:] != ["--selftest"]:
+ raise SystemExit("Usage: python -m core.do_usage --selftest")
+ _selftest()
diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py
index 5553b62f08033aec5e2da696532af4cc0c521449..ac5c6cb060582da8833d5e98c6df5d987724ae52 100644
--- a/platform/core/external_tools.py
+++ b/platform/core/external_tools.py
@@ -9,7 +9,7 @@ import json
from dataclasses import dataclass
from typing import Callable, Optional
-from core import sheets, sheet_structure, sheet_templates, write_execution
+from core import perm_scope, sheets, sheet_structure, sheet_templates, write_execution
@dataclass(frozen=True)
@@ -42,12 +42,19 @@ _STRING = {"type": "string"}
_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
"currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
+# S67-T14 / contract C2: a border side is the legacy `true` (read as a thin black line) or
+# `{style, color}` with one of Excel's seven line styles and a six digit hex colour; `false` or an
+# absent side is no side. `sheets.py::_clean_border_side` refuses anything else by code, and this
+# publishes the same shape so an AI client can draw a styled border at all.
+_BORDER_SIDE = {"oneOf": [{"type": "boolean"}, _object({
+ "style": {"enum": list(sheets.CELL_BORDER_STYLES)},
+ "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"}}, ("style", "color"))]}
_STYLE = _object({
**{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
**{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
"size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
"textRotation": {"type": "number"}, "format": _FORMAT,
- "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}),
+ "border": _object({side: _BORDER_SIDE for side in ("top", "right", "bottom", "left")}),
})
_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
@@ -62,15 +69,25 @@ _BINDING = _object({
"period": _object({"unit": _STRING, "start": _STRING}),
"ending": {"type": "object"},
}, ("database", "aggregate", "scope"))
-_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
- "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
- "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
for key in ("col0", "row0", "col1", "row1")},
("col0", "row0", "col1", "row1"))]}
+# S67-T18: an in-cell sparkline, `{kind, range}` exactly as `sheets.py::_clean_sparkline` stores
+# it; the range is a rectangle on the SAME tab (an A1 range such as "B2:M2", or the dict).
+_SPARKLINE = _object({"kind": {"enum": list(sheets.SPARKLINE_KINDS)}, "range": _RANGE},
+ ("kind", "range"))
+_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
+ "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
+ "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING,
+ "sparkline": _SPARKLINE}, ("kind",))
+# S67-T13: a colour scale's two ends and what each end means. `sheets.py::_clean_rules` refuses
+# a type outside `SHEET_RULE_SCALE_TYPES`; the enum publishes the same list to the caller.
+_SCALE_TYPE = {"enum": list(sheets.SHEET_RULE_SCALE_TYPES)}
_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
"value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
- "scale": {"type": "object"}}, ("range", "op"))
+ "scale": _object({"low": _STRING, "high": _STRING,
+ "minType": _SCALE_TYPE, "maxType": _SCALE_TYPE})},
+ ("range", "op"))
_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
"stacking": _STRING, "source": {"type": "object"},
"series": {"type": "array", "items": {"type": "object"}},
@@ -166,6 +183,16 @@ def _add_tab(ctx, args):
return _put(ctx, args, book, tab["id"])
+def _delete_tab(ctx, args):
+ book = _book(ctx, args, write=True)
+ tab = sheets.find_tab(book, args["tab"])
+ if len(book["sheets"]) <= 1:
+ raise sheets.SheetRefusal(
+ "tab_last", "A workbook keeps at least one sheet, so this one cannot be deleted.")
+ book["sheets"] = [row for row in book["sheets"] if row["id"] != tab["id"]]
+ return _put(ctx, args, book)
+
+
def _save_template(ctx, args):
book = ctx.project_book(_book(ctx, args))
record = sheet_templates.save_sheet_template(
@@ -269,7 +296,7 @@ def _structure(ctx, args):
return _put(ctx, args, shifted, tab["id"])
-def _tool(description, schema, handler, *, write=False, grants=("analyst", "sheets")):
+def _tool(description, schema, handler, *, write=False, grants=("ai_apps", "sheets")):
return {"description": description, "inputSchema": schema, "write": write,
"grants": grants, "handler": handler}
@@ -328,6 +355,9 @@ TOOLS = {
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
"sheet_add_tab": _tool("Add a blank tab to a workbook.",
_schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
+ "sheet_delete_tab": _tool(
+ "Preview deleting a tab by id or name; requires typed confirmation. The last tab cannot be deleted.",
+ _schema({}), _delete_tab, write=True),
"sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
_schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
"sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
@@ -367,10 +397,10 @@ TOOLS = {
("workbook_id", "key", "name", "description"), additionalProperties=False),
_save_template, write=True),
"list_databases": _tool("List the databases permitted or visible to the key's maker.",
- _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("analyst",)),
+ _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)),
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
_object({"database": _STRING}, ("database",), additionalProperties=False),
- lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
+ lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)),
"query_database": _tool(
"Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
_object({
@@ -389,7 +419,7 @@ TOOLS = {
}, ("database",), additionalProperties=False),
lambda ctx, args: ctx.query_database(args["database"], {
key: value for key, value in args.items() if key != "database"
- }), grants=("analyst",)),
+ }), grants=("ai_apps",)),
"view_upsert": _tool(
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
_object({"database": _STRING, "view": _object({
@@ -398,7 +428,7 @@ TOOLS = {
"config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
additionalProperties=False)},
("database", "view"), additionalProperties=False),
- lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
+ lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("ai_apps",)),
"view_share_preview": _tool(
"Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
_object({
@@ -408,21 +438,21 @@ TOOLS = {
"user": _STRING, "role": {"enum": ["view", "edit"]},
}, ("user", "role"), additionalProperties=False)},
}, ("database", "view_id", "entries"), additionalProperties=False),
- lambda ctx, args: ctx.view_share_preview(args), grants=("analyst",)),
+ lambda ctx, args: ctx.view_share_preview(args), grants=("ai_apps",)),
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
additionalProperties=False),
- lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("analyst",)),
+ lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("ai_apps",)),
"app_upsert": _tool(
"Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
_object(_APPLICATION, additionalProperties=False,
anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
{"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
- lambda ctx, args: ctx.app_upsert(args), write=True, grants=("analyst",)),
+ lambda ctx, args: ctx.app_upsert(args), write=True, grants=("ai_apps",)),
"app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
additionalProperties=False),
- lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
+ lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)),
"database_object": _tool(
"Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
_object({
@@ -435,7 +465,7 @@ TOOLS = {
"query": {"type": "object"},
}, ("action",), additionalProperties=False),
lambda ctx, args: _callback(ctx, "database_object", args), write=True,
- grants=("analyst",)),
+ grants=("ai_apps",)),
"view_object": _tool(
"Read or preview-confirm a personal View operation through the ordinary grid writer.",
_object({
@@ -452,7 +482,7 @@ TOOLS = {
ctx.view_share_preview({"database": args["database"],
"view_id": args["view_id"],
"entries": args.get("entries") or []})
- ), write=True, grants=("analyst",)),
+ ), write=True, grants=("ai_apps",)),
**{name: _object_tool(name, actions)
for name, actions in _SHEET_OBJECT_ACTIONS.items()},
"validated_batch": _tool(
@@ -464,12 +494,12 @@ TOOLS = {
additionalProperties=False)}},
("operations",), additionalProperties=False),
lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
- grants=("analyst",)),
+ grants=("ai_apps",)),
"confirm_change": _tool(
"Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
_object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
("confirmation_token",), additionalProperties=False),
- lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("analyst",)),
+ lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("ai_apps",)),
}
@@ -506,7 +536,12 @@ def dispatch(ctx, name, arguments):
if tool is None:
raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
for grant in tool["grants"]:
- ctx.session.require(grant)
+ if grant == perm_scope.AI_APPS:
+ reason = perm_scope.ai_apps_refusal(ctx.session.user, ctx.session.runtime)
+ if reason:
+ raise sheets.SheetRefusal("ai_apps_off", reason)
+ else:
+ ctx.session.require(grant)
schema = tool["inputSchema"]
if not isinstance(arguments, dict):
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py
index f2c308477c631434959a219e36f24acfdf963bda..496b8c568257451ec0014edbbae423f1b7a163ae 100644
--- a/platform/core/grid_events.py
+++ b/platform/core/grid_events.py
@@ -632,6 +632,57 @@ def _grant_marker(ctx, ws):
return _has_grants
+#: S67-T39 / R36: the Application registry's store key. READ here, never imported:
+#: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE).
+_APPLICATIONS_KEY = 'applications_v1'
+
+
+def _live_application_ids(ctx):
+ """The set of App ids the tenant registry holds, or None when that is not KNOWN.
+
+ None (read failed, or no registry document with an `applications` map) means "strip
+ nothing": an unknown answer must never turn a live App binding into a plain view.
+ """
+ try:
+ doc = _store_of(ctx).get(_APPLICATIONS_KEY)
+ except Exception as exc:
+ _tel.error('applications:live', exc)
+ return None
+ apps = doc.get('applications') if isinstance(doc, dict) else None
+ return {str(key) for key in apps} if isinstance(apps, dict) else None
+
+
+def _application_ref_id(view):
+ config = view.get('config') if isinstance(view, dict) else None
+ ref = config.get('application') if isinstance(config, dict) else None
+ return str(ref.get('id') or '') if isinstance(ref, dict) else None
+
+
+def _drop_dead_application_refs(ctx, views):
+ """S67-T39 / R36 / D-2233: a View bound to a DELETED App is served as a plain view of its
+ database (the `application` member dropped, everything else kept), so it opens with no
+ error. Projection only, and it never raises: any failure serves the views unchanged.
+ The registry is read only when some View actually carries a reference.
+ """
+ try:
+ if not any(_application_ref_id(v) is not None for v in (views or {}).values()):
+ return views
+ live = _live_application_ids(ctx)
+ if live is None:
+ return views
+ out = {}
+ for vid, view in views.items():
+ ref_id = _application_ref_id(view)
+ if ref_id is not None and ref_id not in live:
+ config = {k: v for k, v in view['config'].items() if k != 'application'}
+ view = {**view, 'config': config}
+ out[vid] = view
+ return out
+ except Exception as exc:
+ _tel.error('applications:dead_refs', exc)
+ return views
+
+
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
@@ -707,6 +758,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
merged.update(ws.get('views') or {}) # own views win the id collision
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
for vid, v in merged.items()}
+ if ws.get('views'):
+ ws['views'] = _drop_dead_application_refs(ctx, ws['views'])
return ws
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
# API caller has no durable dict and must be told, not quietly served memory.
@@ -1584,7 +1637,13 @@ def handle_one(event, ctx):
_refuse(ctx, 'invalid_application_ref',
'that Application reference is not a valid typed identity', view_id)
return False
- config['application'] = application
+ # S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View
+ # saves as a plain view of its database. Unknown registry state keeps the ref.
+ _live = _live_application_ids(ctx)
+ if _live is not None and application['id'] not in _live:
+ application = None
+ else:
+ config['application'] = application
# Wave-6 item 10: a direct non-grid display mode rides the view config, validated
# structurally (mode whitelist, refs must be this table's fields). A bound Application's
# config remains in the registry and is resolved on read, so an offered clone is dropped.
diff --git a/platform/core/neon_usage.py b/platform/core/neon_usage.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5b25fb03291807bb87dfcb3c3e5d25595ee28b2
--- /dev/null
+++ b/platform/core/neon_usage.py
@@ -0,0 +1,590 @@
+"""Neon management-plane usage. Importing or reading a card never opens a database.
+
+Storage is injected by the API host and called only after an awake observation.
+Unchanged counters are stale observations, not evidence of zero consumption.
+"""
+from __future__ import annotations
+
+import copy
+import json
+import math
+import os
+import re
+import threading
+from datetime import datetime, timezone
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode, urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+
+# Launch pricing: AutoWiki/research/neon-usage-samples/2026-09-25-sample1.json
+# notes[2] and AutoWiki/research/neon-alternative-feasibility.md. The compute
+# allowance is a meter/alert threshold; ALL CU-hours are billed, including it.
+LAUNCH_PLAN = {
+ "allowance": {"transfer_gb": 500.0, "compute_hours": 300.0, "storage_gib": 10.0},
+ "usd": {"transfer_per_gb": 0.10, "compute_per_hour": 0.106,
+ "storage_per_gb_month": 0.35},
+}
+API_BASE = "https://console.neon.tech/api/v2"
+HTTP_TIMEOUT = 5
+DEFAULT_SAMPLE_SECONDS = 21600
+MAX_SAMPLES = 240
+DEFAULT_ALARM_SHARE = 0.70
+STORE_KEY = "platform_neon_usage_samples"
+_FIELDS = ("data_transfer_bytes", "compute_time_seconds", "synthetic_storage_size")
+_METRIC_LABELS = {"transfer_gb": "Transfer", "compute_hours": "Compute", "storage_gib": "Storage"}
+_ALERT_KEYS = {f"{metric}:{kind}" for metric in _METRIC_LABELS
+ for kind in ("share", "projected_overage")}
+_guards = None
+
+
+def set_guards(guards):
+ """The API host publishes its startup checks; this layer never imports the API."""
+ global _guards
+ _guards = copy.deepcopy(guards)
+
+
+class UsageReadError(Exception):
+ """Contains only a fixed, credential-free explanation suitable for the card."""
+
+
+class _NoRedirect(HTTPRedirectHandler):
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ return None
+
+
+def _get(path, key):
+ request = Request(API_BASE + path, headers={"Authorization": "Bearer " + key}, method="GET")
+ try:
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
+ payload = json.loads(response.read(1_000_001))
+ if not isinstance(payload, dict):
+ raise ValueError()
+ return payload
+ except HTTPError as exc:
+ raise UsageReadError(f"Neon management API returned HTTP {exc.code}.") from None
+ except TimeoutError:
+ raise UsageReadError("Neon management API timed out.") from None
+ except URLError as exc:
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
+ raise UsageReadError(f"Neon management API {cause}.") from None
+ except (ValueError, OSError):
+ raise UsageReadError("Neon management API returned an unreadable response.") from None
+
+
+def _identifier(value):
+ return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value) is not None
+
+
+def _pages(path, field, key, params=None):
+ """Bound discovery, and refuse partial discovery rather than guessing uniqueness."""
+ params, seen = dict(params or {}), set()
+ for _ in range(10):
+ payload = _get(path + ("?" + urlencode(params) if params else ""), key)
+ rows = payload.get(field)
+ if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
+ raise UsageReadError("Neon project discovery returned an unreadable response.")
+ # Neon echoes the last cursor even on the terminal empty project page.
+ if not rows:
+ return
+ yield from rows
+ cursor = (payload.get("pagination") or {}).get("cursor")
+ if not cursor:
+ return
+ if cursor in seen:
+ break
+ seen.add(cursor)
+ params["cursor"] = cursor
+ raise UsageReadError("Neon project discovery is incomplete; set NEON_PROJECT_ID.")
+
+
+def resolve_project_id(api_key=None, project_id=None):
+ """Explicit id first; otherwise exactly one project across the visible orgs."""
+ key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
+ if not key:
+ raise UsageReadError("NEON_API_KEY is absent; Neon usage cannot be read.")
+ selected = os.environ.get("NEON_PROJECT_ID", "") if project_id is None else project_id
+ if selected:
+ if not _identifier(selected):
+ raise UsageReadError("NEON_PROJECT_ID is invalid.")
+ return selected
+ projects = set()
+ for org in _pages("/users/me/organizations", "organizations", key):
+ if not _identifier(org.get("id")):
+ raise UsageReadError("Neon organization discovery returned no usable id.")
+ for project in _pages("/projects", "projects", key, {"org_id": org["id"], "limit": 100}):
+ if not _identifier(project.get("id")):
+ raise UsageReadError("Neon project discovery returned no usable id.")
+ projects.add(project["id"])
+ if len(projects) > 1:
+ raise UsageReadError("Multiple Neon projects are visible; set NEON_PROJECT_ID.")
+ if len(projects) != 1:
+ raise UsageReadError("No Neon project is visible; set NEON_PROJECT_ID.")
+ return projects.pop()
+
+
+def _now():
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _stamp(value):
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ return parsed if parsed.tzinfo else None
+ except (ValueError, TypeError):
+ return None
+
+
+def _number(value):
+ return (isinstance(value, (int, float)) and not isinstance(value, bool)
+ and math.isfinite(value) and value >= 0)
+
+
+def read_project(api_key=None, project_id=None):
+ """Consumption comes ONLY from GET /projects/{id}, never branch sums or SQL."""
+ result = {"available": False, "cause": None, "read_at": _now(), "id": None, "name": None,
+ "consumption_period_start": None, "consumption_period_end": None,
+ **dict.fromkeys(_FIELDS)}
+ try:
+ key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
+ selected = resolve_project_id(key, project_id)
+ project = _get("/projects/" + selected, key).get("project")
+ if not isinstance(project, dict) or any(not _number(project.get(k)) for k in _FIELDS):
+ raise UsageReadError("Neon project response is missing valid usage totals.")
+ start, end = (_stamp(project.get("consumption_period_" + k)) for k in ("start", "end"))
+ if not start or not end or end <= start:
+ raise UsageReadError("Neon project response is missing a valid billing period.")
+ if project.get("id") != selected:
+ raise UsageReadError("Neon project response did not match the requested project.")
+ result.update({k: project[k] for k in _FIELDS})
+ result.update(available=True, id=selected, name=project.get("name") or selected,
+ consumption_period_start=project["consumption_period_start"],
+ consumption_period_end=project["consumption_period_end"], read_at=_now())
+ except UsageReadError as exc:
+ result["cause"] = str(exc)
+ except Exception:
+ # Never return exception text: HTTP clients and credentials can occur in it.
+ result["cause"] = "Neon usage could not be read because of an unexpected response."
+ return result
+
+
+def _clean_samples(samples):
+ clean = {}
+ for item in samples if isinstance(samples, list) else []:
+ if not isinstance(item, dict) or not _stamp(item.get("at")) or not _stamp(item.get("period_start")):
+ continue
+ if any(not _number(item.get(k)) for k in ("transfer_bytes", "compute_seconds", "storage_bytes")):
+ continue
+ row = {k: item[k] for k in ("at", "transfer_bytes", "compute_seconds", "storage_bytes", "period_start")}
+ clean[(row["at"], row["period_start"])] = row
+ return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
+
+
+def _rate(samples, field, divisor, *, gauge=False):
+ result = {"per_day": None, "cause": "Waiting for two samples with different values; Neon totals refresh slowly.",
+ "from_at": None, "to_at": None}
+ if len(samples) < 2:
+ return result
+ latest = samples[-1]
+ for prior in reversed(samples[:-1]):
+ days = (_stamp(latest["at"]) - _stamp(prior["at"])).total_seconds() / 86400
+ delta = latest[field] - prior[field]
+ if days <= 0 or delta == 0:
+ continue
+ if delta < 0 and not gauge:
+ result["cause"] = "The counter decreased; waiting for a new pair of increasing samples."
+ return result
+ return {"per_day": delta / divisor / days, "cause": None,
+ "from_at": prior["at"], "to_at": latest["at"]}
+ return result
+
+
+def _alarm_threshold():
+ raw = os.environ.get("AIOS_NEON_ALARM_SHARE")
+ if raw is None:
+ return DEFAULT_ALARM_SHARE, None
+ try:
+ threshold = float(raw)
+ if not math.isfinite(threshold) or not 0 < threshold <= 1.5:
+ raise ValueError()
+ return threshold, None
+ except ValueError:
+ return DEFAULT_ALARM_SHARE, "AIOS_NEON_ALARM_SHARE is invalid; using the default 70% threshold."
+
+
+def _alarm_message(summary, crossings, test):
+ metrics = []
+ for metric, label in _METRIC_LABELS.items():
+ rows = [row for row in crossings if row["metric"] == metric]
+ if rows:
+ kinds = ", ".join("share limit" if row["kind"] == "share" else "projected overage"
+ for row in rows)
+ metrics.append(f"{label} {rows[0]['share']:.1%} used ({kinds})")
+ total = summary.get("projection", {}).get("total")
+ bill = f"USD {total:,.2f}" if _number(total) else "unavailable"
+ return ("TEST " if test else "") + "Neon usage: " + "; ".join(metrics) + f". Projected bill {bill}."
+
+
+def evaluate_alarm(summary, share_threshold=DEFAULT_ALARM_SHARE):
+ """Pure threshold evaluation, independent of delivery or saved alert markers."""
+ if not summary.get("available"):
+ return None
+ if not _number(share_threshold) or not 0 < share_threshold <= 1.5:
+ share_threshold = DEFAULT_ALARM_SHARE
+ crossings = []
+ projection = summary.get("projection", {})
+ bill = projection.get("total")
+ for metric in _METRIC_LABELS:
+ share = summary.get("share", {}).get(metric)
+ if not _number(share):
+ continue
+ projected = projection.get("usage", {}).get(metric)
+ allowance = summary.get("allowance", {}).get(metric)
+ for kind, crossed in (
+ ("share", share >= share_threshold),
+ ("projected_overage", _number(projected) and _number(allowance) and projected > allowance),
+ ):
+ if crossed:
+ crossings.append({"metric": metric, "kind": kind, "share": share,
+ "projected_bill": bill if _number(bill) else None})
+ if not crossings:
+ return None
+ test = share_threshold != DEFAULT_ALARM_SHARE
+ return {"level": "red", "message": _alarm_message(summary, crossings, test),
+ "crossings": crossings, "threshold": share_threshold, "test": test}
+
+
+def send_telegram(text):
+ """Bounded POST; return only a message id or a fixed, credential-free cause."""
+ token = os.environ.get("TELEGRAM_BOT_TOKEN", "")
+ chat_id = os.environ.get("TELEGRAM_CHAT_ID", "")
+ missing = [name for name, value in (("TELEGRAM_BOT_TOKEN", token), ("TELEGRAM_CHAT_ID", chat_id))
+ if not value]
+ if missing:
+ return False, "Telegram alert cannot be sent: missing " + ", ".join(missing) + "."
+ try:
+ body = json.dumps({"chat_id": chat_id, "text": text[:4096]}).encode("utf-8")
+ request = Request("https://api.telegram.org/bot" + token + "/sendMessage", data=body,
+ headers={"Content-Type": "application/json"}, method="POST")
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
+ payload = json.loads(response.read(65_537))
+ if not isinstance(payload, dict):
+ return False, "Telegram returned an unreadable alert response."
+ result = payload.get("result")
+ if (payload.get("ok") is True and isinstance(result, dict)
+ and type(result.get("message_id")) is int):
+ return True, result["message_id"]
+ # Telegram descriptions can echo request data. Never return them.
+ code = payload.get("error_code")
+ suffix = f" (HTTP {code})" if type(code) is int and 100 <= code <= 599 else ""
+ return False, "Telegram rejected the alert" + suffix + "."
+ except HTTPError as exc:
+ code = exc.code
+ suffix = f" HTTP {code}" if type(code) is int and 100 <= code <= 599 else " an HTTP error"
+ return False, "Telegram returned" + suffix + "."
+ except TimeoutError:
+ return False, "Telegram alert timed out."
+ except URLError as exc:
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
+ return False, f"Telegram {cause}."
+ except (ValueError, OSError):
+ return False, "Telegram returned an unreadable alert response."
+ except Exception:
+ return False, "Telegram alert could not be sent."
+
+
+def summarise(project, samples, *, share_threshold=None):
+ """Calculate usage and alarms; earlier periods remain in history, never in rates."""
+ if share_threshold is None:
+ share_threshold, _ = _alarm_threshold()
+ samples = _clean_samples(samples)
+ allowance = dict(LAUNCH_PLAN["allowance"])
+ result = {"available": project["available"], "cause": project.get("cause"),
+ "read_at": project["read_at"], "project": {k: project.get(k) for k in ("id", "name")},
+ "period": {k: project.get("consumption_period_" + k) for k in ("start", "end")},
+ "usage": dict.fromkeys(allowance), "allowance": allowance,
+ "share": dict.fromkeys(allowance), "rate": {},
+ "projection": {"transfer": None, "compute": None, "storage": None, "total": None,
+ "currency": "USD", "basis": {}, "cause": project.get("cause")},
+ "samples": samples, "alarm": None, "guards": copy.deepcopy(_guards)}
+ if not project["available"]:
+ for metric in allowance:
+ result["rate"][metric] = {"per_day": None, "cause": project.get("cause"),
+ "from_at": None, "to_at": None}
+ return result
+ usage = dict(zip(allowance, (project[_FIELDS[0]] / 1e9, project[_FIELDS[1]] / 3600,
+ project[_FIELDS[2]] / 2**30)))
+ result["usage"] = usage
+ result["share"] = {key: usage[key] / value for key, value in allowance.items()}
+ start, end, now = (_stamp(value) for value in (result["period"]["start"], result["period"]["end"],
+ result["read_at"]))
+ current = [row for row in samples if _stamp(row["period_start"]) == start
+ and start <= _stamp(row["at"]) <= min(now, end)]
+ rates = {"transfer_gb": _rate(current, "transfer_bytes", 1e9),
+ "compute_hours": _rate(current, "compute_seconds", 3600),
+ "storage_gib": _rate(current, "storage_bytes", 2**30, gauge=True)}
+ result["rate"] = rates
+ elapsed = (min(now, end) - start).total_seconds() / 86400
+ remaining = max(0, (end - now).total_seconds() / 86400)
+ period_days = (end - start).total_seconds() / 86400
+ if elapsed <= 0:
+ result["projection"]["cause"] = "The billing period has not started; an average is not available."
+ result["alarm"] = evaluate_alarm(result, share_threshold)
+ return result
+ projected, basis = {}, {}
+ for metric in ("transfer_gb", "compute_hours"):
+ rate = rates[metric]["per_day"]
+ basis[metric] = "Two changed samples" if rate is not None else "Period-to-date average"
+ projected[metric] = usage[metric] + remaining * (rate if rate is not None else usage[metric] / elapsed)
+ # Storage is a gauge, not a cumulative counter: dividing its size by elapsed
+ # days would invent growth. Integrate the observed sizes over the period,
+ # carrying the first size back to its start when no older observation exists.
+ cursor, size, size_days = start, (current[0]["storage_bytes"] / 2**30 if current else usage["storage_gib"]), 0.0
+ for row in current:
+ at = _stamp(row["at"])
+ size_days += size * (at - cursor).total_seconds() / 86400
+ cursor, size = at, row["storage_bytes"] / 2**30
+ size_days += size * (min(now, end) - cursor).total_seconds() / 86400
+ storage_average = size_days / elapsed
+ storage_rate = rates["storage_gib"]["per_day"]
+ projected["storage_gib"] = (max(0, usage["storage_gib"] + remaining * storage_rate)
+ if storage_rate is not None else storage_average)
+ future_average = ((usage["storage_gib"] + projected["storage_gib"]) / 2
+ if storage_rate is not None else storage_average)
+ billed_storage_gb = (size_days + remaining * future_average) / period_days * 2**30 / 1e9
+ basis["storage_gib"] = ("Two changed samples; estimated average storage" if storage_rate is not None
+ else "Period-to-date average storage size; held constant")
+ prices = LAUNCH_PLAN["usd"]
+ costs = {"transfer": max(0, projected["transfer_gb"] - allowance["transfer_gb"]) * prices["transfer_per_gb"],
+ "compute": projected["compute_hours"] * prices["compute_per_hour"],
+ "storage": billed_storage_gb * prices["storage_per_gb_month"]}
+ result["projection"] = {**costs, "total": sum(costs.values()), "currency": "USD",
+ "basis": basis, "usage": projected, "cause": None}
+ result["alarm"] = evaluate_alarm(result, share_threshold)
+ return result
+
+
+def store_is_awake(project=None, database_url=None, backend=None):
+ """No SQL probe. Unknown host/state fails closed, including history loading."""
+ backend = os.environ.get("STORE_BACKEND", "hf") if backend is None else backend
+ if backend != "pg":
+ return True, None
+ try:
+ host = urlsplit(os.environ.get("DATABASE_URL", "") if database_url is None else database_url).hostname
+ except ValueError:
+ host = None
+ if not host:
+ return False, "Sample history is pending: the store host is not configured."
+ if not (host.endswith(".neon.tech") or host.endswith(".neon.build")):
+ return True, None
+ # Other management-plane samplers share this guard without reading Neon
+ # at all when their history store is hosted elsewhere.
+ if project is None:
+ project = read_project()
+ if not project["available"]:
+ return False, project["cause"]
+ try:
+ endpoints = _get("/projects/" + project["id"] + "/endpoints", os.environ.get("NEON_API_KEY", ""))["endpoints"]
+ for endpoint in endpoints:
+ endpoint_id = endpoint.get("id")
+ if _identifier(endpoint_id) and (host.startswith(endpoint_id + ".") or host.startswith(endpoint_id + "-pooler.")):
+ if endpoint.get("current_state") == "active":
+ return True, None
+ return False, "Sample history is pending while the store compute is idle."
+ return False, "Sample history is pending: the store endpoint was not found in this project."
+ except UsageReadError as exc:
+ return False, str(exc)
+ except Exception:
+ return False, "Sample history is pending: the store compute state could not be read."
+
+
+_lock = threading.RLock()
+_thread = None
+_stop = threading.Event()
+_samples = []
+_loaded = False
+_project_id = None
+_alerted = {}
+_status = {"every_s": 0, "last_at": None, "pending": 0, "persisted": None,
+ "cause": "Scheduled sampling is disabled; AIOS_NEON_SAMPLE_SECONDS is unset or zero."}
+
+
+def snapshot():
+ with _lock:
+ return copy.deepcopy(_samples), dict(_status)
+
+
+def _sampler_cause(cause=None):
+ _, config_cause = _alarm_threshold()
+ if config_cause and config_cause not in (cause or ""):
+ return " ".join(item for item in (cause, config_cause) if item)
+ return cause or None
+
+
+def _period_key(value):
+ stamp = _stamp(value)
+ if stamp is None:
+ raise ValueError("Invalid alert period")
+ return stamp.astimezone(timezone.utc).isoformat(timespec="seconds")
+
+
+def _clean_alerted(alerted, period_start, samples):
+ """At most six keys each for the current and previous monthly billing period."""
+ if not isinstance(alerted, dict):
+ raise ValueError("Unreadable alert markers")
+ current = _period_key(period_start)
+ clean = {}
+ for period, keys in alerted.items():
+ period = _period_key(period)
+ if not isinstance(keys, list) or any(not isinstance(key, str) or key not in _ALERT_KEYS for key in keys):
+ raise ValueError("Unreadable alert markers")
+ clean.setdefault(period, set()).update(keys)
+ periods = set(clean) | {_period_key(row["period_start"]) for row in samples}
+ start = _stamp(current)
+ month = start.year * 12 + start.month
+ previous = max((period for period in periods if period < current
+ and month - (_stamp(period).year * 12 + _stamp(period).month) <= 1), default=None)
+ return {period: sorted(clean.get(period, set())) for period in (previous, current) if period}
+
+
+def report():
+ """Card path: management API plus memory only, never the store callbacks."""
+ project = read_project()
+ samples, status = snapshot()
+ status["cause"] = _sampler_cause(status["cause"])
+ return {**summarise(project, samples), "sampler": status}
+
+
+def _sample_once(load, persist, *, sender=None):
+ global _loaded, _samples, _alerted, _project_id
+ project = read_project()
+ if not project["available"]:
+ with _lock:
+ _status["cause"] = _sampler_cause(project["cause"])
+ return
+ sample = {"at": project["read_at"], "transfer_bytes": project[_FIELDS[0]],
+ "compute_seconds": project[_FIELDS[1]], "storage_bytes": project[_FIELDS[2]],
+ "period_start": project["consumption_period_start"]}
+ with _lock:
+ if _project_id != project["id"]:
+ _project_id, _loaded, _samples, _alerted = project["id"], False, [], {}
+ _status.update(pending=0, persisted=None)
+ _samples = _clean_samples([*_samples, sample])
+ _status.update(last_at=sample["at"], pending=min(MAX_SAMPLES, _status["pending"] + 1))
+ awake, cause = store_is_awake(project)
+ if not awake:
+ with _lock:
+ _status["cause"] = _sampler_cause(cause)
+ return
+ try:
+ if not _loaded:
+ saved = load()
+ if saved is None:
+ saved = {}
+ with _lock:
+ # Never mix a saved ring from a different configured project.
+ matching = saved.get("project_id") == project["id"]
+ history = saved.get("samples", []) if matching else []
+ _samples = _clean_samples([*history, *_samples])
+ _alerted = _clean_alerted(saved.get("alerted", {}) if matching else {},
+ sample["period_start"], _samples)
+ _loaded = True
+ except Exception:
+ with _lock:
+ _status["cause"] = _sampler_cause("Sample history and alert markers could not be loaded; alerts deferred.")
+ return
+ sent = False
+ try:
+ # Loading/initializing the runtime may take time. Re-observe immediately
+ # before the write rather than relying on its earlier active state.
+ awake, cause = store_is_awake(project)
+ if not awake:
+ with _lock:
+ _status["cause"] = _sampler_cause(cause)
+ return
+ samples, _ = snapshot()
+ with _lock:
+ _alerted = _clean_alerted(_alerted, sample["period_start"], samples)
+ alerted = copy.deepcopy(_alerted)
+ # Confirm storage works before sending. This also flushes any markers
+ # left in memory by a successful send whose subsequent save failed.
+ persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
+ with _lock:
+ _status.update(pending=0, persisted=_now(), cause=_sampler_cause())
+ summary = summarise(project, samples)
+ alarm = summary["alarm"]
+ period = _period_key(sample["period_start"])
+ crossings = [row for row in alarm["crossings"]
+ if f"{row['metric']}:{row['kind']}" not in alerted[period]] if alarm else []
+ if not crossings:
+ return
+ text = (_alarm_message(summary, crossings, alarm["test"])
+ + f"\nPeriod {period} to {_period_key(summary['period']['end'])} (end exclusive).")
+ try:
+ ok, delivery = (sender or send_telegram)(text)
+ except Exception:
+ ok, delivery = False, "Telegram alert could not be sent."
+ if not ok:
+ with _lock:
+ _status["cause"] = _sampler_cause(delivery)
+ return
+ sent = True
+ with _lock:
+ _alerted[period] = sorted(set(_alerted[period])
+ | {f"{row['metric']}:{row['kind']}" for row in crossings})
+ alerted = copy.deepcopy(_alerted)
+ # Keep successful deliveries in memory even if this save fails. The
+ # next tick flushes them before considering another message.
+ awake, cause = store_is_awake(project)
+ if not awake:
+ with _lock:
+ _status["cause"] = _sampler_cause("Telegram alert sent; alert markers await storage. " + (cause or ""))
+ return
+ persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
+ with _lock:
+ _status.update(persisted=_now(), cause=_sampler_cause())
+ except Exception:
+ with _lock:
+ _status["cause"] = _sampler_cause(
+ "Telegram alert sent; alert markers could not be stored and remain pending in memory." if sent
+ else "Sample history could not be stored; samples and alert markers remain pending in memory.")
+
+
+def start_sampler(*, load=None, persist=None, sender=None):
+ """One daemon per app process; unset/zero is OFF in imports, gates and tests.
+
+ Deploy explicitly installs DEFAULT_SAMPLE_SECONDS. Callbacks are mandatory
+ so this module never imports a database implementation or a tenant runtime.
+ """
+ global _thread
+ try:
+ every_s = int(os.environ.get("AIOS_NEON_SAMPLE_SECONDS", "0"))
+ except ValueError:
+ every_s = 0
+ if every_s <= 0 or load is None or persist is None:
+ return False
+ # Never turn a typo into a tight management-API polling loop.
+ every_s = max(60, every_s)
+ with _lock:
+ if _thread is not None and _thread.is_alive():
+ return True
+ _status.update(every_s=every_s, cause=_sampler_cause("Waiting for the first scheduled sample."))
+ _stop.clear()
+
+ def run():
+ while not _stop.is_set():
+ try:
+ _sample_once(load, persist, sender=sender)
+ except Exception:
+ with _lock:
+ _status["cause"] = _sampler_cause("The scheduled sample could not be completed; it will retry.")
+ if _stop.wait(every_s):
+ break
+
+ _thread = threading.Thread(target=run, name="neon-usage-sampler", daemon=True)
+ _thread.start()
+ return True
+
+
+def stop_sampler():
+ _stop.set()
diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py
index f7494bdcc99bff23462183cbbd6818f34fab7ef5..bb82e8790330ec3d9fa76a02f3ef0b4770d0ac51 100644
--- a/platform/core/perm_scope.py
+++ b/platform/core/perm_scope.py
@@ -51,6 +51,9 @@ import core.perms as perms
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
PERMS_VERSION = 1
+AI_APPS = "ai_apps"
+AI_APPS_OFF = ("AI apps (MCP) is switched off for your account. "
+ "An administrator can switch it on in Settings > Manage users.")
def _rec(user):
@@ -118,6 +121,32 @@ def may_metrics(user, module):
return not (isinstance(e, dict) and e.get('metrics') is False)
+def may_ai_apps(user):
+ """R3/C6: AI apps are a separate capability, granted unless explicitly switched off.
+
+ Like may_metrics, this deliberately differs from may_access: no existing migrated or
+ legacy account could declare this new key. Absence must not revoke everybody's gateway
+ access. Only access: False denies; the in-app Assistant still uses analyst.
+ """
+ if perms.is_admin(user):
+ return True
+ e = entry(user, AI_APPS)
+ return not (e is not None and e.get('access') is False)
+
+
+def ai_apps_refusal(user, runtime):
+ """Return the account or tenant refusal, or None. Shared by OAuth and every tool call.
+
+ Do not route this through Session.require: its literal tenant-list check cannot recognise
+ the existing analyst tenant entitlement that tenant_has_module preserves for this gateway.
+ """
+ if not may_ai_apps(user):
+ return AI_APPS_OFF
+ if not perms.tenant_has_module(runtime, AI_APPS):
+ return "AI apps (MCP) is not available in this workspace."
+ return None
+
+
def nav_may_open(user, key, st=None):
"""May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question.
@@ -136,6 +165,8 @@ def nav_may_open(user, key, st=None):
1. an admin sees everything (break-glass, as everywhere else);
2. an EXPLICIT `access: false` DENIES, on any key — an administrator unticking a box must
take the row off the nav, and before this it did not;
+ R3/C6's ai_apps capability then grants by absence, including migrated and legacy-list
+ accounts. It is an editor surface, not a registry page or a rail entry;
3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` — creator, admin or
a `core.shares` grant — and **an `access: true` entry may NEVER widen past it**. The
editor must not become a way to hand somebody another user's private table;
@@ -149,11 +180,12 @@ def nav_may_open(user, key, st=None):
registry topic and fatal for exactly one key family. Every door that can be handed a
`ut_*` key now threads its session's own runtime: `deps.Session.require`,
`routes_charts._require_database`, `routes_nav.nav` and `routes_tables` (both lending
- the rows-free projection) and `routes_admin.get_perms`. The two callers that still omit
- `st` are `perms.nav_pages`, which iterates `core.registry.REGISTRY` (a compiled literal
+ the rows-free projection) and `routes_admin.get_perms`. The caller that still omits
+ `st` is `perms.nav_pages`, which iterates `core.registry.REGISTRY` (a compiled literal
carrying no `ut_` key — `verify_perm_scope.py` section K pins that, so the fence can go
- red rather than merely being asserted here), and `routes_admin`'s surface DEFAULT, which
- its own `m.get("surface")` test fences to registry rows. Neither can reach this leg;
+ red rather than merely being asserted here). It cannot reach this leg. The editor's
+ surface default now also threads its runtime; its catalogue includes R3/C6's gateway
+ capability as well as registry pages;
4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the
editor IS the authority: `_clean_perms` writes an entry for every governed topic on
@@ -169,8 +201,11 @@ def nav_may_open(user, key, st=None):
if perms.is_admin(user):
return True
e = entry(user, key)
- if e is not None and not bool(e.get('access', True)):
+ if e is not None and (e.get('access') is False if key == AI_APPS
+ else not bool(e.get('access', True))):
return False
+ if key == AI_APPS:
+ return True
if str(key or '').startswith(_ut().KEY_PREFIX):
return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
if e is not None:
@@ -186,12 +221,39 @@ def assistant_entry(user, module):
that reader must be able to name the migrated grant that admitted a database. In particular,
a store-outage admin identity with no ``perms`` document is not an unresolved permission that
may be widened into data access.
+
+ ⭐ S67-T34 / D-276: ONE NAMED EXCEPTION, the grant the NAVIGATION uses. For the two registry
+ topics in ``ASSISTANT_NAV_TOPICS`` an administrator account is admitted unfiltered (unless
+ explicitly unticked), because ``nav_may_open`` leg 1 opens them to it; the AI list must equal
+ the nav (R33). See the comment at that leg for why the outage identity no longer reaches it.
"""
e = entry(user, module)
- if (not is_migrated(user) or not isinstance(e, dict)
- or not bool(e.get('access', True)) or not may_access(user, module)):
- return None
- return e
+ if (is_migrated(user) and isinstance(e, dict)
+ and bool(e.get('access', True)) and may_access(user, module)):
+ return e
+ # ⭐⭐ S67-T34 / D-276 / R26: THE ADMINISTRATOR THE NAVIGATION ADMITS. `nav_may_open` leg 1
+ # opens every registry topic to an administrator, and `_clean_perms` writes no per-topic
+ # entry for one, so an administrator saw `customer_data` and `product_data` (Odoo products,
+ # R26's Inventory source) in the nav and had them refused here. MEASURED on staging
+ # 2026-09-28: aaron (admin, no `perms_v`) had both in the nav and neither in the AI list. The
+ # answer is the grant the nav already uses, admitted for exactly these topics: an account
+ # whose ROLE is admin, minus an explicit `access: false` (narrower than the nav, never wider).
+ # Unfiltered, as the nav's break-glass is.
+ # ⚠ THE STORE-OUTAGE IDENTITY the docstring names is no longer built by `deps._user_for`
+ # (it admits a stored record or the QA runner, nothing else), so the role is read off a
+ # stored account here. If a synthetic admin identity is ever reintroduced, it must be kept
+ # out of this leg.
+ # ⛔ NOT A MEMBER. A non-administrator still needs the explicit migrated grant above, so a
+ # user without it does not see the database (the NC).
+ if (module in ASSISTANT_NAV_TOPICS and perms.is_admin(user)
+ and not (isinstance(e, dict) and e.get('access') is False)):
+ return dict(e) if isinstance(e, dict) else {'access': True, 'filter': None}
+ return None
+
+
+#: S67-T34: the registry topics the navigation opens to an administrator by break-glass, which
+#: `assistant_entry` therefore admits for a migrated administrator without a per-topic entry.
+ASSISTANT_NAV_TOPICS = frozenset({'customer_data', 'product_data'})
# ── FIELDS ───────────────────────────────────────────────────────────────────────────────────
@@ -1672,7 +1734,9 @@ def _read_user_table(table_key, st, want_rows=True):
'reading a read-through database, or read it through the API'))
keys = {f['key'] for f in fields if f.get('key')}
return fields, list(_MIRROR_READER(table_key, keys, st) or ())
- whole = ut.get(table_key, st=st)
+ # HOTFIX 2026-09-28: this table's entry alone where the store can (`get_one`), not a deep
+ # copy of every table in the tenant. Amazon link cells read listings here on every page.
+ whole = ut.get_one(table_key, st=st)
if whole is None:
# Deleted between the wall and here. The same refusal, not an empty table.
raise UnknownTable(f"no database named '{table_key}'")
diff --git a/platform/core/perms.py b/platform/core/perms.py
index 325196813dfa53dbbf0212c29fd13194edfaddb5..f7486c683a71c597408a48be3f5662f1eee0fc71 100644
--- a/platform/core/perms.py
+++ b/platform/core/perms.py
@@ -253,6 +253,10 @@ def tenant_has_module(runtime, key):
allowed = tenant_modules(runtime)
if allowed is None or key in allowed:
return True
+ # R3/C6: tenants provisioned with Assistant already had the external gateway. Its new
+ # account-level grant must not remove that entitlement from existing tenant catalogues.
+ if key == 'ai_apps':
+ return 'analyst' in allowed
parent = registry.BY_KEY.get(key, {}).get('parent')
return parent in allowed if parent else False
@@ -336,17 +340,22 @@ def governable_surface_modules():
filter or hiddenFields against an empty vocabulary, so a client cannot store a rule nothing
would apply.
- ⭐ DERIVED, NEVER LISTED: every non-archived, non-`group_only` registry row with NO
+ Page surfaces are DERIVED: every non-archived, non-`group_only` registry row with NO
`topic` that `nav_pages` would offer (not `validate_only`, not `api_surface: False`, not
`settings`). Today that is `analyst` (Assistant) and `automation` (Agents); a surface added
to the registry tomorrow is governable with no edit here — the same rule
`tenant_governable_modules` states for databases, applied to the other kind of row.
+
+ R3/C6 adds one listed capability: ai_apps, labelled AI apps (MCP). It governs the external
+ gateway, not a page, so it belongs in the account editor without entering REGISTRY or the
+ navigation rail. registry.py is outside this ticket's fence. Both consumers here are editor
+ helpers (the catalogue and empty field schema); neither turns this row into navigation.
"""
return [{'key': m['key'], 'label': m['label']}
for m in registry.REGISTRY
if not m.get('archived') and not m.get('group_only') and not m.get('topic')
and not m.get('validate_only') and m.get('api_surface') is not False
- and m['key'] != 'settings']
+ and m['key'] != 'settings'] + [{'key': 'ai_apps', 'label': 'AI apps (MCP)'}]
#: ⛔ `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
diff --git a/platform/core/registry.py b/platform/core/registry.py
index ae3c90ed139055dca5823c8d1b6c4b4b95af3310..ad4901aeb538a3ddb2752d5f474c8cf52e5bc8e6 100644
--- a/platform/core/registry.py
+++ b/platform/core/registry.py
@@ -203,7 +203,12 @@ REGISTRY = [
'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
'them") made that literal: `validate_only` takes it out of the account menu too, '
'which is the last place it was still visible. The proof survives; the door does not.'},
+ # S67-T36 (owner instr 24, R25, 2026-09-28): Agents is TABLED. `archived` hides the UI only:
+ # no rail row, no Settings module toggle, no Home section, no agent templates. The
+ # `/automations` APIs, stored automations and their schedules are untouched and keep running.
+ # ⭐ RESTORING IS THIS ONE LINE: delete `'archived': True` below and every door comes back.
{'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
+ 'archived': True,
'nav': True,
'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
'(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
@@ -280,6 +285,11 @@ def brand_filterable(key) -> bool:
return bool(BY_KEY.get(key, {}).get('brand'))
+def is_archived(key) -> bool:
+ """True when the registry row `key` is archived (tabled for everyone). Unknown keys: False."""
+ return bool(BY_KEY.get(key, {}).get('archived'))
+
+
# ═════════════════════════════════════════════════════════════════════════════════════════════
# ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE
# ═════════════════════════════════════════════════════════════════════════════════════════════
diff --git a/platform/core/sheet_binding.py b/platform/core/sheet_binding.py
index 9dbbe8dafe122af327512c95a3d5bc2cacab5008..511b84471bf6465726e67aee02f3e84ecebe08f4 100644
--- a/platform/core/sheet_binding.py
+++ b/platform/core/sheet_binding.py
@@ -163,11 +163,13 @@ class BindingMetadataRefusal(Exception):
def clean_related(raw):
- """One declared edge key and a structurally valid, losslessly copied filter tree.
+ """Read the legacy edge bag for its one-release migration to ordinary filter leaves.
Absence is handled by the caller. An explicitly supplied null is malformed, as are
extra relationship bags and nested relationships. Operator support belongs to the
query compiler, which reports unpushable conditions rather than dropping metadata.
+ The binding cleaner qualifies this copied tree and intersects it with the local scope;
+ neither this reader nor that rewrite mutates the saved document.
"""
def refuse():
raise BindingMetadataRefusal("related_invalid")
@@ -214,6 +216,63 @@ def clean_related(raw):
return {"via": via.strip(), "filter": deepcopy(tree)}
+def rewrite_legacy_related(binding, *, linked=False):
+ """Copy a binding, moving its legacy bag into the ordinary filter tree.
+
+ This is a lossless metadata rewrite, not a binding cleaner or a relationship lookup.
+ The scoped resolver still proves every named edge. Migration may also replace the
+ linked spelling with its explicit bridge identity; normal reads leave those aliases
+ to the API's permission-scoped declaration lookup.
+ """
+ out = deepcopy(binding)
+ if "related" in out:
+ legacy = clean_related(out.pop("related"))
+
+ def parts(tree):
+ if isinstance(tree, dict):
+ nodes = tree.get("nodes")
+ return (nodes if isinstance(nodes, list) else []), (
+ "or" if tree.get("conj") == "or" else "and")
+ return (tree if isinstance(tree, list) else []), "and"
+
+ def qualify(node):
+ if isinstance(node.get("children"), list):
+ return {**node, "children": [qualify(child) for child in node["children"]]}
+ leaf = {**node, "colId": f"related:{legacy['via']}:{node['colId']}"}
+ if isinstance(node.get("rhs"), dict) and node["rhs"].get("colId"):
+ leaf["rhs"] = {**node["rhs"],
+ "colId": f"related:{legacy['via']}:{node['rhs']['colId']}"}
+ return leaf
+
+ scope = out.setdefault("scope", {})
+ if not isinstance(scope, dict):
+ raise BindingMetadataRefusal("related_invalid")
+ nodes, conj = parts(legacy["filter"])
+ original, original_conj = parts(scope.get("filter"))
+ original = ([{"conj": original_conj, "children": original}]
+ if original and original_conj == "or" else list(original))
+ scope["filter"] = {"conj": "and", "nodes": [*original,
+ {"conj": conj, "children": [qualify(node) for node in nodes]}]}
+
+ def rewrite_links(value):
+ if isinstance(value, dict):
+ key = value.get("colId")
+ if isinstance(key, str) and key.startswith("linked:"):
+ parts = key.split(":")
+ if len(parts) != 3 or not all(parts[1:]):
+ raise BindingMetadataRefusal("related_invalid", key)
+ value["colId"] = "related:" + key[len("linked:"):]
+ for child in value.values():
+ rewrite_links(child)
+ elif isinstance(value, list):
+ for child in value:
+ rewrite_links(child)
+
+ if linked:
+ rewrite_links(out)
+ return out
+
+
def clean_period(raw):
"""Return one canonical Sheet period, or ``None`` when no window was named."""
if not isinstance(raw, dict):
diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py
index 1bc4d6ce5726f9cf67e7db286b78602b3944eeb1..ec057f106cf47a08b0f589b77aee9b29ce004ce6 100644
--- a/platform/core/sheet_import.py
+++ b/platform/core/sheet_import.py
@@ -43,9 +43,11 @@ from __future__ import annotations
import re
import zipfile
import hashlib
+import io
import json
import math
import os
+import posixpath
import tempfile
import xml.etree.ElementTree as ET
from datetime import date, datetime, time, timezone
@@ -392,12 +394,11 @@ def style_of(cell, notes):
if border is not None:
sides = {}
for side in sheets._BORDER_SIDES:
- edge = getattr(border, side, None)
- if edge is not None and getattr(edge, "style", None):
- sides[side] = True
+ line = border_line(getattr(border, side, None), notes)
+ if line is not None:
+ sides[side] = line
if sides:
out["border"] = sides
- notes.add("a border keeps its side but not its weight, style or colour")
fmt, note = classify_format(getattr(cell, "number_format", ""))
if fmt:
@@ -407,6 +408,47 @@ def style_of(cell, notes):
return out
+#: S67-T14: the OOXML line styles that are not one of C2's seven, each drawn as its nearest.
+#: The dash and dot variants keep their PATTERN over their weight, because a patterned line drawn
+#: solid reads as a different border while a medium dash drawn at 1px still reads as dashed.
+_BORDER_NEAREST = {
+ "mediumDashed": "dashed", "dashDot": "dashed", "dashDotDot": "dashed",
+ "mediumDashDot": "dashed", "mediumDashDotDot": "dashed", "slantDashDot": "dashed",
+}
+
+
+def border_line(edge, notes):
+ """One openpyxl border side as C2's `{style, color}`, or None when the side draws nothing.
+
+ The seven Excel names C2 shares pass through unchanged; the dash dot variants become their
+ nearest with a note naming the approximation. The colour is the explicit RGB when there is one;
+ an automatic colour is black, which is what Excel draws; a theme or indexed colour is drawn in
+ black too, with a note, because a colour nobody chose is never guessed (see `argb_of`).
+ """
+ style = str(getattr(edge, "style", None) or "") if edge is not None else ""
+ if not style or style == "none":
+ return None
+ if style not in sheets.CELL_BORDER_STYLES:
+ nearest = _BORDER_NEAREST.get(style, "thin")
+ notes.add("a " + style + " border is drawn as " + nearest)
+ style = nearest
+ colour = getattr(edge, "color", None)
+ argb = argb_of(colour)
+ if argb:
+ hex_colour = "#" + argb.lower()
+ else:
+ hex_colour = "#000000"
+ kind = getattr(colour, "type", "") if colour is not None else ""
+ # ⚠ `type`, never `.auto`: openpyxl answers an unset descriptor with a (truthy) error
+ # STRING, so `colour.auto` reads as set on every theme colour.
+ automatic = (colour is None or kind == "auto"
+ or (kind == "indexed" and getattr(colour, "indexed", None) == 64)
+ or (kind == "rgb" and str(getattr(colour, "rgb", "")).upper() == "00000000"))
+ if not automatic:
+ notes.add("a theme or indexed border colour was drawn in black")
+ return {"style": style, "color": hex_colour}
+
+
# ── values ───────────────────────────────────────────────────────────────────────────────────
def json_value(value):
@@ -1037,6 +1079,134 @@ def guard_import_archive(path):
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
+# ── sparklines (S67-T18) ─────────────────────────────────────────────────────────────────────
+
+#: ⭐ S67-T18: Excel keeps sparkline groups in the worksheet's x14 extension, which openpyxl drops,
+#: so they are read from the worksheet PART itself. A group with no `type` draws lines, `column`
+#: draws columns and `stacked` draws win/loss bars (Excel's name for `sheets.SPARKLINE_KINDS`'s
+#: `winloss`). The part was already admitted by `guard_import_archive`, so this second read adds
+#: no new exposure.
+_X14 = "{http://schemas.microsoft.com/office/spreadsheetml/2009/9/main}"
+_XM = "{http://schemas.microsoft.com/office/excel/2006/main}"
+_MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+_DOC_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
+_PKG_REL_NS = "{http://schemas.openxmlformats.org/package/2006/relationships}"
+_SPARKLINE_KIND_OF = {"line": "line", "column": "column", "stacked": "winloss"}
+
+
+def _worksheet_parts(archive):
+ """`{tab name: worksheet part}` through `xl/workbook.xml` and its relationships, never by index."""
+ names = set(archive.namelist())
+ if "xl/workbook.xml" not in names or "xl/_rels/workbook.xml.rels" not in names:
+ return {}
+ targets = {}
+ for rel in ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")).iter(_PKG_REL_NS + "Relationship"):
+ target = str(rel.get("Target") or "")
+ targets[rel.get("Id")] = (target.lstrip("/") if target.startswith("/")
+ else posixpath.normpath("xl/" + target))
+ out = {}
+ for sheet in ET.fromstring(archive.read("xl/workbook.xml")).iter(_MAIN_NS + "sheet"):
+ part = targets.get(sheet.get(_DOC_REL_NS + "id"))
+ if part in names:
+ out[str(sheet.get("name") or "")] = part
+ return out
+
+
+def _sparkline_groups(stream):
+ """`[(type, [(formula, sqref), ...]), ...]` for one worksheet part's x14 sparkline groups."""
+ groups = []
+ inside = 0 # a group's subtree is kept until the group itself ends
+ for event, element in ET.iterparse(stream, events=("start", "end")):
+ if element.tag == _X14 + "sparklineGroup":
+ if event == "start":
+ inside += 1
+ continue
+ inside -= 1
+ entries = [((spark.findtext(_XM + "f") or "").strip(),
+ (spark.findtext(_XM + "sqref") or "").strip())
+ for spark in element.iter(_X14 + "sparkline")]
+ groups.append((str(element.get("type") or "line"), entries))
+ element.clear()
+ elif event == "end" and not inside:
+ element.clear()
+ return groups
+
+
+def _sparklines_of(path, notes):
+ """`{tab name: groups}` for every worksheet that carries x14 sparklines; `{}` for the rest.
+
+ A part whose bytes never mention a sparkline group is not parsed at all, so a workbook with
+ no sparklines imports exactly as it did before S67-T18.
+ """
+ out = {}
+ try:
+ with zipfile.ZipFile(path) as archive:
+ for name, part in _worksheet_parts(archive).items():
+ raw = archive.read(part)
+ if b"sparklineGroup" not in raw:
+ continue
+ groups = _sparkline_groups(io.BytesIO(raw))
+ if groups:
+ out[name] = groups
+ except (KeyError, OSError, ET.ParseError, zipfile.BadZipFile):
+ notes.add("the workbook's sparklines could not be read, so none were imported")
+ return {}
+ return out
+
+
+def _count_phrase(count, one, many):
+ return (one if count == 1 else str(count) + " " + many)
+
+
+def _apply_sparklines(tab_name, cells, groups, blank_styles, notes):
+ """Write `cell.sparkline {kind, range}` on each location cell of this tab's groups.
+
+ `sheets._clean_sparkline` stores a rectangle LOCAL to its tab, so a sparkline drawn from
+ another tab's data cannot be carried: it is left out and NAMED, never dropped in silence
+ (the module note). A location that is not one cell, or data that is not a range, is named too.
+ A blank location becomes an `empty` cell holding only the sparkline, which is exactly what
+ `clean_cell`'s empty branch keeps; its compact blank-style reference moves onto it.
+ """
+ other_tab = unreadable = 0
+ for kind_name, entries in groups:
+ kind = _SPARKLINE_KIND_OF.get(kind_name)
+ if kind is None:
+ unreadable += len(entries)
+ continue
+ for formula, sqref in entries:
+ source = sheet_of_ref(formula)
+ if source and source.casefold() != tab_name.casefold():
+ other_tab += 1
+ continue
+ rect = rect_of(formula)
+ places = sqref.replace("$", "").upper().split()
+ if (rect is None or len(places) != 1 or ":" in places[0]
+ or not _address_in_document(places[0])):
+ unreadable += 1
+ continue
+ addr = places[0]
+ spark = {"kind": kind, "range": rect}
+ cell = cells.get(addr)
+ if cell is None:
+ if len(cells) >= sheets._MAX_CELLS_PER_SHEET:
+ unreadable += 1
+ continue
+ cell = {"kind": "empty", "sparkline": spark}
+ if addr in blank_styles:
+ cell["excel"] = {"styleId": blank_styles.pop(addr)}
+ cells[addr] = cell
+ else:
+ cell["sparkline"] = spark
+ if other_tab:
+ notes.add(_count_phrase(other_tab, "a sparkline", "sparklines")
+ + " drawn from another tab's data " + ("was" if other_tab == 1 else "were")
+ + " left out")
+ if unreadable:
+ notes.add(_count_phrase(unreadable, "a sparkline", "sparklines")
+ + " with a location or data range that could not be read "
+ + ("was" if unreadable == 1 else "were") + " left out")
+
+
def _print_of(ws):
"""Store print setup and inclusive print areas; export executes these settings later."""
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
@@ -1148,6 +1318,7 @@ def import_workbook(path, *, source="", progress=None):
counts = {"cells": 0, "formulas": 0}
book_notes = set()
+ sparklines = _sparklines_of(path, book_notes) # S67-T18
authored = _authored_chart_document(path, book_notes)
chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
if authored and len(chart_tabs) != len(authored["sheets"]):
@@ -1209,6 +1380,8 @@ def import_workbook(path, *, source="", progress=None):
cells = _cells_of(ws, cached, notes, counts, excel)
_say("formats", 0.7 + 0.25 * (index / total))
_validations_of(ws, cells, notes)
+ if name in sparklines: # S67-T18, before usedExtent
+ _apply_sparklines(name, cells, sparklines[name], excel["blankStyles"], notes)
cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
"cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
diff --git a/platform/core/sheet_structure.py b/platform/core/sheet_structure.py
index 0b22abddfa84552de34cf00cd545516c9d0ec1af..39495d18a91b1bb74e73fd22810e68e985c12ade 100644
--- a/platform/core/sheet_structure.py
+++ b/platform/core/sheet_structure.py
@@ -11,7 +11,8 @@ import math
import re
from collections.abc import Callable
-from core.sheets import SheetRefusal
+from core.sheets import (DEFAULT_SHEET_COLS, DEFAULT_SHEET_ROWS, MAX_SHEET_COLS,
+ MAX_SHEET_ROWS, SheetRefusal)
REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
@@ -162,8 +163,7 @@ def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool =
return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
- # Insertions keep the existing per-reference scan; range handling is deletion-only.
- return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None)
+ return rewrite_refs(text, rewrite, rewrite_range)
def _moved_index(value: int, at: int, delta: int) -> int | None:
@@ -185,6 +185,11 @@ def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool:
if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
changed = value[key] != child or changed
+ elif key in BINDING_REF_KEYS and isinstance(child, str) and child.strip().startswith("="):
+ # HOTFIX 2026-09-29: a filter value slot may hold a FORMULA (`=DATE($E$24,1,1)`);
+ # its references follow a row or column edit exactly as a formula cell's do.
+ value[key] = shift_for_structure(child, shift, own_sheet)
+ changed = value[key] != child or changed
elif isinstance(child, (dict, list)):
changed = _reanchor_binding(child, shift, own_sheet) or changed
return changed
@@ -206,7 +211,6 @@ def _reanchor_cells(tab: dict, shift: dict) -> None:
href = link.get("href")
if isinstance(href, str) and parse_ref(href):
link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
- changed = link["href"] != href or changed
if changed and cell.get("kind") in ("formula", "bound"):
cell["state"] = "stale"
@@ -239,6 +243,28 @@ def _move_tab(tab: dict, op: dict) -> None:
continue
moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
if moved is not None:
+ binding = cell.get("binding") or {}
+ if (cell.get("kind") == "bound" and binding.get("mode") == "view"
+ and binding.get("viewType") != "grid" and isinstance(binding.get("block"), dict)):
+ block = binding["block"]
+ size = {key: value if type(value := block.get(key)) is int and value > 0 else fallback
+ for key, fallback in (("cols", 8), ("rows", 20))}
+ key = "rows" if axis == "row" else "cols"
+ value = ref["row"] if axis == "row" else col_to_index(ref["col"])
+ end = value + size[key]
+ change = (delta if value < at < end else 0) if delta > 0 else -max(
+ 0, min(end, at - delta) - max(value, at))
+ if change:
+ binding["block"] = {**size, key: max(1, size[key] + change)}
+ child = cell.get("viewChild")
+ origin = parse_ref(child.get("origin")) if isinstance(child, dict) else None
+ if origin:
+ index = _moved_index(origin["row"] if axis == "row" else col_to_index(origin["col"]), at, delta)
+ if index is None:
+ cell.pop("viewChild", None)
+ else:
+ origin[axis] = index if axis == "row" else index_to_col(index)
+ child["origin"] = format_ref(origin)
ref[axis] = moved if axis == "row" else index_to_col(moved)
cells[format_ref(ref)] = cell
tab["cells"] = cells
@@ -262,26 +288,78 @@ def _move_tab(tab: dict, op: dict) -> None:
if "rules" in tab:
rules = []
for rule in tab["rules"]:
- rect = _move_range(rule["range"], op, drop_deleted=True)
+ rect = rule["range"]
+ if isinstance(rect, str):
+ rect = shift_for_structure(rect, {**op, "sheet": tab["name"]}, tab["name"])
+ if rect == "#REF!":
+ continue
+ else:
+ rect = _move_range(rect, op, drop_deleted=True)
if rect is not None:
rules.append({**rule, "range": rect})
tab["rules"] = rules
+ if axis == "row" and "rowGroups" in tab:
+ groups = []
+ for group in tab["rowGroups"]:
+ rect = _move_range({"row0": group["start"] - 1, "row1": group["end"] - 1},
+ op, drop_deleted=True)
+ if rect is not None:
+ groups.append({**group, "start": rect["row0"] + 1, "end": rect["row1"] + 1})
+ if groups:
+ tab["rowGroups"] = groups
+ else:
+ tab.pop("rowGroups", None)
+
+
+def _move_chart_range(rect: dict, op: dict, extent: dict) -> dict:
+ """Match types.ts::chartRangeAfterStructure, including its viewport clamp."""
+ start, end = op["axis"] + "0", op["axis"] + "1"
+ lo, hi = max(0, min(rect[start], rect[end])), max(0, max(rect[start], rect[end]))
+ moved = _move_range({**rect, start: lo, end: hi}, op)
+ lo, hi = moved[start], moved[end]
+ ceiling = max(0, extent["rows" if op["axis"] == "row" else "cols"] - 1)
+ if hi > ceiling:
+ lo, hi = max(0, lo - (hi - ceiling)), ceiling
+ return {**moved, start: min(lo, ceiling), end: min(max(lo, hi), ceiling)}
-def _move_charts(tab: dict, tab_id: str, op: dict) -> None:
+def _move_charts(tab: dict, tab_id: str, op: dict, extent: dict) -> None:
for chart in tab.get("charts", []):
source = chart.get("source") or {}
+ previous = chart.get("geometry") or {}
+ geometry = {"col": previous.get("col", min(chart["range"]["col0"], chart["range"]["col1"])),
+ "row": previous.get("row", min(chart["range"]["row0"], chart["range"]["row1"])),
+ "width": previous.get("width", 640), "height": previous.get("height", 320)}
+ if tab["id"] == tab_id:
+ point = _move_chart_range({"col0": geometry["col"], "col1": geometry["col"],
+ "row0": geometry["row"], "row1": geometry["row"]}, op, extent)
+ if point["col0"] != geometry["col"] or point["row0"] != geometry["row"]:
+ chart["geometry"] = {**geometry, "col": point["col0"], "row": point["row0"]}
+ anchor = chart.get("anchor")
+ if isinstance(anchor, dict) and op["axis"] in anchor:
+ moved = shift_index(anchor[op["axis"]] + 1, op["at"], op["delta"])
+ anchor[op["axis"]] = (op["at"] if moved is None else moved) - 1
if source.get("sheetId", tab["id"]) == tab_id:
- chart["range"] = _move_range(chart["range"], op)
+ rect = _move_chart_range(chart["range"], op, extent)
+ if (tab["id"] != tab_id and "geometry" not in chart
+ and (rect["col0"] != chart["range"]["col0"] or rect["row0"] != chart["range"]["row0"])):
+ chart["geometry"] = geometry
+ chart["range"] = rect
if "ranges" in source:
- source["ranges"] = [_move_range(rect, op) for rect in source["ranges"]]
- if tab["id"] == tab_id:
- # Stored geometry anchors are zero-based too; pixel offsets and sizes stay intact.
- for key in ("geometry", "anchor"):
- geometry = chart.get(key)
- if isinstance(geometry, dict) and op["axis"] in geometry:
- moved = shift_index(geometry[op["axis"]] + 1, op["at"], op["delta"])
- geometry[op["axis"]] = (op["at"] if moved is None else moved) - 1
+ source["ranges"] = [_move_chart_range(rect, op, extent) for rect in source["ranges"]]
+ presentation = chart.get("presentation") or {}
+ if "series" in presentation:
+ series = []
+ for setting in presentation["series"]:
+ match = re.fullmatch(r"(column|row):([A-Z]+|\d+)", setting["key"])
+ if match and ("col" if match[1] == "column" else "row") == op["axis"]:
+ index = col_to_index(match[2]) if match[1] == "column" else int(match[2])
+ moved = _moved_index(index, op["at"], op["delta"])
+ if moved is None:
+ continue
+ setting["key"] = match[1] + ":" + (index_to_col(moved) if match[1] == "column" else str(moved))
+ series.append(setting)
+ presentation["series"] = series
def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
@@ -308,9 +386,31 @@ def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
if not op["delta"]:
return result
shift = {**op, "sheet": target["name"]}
+ extent = {"cols": min(MAX_SHEET_COLS, max(DEFAULT_SHEET_COLS, target.get("cols", DEFAULT_SHEET_COLS))),
+ "rows": min(MAX_SHEET_ROWS, max(DEFAULT_SHEET_ROWS, target.get("rows", DEFAULT_SHEET_ROWS)))}
for tab in result["sheets"]:
_reanchor_cells(tab, shift)
- _move_charts(tab, tab_id, op)
+ _move_charts(tab, tab_id, op, extent)
+ for rule in tab.get("rules", []):
+ if rule.get("kind") == "formula":
+ for key in ("value", "value2"):
+ if isinstance(rule.get(key), str):
+ rule[key] = shift_for_structure(rule[key], shift, tab["name"])
if tab["id"] == tab_id:
_move_tab(tab, op)
+ names = result.get("namedRanges") or []
+ # The lossless document accepts the older name -> ref dictionary as well as named rows.
+ if isinstance(names, dict):
+ for name, value in names.items():
+ if isinstance(value, str):
+ names[name] = shift_for_structure(value, shift, result["sheets"][0]["name"])
+ names = [value for value in names.values() if isinstance(value, dict)]
+ for named_range in names:
+ scope = named_range.get("sheet")
+ own_sheet = next((tab["name"] for tab in result["sheets"]
+ if tab["id"] == scope or tab["name"] == scope),
+ scope if scope is not None else result["sheets"][0]["name"])
+ for key in ("ref", "base"):
+ if isinstance(named_range.get(key), str):
+ named_range[key] = shift_for_structure(named_range[key], shift, own_sheet)
return result
diff --git a/platform/core/sheets.py b/platform/core/sheets.py
index 097b64011319254cdc743bc8596042c64f7288b7..62d9c4584e22dbd8352f2833cd0098a58a2e08bd 100644
--- a/platform/core/sheets.py
+++ b/platform/core/sheets.py
@@ -152,6 +152,14 @@ MAX_FONT_PT = 96
_STYLE_MARKS = ("bold", "italic", "underline")
_BORDER_SIDES = ("top", "right", "bottom", "left")
+#: ⭐⭐ S67-T14 / CONTRACT C2: Excel's seven border line styles, name for name. Mirrored in
+#: `aios-web/web/src/sheet/types.ts::CELL_BORDER_STYLES`, and `verify_sheet.py` compares the two
+#: lists. A side is either the legacy `True` (read as a thin black line and stored unchanged, so a
+#: document written before S67 reads back byte for byte) or `{"style": ,
+#: "color": "#rrggbb"}`.
+CELL_BORDER_STYLES = ("thin", "medium", "thick", "dashed", "dotted", "double", "hair")
+_BORDER_COLOUR = re.compile(r"^#[0-9a-f]{6}$")
+
#: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
#: outlives every layout change, exactly as a stored hex outlives a palette.
_MAX_INDENT = 8
@@ -204,6 +212,12 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
#: every rule written before this ticket already is, so writing the word onto them would make a
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
+#: ⭐ S67-T13: Excel's colour scale point types, one list for both ends. `lowest` / `highest` are
+#: measured from the range at paint, so the scale re-spreads as the data changes; the other three
+#: read their number from the rule's `value` (low end) or `value2` (high end). A type outside this
+#: list is REFUSED by name, never dropped: a dropped type reads back as a different scale. Mirrored
+#: in `web/src/sheet/types.ts::SheetRuleScaleType`.
+SHEET_RULE_SCALE_TYPES = ("lowest", "highest", "number", "percent", "percentile")
# The client compiler accepts 500 characters per condition.
MAX_RULE_FORMULA_LENGTH = 500
@@ -267,10 +281,8 @@ _MAX_UNSUPPORTED = 200
CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
_CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
-#: R2 / C1: CELL `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard stays on
-#: `_clean_rule_style` and `_clean_scale`: their RULE_FILLS / RULE_INKS galleries remain token
-#: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
-#: the conditional-format palettes or make either guard dead.
+#: Cell and rule `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard now serves
+#: `_clean_rule_scale` only; the scale's existing storage allowance is unchanged.
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
_MAX_DECIMALS = 10
@@ -654,13 +666,44 @@ def _clean_style(raw):
border = raw.get("border")
if isinstance(border, dict):
- sides = {side: True for side in _BORDER_SIDES if border.get(side) is True}
+ sides = {}
+ for side in _BORDER_SIDES:
+ line = _clean_border_side(border.get(side))
+ if line is not None:
+ sides[side] = line
if sides:
out["border"] = sides
return out or None
+def _clean_border_side(raw):
+ """One C2 border side: `True`, a `{style, color}` dict, or None when the side is absent.
+
+ ⛔⛔ THE SECOND DOCUMENTED EXCEPTION TO "UNKNOWN KEYS ARE DROPPED IN SILENCE". A border side
+ that names a line style outside the seven, or a colour that is not a six digit hex, is
+ REFUSED by name. Dropping it would store a cell with no line where the person drew one, and
+ the caller could not learn its request went nowhere: `_clean_format`'s reason, one key over.
+ A side that is `False`, None or absent is simply no side. A legacy `True` stays `True`.
+ """
+ if raw is None or raw is False:
+ return None
+ if raw is True:
+ return True
+ if not isinstance(raw, dict):
+ return None
+ style = raw.get("style")
+ colour = raw.get("color")
+ if isinstance(colour, str):
+ colour = colour.strip().lower()
+ if (not isinstance(style, str) or style not in CELL_BORDER_STYLES
+ or not isinstance(colour, str) or not _BORDER_COLOUR.match(colour)):
+ raise SheetRefusal(
+ "cell_border_invalid",
+ "A border needs a line style such as thin and a six digit hex colour such as #000000.")
+ return {"style": style, "color": colour}
+
+
# ── the cell ─────────────────────────────────────────────────────────────────────────────────
def _clean_validation(raw):
@@ -1100,9 +1143,22 @@ def _clean_rule_style(raw):
return {}
out = {}
for key in _RULE_STYLE_COLOURS:
- token = _text(raw.get(key), 40)
- if token and _COLOUR.match(token):
- out[key] = token
+ token = raw.get(key)
+ if token is None or token == "":
+ continue
+ if isinstance(token, str):
+ token = token.strip()
+ if not token:
+ continue
+ if token.startswith("#"):
+ token = token.lower()
+ # Validate the whole value before normalising: neither truncation nor the cell
+ # cleaner's three digit import rescue may turn an invalid rule colour into a choice.
+ if not isinstance(token, str) or not _CELL_COLOUR.fullmatch(token):
+ raise SheetRefusal(
+ "sheet_rule_colour_invalid",
+ "A rule colour must be a palette colour or a six digit hex colour such as #c8a24b.")
+ out[key] = _normalise_colour(token)
for mark in _RULE_STYLE_MARKS:
if raw.get(mark) is True:
out[mark] = True
@@ -1114,21 +1170,37 @@ def _clean_rule_scale(raw):
⛔ BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would
leave the client looking up a pair that cannot match and falling back to a default scale the
- person never chose. The tokens go through the same `_COLOUR` guard `_clean_rule_style` uses,
- so nothing free form reaches a canvas `fillStyle` from a stored document.
+ person never chose. The tokens keep their existing `_COLOUR` guard, so nothing free form
+ reaches a canvas `fillStyle` from a stored document.
⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
`_clean_rules` already persists for every op that takes operands, so a colour scale needed no
new numeric slot at all.
+
+ ⭐ S67-T13: `minType` / `maxType` say what each end MEANS (`SHEET_RULE_SCALE_TYPES`). Each is
+ kept only when present and never injected, so a stored boundless scale reads back byte for
+ byte, and input with no types returns exactly the `{low, high}` it always did. An unknown type
+ is refused by name.
"""
if not isinstance(raw, dict):
return None
+ types = {}
+ for key in ("minType", "maxType"):
+ kind = raw.get(key)
+ if kind is None or kind == "":
+ continue
+ if kind not in SHEET_RULE_SCALE_TYPES:
+ raise SheetRefusal(
+ "sheet_rule_scale_type_unknown",
+ "A colour scale end must be one of " + ", ".join(SHEET_RULE_SCALE_TYPES) + ".")
+ types[key] = kind
out = {}
for key in ("low", "high"):
token = _text(raw.get(key), 40)
if not token or not _COLOUR.match(token):
return None
out[key] = token
+ out.update(types)
return out
@@ -1699,10 +1771,31 @@ def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""):
tab["writes"] = writes[-WRITE_LOG_REVS:]
+def _read_workbook_doc(st, key):
+ """A workbook document, `{}` when it is absent, and a REFUSAL when the store could not answer.
+
+ HOTFIX 2026-09-29: `_read` swallows every store error into `{}`, so an overloaded database
+ (staging right after a restart: a pool timeout, a 503 on the next request) was reported as
+ "That workbook no longer exists." and the browser dropped the owner's open tab. A store that
+ fails is not a workbook that is gone; the reader is told to try again and nothing is dropped.
+ """
+ try:
+ projected = getattr(st, "get_projection", None)
+ if callable(projected):
+ return dict(projected(key, drop=()) or {})
+ return dict(st.get(key) or {})
+ except (KeyError, FileNotFoundError):
+ return {}
+ except Exception as exc: # noqa: BLE001
+ raise SheetRefusal("workbook_unavailable",
+ "This workbook could not be read just now, so nothing was changed. "
+ "Try again in a moment.") from exc
+
+
def get_workbook(st, workbook_id):
"""One workbook, or a named refusal when the document is not there."""
wid = _slug_id(workbook_id)
- doc = _read(st, workbook_key(wid))
+ doc = _read_workbook_doc(st, workbook_key(wid))
if not doc or not doc.get("id"):
raise SheetRefusal("workbook_missing",
"That workbook no longer exists.")
diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py
index eb698d5ef88d363ba61ae1ad1ccde06c01e0bafa..9e25e9bdf1797743eaa35650bdc94fa85e1e7aef 100644
--- a/platform/core/store_pg.py
+++ b/platform/core/store_pg.py
@@ -36,16 +36,25 @@ worth hiding: a connection failure RAISES from `get()` here. Callers that must n
should catch it — silently returning `{}` from a database that is simply unreachable is how an
empty dict gets written back over real data.
"""
+import atexit
+import base64
+import binascii
import json
import os
+import ssl
+import tempfile
import threading
+from functools import lru_cache
+from pathlib import Path
from urllib.parse import urlparse
import core.data_binding as data_binding
+from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
_URL_ENV = 'DATABASE_URL'
_LOCK = threading.RLock()
_POOL = {'pool': None, 'url': None, 'schema': None}
+_DATABASE_CA_ENV = 'AIOS_DATABASE_CA_PEM_B64'
def _url():
@@ -149,15 +158,25 @@ def sandbox_tenants():
def isolated_staging_database():
- """Whether this process is staging on the exact isolated database host it was handed."""
+ """Match the exact staging database; legacy host stamps are valid only on port 5432."""
if data_binding.deployment_id() != _STAGING_SPACE:
return False
- allowed_host = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip().lower()
+ stamp = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip()
try:
- current_host = (urlparse(str(os.environ.get('DATABASE_URL') or '')).hostname or '').lower()
+ current = _database_endpoint(_url() or '')
+ if not stamp or not current[0]:
+ return False
+ if '/' in stamp or ':' in stamp:
+ return current == _database_endpoint('postgresql://' + stamp)
+ return current[1] == 5432 and current[0] == stamp.lower()
except ValueError:
return False
- return bool(allowed_host and current_host == allowed_host)
+
+
+def _database_endpoint(url):
+ parsed = urlparse(url)
+ return ((parsed.hostname or '').lower(), parsed.port or 5432,
+ parsed.path.lstrip('/').split('/', 1)[0])
def write_refusal(tenant_slug=None):
@@ -172,7 +191,7 @@ def write_refusal(tenant_slug=None):
4. anything else -> REFUSED
Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant,
- and a DATABASE_URL host matching the isolated host stamped by the deploy must all agree.
+ and a DATABASE_URL identity matching the isolated database stamped by the deploy must agree.
"""
slug = _effective_slug(tenant_slug)
if data_binding.is_production_deployment():
@@ -202,6 +221,29 @@ def write_refusal(tenant_slug=None):
f'intended.')
+def gate_write_refusal(tenant_slug=None):
+ """`None` unless a GATE process is writing a production tenant, else why not. W-219.
+
+ ⛔⛔ CONSULTED BEFORE `write_refusal`, AND IT IS THE HALF THAT CLOSES W-219. A lane container is
+ the exact isolated staging posture `write_refusal` rule 2 ALLOWS for tenant #0, so a gate run
+ there wrote staging's `t_royal_imports` (SDLC 61: `odoo_connector_config` `syncEvery=manual`,
+ `ut_odoo_invoices` disabled, four fixture keychain credentials). A gate writes fixture tenants
+ (`qa-b`, `qa-pg`) or nothing, on every deployment, whatever the environment grants: see
+ `data_binding.gate_process` for who counts as a gate and why the environment cannot opt out.
+ `write_refusal` itself is unchanged, so the product rule stays testable on its own.
+ """
+ who = data_binding.gate_process()
+ if not who:
+ return None
+ slug = _effective_slug(tenant_slug)
+ if slug not in PRODUCTION_TENANTS:
+ return None
+ return (f'this process is a verification gate ({who}), and a gate never writes tenant '
+ f'{slug!r} ({_schema_for(slug)}), which holds real customer data, not even on isolated '
+ f'staging. A gate writes fixture tenants only: run it with STORE_BACKEND=hf and no '
+ f'DATABASE_URL (W-219).')
+
+
def check_write(tenant_slug=None, operation='write'):
"""Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
@@ -212,12 +254,44 @@ def check_write(tenant_slug=None, operation='write'):
a session-scoped fallback and tell the user their change was saved. A new type would have had
to re-earn both properties, and would have failed the second one silently.
"""
- reason = write_refusal(tenant_slug)
+ reason = gate_write_refusal(tenant_slug) or write_refusal(tenant_slug)
if reason:
raise data_binding.StoreWriteRefused(
f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
+class DatabaseCAError(RuntimeError):
+ """The configured database CA cannot be used; TLS must never silently fall back."""
+
+
+@lru_cache(maxsize=1)
+def _database_ca_file(encoded):
+ """Decode a configured CA once per process and retain its private file for libpq."""
+ try:
+ pem = base64.b64decode(encoded, validate=True)
+ if not (pem.strip().startswith(b'-----BEGIN CERTIFICATE-----')
+ and pem.strip().endswith(b'-----END CERTIFICATE-----')):
+ raise ValueError
+ ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=pem.decode('ascii'))
+ except (ValueError, binascii.Error, ssl.SSLError):
+ raise DatabaseCAError(
+ f'{_DATABASE_CA_ENV} must be base64-encoded PEM certificate data') from None
+ try:
+ fd, name = tempfile.mkstemp(prefix='aios-database-ca-', suffix='.crt')
+ path = Path(name)
+ try:
+ with os.fdopen(fd, 'wb') as target:
+ os.chmod(path, 0o600)
+ target.write(pem)
+ except BaseException:
+ path.unlink(missing_ok=True)
+ raise
+ except OSError:
+ raise DatabaseCAError(f'{_DATABASE_CA_ENV} could not create a private CA file') from None
+ atexit.register(path.unlink, missing_ok=True)
+ return str(path)
+
+
def _pool():
"""The psycopg connection pool, opened once per process.
@@ -229,7 +303,10 @@ def _pool():
if not url:
raise RuntimeError(
f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.')
- if _POOL['pool'] is not None and _POOL['url'] == url:
+ ca = os.environ.get('AIOS_DATABASE_CA_PEM_B64')
+ sslrootcert = _database_ca_file(ca) if ca is not None else None
+ if (_POOL['pool'] is not None and _POOL['url'] == url
+ and _POOL.get('sslrootcert') == sslrootcert):
return _POOL['pool']
try:
from psycopg_pool import ConnectionPool
@@ -249,9 +326,11 @@ def _pool():
# real requests, not be held awake by this process's idle pool for psycopg's 10-minute
# default. Thirty seconds still reuses a normal request burst without creating a resident
# database client.
+ options = {'kwargs': {'sslrootcert': sslrootcert}} if sslrootcert is not None else {}
_POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0,
- max_idle=30.0)
+ max_idle=30.0, **options)
_POOL['url'] = url
+ _POOL['sslrootcert'] = sslrootcert
return _POOL['pool']
@@ -389,6 +468,22 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
}
+def _get_entry(name, entry, tenant_slug=None):
+ """ONE top-level entry of document `name` (`value -> entry`), or None when either is absent.
+
+ HOTFIX 2026-09-28. Private for `_get_projections`' reason: an optimization offered by a bound
+ `PgStore` handle, never a store operation, so callers keep their whole-read fallback. A
+ stored `ut_*` table read this way crosses the wire as that table alone, not the whole
+ `user_tables` document (~60-100 MB on staging, ~3 s per read, holding a pool connection).
+ """
+ with _pool().connection() as con:
+ row = con.execute(
+ _q('SELECT value -> %s FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
+ (str(entry), str(name))).fetchone()
+ value = row[0] if row else None
+ return dict(value) if isinstance(value, dict) else None
+
+
def _get_many(names, tenant_slug=None):
"""Several ordinary documents in one round trip, keyed by requested store key."""
keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
@@ -435,6 +530,32 @@ def exists(name, tenant_slug=None):
DOCUMENT_CEILING_BYTES = 128 * 1024 * 1024
+def byte_budget_status():
+ """Inspect the shipped limits and enforcer without reading or writing a store."""
+ from core import user_tables as ut
+
+ reasons = []
+ limits = (
+ ("MAX_TABLE_BYTES", ut.MAX_TABLE_BYTES, 48 * 1024 * 1024),
+ ("MAX_DOCUMENT_BYTES", ut.MAX_DOCUMENT_BYTES, 128 * 1024 * 1024),
+ ("DOCUMENT_CEILING_BYTES", DOCUMENT_CEILING_BYTES, 128 * 1024 * 1024),
+ )
+ for name, value, shipped in limits:
+ # bool is an int subclass, but is not a byte limit.
+ if type(value) is not int or value <= 0:
+ reasons.append(f"{name} must be a positive integer")
+ elif value > shipped:
+ reasons.append(f"{name} exceeds the shipped {shipped // 1048576} MB limit")
+ if (ut.enforce_byte_budget is not _BYTE_BUDGET_ENFORCER
+ or getattr(_BYTE_BUDGET_ENFORCER, "__module__", None) != "core.user_tables"
+ or getattr(_BYTE_BUDGET_ENFORCER, "__name__", None) != "enforce_byte_budget"):
+ reasons.append("user_tables.enforce_byte_budget is not the original enforcer")
+ return {"armed": not reasons, "reasons": reasons,
+ "max_table_bytes": ut.MAX_TABLE_BYTES,
+ "max_document_bytes": ut.MAX_DOCUMENT_BYTES,
+ "document_ceiling_bytes": DOCUMENT_CEILING_BYTES}
+
+
class StoreDocumentTooLarge(RuntimeError):
"""A write refused because the document would grow past `DOCUMENT_CEILING_BYTES`."""
@@ -733,6 +854,10 @@ class PgStore:
"""Bound multi-key read used by request-scoped read snapshots."""
return _get_many(names, tenant_slug=self.tenant_slug)
+ def get_entry(self, name, entry):
+ """Bound single-entry read; None means absent, and the caller falls back to `get`."""
+ return _get_entry(name, entry, tenant_slug=self.tenant_slug)
+
def _read_strict(self, name):
"""Interface parity with the HF Store. There, `get` is lenient (swallows a transient
failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py
index 41776f5a2414008d41e6f37bd785e5ee093bbda1..6374ced8a6498ba669f2f15658b7e0f428191486 100644
--- a/platform/core/user_tables.py
+++ b/platform/core/user_tables.py
@@ -46,6 +46,7 @@ inherits C1 by construction — there is no list to add it to, which is R6's act
"""
import datetime as _dt
import hashlib
+import inspect
import json as _json
import math
import re
@@ -278,6 +279,11 @@ def is_connected(table_key, st=None):
table = all_defs(st).get(key) or {}
if not table:
return False
+ return _connected_definition(key, table)
+
+
+def _connected_definition(key, table):
+ """`is_connected`'s answers 2 and 3, read off a definition the caller already holds."""
if table.get('connected') is True:
return True
return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
@@ -285,6 +291,26 @@ def is_connected(table_key, st=None):
and table.get('recordMode') == AUTOMATION_RECORD_MODE)
+def counts_toward_table_cap(table_key, definition=None):
+ """Does this database take one of the tenant's `MAX_TABLES` slots?
+
+ ⭐⭐ OWNER RULING 2026-09-27 (D-3284): CONNECTOR-OWNED DATABASES DO NOT. The cap bounds what
+ PEOPLE create. A connector spawns one database per report (Amazon alone about sixty), and
+ counting those hid ten Amazon databases behind the cap with their rows sitting unreachable in
+ the mirror. The same answers as `is_connected`, off a definition the caller holds, because a
+ spawn's updater holds the raw document rather than a store.
+ """
+ key = str(table_key or '').strip()
+ if key in _CONNECTED:
+ return False
+ return not _connected_definition(key, definition if isinstance(definition, dict) else {})
+
+
+def capped_table_count(tables):
+ """How many `MAX_TABLES` slots `{key: definition}` uses. Connector databases take none."""
+ return sum(1 for k, d in (tables or {}).items() if counts_toward_table_cap(k, d))
+
+
#: ⭐⭐ W30-T31 (D-87) — table keys whose rows DO NOT LIVE HERE AT ALL.
#:
#: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the
@@ -1572,6 +1598,28 @@ def get(table_key, st=None):
return all_tables(st).get(str(table_key))
+def get_one(table_key, st=None):
+ """`get`, reading ONLY this table's entry when the bound store can (HOTFIX 2026-09-28).
+
+ `get` deep-copies the whole tenant document to answer for one table. On a Postgres-backed
+ runtime `get_entry` reads `value -> table_key` instead, so a 1,168-row stored table no longer
+ costs a ~60-100 MB read. ⛔ ONLY A CLASS THAT DEFINES `get_entry` QUALIFIES (`getattr_static`),
+ never one that FORWARDS it: `_Lent`, the external-API draft runtime and the other snapshot
+ wrappers serve documents of their own and pass the rest through `__getattr__`, so a plain
+ `getattr` would read the live store past the snapshot (or the drafted rows) they promised.
+ Anything else (no capability, an absent entry, a failed read) falls back to `get` unchanged.
+ """
+ if (st is not None and not isinstance(st, _Lent)
+ and callable(inspect.getattr_static(type(st), 'get_entry', None))):
+ try:
+ one = st.get_entry(STORE_KEY, str(table_key))
+ except Exception: # noqa: BLE001
+ one = None
+ if isinstance(one, dict):
+ return one
+ return get(table_key, st)
+
+
# --- DURABLE MAP ZONES -------------------------------------------------------
# Zones intentionally do not live in a view config: a view is a person's presentation of rows,
# while a named geographic boundary is tenant data used by several views and routes. Keeping a
@@ -1807,7 +1855,7 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non
if not label:
return None
existing = all_tables(st)
- if len(existing) >= MAX_TABLES:
+ if capped_table_count(existing) >= MAX_TABLES: # connector databases take no slot (D-3284)
return None
key = KEY_PREFIX + _slug(label)
from core import route_system
@@ -5177,6 +5225,23 @@ def unread_note_subjects(subjects, viewer, *, st=None):
return out
+def database_source_group(key, source=None):
+ """Default origin folder; a user's placement is applied by the Data index consumer."""
+ from core.connector_spec import connector_of
+
+ connector = connector_of(key)
+ if connector is None:
+ # Connector registries can lag newly synced tables. Monid's ut_amazon_ keys
+ # are Loopable-made, so only the connector-owned ut_amzn_ prefix belongs here.
+ connector = next((name for prefix, name in (
+ ('ut_amzn_', 'amazon'), ('ut_meta_', 'meta'), ('ut_odoo_', 'odoo'),
+ ) if key.startswith(prefix)), None)
+ if connector is None and (key in {'customer_data', 'product_data'}
+ or str(source or '').lower() == 'odoo'):
+ connector = 'odoo'
+ return {'amazon': 'Amazon', 'meta': 'Meta Ads', 'odoo': 'Odoo'}.get(connector, 'Loopable')
+
+
def nav_entries(viewer=None, is_admin=False, st=None):
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
@@ -5206,8 +5271,7 @@ def nav_entries(viewer=None, is_admin=False, st=None):
if route_system.is_managed(t) or route_system.is_managed(key):
continue
source = str(t.get('source') or BLANK_SOURCE)
- source_group = ('Odoo' if source.lower() == 'odoo' or key.startswith('ut_odoo_')
- else 'Custom' if source == BLANK_SOURCE else source)
+ source_group = database_source_group(key, source)
out.append({'key': key, 'label': t.get('label') or key,
'source': source, 'sourceGroup': source_group,
'user_table': True, 'nav': True, 'validate': False})
diff --git a/platform/core/users.py b/platform/core/users.py
index 85cdd5a4241bb01a5799ff7ca5d11538f246365a..11be0a103fd7d12d20125acd53a4ca1f4620de14 100644
--- a/platform/core/users.py
+++ b/platform/core/users.py
@@ -566,6 +566,15 @@ def registry():
return store.get('users')
+class RegistryUnavailable(RuntimeError):
+ """The account registry could not be READ, which is not the same fact as "no such account".
+
+ HOTFIX 2026-09-28: `verify` used to fold a failed read into `{}`, so a busy store (a
+ `PoolTimeout` on staging) answered a CORRECT password with "that username and password do
+ not match" and charged the attempt to the lockout budget. Still fail-closed: the caller
+ answers 503, it never admits anyone."""
+
+
def ensure_bootstrap():
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
@@ -700,6 +709,7 @@ def verify(username, pw):
``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
+ An unreadable registry raises `RegistryUnavailable` rather than reading as an unknown user.
"""
username = (username or '').strip().lower()
if not username or not pw:
@@ -707,8 +717,8 @@ def verify(username, pw):
try:
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
reg = store.get('users', fresh=True)
- except Exception:
- reg = {}
+ except Exception as exc:
+ raise RegistryUnavailable(f'{type(exc).__name__}: {exc}') from exc
username, u = _resolve_identifier(reg, username)
# ⛔ KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py
index ca274b336b7f9e8a162d0a32df236aff23c29506..39707a197c8e0955cf7783c7dd2b3c2b833033de 100644
--- a/platform/harness/amazon_client.py
+++ b/platform/harness/amazon_client.py
@@ -21,19 +21,19 @@ disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a pl
signing requests is describing a dead API.
· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
-⛔ **READ-ONLY, ENFORCED HERE, WITH EXACTLY TWO NAMED EXCEPTIONS.** `_request` refuses any method but
+⛔ **READ-ONLY, ENFORCED HERE, WITH EXACTLY THREE NAMED EXCEPTIONS.** `_request` refuses any method but
GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
none of that is reachable from this module, and the refusal sits below every route and payload so no
caller can opt out.
-⚠ **The exceptions are `_create_report` and `_competitive_summary_batch`, and both are declared
+⚠ **The exceptions are `_create_report`, `_competitive_summary_batch` and `_create_query`, all declared
rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
-`getCompetitiveSummary` is a POST-wrapped batch of read-only GET subrequests. Both functions have
+`getCompetitiveSummary` wraps read-only GETs, and Data Kiosk generates a fixed economics query. These functions have
their path as a module constant rather than a parameter, so no caller can redirect either one. They
cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
reasoning sits on their path constants.
-⛔ **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
+⛔ **REPORTS AND DATA KIOSK ARE THE BULK PATHS; FINANCES FOLLOWS EVERY PAGE.** SP-API is token-bucket rate limited per
operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's
records, and its second half is what applies: a limit that genuinely cannot be removed must be
@@ -47,8 +47,10 @@ from __future__ import annotations
import contextvars
import json
import random
+import re
import threading
import time
+from datetime import date, datetime, timezone, timedelta
import urllib.error
import urllib.parse
import urllib.request
@@ -100,6 +102,62 @@ _report_pacer = {
_entity_pacers = {
"competitive_summary": {"last": None, "lock": threading.Lock()},
"awd_inventory": {"last": None, "lock": threading.Lock()},
+ "transactions": {"last": None, "lock": threading.Lock()},
+ "getOrderMetrics": {"last": None, "lock": threading.Lock()},
+ "getInventorySummaries": {"last": None, "lock": threading.Lock()},
+ "searchListingsItems": {"last": None, "lock": threading.Lock()},
+ "searchCatalogItems": {"last": None, "lock": threading.Lock()},
+}
+
+# Data Kiosk has its own quota, independent of createReport.
+QUERY_BURST = 15
+QUERY_SUSTAINED_SECONDS = 60.0
+_query_pacer = {"tokens": float(QUERY_BURST), "updated": None, "lock": threading.Lock()}
+_QUERIES_CREATE_PATH = "dataKiosk/2023-11-15/queries"
+_TRANSACTIONS_PATH = "finances/2024-06-19/transactions"
+TRANSACTIONS_INTERVAL = 2.0
+
+# Selection pinned to Amazon's published analytics_economics_2024_03_15.graphql,
+# read 2026-09-25. RANGE avoids MONTH's automatic rounding of partial months.
+# Only dates, a validated marketplace ID and one of two fixed grains are substituted.
+_ECONOMICS_QUERY = '''query {
+ analytics_economics_2024_03_15 {
+ economics(startDate: "__START__", endDate: "__END__",
+ marketplaceIds: ["__MARKETPLACE__"], aggregateBy: {date: __DATE__, productId: __PRODUCT__}) {
+ startDate endDate marketplaceId msku fnsku childAsin parentAsin
+ sales {
+ orderedProductSales { amount currencyCode } unitsOrdered
+ refundedProductSales { amount currencyCode } unitsRefunded
+ netProductSales { amount currencyCode } netUnitsSold
+ averageSellingPrice { amount currencyCode }
+ }
+ fees { feeTypeName charges { startDate endDate identifier aggregatedDetail {
+ amount { amount currencyCode } totalAmount { amount currencyCode }
+ amountPerUnit { amount currencyCode } amountPerUnitDelta { amount currencyCode }
+ promotionAmount { amount currencyCode } taxAmount { amount currencyCode } quantity
+ } } }
+ ads { adTypeName charge {
+ amount { amount currencyCode } totalAmount { amount currencyCode }
+ amountPerUnit { amount currencyCode } promotionAmount { amount currencyCode }
+ taxAmount { amount currencyCode } quantity
+ } }
+ netProceeds { total { amount currencyCode } perUnit { amount currencyCode } }
+ cost {
+ costOfGoodsSold { amount currencyCode } miscellaneousCost { amount currencyCode }
+ fbaCost { shippingToAmazonCost { amount currencyCode } }
+ mfnCost { fulfillmentCost { amount currencyCode } storageCost { amount currencyCode } }
+ }
+ }
+ }
+}'''
+_ECONOMICS_TEMPLATES = {
+ False: _ECONOMICS_QUERY.replace("__DATE__", "DAY").replace("__PRODUCT__", "MSKU"),
+ # S66-T40 (lead, measured live 2026-09-25, window 2026-09-20..24): RANGE x MSKU equals the
+ # sum of our DAY x MSKU rows to the cent (fees 38,009.94, net 38,474.335), while ANY
+ # PARENT_ASIN grain carries 37.75 of fees Amazon attributes to no MSKU. The reconciliation
+ # checks OUR date roll-up at the SAME product grain; comparing across grains refused a
+ # correct transform.
+ True: _ECONOMICS_QUERY.replace("__DATE__", "RANGE").replace("__PRODUCT__", "MSKU"),
}
#: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
@@ -179,6 +237,9 @@ CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "seller_id",
#: production host, whereas a production credential wrongly marked sandbox would quietly read mocks.
OPTIONAL_FIELDS = ("sandbox", "region")
+SQP_ASIN_OPTION_MAX_CHARS = 200
+SQP_PERIOD = "MONTH"
+
#: ⭐ THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names
#: the table, the bulk report
#: type that fills it, and the summary aggregate its `validate()` reconciles against.
@@ -230,6 +291,7 @@ REPORT_TYPES = {
},
"listings": {
"report": "GET_MERCHANT_LISTINGS_ALL_DATA",
+ "snapshot": True,
"validates_against": "listing report row count vs our listing rows",
"role": "Product Listing",
},
@@ -254,6 +316,8 @@ REPORT_TYPES = {
},
"inventory": {
"report": "GET_LEDGER_SUMMARY_VIEW_DATA",
+ "snapshot": True,
+ "options": {"aggregateByTimePeriod": "DAILY", "aggregateByLocation": "COUNTRY"},
"entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
"validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
"role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
@@ -273,6 +337,7 @@ REPORT_TYPES = {
# page loop over a full catalogue is precisely what rule 1 forbids.
"pricing": {
"report": "GET_SALES_AND_TRAFFIC_REPORT",
+ "json_root": "salesAndTrafficByAsin",
"report_covers": "buy box SHARE only; PRICE has no bulk path",
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
"validates_against": "sales and traffic report totals and competitive ASIN responses",
@@ -283,11 +348,630 @@ REPORT_TYPES = {
# wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows
# for any registered brand, else demote to a declared-but-unsynced stub.
"report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
+ "json_root": "dataByDepartmentAndSearchTerm",
"validates_against": "report period totals vs our search-term rows",
"role": "Brand Analytics",
},
+ "economics": {
+ "api": "createQuery",
+ "validates_against": "DAY x MSKU sales, fees and net proceeds vs Amazon RANGE x MSKU totals per currency",
+ "role": "Selling Partner Insights",
+ },
+ "transactions": {
+ "api": "listTransactions",
+ "validates_against": "Amazon transaction count and totalAmount per currency vs stored items, and breakdowns vs transaction totals",
+ "role": "Finance and Accounting",
+ },
+ # snapshot forbids dataStartTime/dataEndTime; min_interval_s is a successful
+ # refresh floor (also enforced on manual sync). Monthly history is separate.
+ "fba_fee_estimates": {
+ "report": "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", "snapshot": True,
+ "min_interval_s": 86400,
+ "validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
+ "role": "Amazon Fulfillment",
+ },
+ "referral_fees": {
+ "report": "GET_REFERRAL_FEE_PREVIEW_REPORT", "snapshot": True,
+ "min_interval_s": 86400,
+ "validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
+ "role": "Finance and Accounting",
+ },
+ "storage_fees": {
+ "report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
+ "min_interval_s": 30 * 86400,
+ "validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
+ "role": "Amazon Fulfillment",
+ },
+ "search_query_performance": {
+ "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
+ "role": "Brand Analytics",
+ "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
+ "json_root": "dataByAsin", "monthly": True,
+ "period": SQP_PERIOD, "asin_option_max_chars": SQP_ASIN_OPTION_MAX_CHARS,
+ },
+}
+
+# S66-T42: inventory sections 2.2 and 2.3. Fields are measured at sync, including
+# JSON documents. Row roots were read from amzn/selling-partner-api-models/schemas/reports
+# on 2026-09-25 at commit 713565ff394d136629a342120872e65cb073d162.
+# json_schema names the public source, without changing the pinned proto.
+GROUP_A_TABLES = {
+ "long_term_storage_fees": {
+ "label": "Amazon long term storage fees",
+ "report": "GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
+ "monthly": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "storage_overage_fees": {
+ "label": "Amazon storage overage fees",
+ "report": "GET_FBA_OVERAGE_FEE_CHARGES_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
+ "monthly": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "reimbursements": {
+ "label": "Amazon reimbursements",
+ "report": "GET_FBA_REIMBURSEMENTS_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "fba_customer_returns": {
+ "label": "Amazon FBA customer returns",
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "replacements": {
+ "label": "Amazon replacements",
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "fba_shipped_sales": {
+ "label": "Amazon FBA shipped sales",
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "fba_shipment_promotions": {
+ "label": "Amazon FBA shipment promotions",
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "inventory_ledger_detail": {
+ "label": "Amazon inventory ledger detail",
+ "report": "GET_LEDGER_DETAIL_VIEW_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "inventory_planning": {
+ "label": "Amazon inventory planning",
+ "report": "GET_FBA_INVENTORY_PLANNING_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "manage_inventory": {
+ "label": "Amazon manage inventory",
+ "report": "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "reserved_inventory": {
+ "label": "Amazon reserved inventory",
+ "report": "GET_RESERVED_INVENTORY_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "restock_recommendations": {
+ "label": "Amazon restock recommendations",
+ "report": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "stranded_inventory": {
+ "label": "Amazon stranded inventory",
+ "report": "GET_STRANDED_INVENTORY_UI_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "inbound_noncompliance": {
+ "label": "Amazon inbound noncompliance",
+ "report": "GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "inventory_expiration_dates": {
+ "label": "Amazon inventory expiration dates",
+ "report": "GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "recommended_removals": {
+ "label": "Amazon recommended removals",
+ "report": "GET_FBA_RECOMMENDED_REMOVAL_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "removal_orders": {
+ "label": "Amazon removal orders",
+ "report": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "removal_shipments": {
+ "label": "Amazon removal shipments",
+ "report": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "subscribe_save_forecast": {
+ "label": "Amazon Subscribe and Save forecast",
+ "report": "GET_FBA_SNS_FORECAST_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "subscribe_save_performance": {
+ "label": "Amazon Subscribe and Save performance",
+ "report": "GET_FBA_SNS_PERFORMANCE_DATA",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "remote_fulfillment_eligibility": {
+ "label": "Amazon remote fulfillment eligibility",
+ "report": "GET_REMOTE_FULFILLMENT_ELIGIBILITY",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 86400,
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "search_catalog_performance": {
+ "label": "Amazon search catalog performance",
+ "report": "GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
+ "role": "Brand Analytics",
+ "min_interval_s": 86400,
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
+ "monthly": True,
+ "json_root": "dataByAsin",
+ "json_schema": "sellingPartnerSearchCatalogPerformanceReport.json",
+ "period": "MONTH",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "market_basket": {
+ "label": "Amazon market basket",
+ "report": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
+ "role": "Brand Analytics",
+ "min_interval_s": 86400,
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
+ "monthly": True,
+ "json_root": "dataByAsin",
+ "json_schema": "sellingPartnerMarketBasketAnalysisReport.json",
+ "period": "MONTH",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "repeat_purchase": {
+ "label": "Amazon repeat purchase",
+ "report": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
+ "role": "Brand Analytics",
+ "min_interval_s": 86400,
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
+ "monthly": True,
+ "json_root": "dataByAsin",
+ "json_schema": "sellingPartnerRepeatPurchaseReport.json",
+ "period": "MONTH",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "promotion_performance": {
+ "label": "Amazon promotion performance",
+ "report": "GET_PROMOTION_PERFORMANCE_REPORT",
+ "role": "Selling Partner Insights",
+ "min_interval_s": 86400,
+ "cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
+ "json_root": "promotions",
+ "json_schema": "promotionReport.json",
+ "date_options": ["promotionStartDateFrom","promotionStartDateTo"],
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "coupon_performance": {
+ "label": "Amazon coupon performance",
+ "report": "GET_COUPON_PERFORMANCE_REPORT",
+ "role": "Brand Analytics",
+ "min_interval_s": 86400,
+ "cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
+ "json_root": "coupons",
+ "json_schema": "sellerCouponReport.json",
+ "date_options": ["couponStartDateFrom","couponStartDateTo"],
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "account_health": {
+ "label": "Amazon account health",
+ "report": "GET_V2_SELLER_PERFORMANCE_REPORT",
+ "role": "Selling Partner Insights",
+ "min_interval_s": 86400,
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "snapshot": True,
+ "json_root": "performanceMetrics",
+ "json_schema": "accountHealthReport-2020-11-18.json",
+ "json_extra_roots": ["accountStatuses"],
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "inactive_listings": {
+ "label": "Amazon inactive listings",
+ "report": "GET_MERCHANT_LISTINGS_INACTIVE_DATA",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "cancelled_listings": {
+ "label": "Amazon cancelled listings",
+ "report": "GET_MERCHANT_CANCELLED_LISTINGS_DATA",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "suppressed_listings": {
+ "label": "Amazon suppressed listings",
+ "report": "GET_MERCHANTS_LISTINGS_FYP_REPORT",
+ "role": "Product Listing",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "listing_defects": {
+ "label": "Amazon listing defects",
+ "report": "GET_MERCHANT_LISTINGS_DEFECT_DATA",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "sold_listings": {
+ "label": "Amazon sold listings",
+ "report": "GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "mfn_returns": {
+ "label": "Amazon MFN returns",
+ "report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
+ "max_window_days": 60,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "b2b_recommended": {
+ "label": "Amazon B2B product opportunities",
+ "report": "GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
+ "role": "Product Listing",
+ "min_interval_s": 604800,
+ "cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
+ "snapshot": True,
+ "json_root": "recommendations",
+ "json_schema": "b2bProductOpportunitiesRecommendedForYouReport-2020-11-19.json",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "b2b_not_yet_on_amazon": {
+ "label": "Amazon B2B products not yet on Amazon",
+ "report": "GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
+ "role": "Product Listing",
+ "min_interval_s": 604800,
+ "cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
+ "snapshot": True,
+ "json_root": "recommendations",
+ "json_schema": "b2bProductOpportunitiesNotYetOnAmazonReport-2020-11-19.json",
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "geo_opportunities": {
+ "label": "Amazon geo opportunities",
+ "report": "GET_FLAT_FILE_GEO_OPPORTUNITIES",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 86400,
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "out_of_stock_mfn": {
+ "label": "Amazon out of stock MFN listings",
+ "report": "GET_OUT_OF_STOCK_MFN_INFORMATION",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 14400,
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "b2b_quotes": {
+ "label": "Amazon B2B quantity discounts",
+ "report": "RFQD_BULK_DOWNLOAD",
+ "role": "Product Listing",
+ "min_interval_s": 86400,
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "fee_discounts": {
+ "label": "Amazon fee discounts",
+ "report": "FEE_DISCOUNTS_REPORT",
+ "role": "Product Listing",
+ "min_interval_s": 86400,
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "order_metrics": {
+ "label": "Amazon order metrics",
+ "api": "getOrderMetrics",
+ "role": "Inventory and Order Tracking",
+ "min_interval_s": 1800,
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
+ "validates_against": "Amazon daily order, unit and sales totals vs stored metrics; order counts independently reconcile orders",
+ },
+ "fba_inventory": {
+ "label": "Amazon FBA inventory summaries",
+ "api": "getInventorySummaries",
+ "role": "Amazon Fulfillment",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "listing_items": {
+ "label": "Amazon listing status and issues",
+ "api": "searchListingsItems",
+ "role": "Product Listing",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+ "catalog_items": {
+ "label": "Amazon catalog attributes",
+ "api": "searchCatalogItems",
+ "role": "Product Listing",
+ "min_interval_s": 1800,
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
+ "snapshot": True,
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
+ },
+}
+REPORT_TYPES.update(GROUP_A_TABLES)
+
+# These are visible declarations, never enablement keys or request allow-list entries.
+DECLARED_DATASETS = {
+ "manage_inventory_variants": {
+ "label": "Amazon manage inventory alternatives",
+ "report_types": ["GET_FBA_MYI_ALL_INVENTORY_DATA","GET_AFN_INVENTORY_DATA"],
+ "role": "Amazon Fulfillment",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "The connector uses the unsuppressed manage inventory report for this dataset.",
+ },
+ "stranded_inventory_loader": {
+ "label": "Amazon stranded inventory loader",
+ "report_types": ["GET_STRANDED_INVENTORY_LOADER_DATA"],
+ "role": "Amazon Fulfillment",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "The connector uses the stranded inventory UI report for this dataset.",
+ },
+ "listing_variants": {
+ "label": "Amazon listing report variants",
+ "report_types": ["GET_MERCHANT_LISTINGS_DATA","GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT","GET_MERCHANT_LISTINGS_DATA_LITE","GET_MERCHANT_LISTINGS_DATA_LITER","GET_FLAT_FILE_OPEN_LISTINGS_DATA"],
+ "role": "Product Listing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "These listing variants overlap Amazon listings, which reads the all listings report.",
+ },
+ "pending_orders": {
+ "label": "Amazon pending order reports",
+ "report_types": ["GET_FLAT_FILE_PENDING_ORDERS_DATA","GET_PENDING_ORDERS_DATA","GET_CONVERGED_FLAT_FILE_PENDING_ORDERS_DATA"],
+ "role": "Inventory and Order Tracking",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Pending orders are covered by Amazon orders.",
+ },
+ "archived_orders": {
+ "label": "Amazon archived orders",
+ "report_types": ["GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE"],
+ "role": "Inventory and Order Tracking",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Archived orders before the normal history window are not built in this release.",
+ },
+ "xml_orders": {
+ "label": "Amazon XML order reports",
+ "report_types": ["GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL","GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL"],
+ "role": "Inventory and Order Tracking",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "The XML order reports duplicate the flat file reports used by Amazon orders.",
+ },
+ "mfn_return_variants": {
+ "label": "Amazon MFN return report variants",
+ "report_types": ["GET_CSV_MFN_PRIME_RETURNS_REPORT","GET_XML_MFN_PRIME_RETURNS_REPORT","GET_FLAT_FILE_MFN_SKU_RETURN_ATTRIBUTES_REPORT","GET_XML_MFN_SKU_RETURN_ATTRIBUTES_REPORT"],
+ "role": "Inventory and Order Tracking",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Prime return, return attribute and attribute feed record variants are not built; Amazon MFN returns reads returns by return date.",
+ },
+ "browse_tree": {
+ "label": "Amazon browse tree",
+ "report_types": ["GET_XML_BROWSE_TREE_DATA"],
+ "role": "Product Listing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "The XML browse tree is category reference data and is not built in this release.",
+ },
+ "sales_traffic_kiosk": {
+ "label": "Amazon sales and traffic query",
+ "operations": ["analytics_salesAndTraffic_2024_04_24"],
+ "role": "Brand Analytics",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Same rows as Amazon pricing and Buy Box, which reads the report twin.",
+ },
+ "financial_events": {
+ "label": "Amazon legacy financial events",
+ "operations": ["listFinancialEvents","listFinancialEventGroups"],
+ "role": "Finance and Accounting",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Superseded by Amazon financial transactions.",
+ },
+ "payouts": {
+ "label": "Amazon payouts",
+ "operations": ["listPayouts","listExpectedPayouts"],
+ "role": "Finance and Accounting",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Payout timing from the Transfers API is not built in this release.",
+ },
+ "fee_estimates_batch": {
+ "label": "Amazon batch fee estimates",
+ "operations": ["getMyFeesEstimates"],
+ "role": "Pricing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Current price fee estimates are covered by Amazon FBA fee estimates and referral fee preview; hypothetical prices are not built.",
+ },
+ "offer_prices": {
+ "label": "Amazon offer price alternatives",
+ "operations": ["getItemOffersBatch","getListingOffersBatch","getFeaturedOfferExpectedPriceBatch"],
+ "role": "Pricing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Amazon pricing and Buy Box reads competitive summaries; other offer price operations are not built in this release.",
+ },
+ "inbound_shipments": {
+ "label": "Amazon inbound shipments",
+ "operations": ["listInboundPlans","listShipmentItems","listInboundShipments"],
+ "role": "Amazon Fulfillment",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "FBA and AWD inbound shipment plans and items are not built in this release.",
+ },
+ "replenishment": {
+ "label": "Amazon replenishment metrics",
+ "operations": ["listOfferMetrics","getSellingPartnerMetrics"],
+ "role": "Brand Analytics",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Only returns rows for Subscribe and Save offers; API offer metrics are not built in this release.",
+ },
+ "review_topics": {
+ "label": "Amazon review topics and trends",
+ "operations": ["getItemReviewTopics","getItemReviewTrends","getBrowseNodeReturnTopics","getBrowseNodeReturnTrends"],
+ "role": "Selling Partner Insights",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Per ASIN review and return topics are not built in this release; the limit is Amazon's per ASIN request rate.",
+ },
+ "promotions": {
+ "label": "Amazon promotion calendar",
+ "operations": ["searchPromotions","getPromotion"],
+ "role": "Pricing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "The Promotions API calendar is not built in this release; promotion results are in Amazon promotion performance.",
+ },
+ "aplus_content": {
+ "label": "Amazon A plus content",
+ "operations": ["searchContentDocuments"],
+ "role": "Brand Analytics",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "A plus content reference data is not built in this release.",
+ },
+ "listing_restrictions": {
+ "label": "Amazon listing restrictions",
+ "operations": ["getListingsRestrictions"],
+ "role": "Product Listing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Listing restriction reference data is not built in this release.",
+ },
+ "product_types": {
+ "label": "Amazon product type definitions",
+ "operations": ["searchDefinitionsProductTypes","getDefinitionsProductType"],
+ "role": "Product Listing",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Product type definition reference data is not built in this release.",
+ },
+ "supply_sources": {
+ "label": "Amazon supply sources",
+ "operations": ["getSupplySources"],
+ "role": "Selling Partner Insights",
+ "cadence_note": "Not requested by this connector.",
+ "reason": "Supply source reference data is not built in this release.",
+ },
}
+# Include the existing seven base tables and T40/T41 datasets in the same inventory.
+GROUP_A = {
+ key: {"key": key, "label": spec.get("label") or "Amazon " + key.replace("_", " "),
+ "report_types": tuple(spec[k] for k in ("report", "report_incremental", "report_discover") if spec.get(k)),
+ "operation": spec.get("api"), "role": spec["role"],
+ "cadence_note": spec.get("cadence_note") or "Uses the existing connector cadence."}
+ for key, spec in REPORT_TYPES.items()
+}
+GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
+
+
+# Inventory section 2.5: immutable refusal even if a future report roster admits one.
+EXCLUDED_REPORT_TYPES = frozenset({
+ "GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
+ "GET_FLAT_FILE_ORDER_REPORT_DATA_SHIPPING", "GET_FLAT_FILE_ORDER_REPORT_DATA_TAX",
+ "GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_SHIPPING", "GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_TAX",
+ "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING", "GET_ORDERS_DATA",
+ "GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_ALL", "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_TAX",
+ "GET_FLAT_FILE_SALES_TAX_DATA", "GET_FLAT_FILE_VAT_INVOICE_DATA_REPORT",
+ "GET_XML_VAT_INVOICE_DATA_REPORT", "GET_VAT_TRANSACTION_DATA",
+ "GET_GST_MTR_B2B_CUSTOM", "GET_GST_MTR_B2C_CUSTOM", "GET_EASYSHIP_DOCUMENTS",
+ "GET_EASYSHIP_DOCUMENTS_PICKEDUP", "GET_EASYSHIP_DOCUMENTS_WAITING_FOR_PICKUP",
+ "GET_MONTHLY_WARRANTIES", "WARRANTY_BASE_ITEM_INFO",
+ "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", "GET_SELLER_FEEDBACK_DATA",
+})
+
#: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation
#: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those
#: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an
@@ -312,13 +996,14 @@ REPORT_ROLES = {
}
_REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1}
_BULK_REPORT_ROLES = tuple(dict.fromkeys(
- roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()))
+ roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()
+ if REPORT_TYPES[table].get("report") or REPORT_TYPES[table].get("report_discover")))
_APPROVED_REPORT_TYPES = tuple(dict.fromkeys(
spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental")
if spec.get(key)))
#: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query,
-#: escaped slash or traversal. The two POSTs only request a report or wrap fixed read-only GETs.
+#: escaped slash or traversal. The three POSTs generate reports/queries or wrap fixed read-only GETs.
#: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API
#: operations. The gate enumerates those transports separately; they confer no operation scope.
OPERATIONS = {
@@ -353,6 +1038,22 @@ OPERATIONS = {
"name": "listInventory", "roles": ("Amazon Warehousing and Distribution",),
"method": "GET", "path": "awd/2024-05-09/inventory",
},
+ "createQuery": {"name": "createQuery", "roles": ("Selling Partner Insights",),
+ "method": "POST", "path": _QUERIES_CREATE_PATH},
+ "getQuery": {"name": "getQuery", "roles": ("Selling Partner Insights",),
+ "method": "GET", "path": "dataKiosk/2023-11-15/queries/{queryId}"},
+ "getDocument": {"name": "getDocument", "roles": ("Selling Partner Insights",),
+ "method": "GET", "path": "dataKiosk/2023-11-15/documents/{documentId}"},
+ "listTransactions": {"name": "listTransactions", "roles": ("Finance and Accounting",),
+ "method": "GET", "path": _TRANSACTIONS_PATH},
+ "getOrderMetrics": {"name": "getOrderMetrics", "roles": ("Inventory and Order Tracking",),
+ "method": "GET", "path": "sales/v1/orderMetrics"},
+ "getInventorySummaries": {"name": "getInventorySummaries", "roles": ("Amazon Fulfillment",),
+ "method": "GET", "path": "fba/inventory/v1/summaries"},
+ "searchListingsItems": {"name": "searchListingsItems", "roles": ("Product Listing",),
+ "method": "GET", "path": "listings/2021-08-01/items/{sellerId}"},
+ "searchCatalogItems": {"name": "searchCatalogItems", "roles": ("Product Listing",),
+ "method": "GET", "path": "catalog/2022-04-01/items"},
}
#: Answer "what does this approved role let this connector read?" directly from data.
@@ -444,6 +1145,26 @@ class AmazonError(RuntimeError):
"""Any refusal from LWA or SP-API, already stripped of anything secret."""
+class AmazonReportError(AmazonError):
+ """Vendor report refusal, with its complete reason available to the batch coordinator."""
+
+ def __init__(self, message, reason, status):
+ self.vendor_reason = str(reason).replace("\u2014", "; ").replace("\u2013", " to ")
+ self.status = status
+ super().__init__(f"{message}: {self.vendor_reason}")
+
+
+def sqp_refused_asins(error, asins):
+ """Only Amazon's named brand refusals permit removing an ASIN from one retry."""
+ if not isinstance(error, AmazonReportError) or error.status not in (400, "FATAL"):
+ return []
+ reason = error.vendor_reason
+ if not re.search(r"not\s+(?:part|associated|owned|eligible|authorized|belong|registered)|invalid\s+asin|outside.*brand", reason, re.I):
+ return []
+ named = set(re.findall(r"\b[A-Z0-9]{10}\b", reason.upper()))
+ return [asin for asin in asins if asin in named]
+
+
class AmazonOperationRefused(AmazonError):
"""A local scope refusal, never a throttle and never eligible for throttling retries."""
@@ -497,6 +1218,8 @@ def operation_for(method, path):
def require_operation(method, path, *, report_type=None):
"""Fail closed before constructing any URL or touching a transport."""
+ if report_type in EXCLUDED_REPORT_TYPES:
+ raise AmazonOperationRefused(f"excluded report {report_type}")
operation = operation_for(method, path)
if operation is None:
raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}")
@@ -564,6 +1287,252 @@ def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic):
return wait
+def _pace_query_create(*, sleep=time.sleep, now=time.monotonic):
+ """Reserve from Data Kiosk's independent burst-15, one-per-minute bucket."""
+ with _query_pacer["lock"]:
+ stamp = now()
+ updated = _query_pacer["updated"]
+ elapsed = max(0.0, stamp - updated) if updated is not None else 0.0
+ tokens = min(QUERY_BURST, _query_pacer["tokens"] + elapsed / QUERY_SUSTAINED_SECONDS)
+ wait = max(0.0, (1.0 - tokens) * QUERY_SUSTAINED_SECONDS)
+ if wait:
+ sleep(wait)
+ issued = max(stamp + wait, now())
+ _query_pacer.update(tokens=min(QUERY_BURST, tokens +
+ (issued - stamp) / QUERY_SUSTAINED_SECONDS) - 1.0, updated=issued)
+ return wait
+
+
+def economics_query(start, end, marketplace_id, *, summary=False):
+ """Build only the pinned economics query; never accept caller GraphQL."""
+ for value in (start, end):
+ if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value):
+ raise AmazonError("Economics dates must be YYYY-MM-DD")
+ try:
+ date.fromisoformat(value)
+ except ValueError:
+ raise AmazonError("Economics dates must be valid YYYY-MM-DD dates") from None
+ if start > end or type(summary) is not bool:
+ raise AmazonError("Economics needs an ordered date range and a boolean summary selection")
+ if not isinstance(marketplace_id, str) or not re.fullmatch(r"[A-Z0-9]+", marketplace_id):
+ raise AmazonError("Economics needs an alphanumeric marketplace ID")
+ return (_ECONOMICS_TEMPLATES[summary].replace("__START__", start)
+ .replace("__END__", end).replace("__MARKETPLACE__", marketplace_id))
+
+
+def _create_query(token, start, end, marketplace_id, *, summary=False,
+ region=DEFAULT_REGION, timeout=_TIMEOUT, sandbox=False):
+ """The third declared POST wrapper, fixed to a read-only economics query."""
+ require_operation("POST", _QUERIES_CREATE_PATH)
+ body = {"query": economics_query(start, end, marketplace_id, summary=summary)}
+
+ def attempt():
+ base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(str(region).lower())
+ if not base:
+ raise AmazonError(f"unknown region `{region}`")
+ url = f"{base}/{_QUERIES_CREATE_PATH}"
+ req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST")
+ req.add_header("x-amz-access-token", token)
+ req.add_header("Content-Type", "application/json")
+ req.add_header("User-Agent", _UA)
+ try:
+ with urllib.request.urlopen(req, timeout=timeout) as response:
+ got = json.loads(response.read().decode("utf-8") or "{}")
+ except urllib.error.HTTPError as exc:
+ if exc.code == 429:
+ raise AmazonThrottled("createQuery throttled (429); Data Kiosk permits a burst of "
+ "15, then one query per 60 seconds",
+ retry_after=(exc.headers or {}).get("Retry-After")) from None
+ if exc.code == 403:
+ raise AmazonError("createQuery answered 403: needs the Selling Partner Insights role") from None
+ raise AmazonError(f"createQuery answered {exc.code}") from None
+ except Exception as exc:
+ raise AmazonError(f"createQuery unreachable: {type(exc).__name__}") from None
+ if not got.get("queryId"):
+ raise AmazonError("createQuery returned no queryId")
+ return str(got["queryId"])
+
+ _pace_query_create()
+ return _retry(attempt, _QUERIES_CREATE_PATH,
+ exhausted_fix="reduce Data Kiosk query frequency and retry the window")
+
+
+def _kiosk_document(token, document_id, *, region, sandbox, timeout):
+ meta = _request(f"dataKiosk/2023-11-15/documents/{document_id}", token,
+ region=region, sandbox=sandbox, timeout=timeout)
+ url = str(meta.get("documentUrl") or "")
+ if not url:
+ raise AmazonError("Data Kiosk document returned no documentUrl")
+ text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
+ return text
+
+
+def economics_document(fields, start, end, *, summary=False, max_wait=900,
+ poll_seconds=15, timeout=_TIMEOUT, sleep=time.sleep):
+ """Request, poll and download JSONL, including the vendor error document on failure."""
+ marketplace = str(fields.get("marketplace_id") or US_MARKETPLACE_ID)
+ economics_query(start, end, marketplace, summary=summary) # Validate before LWA.
+ token = access_token(fields, timeout=timeout)
+ region, sandbox = str(fields.get("region") or DEFAULT_REGION), is_sandbox(fields)
+ query_id = _create_query(token, start, end, marketplace, summary=summary,
+ region=region, sandbox=sandbox, timeout=timeout)
+ waited = 0
+ while waited <= max_wait:
+ got = _request(f"dataKiosk/2023-11-15/queries/{query_id}", token,
+ region=region, sandbox=sandbox, timeout=timeout)
+ state = got.get("processingStatus")
+ if got.get("errorDocumentId"):
+ detail = _kiosk_document(token, got["errorDocumentId"], region=region,
+ sandbox=sandbox, timeout=timeout)
+ detail = detail.replace("\u2014", "; ").replace("\u2013", " to ")
+ raise AmazonError(f"Economics query {query_id} {state}: {detail[:1200]}")
+ if state == "DONE":
+ doc_id = got.get("dataDocumentId")
+ if not doc_id:
+ raise AmazonError(f"Economics query {query_id} returned no dataDocumentId")
+ return (_kiosk_document(token, doc_id, region=region, sandbox=sandbox, timeout=timeout),
+ {"queryId": query_id, "reportDocumentId": doc_id})
+ if state in ("FATAL", "CANCELLED"):
+ raise AmazonError(f"Economics query {query_id} ended as {state}")
+ sleep(poll_seconds)
+ waited += poll_seconds
+ if waited and waited % 1800 == 0:
+ token = access_token(fields, timeout=timeout)
+ raise AmazonError(f"Economics query {query_id} still pending after {max_wait}s; "
+ "increase the query wait before retrying the window")
+
+
+def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
+ """Follow every Finances page at 0.5/s, including empty pages carrying a nextToken."""
+ token, issued = access_token(fields, timeout=timeout), now()
+ following, seen, rows = None, set(), []
+ while True:
+ _pace_entity("transactions", TRANSACTIONS_INTERVAL, sleep=sleep, now=now)
+ if now() - issued >= 3000:
+ token, issued = access_token(fields, timeout=timeout), now()
+ got = _request(_TRANSACTIONS_PATH, token,
+ region=str(fields.get("region") or DEFAULT_REGION),
+ sandbox=is_sandbox(fields), timeout=timeout,
+ postedAfter=start, postedBefore=end,
+ marketplaceId=fields.get("marketplace_id") or US_MARKETPLACE_ID,
+ nextToken=following,
+ throttle_fix="resume the same Finances window at its 0.5 requests/second rate")
+ payload = got.get("payload") or {}
+ page = payload.get("transactions")
+ if not isinstance(page, list):
+ raise AmazonError("listTransactions returned no transactions list")
+ rows.extend(page)
+ following = payload.get("nextToken")
+ if not following:
+ return rows
+ if following in seen:
+ raise AmazonError("listTransactions repeated a nextToken; cannot prove complete pagination; "
+ "retry the window after Amazon corrects the cursor")
+ seen.add(following)
+
+
+ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
+ "searchListingsItems": 0.2, "searchCatalogItems": 0.5}
+
+
+def group_a_records(fields, operation, *, start=None, end=None, asins=(),
+ timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
+ """Fixed GET readers, every page and every own-ASIN batch; no record ceiling."""
+ if operation not in ENTITY_INTERVALS:
+ raise AmazonOperationRefused("unapproved group (a) reader")
+ marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
+ path = OPERATIONS[operation]["path"]
+ params = {"marketplaceIds": marketplace}
+ if operation == "getOrderMetrics":
+ params.update(interval=f"{start}--{end}", granularity="Day", granularityTimeZone="UTC")
+ root, payload_key, next_key = None, "payload", None
+ elif operation == "getInventorySummaries":
+ params.update(details="true", granularityType="Marketplace", granularityId=marketplace)
+ root, payload_key, next_key = "inventorySummaries", "payload", "nextToken"
+ elif operation == "searchListingsItems":
+ seller = str(fields.get("seller_id") or "")
+ if not re.fullmatch(r"[A-Za-z0-9]+", seller):
+ raise AmazonError("Listing status needs the seller ID saved with the connection")
+ path = path.replace("{sellerId}", seller)
+ params.update(includedData="summaries,issues,offers,fulfillmentAvailability,attributes", pageSize=20)
+ root, payload_key, next_key = "items", None, "pageToken"
+ else:
+ # `relationships` (S67-T29, O23): same Catalog Items endpoint and Product Listing role;
+ # it carries VARIATION parentAsins so the mirror can map every own ASIN to its parent.
+ params.update(identifiersType="ASIN",
+ includedData="summaries,salesRanks,dimensions,attributes,relationships",
+ pageSize=20)
+ root, payload_key, next_key = "items", None, "pageToken"
+ batches = [None]
+ if operation == "searchCatalogItems":
+ identifiers = sorted(set(asins))
+ if any(not re.fullmatch(r"[A-Z0-9]{10}", str(a)) for a in identifiers):
+ raise AmazonError("Catalog contains an invalid ASIN; correct the listings or pricing mirror and retry")
+ batches = [identifiers[i:i + 20] for i in range(0, len(identifiers), 20)]
+ if not batches:
+ raise AmazonError("Catalog attributes are waiting for our ASINs; sync Amazon listings first")
+ token, issued = access_token(fields, timeout=timeout), now()
+ rows = []
+ listing_end = datetime.now(timezone.utc)
+ # Date partitions bypass Listings search's 1,000-item pagination ceiling.
+ # Inclusive boundaries are deduplicated by SKU after every partition has finished.
+ if operation == "searchListingsItems":
+ batches = [(datetime(1970, 1, 1, tzinfo=timezone.utc), listing_end)]
+ for batch in batches:
+ query, seen = dict(params), set()
+ if operation == "searchListingsItems":
+ lower, upper = batch
+ query.update(createdAfter=lower.isoformat(), createdBefore=upper.isoformat())
+ elif batch is not None:
+ query["identifiers"] = ",".join(batch)
+ page_rows, expected = [], None
+ while True:
+ _pace_entity(operation, ENTITY_INTERVALS[operation], sleep=sleep, now=now)
+ if now() - issued >= 3000:
+ token, issued = access_token(fields, timeout=timeout), now()
+ got = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
+ sandbox=is_sandbox(fields), timeout=timeout, **query,
+ throttle_fix=f"continue every page at Amazon's {1 / ENTITY_INTERVALS[operation]:g} requests per second")
+ body = got.get(payload_key) if payload_key else got
+ if operation == "searchListingsItems" and isinstance(body, dict):
+ count = body.get("numberOfResults")
+ if isinstance(count, int) and count > 1000:
+ midpoint = lower + (upper - lower) / 2
+ if (upper - lower) <= timedelta(milliseconds=1):
+ raise AmazonError("Amazon Listings search exceeds its 1,000 SKU limit within one millisecond. "
+ "No partial snapshot was stored. Ask Amazon to raise this search limit or use a SKU-filtered export.")
+ batches.extend(((lower, midpoint), (midpoint, upper)))
+ page_rows = []
+ break
+ page = body.get(root) if root and isinstance(body, dict) else body
+ if not isinstance(page, list) or any(not isinstance(row, dict) for row in page):
+ raise AmazonError(f"{operation} returned no valid row list; retry after Amazon restores its response")
+ page_rows.extend(page)
+ if isinstance(body, dict) and type(body.get("numberOfResults")) is int:
+ expected = max(expected or 0, body["numberOfResults"])
+ following = (got.get("pagination") or {}).get("nextToken")
+ if isinstance(body, dict):
+ following = following or (body.get("pagination") or {}).get("nextToken") or body.get("nextToken")
+ if not following:
+ if expected is not None and len(page_rows) != expected:
+ raise AmazonError(f"{operation} reported {expected} rows but returned {len(page_rows)}. "
+ "No partial snapshot was stored; retry after Amazon restores complete pagination.")
+ break
+ if not next_key or following in seen:
+ raise AmazonError(f"{operation} repeated or cannot follow its nextToken; retry after Amazon fixes pagination")
+ seen.add(following)
+ query[next_key] = following
+ rows.extend(page_rows)
+ if operation == "searchListingsItems":
+ unique = {}
+ for row in rows:
+ if not row.get("sku"):
+ raise AmazonError("Amazon listing search omitted a SKU; cannot reconcile date partition boundaries")
+ unique[row["sku"]] = row
+ rows = list(unique.values())
+ return rows
+
+
def _post_form(url, data, timeout=_TIMEOUT):
# LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
# LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
@@ -850,6 +1819,9 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
is that this function can reach exactly one operation and a caller cannot redirect it.
"""
require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type))
+ if any(s.get("snapshot") and s.get("report") == report_type for s in REPORT_TYPES.values()):
+ if start is not None or end is not None:
+ raise AmazonError("Snapshot reports cannot take a data window")
def attempt():
"""One raw attempt at the declared report-creation POST."""
@@ -877,10 +1849,12 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
except urllib.error.HTTPError as e:
detail = ""
try:
- errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or [])
- detail = str((errs[0] or {}).get("message") or "")[:200] if errs else ""
+ detail = (e.read() or b"").decode("utf-8", errors="replace")
+ errs = json.loads(detail or "{}").get("errors") or []
+ detail = " ".join(str(err.get(part) or "") for err in errs
+ for part in ("message", "details") if err.get(part)) or detail
except Exception: # noqa: BLE001
- detail = ""
+ pass
if e.code == 429:
raise AmazonThrottled(
f"{_safe(url)} was throttled (429) creating a {report_type} report. Report "
@@ -891,8 +1865,11 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
raise AmazonError(
f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this "
"report needs a role the application was not granted") from None
+ if e.code == 400 and report_type == REPORT_TYPES["search_query_performance"]["report"]:
+ raise AmazonReportError("Amazon refused Search Query Performance (400)",
+ detail, 400) from None
raise AmazonError(f"{_safe(url)} answered {e.code}"
- + (f": {detail}" if detail else "")) from None
+ + (f": {detail[:200]}" if detail else "")) from None
except Exception as e: # noqa: BLE001
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
rid = str(got.get("reportId") or "")
@@ -954,7 +1931,41 @@ def report_document(fields, report_type, start=None, end=None, options=None,
doc_id = str(got.get("reportDocumentId") or "")
break
if state in _REPORT_DEAD:
- raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})")
+ if state == "FATAL" and report_type == REPORT_TYPES["search_query_performance"]["report"]:
+ error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
+ reason = str(got.get("errorMessage") or "No error document was supplied by Amazon")
+ if error_id:
+ meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
+ region=region, timeout=timeout, sandbox=sbx)
+ if meta.get("url"):
+ reason = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
+ raise AmazonReportError(f"Amazon ended Search Query Performance as FATAL (reportId {rid})",
+ reason, state)
+ if state == "CANCELLED":
+ # S66-T42 (lead, observed live 2026-09-25): Amazon documents CANCELLED as also
+ # meaning "the report had no data to return". Raising turned an empty period into
+ # a role-shaped refusal on screen and left the window to retry forever. An empty
+ # document lets the window complete and the connector detail name the empty period.
+ say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
+ return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True,
+ "dataStartTime": got.get("dataStartTime"),
+ "dataEndTime": got.get("dataEndTime"),
+ "createdTime": got.get("createdTime"), "compression": None}
+ # Every FATAL report now carries Amazon's OWN reason when it supplied an error document,
+ # so the connector detail names why, never only a generic role hint.
+ detail = ""
+ error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
+ if error_id:
+ try:
+ meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
+ region=region, timeout=timeout, sandbox=sbx)
+ if meta.get("url"):
+ detail = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
+ except AmazonError:
+ detail = ""
+ detail = " ".join(str(detail).replace("\u2014", "; ").replace("\u2013", " to ").split())[:240]
+ raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})"
+ + (f": {detail}" if detail else ""))
time.sleep(poll_seconds)
waited += poll_seconds
# ⛔ The token is ~1 hour and a long report can outlive it. Re-mint rather than 403.
@@ -973,6 +1984,8 @@ def report_document(fields, report_type, start=None, end=None, options=None,
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
say(f" {report_type}: {len(text):,} chars")
return text, {"reportId": rid, "reportDocumentId": doc_id,
+ "dataStartTime": got.get("dataStartTime"),
+ "dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
"compression": meta.get("compressionAlgorithm")}
@@ -1039,6 +2052,31 @@ def settlement_document(fields, document_id, *, timeout=_TIMEOUT):
return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
+def _workbook_as_tsv(raw):
+ """An Excel workbook document, as the tab-separated text every flat-file parser reads.
+
+ S66-T42 (lead, observed live 2026-09-25): Amazon delivers some reports (FEE_DISCOUNTS_REPORT,
+ GET_REMOTE_FULFILLMENT_ELIGIBILITY) as an .xlsx workbook, a ZIP container. Decoded as text it
+ became garbage column names carrying NUL bytes, and Postgres refused the whole Amazon write.
+ The first sheet is the report; its first row is the header, like a flat file. Tabs and line
+ breaks inside a cell become spaces so the TSV shape holds.
+ """
+ import io
+ try:
+ import openpyxl
+ book = openpyxl.load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
+ except Exception as e: # noqa: BLE001
+ raise AmazonError(f"the report document is a workbook that did not open: {type(e).__name__}") from None
+ sheet = book.worksheets[0] if book.worksheets else None
+ lines = []
+ for values in (sheet.iter_rows(values_only=True) if sheet is not None else ()):
+ cells = ["" if v is None else " ".join(str(v).replace("\x00", "").split()) for v in values]
+ if any(cells):
+ lines.append("\t".join(cells))
+ book.close()
+ return "\n".join(lines)
+
+
def _download_document(url, compression="", timeout=120):
"""Fetch one presigned report document and decode it. -> text.
@@ -1057,12 +2095,20 @@ def _download_document(url, compression="", timeout=120):
raw = r.read()
except Exception as e: # noqa: BLE001
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
+ # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
+ # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
+ # algorithm the bytes were decoded as text and every economics window failed to parse. The
+ # gzip magic number is unambiguous (no text report starts with 0x1f 0x8b), so it decides.
+ if not compression and raw[:2] == b"\x1f\x8b":
+ compression = "GZIP"
if str(compression).upper() == "GZIP":
import gzip
try:
raw = gzip.decompress(raw)
except Exception as e: # noqa: BLE001
raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
+ if raw[:4] == b"PK\x03\x04":
+ return _workbook_as_tsv(raw)
for enc in ("utf-8", "cp1252"):
try:
return raw.decode(enc)
diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py
index fc14c2db3ee589ca1405403a7f9c82b1b2fc0117..864d0ff450c117a9e1bbe8304fc47894727ca048 100644
--- a/platform/harness/amazon_schema.py
+++ b/platform/harness/amazon_schema.py
@@ -53,6 +53,9 @@ from pathlib import Path
_HERE = Path(__file__).resolve().parent
CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json"
+if str(_HERE.parent) not in sys.path:
+ sys.path.insert(0, str(_HERE.parent))
+from harness.amazon_client import GROUP_A_TABLES, DECLARED_DATASETS
#: The vendor repository, pinned. ⚠ `main` is what `--refresh` reads, and the resulting file
#: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column
@@ -81,14 +84,24 @@ PROVENANCE = {
"inventory": "measured_at_sync",
"pricing": "schema_artifact",
"brand_analytics": "schema_artifact",
+ "economics": "measured_at_sync",
+ "transactions": "measured_at_sync",
+ "fba_fee_estimates": "measured_at_sync",
+ "referral_fees": "measured_at_sync",
+ "storage_fees": "measured_at_sync",
+ "search_query_performance": "measured_at_sync",
}
+PROVENANCE.update(dict.fromkeys(GROUP_A_TABLES, "measured_at_sync"))
+PROVENANCE.update(dict.fromkeys(DECLARED_DATASETS, "declared"))
+
#: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
#: screen and standing rule 2 governs it.
PROVENANCE_COPY = {
+ "declared": "This dataset is listed for completeness; its named reason explains why no rows are loaded.",
"schema_artifact": "Columns come from Amazon's published report schema, pinned below.",
- "measured_at_sync": ("Amazon publishes no schema for this report. Its columns are read from "
- "the report's own header row the first time it syncs."),
+ "measured_at_sync": ("Columns are measured from the document header or JSON record keys "
+ "when this database syncs; no column artifact is pinned here."),
}
_JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"}
@@ -254,9 +267,23 @@ def refresh(log=print):
def catalogue():
"""The pinned catalogue off disk, or an empty shell when it has never been refreshed."""
try:
- return json.loads(CATALOGUE.read_text(encoding="utf-8"))
+ result = json.loads(CATALOGUE.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
- return {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}}
+ result = {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}}
+ from core.connector_spec import AMAZON_TABLES
+ descriptors = {row.key: row for row in AMAZON_TABLES}
+ for name, provenance in PROVENANCE.items():
+ key = "ut_amzn_" + name
+ table = result.setdefault("tables", {}).setdefault(name, {})
+ table["provenance"] = provenance
+ if name in DECLARED_DATASETS:
+ declared = DECLARED_DATASETS[name]
+ label = declared["label"]
+ native = ", ".join(declared.get("report_types") or declared.get("operations") or ())
+ else:
+ label, native = descriptors[key].label, descriptors[key].native
+ table[key] = {"label": label, "native": native}
+ return result
def columns_for(table):
@@ -282,7 +309,7 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
"provenance": PROVENANCE[table],
"assumption": (
"A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. "
- "The five measured_at_sync tables' columns are re-measured against the first "
+ "Measured-at-sync columns are re-measured against the first "
"production sync during the deferred QI13 hotfix (intake RDA6)."),
},
"row_identity": identity,
@@ -358,9 +385,122 @@ TABLE_SPECS = {
"brand_analytics": _table_spec("brand_analytics", (
"department_name", "search_term", "clicked_asin", "click_share_rank",
)),
+ "economics": _table_spec("economics", ("start_date", "msku", "marketplace_id"), (
+ ("msku", "listings", "seller_sku", "many_to_one"),
+ ("child_asin", "listings", "asin1", "many_to_one"),
+ ), grain="day_msku"),
+ "transactions": _table_spec("transactions", ("transaction_id", "item_position"), (
+ ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),
+ ("sku", "listings", "seller_sku", "many_to_one"),
+ ), grain="transaction_item"),
+ # S66-T40 (lead, measured live): the fee estimate snapshot has one row per SKU per Amazon
+ # store, and the referral fee preview spells its SKU `seller_sku`.
+ "fba_fee_estimates": _table_spec("fba_fee_estimates", ("_snapshot_date", "sku", "amazon_store"), (
+ ("sku", "listings", "seller_sku", "many_to_one"),
+ ), grain="snapshot_sku"),
+ "referral_fees": _table_spec("referral_fees", ("_snapshot_date", "seller_sku"), (
+ ("seller_sku", "listings", "seller_sku", "many_to_one"),
+ ), grain="snapshot_sku"),
+ "storage_fees": _table_spec("storage_fees", ("month", "asin", "fnsku", "fulfillment_center"), (
+ ("asin", "listings", "asin1", "many_to_one"),
+ ), grain="month_product_center"),
+ "search_query_performance": _table_spec("search_query_performance", (
+ "start_date", "end_date", "asin", "search_query_data_search_query"), (
+ ("asin", "listings", "asin1", "many_to_one"),
+ ), grain="month_asin_search_query"),
}
+TABLE_SPECS.update({
+ "long_term_storage_fees": _table_spec("long_term_storage_fees", ("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
+ "storage_overage_fees": _table_spec("storage_overage_fees", ("_period_start", "storage_type", "_report_row_number"),
+ (), grain="month_row"),
+ "reimbursements": _table_spec("reimbursements", ("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"), ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),), grain="window_row"),
+ "fba_customer_returns": _table_spec("fba_customer_returns", ("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
+ (("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "replacements": _table_spec("replacements", ("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "fba_shipped_sales": _table_spec("fba_shipped_sales", ("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
+ (("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "fba_shipment_promotions": _table_spec("fba_shipment_promotions", ("_period_start", "amazon_order_id", "sku", "_report_row_number"),
+ (("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
+ "inventory_ledger_detail": _table_spec("inventory_ledger_detail", ("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
+ (("msku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "inventory_planning": _table_spec("inventory_planning", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "manage_inventory": _table_spec("manage_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "reserved_inventory": _table_spec("reserved_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "restock_recommendations": _table_spec("restock_recommendations", ("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "stranded_inventory": _table_spec("stranded_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "inbound_noncompliance": _table_spec("inbound_noncompliance", ("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "inventory_expiration_dates": _table_spec("inventory_expiration_dates", ("_snapshot_date", "sku", "fnsku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "recommended_removals": _table_spec("recommended_removals", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "removal_orders": _table_spec("removal_orders", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
+ "removal_shipments": _table_spec("removal_shipments", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
+ "subscribe_save_forecast": _table_spec("subscribe_save_forecast", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "subscribe_save_performance": _table_spec("subscribe_save_performance", ("_period_start", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "remote_fulfillment_eligibility": _table_spec("remote_fulfillment_eligibility", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "search_catalog_performance": _table_spec("search_catalog_performance", ("_period_start", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
+ "market_basket": _table_spec("market_basket", ("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
+ "repeat_purchase": _table_spec("repeat_purchase", ("_period_start", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
+ "promotion_performance": _table_spec("promotion_performance", ("_period_start", "promotion_id", "_report_row_number"),
+ (), grain="window_row"),
+ "coupon_performance": _table_spec("coupon_performance", ("_period_start", "coupon_id", "_report_row_number"),
+ (), grain="window_row"),
+ "account_health": _table_spec("account_health", ("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
+ (), grain="snapshot_row"),
+ "inactive_listings": _table_spec("inactive_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
+ (("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "cancelled_listings": _table_spec("cancelled_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
+ (("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "suppressed_listings": _table_spec("suppressed_listings", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "listing_defects": _table_spec("listing_defects", ("_snapshot_date", "sku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "sold_listings": _table_spec("sold_listings", ("_snapshot_date", "sku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "mfn_returns": _table_spec("mfn_returns", ("_period_start", "order_id", "sku", "asin", "_report_row_number"),
+ (("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
+ "b2b_recommended": _table_spec("b2b_recommended", ("_snapshot_date", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "b2b_not_yet_on_amazon": _table_spec("b2b_not_yet_on_amazon", ("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
+ (), grain="snapshot_row"),
+ "geo_opportunities": _table_spec("geo_opportunities", ("_snapshot_date", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "out_of_stock_mfn": _table_spec("out_of_stock_mfn", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "b2b_quotes": _table_spec("b2b_quotes", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "fee_discounts": _table_spec("fee_discounts", ("_snapshot_date", "sku", "asin", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "order_metrics": _table_spec("order_metrics", ("_period_start", "interval", "_report_row_number"),
+ (), grain="window_row"),
+ "fba_inventory": _table_spec("fba_inventory", ("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
+ (("seller_sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+ "listing_items": _table_spec("listing_items", ("_snapshot_date", "sku", "_report_row_number"),
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
+ "catalog_items": _table_spec("catalog_items", ("_snapshot_date", "asin", "_report_row_number"),
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
+})
+
+
def row_identity_for(table, row):
"""Evaluate the declared full composite. Missing lineage is an error, never a parent ID.
@@ -469,7 +609,8 @@ def selftest(log=print):
ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan)
# NC-AS7 — the seven tables of R4, no more and no fewer.
- ck("NC-AS7 seven tables are declared", len(PROVENANCE) == 7, sorted(PROVENANCE))
+ ck("NC-AS7 table and declared dataset keys cover group (a)",
+ set(PROVENANCE) == set(TABLE_SPECS) | set(DECLARED_DATASETS), sorted(PROVENANCE))
# NC-AS8 — standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen.
dashed = [k for k, v in PROVENANCE_COPY.items() if "—" in v or "–" in v]
@@ -477,8 +618,8 @@ def selftest(log=print):
# NC-AS9 — disclosure() names every table and carries the sync stamp key even when unpinned.
d = disclosure()
- ck("NC-AS9 disclosure covers all seven and carries a synced field",
- len(d["tables"]) == 7 and "synced" in d and all(r["table"].startswith("ut_amzn_")
+ ck("NC-AS9 disclosure covers all datasets and carries a synced field",
+ len(d["tables"]) == len(PROVENANCE) and "synced" in d and all(r["table"].startswith("ut_amzn_")
for r in d["tables"]))
# NC-AS10 — a measured_at_sync table must ship ZERO columns. An invented column is the defect.
diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py
index a1088eb0f319150825a187712d99f9bc76226327..2bc94de4e1557e3a35c099519802bfad0b349c80 100644
--- a/platform/harness/amazon_store.py
+++ b/platform/harness/amazon_store.py
@@ -42,13 +42,16 @@ from __future__ import annotations
import argparse
import csv
+import hashlib
import io
import json
+import re
import os
import shutil
import sys
import tempfile
import time
+import threading
import uuid
from contextlib import closing
from datetime import datetime, timedelta, timezone
@@ -68,15 +71,24 @@ PREFIX = "amzn_"
#: report document as `orders` rather than costing a second report: the flat-file order report is
#: already one row per order ITEM, so orders is the deduplicated head and order_items the lines.
TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
- "pricing", "brand_analytics")
+ "pricing", "brand_analytics", "economics", "transactions",
+ "fba_fee_estimates", "referral_fees", "storage_fees", "search_query_performance")
+MARGIN_TABLES = ("economics", "transactions", "fba_fee_estimates", "referral_fees", "storage_fees")
+SQP_TABLE = "search_query_performance"
+SQP_LIMIT_KEY = "amzn_limit_search_query_performance"
+GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
+TABLES += GROUP_A_TABLES
+PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES
+LEGACY_SNAPSHOT_TABLES = ("listings", "inventory")
#: Head -> lines: both tables read the same document; a second report buys no new rows.
SHARED_REPORT_SIBLINGS = {"orders": "order_items"}
# C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity.
ENABLEMENT_KEY = "amazon_connector_enablement"
-# Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing.
-SETTLEMENT_SYNC_SECONDS = 14 * 24 * 60 * 60
+MIRROR_PREFIX = "amazon_mirror/"
+_MIRROR_CACHE = {}
+_MIRROR_LOCK = threading.RLock()
def read_database_enablement(rt):
@@ -100,23 +112,22 @@ def read_database_enablement(rt):
def sync_plan(states, tables=None, *, now=None):
"""Pure cadence decision: one oldest due table per tick, with no stagger sleep.
- Attempt markers live in this tenant's mirror, so a replacement mirror starts fresh. Failed
- tables rotate behind their peers and retry on a later tick, including failed settlements.
+ Attempt markers survive in the tenant's saved mirror. Failed tables rotate behind peers.
+ Settlement LIST discovery runs separately on every resync and consumes no report slot.
The client still owns pacing/retries within a table; this spreads tables across caller ticks.
"""
from harness.amazon_client import REPORT_STAGGER_SECONDS
now = _utc(now)
want = list(TABLES if tables is None else tables)
- stamps = {table: _utc(state["updated_at"]) for table, state in states.items() if state}
+ stamps = {table: _utc(state["updated_at"]) for table, state in states.items()
+ if state and table != "settlements"}
deferred = {}
due = []
for table in want:
- stamp = stamps.get(table)
- if (table == "settlements" and stamp is not None
- and states[table]["state"] == "done"
- and (now - stamp).total_seconds() < SETTLEMENT_SYNC_SECONDS):
- deferred[table] = "settlements are on a 14-day cycle; next due " + _iso(
- stamp + timedelta(seconds=SETTLEMENT_SYNC_SECONDS))
+ if table == "settlements":
+ continue
+ if (reason := _interval_reason(table, states.get(table), now)):
+ deferred[table] = reason
else:
due.append(table)
# Boot and the resident first pass can be adjacent. Report the delay instead of sleeping.
@@ -128,7 +139,30 @@ def sync_plan(states, tables=None, *, now=None):
if table != selected:
deferred[table] = ("table stagger: waiting for a later cadence tick; "
f"minimum spacing {REPORT_STAGGER_SECONDS:g}s")
- return {"tables": [selected] if selected else [], "deferred": deferred}
+ return {"tables": ([selected] if selected else []) + (["settlements"] if "settlements" in want else []),
+ "deferred": deferred}
+
+
+def _readable_duration(seconds):
+ """`86400` -> "24 hours", `2592000` -> "30 days". Screen copy, never raw seconds (S66-T42)."""
+ seconds = int(seconds)
+ for size, unit in ((86400, "day"), (3600, "hour"), (60, "minute")):
+ if seconds >= size and seconds % size == 0:
+ count = seconds // size
+ if unit == "day" and count <= 2:
+ count, unit = count * 24, "hour"
+ return f"{count} {unit}" + ("s" if count != 1 else "")
+ return f"{seconds} seconds"
+
+
+def _interval_reason(table, state, now):
+ seconds = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s")
+ if seconds and state and state["state"] == "done":
+ due = _utc(state["updated_at"]) + timedelta(seconds=seconds)
+ if _utc(now) < due:
+ return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; "
+ f"next due {_iso(due)}")
+ return None
#: Durable proof that the two non-report role operations actually ran. A report-backed table can
#: contain rows while one of these role grants is still missing, so table presence alone is not
@@ -172,7 +206,12 @@ FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
_ORIGIN_TYPES = {"_report_document_id": "VARCHAR", "_report_row_number": "BIGINT",
"_data_origin": "VARCHAR", "_report_type": "VARCHAR"}
DELETION_DAYS = 30
+# Allow re-add for 25 days, leaving five days inside Amazon's 30-day deadline.
+# Even twice-daily resident resync checks then have ample time to complete deletion.
+PURGE_AFTER_DAYS = 25
DELETION_TRIGGERS = ("revocation", "authorization_loss", "termination", "notice")
+_RETENTION_CHECKED = {}
+_RETENTION_LOCK = threading.RLock()
def _origin(row, document_id, row_number, report_type):
@@ -188,6 +227,10 @@ def _origin(row, document_id, row_number, report_type):
_FORCE_TEXT = {"amazon_order_id", "merchant_order_id", "order_id", "asin", "parent_asin",
"child_asin", "sku", "seller_sku", "settlement_id", "shipment_id", "order_item_id",
"merchant_order_item_id", "fnsku", "product_id", "marketplace_id", "seller_id"}
+_FORCE_TEXT.update({"msku", "transaction_id", "currency_code", "month", "fulfillment_center"})
+_FORCE_TEXT.add("search_query_data_search_query")
+_FORCE_TEXT.update({"ean", "upc", "isbn", "part_number", "model_number", "promotion_id", "coupon_id",
+ "reimbursement_id", "replacement_amazon_order_id", "original_amazon_order_id"})
def _snake(name):
@@ -231,6 +274,27 @@ def _dedupe(cols):
return out
+def buyer_column(name):
+ """Deny buyer identity and destination address columns, regardless of spelling."""
+ compact = re.sub(r"[^a-z0-9]", "", str(name).lower())
+ return (bool(re.search(r"(?:buyer|customer|recipient)[a-z0-9]*(?:name|email|phone|telephone)", compact))
+ or any(word in compact for word in ("buyername", "buyeremail", "buyerphone",
+ "customername", "customeremail", "customerphone", "recipientname", "recipientphone",
+ "recipientemail", "rateremail", "shipaddress", "shippingaddress", "shiptoaddress",
+ "destinationaddress", "removaladdress", "addressline"))
+ or compact in {"email", "emailaddress", "phone", "phonenumber", "address", "address1",
+ "address2", "address3", "shiptoname", "shiptophone", "shiptoemail"})
+
+
+def _without_buyer_data(value):
+ # Lists can be serialized into one cell, so sanitize recursively before flattening.
+ if isinstance(value, dict):
+ return {k: _without_buyer_data(v) for k, v in value.items() if not buyer_column(k)}
+ if isinstance(value, list):
+ return [_without_buyer_data(v) for v in value]
+ return value
+
+
def parse_flat_file(text, *, document_id=None, report_type=None):
"""One Amazon flat-file report document -> `(columns, rows)`, columns MEASURED from its header.
@@ -262,9 +326,10 @@ def parse_flat_file(text, *, document_id=None, report_type=None):
if not raw or (len(raw) == 1 and not str(raw[0]).strip()):
continue
vals = list(raw[:width]) + [None] * max(0, width - len(raw))
- row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)}
+ row = {c: (v if (v is None or str(v) != "") else None) for c, v in zip(cols, vals)
+ if not buyer_column(c)}
rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
- return cols + (list(_ORIGIN_TYPES) if attributed else []), rows
+ return [c for c in cols if not buyer_column(c)] + (list(_ORIGIN_TYPES) if attributed else []), rows
def parse_json_report(text, root, *, document_id=None, report_type=None):
@@ -277,15 +342,19 @@ def parse_json_report(text, root, *, document_id=None, report_type=None):
"""
try:
doc = json.loads(str(text or "") or "{}")
- except Exception: # noqa: BLE001
- return [], []
- body = doc.get(root) or []
+ except (ValueError, TypeError) as exc:
+ raise ValueError("Amazon returned malformed JSON; no rows were replaced") from exc
+ if not isinstance(doc, dict) or root not in doc:
+ raise ValueError(f"Amazon JSON report omitted its {root} row root; no rows were replaced")
+ body = doc[root]
if isinstance(body, dict):
body = [body]
+ if not isinstance(body, list) or any(not isinstance(item, dict) for item in body):
+ raise ValueError(f"Amazon JSON report has an invalid {root} row list")
rows = []
attributed = document_id is not None or report_type is not None
for row_number, item in enumerate(body, 1):
- row = _flatten_row(item or {})
+ row = _flatten_row(_without_buyer_data(item))
rows.append(_origin(row, document_id, row_number, report_type) if attributed else row)
return sorted({k for r in rows for k in r}), rows
@@ -300,17 +369,173 @@ def _flatten_row(obj, prefix=""):
out = {}
for k, v in (obj or {}).items():
path = f"{prefix}.{k}" if prefix else k
+ if buyer_column(path):
+ continue
if isinstance(v, dict):
out.update(_flatten_row(v, path))
else:
name = amazon_schema._snake(path)
if name in _ORIGIN_TYPES:
raise ValueError("Amazon JSON field collides with reserved origin metadata")
- out[name] = json.dumps(v, separators=(",", ":"), ensure_ascii=False) if isinstance(
+ out[name] = json.dumps(_without_buyer_data(v), separators=(",", ":"), ensure_ascii=False) if isinstance(
v, list) else v
return out
+def _money(value):
+ """The two Amazon money shapes, with missing values kept distinct from zero."""
+ if not isinstance(value, dict):
+ raise ValueError("Amazon monetary summary is missing")
+ amount = value.get("amount", value.get("currencyAmount"))
+ currency = value.get("currencyCode")
+ if amount is None or not currency:
+ raise ValueError("Amazon monetary summary needs an amount and currency")
+ number = float(amount)
+ if not -float("inf") < number < float("inf"):
+ raise ValueError("Amazon monetary summary must be finite")
+ return number, str(currency)
+
+
+def economics_records(text):
+ """Data Kiosk JSONL records, failing loudly on malformed or error payloads."""
+ records = []
+ for line in str(text or "").splitlines():
+ if not line.strip():
+ continue
+ record = json.loads(line)
+ if not isinstance(record, dict) or "startDate" not in record:
+ raise ValueError("Economics JSONL record has no startDate")
+ records.append(record)
+ return records
+
+
+def economics_rows(records, document_id):
+ """Pivot all fee/ads types into numeric columns; costs and sales flatten normally."""
+ rows = []
+ for position, record in enumerate(records, 1):
+ row = _flatten_row({k: v for k, v in record.items() if k not in ("fees", "ads")})
+ currencies = set()
+ fee_total = 0.0
+ charge_counts = {}
+ for family, label in (("fees", "fee"), ("ads", "ads")):
+ names = {}
+ for entry in record.get(family) or []:
+ raw_name = entry.get("feeTypeName" if family == "fees" else "adTypeName")
+ name = _snake(amazon_schema._snake(raw_name or ""))
+ if not name or (name in names and names[name] != raw_name):
+ raise ValueError("Amazon fee names cannot be mapped to distinct columns")
+ names[name] = raw_name
+ prefix = f"{label}_{name}"
+ row[prefix + "_type_name"] = raw_name
+ charges = ((entry.get("charges") or ([entry["charge"]] if entry.get("charge") else []))
+ if family == "fees" else [entry.get("charge")])
+ for charge in charges:
+ if not charge:
+ continue
+ detail = charge.get("aggregatedDetail", charge)
+ amount, currency = _money(detail.get("totalAmount"))
+ currencies.add(currency)
+ if family == "fees":
+ fee_total += amount
+ row[prefix + "_amount"] = row.get(prefix + "_amount", 0.0) + amount
+ # Rate-card changes can split charges within a day. Preserve each charge's
+ # per-unit amounts separately; adding those would invent a per-unit rate.
+ charge_counts[prefix] = charge_counts.get(prefix, 0) + 1
+ charge_prefix = f"{prefix}_charge_{charge_counts[prefix]}"
+ row.update(_flatten_row(charge, charge_prefix))
+ for source in (record.get("sales", {}).get("orderedProductSales"),
+ record.get("netProceeds", {}).get("total")):
+ if source:
+ currencies.add(_money(source)[1])
+ if len(currencies) != 1:
+ raise ValueError("Economics row must carry one currency")
+ row["currency_code"] = next(iter(currencies))
+ row["fees_total_amount"] = fee_total
+ rows.append(_origin(row, document_id, position, "analytics_economics_2024_03_15"))
+ return rows
+
+
+def economics_summary(records):
+ """Sum the separate coarser vendor response, independently of the detail flattener."""
+ totals = {}
+ for record in records:
+ sales, currency = _money(record["sales"]["orderedProductSales"])
+ net, net_currency = _money(record["netProceeds"]["total"])
+ if net_currency != currency:
+ raise ValueError("Economics summary mixes currencies")
+ fee_total = 0.0
+ for fee in record.get("fees") or []:
+ for charge in fee.get("charges") or ([fee["charge"]] if fee.get("charge") else []):
+ fee_amount, fee_currency = _money(charge["aggregatedDetail"]["totalAmount"])
+ if fee_currency != currency:
+ raise ValueError("Economics summary fee currency differs")
+ fee_total += fee_amount
+ group = totals.setdefault(currency, {"sales": 0.0, "fees": 0.0, "net": 0.0})
+ for key, value in (("sales", sales), ("fees", fee_total), ("net", net)):
+ group[key] += value
+ return totals
+
+
+def _breakdowns(value):
+ # Amazon's published example wraps this list; its model declares a direct array.
+ return (value.get("breakdowns") if isinstance(value, dict) else value) or []
+
+
+def _pivot_breakdowns(value, row, prefix):
+ for part in _breakdowns(value):
+ name = _snake(amazon_schema._snake(part["breakdownType"]))
+ amount, currency = _money(part["breakdownAmount"])
+ column = f"{prefix}_breakdown_{name}_amount"
+ row[column] = row.get(column, 0.0) + amount
+ row[column + "_currency_code"] = currency
+ for child in _breakdowns(part.get("breakdowns")):
+ child_name = _snake(amazon_schema._snake(child["breakdownType"]))
+ child_amount, child_currency = _money(child["breakdownAmount"])
+ column = f"{prefix}_sub_breakdown_{name}__{child_name}_amount"
+ row[column] = row.get(column, 0.0) + child_amount
+ row[column + "_currency_code"] = child_currency
+
+
+def transaction_rows(transactions, operation_id):
+ """One row per item. Transaction totals/breakdowns live only on item position zero."""
+ rows, ids = [], set()
+ for transaction in transactions:
+ transaction_id = transaction.get("transactionId")
+ if not transaction_id or transaction_id in ids:
+ raise ValueError("Finances returned a missing or duplicate transactionId")
+ ids.add(transaction_id)
+ amount, currency = _money(transaction.get("totalAmount"))
+ order_id = next((x.get("relatedIdentifierValue") for x in transaction.get("relatedIdentifiers") or []
+ if x.get("relatedIdentifierName") == "ORDER_ID"), None)
+ for position, item in enumerate(transaction.get("items") or [{}]):
+ row = _flatten_row({k: v for k, v in transaction.items()
+ if k not in ("items", "breakdowns", "totalAmount")})
+ row.update(transaction_id=str(transaction_id), item_position=position,
+ amazon_order_id=order_id, total_amount=amount if position == 0 else None,
+ currency_code=currency,
+ transaction_item_count=len(transaction.get("items") or [{}]) if position == 0 else None)
+ row["posted_date"] = _utc(transaction["postedDate"]).isoformat().replace("+00:00", "Z")
+ row.update(_flatten_row({k: v for k, v in item.items() if k != "breakdowns"}, "item"))
+ context = next((c for c in item.get("contexts") or []
+ if c.get("contextType") == "ProductContext" or c.get("sku")), {})
+ row.update(sku=context.get("sku"), asin=context.get("asin"))
+ if position == 0:
+ _pivot_breakdowns(transaction.get("breakdowns"), row, "transaction")
+ _pivot_breakdowns(item.get("breakdowns"), row, "item")
+ rows.append(_origin(row, operation_id, len(rows) + 1, "listTransactions"))
+ return rows
+
+
+def transaction_summary(transactions):
+ totals = {}
+ for transaction in transactions:
+ amount, currency = _money(transaction.get("totalAmount"))
+ group = totals.setdefault(currency, {"amount": 0.0, "count": 0})
+ group["amount"] += amount
+ group["count"] += 1
+ return totals
+
+
def _competitive_rows(responses):
"""Amazon batch responses -> complete flattened enrichments keyed by returned ASIN."""
out = []
@@ -582,11 +807,418 @@ def _windows(start, end, size=MAX_WINDOW_DAYS):
return out
-def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS):
+def _month_windows(start, end):
+ """Completed calendar months only; never a sliding 30-day storage-fee period."""
+ current = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ stop = _utc(end).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ out = []
+ while current < stop:
+ following = (current.replace(day=28) + timedelta(days=4)).replace(day=1)
+ out.append((current, following))
+ current = following
+ return out
+
+
+def sqp_batches(asins, max_chars=200):
+ """Stable, lossless space-joined options within Amazon's character ceiling."""
+ if type(max_chars) is not int or max_chars < 1:
+ raise ValueError("Search Query Performance needs a positive ASIN option limit")
+ unique = sorted(set(asins))
+ batches, current = [], ""
+ for asin in unique:
+ if not isinstance(asin, str) or not asin or any(c.isspace() for c in asin):
+ raise ValueError(f"Invalid catalogue ASIN for Search Query Performance: {asin!r}")
+ if len(asin) > max_chars:
+ raise ValueError(f"ASIN {asin} exceeds Amazon's {max_chars}-character option limit")
+ candidate = f"{current} {asin}" if current else asin
+ if len(candidate) > max_chars:
+ batches.append(current)
+ current = asin
+ else:
+ current = candidate
+ return batches + ([current] if current else [])
+
+
+def _sqp_catalogue(con):
+ """Our mirror only: prefer the listings catalogue, fall back to pricing if empty."""
+ for table, column in (("amzn_listings", "asin1"), ("amzn_pricing", "child_asin")):
+ columns = {r[0] for r in con.execute(
+ "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()}
+ if column in columns:
+ asins = sorted({str(r[0]).strip() for r in con.execute(
+ f'SELECT DISTINCT "{column}" FROM "{table}" WHERE "{column}" IS NOT NULL').fetchall()
+ if str(r[0]).strip()})
+ if asins:
+ return asins
+ return []
+
+
+_ASIN_RE = re.compile(r"[A-Z0-9]{10}")
+
+
+def _columns_of(con, table):
+ return {r[0] for r in con.execute(
+ "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()}
+
+
+_NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number")
+
+
+def _newest_child_parent_sql(columns, table, only=None):
+ """SELECT child_asin, parent_asin: one row per child, the newest period's parent; None if absent.
+
+ `only` is an optional SQL SELECT of the child ASINs wanted. It filters BEFORE the window, which
+ keeps or drops whole `child_asin` partitions, so every kept child's answer is unchanged. Once
+ the catalog carries parents (its first sync after S67-T29), `asin_parents_sql` passes only the
+ few own ASINs the catalog leaves without one, so S67-T30's per-page lookup stops ranking all of
+ economics (572,320 rows on staging, about 100 ms a query) to answer them.
+ """
+ if not {"child_asin", "parent_asin"} <= columns:
+ return None
+ order = ", ".join(f'"{c}" DESC NULLS LAST' for c in _NEWEST_ORDER if c in columns) or "1"
+ wanted = f" AND TRIM(CAST(child_asin AS VARCHAR)) IN ({only})" if only else ""
+ return (f'SELECT child_asin, parent_asin FROM (SELECT TRIM(CAST(child_asin AS VARCHAR)) AS child_asin, '
+ f'TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, ROW_NUMBER() OVER ('
+ f'PARTITION BY TRIM(CAST(child_asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM "{table}" '
+ f"WHERE child_asin IS NOT NULL AND parent_asin IS NOT NULL "
+ f"AND TRIM(CAST(parent_asin AS VARCHAR)) <> ''{wanted}) WHERE _rn = 1")
+
+
+def _child_parent_map(con, table):
+ """child ASIN -> parent ASIN from the newest rows of `table`; {} when it has no child_asin."""
+ sql = _newest_child_parent_sql(_columns_of(con, table), table)
+ if sql is None:
+ return {}
+ return {str(c): str(p) for c, p in con.execute(sql).fetchall()
+ if _ASIN_RE.fullmatch(str(c)) and _ASIN_RE.fullmatch(str(p))}
+
+
+def _pricing_parent_map(con):
+ """O23 fallback. Used only when pricing HAS child_asin: the live Sales and Traffic mirror is
+ stored at parent granularity (parent_asin only), so this is {} there and economics carries on."""
+ return _child_parent_map(con, "amzn_pricing")
+
+
+def _economics_parent_map(con):
+ """Data Kiosk economics child_asin -> parent_asin, newest start_date wins."""
+ return _child_parent_map(con, "amzn_economics")
+
+
+def _catalog_parent(item, asin, pricing_map, marketplace, economics_map=None):
+ """One item's (parent_asin, source); never raises on a missing or malformed `relationships`."""
+ parents, is_parent = [], False
+ groups = item.get("relationships")
+ for group in groups if isinstance(groups, list) else ():
+ if not isinstance(group, dict):
+ continue
+ if marketplace and group.get("marketplaceId") not in (None, marketplace):
+ continue
+ entries = group.get("relationships")
+ for rel in entries if isinstance(entries, list) else ():
+ # Only VARIATION defines a parent; PACKAGE_HIERARCHY is a pack, not a family.
+ if not isinstance(rel, dict) or rel.get("type") != "VARIATION":
+ continue
+ found = rel.get("parentAsins")
+ parents += [str(a).strip() for a in (found if isinstance(found, list) else ())
+ if _ASIN_RE.fullmatch(str(a).strip())]
+ if isinstance(rel.get("childAsins"), list) and rel["childAsins"]:
+ is_parent = True
+ if parents:
+ return parents[0], "relationships"
+ if is_parent:
+ return asin, "relationships"
+ for source, mapping in (("pricing", pricing_map), ("economics", economics_map)):
+ fallback = str((mapping or {}).get(asin) or "").strip()
+ if _ASIN_RE.fullmatch(fallback):
+ return fallback, source
+ return asin, "self"
+
+
+def catalog_parent_map(items, pricing_map=None, marketplace=None, economics_map=None):
+ """S67-T29 (O23): own ASIN -> (parent ASIN, source) for every catalog item.
+
+ Source order: the catalog VARIATION `parentAsins` for the requested marketplace; an item that
+ is itself a variation parent maps to itself; else pricing's `parent_asin` for that ASIN as
+ `child_asin` (O23); else economics child_asin -> parent_asin; else itself. A standalone ASIN
+ therefore maps to itself, never an error.
+ """
+ out = {}
+ for item in items or ():
+ if not isinstance(item, dict):
+ continue
+ asin = str(item.get("asin") or "").strip()
+ if asin:
+ out[asin] = _catalog_parent(item, asin, pricing_map, marketplace, economics_map)
+ return out
+
+
+#: S67 lane D (S67-T29 finding): Sales and Traffic defaults `asinGranularity` to PARENT, so the
+#: saved `amzn_pricing` mirror carried `parent_asin` with no `child_asin` and no `sku`, and
+#: `connector_spec`'s pricing identity `(child_asin, parent_asin)` and its two listings links never
+#: formed. SKU grain returns parentAsin, childAsin and sku per row: the SAME report type and role
+#: (the PRD's Amazon compliance line: no new endpoint or role).
+REPORT_OPTIONS = {"pricing": {"asinGranularity": "SKU"}}
+
+
+def _pricing_regrain(con, mirror, rows, log=print):
+ """Clear a PARENT-grain pricing mirror before the first SKU-grain rows land; True if cleared.
+
+ `amzn_pricing` keeps history across incremental passes, so without this the old parent-grain
+ days would sit beside SKU-grain days and any sum across days would count twice. A mirror that
+ already holds any `child_asin`, an empty or absent mirror, or rows without `child_asin` are
+ left alone.
+ """
+ if not any((row or {}).get("child_asin") for row in rows):
+ return False
+ have = _columns_of(con, mirror) # empty for an absent table (first load), never raises
+ if not have:
+ return False
+ if "child_asin" in have and con.execute(
+ f'SELECT COUNT(*) FROM {mirror} WHERE "child_asin" IS NOT NULL').fetchone()[0]:
+ return False
+ stale = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
+ if not stale:
+ return False
+ con.execute(f"DELETE FROM {mirror} WHERE 1=1")
+ log(f" pricing: replaced {stale} parent-grain rows with SKU-grain rows (one grain per table)")
+ return True
+
+
+def asin_parents_sql(con):
+ """S67-T29: ONE read-time resolver, one row per own ASIN: `(asin, parent_asin, parent_asin_source)`.
+
+ Own ASINs = DISTINCT listings `asin1` UNION the LATEST catalog `_snapshot_date`'s `asin` (the
+ snapshot tables keep one copy per sync day, D-3031, so only the newest day counts). Parent =
+ that latest catalog row's `parent_asin` (source as stamped at sync), else pricing child_asin ->
+ parent_asin when the pricing mirror carries child_asin (source `pricing`, O23), else economics
+ child_asin -> parent_asin (newest start_date, source `economics`), else the ASIN itself
+ (`self`). Covers own ASINs Amazon's catalog search returns no item for, without stub catalog
+ rows. Degrades with absent tables or columns; None only when no own-ASIN source exists.
+ Reusable verbatim by a SQL join (S67-T30): wrap it as a subquery.
+ """
+ listings, catalog = _columns_of(con, "amzn_listings"), _columns_of(con, "amzn_catalog_items")
+ latest = ('(SELECT MAX("_snapshot_date") FROM amzn_catalog_items)')
+ own = []
+ if "asin1" in listings:
+ own.append("SELECT TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM amzn_listings WHERE asin1 IS NOT NULL")
+ has_catalog = {"asin", "_snapshot_date"} <= catalog
+ if has_catalog:
+ own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin FROM amzn_catalog_items "
+ f'WHERE asin IS NOT NULL AND "_snapshot_date" = {latest}')
+ if not own:
+ return None
+ ctes = ["own AS (SELECT DISTINCT asin FROM (" + " UNION ".join(own) + ") WHERE asin <> '')"]
+ joins, parents, sources = [], [], []
+ if has_catalog and "parent_asin" in catalog:
+ source = ("CAST(parent_asin_source AS VARCHAR)" if "parent_asin_source" in catalog
+ else "'relationships'")
+ order = '"_report_row_number"' if "_report_row_number" in catalog else "1"
+ ctes.append(
+ "cat AS (SELECT asin, parent_asin, parent_asin_source FROM (SELECT TRIM(CAST(asin AS VARCHAR)) AS asin, "
+ "TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, "
+ f"COALESCE({source}, 'relationships') AS parent_asin_source, ROW_NUMBER() OVER ("
+ f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM amzn_catalog_items "
+ f"WHERE \"_snapshot_date\" = {latest} AND parent_asin IS NOT NULL "
+ "AND TRIM(CAST(parent_asin AS VARCHAR)) <> '') WHERE _rn = 1)")
+ joins.append("LEFT JOIN cat ON cat.asin = own.asin")
+ parents.append("cat.parent_asin")
+ sources.append("WHEN cat.parent_asin IS NOT NULL THEN cat.parent_asin_source")
+ # Economics is only the fallback, so rank only the own ASINs the catalog left without a parent.
+ only = "SELECT asin FROM own" + (" WHERE asin NOT IN (SELECT asin FROM cat)" if joins else "")
+ # O23: pricing is the fallback BEFORE economics. None (no step) while the pricing mirror has no
+ # `child_asin`, which is the live Sales and Traffic mirror today (S67-T29 finding).
+ pricing = _newest_child_parent_sql(_columns_of(con, "amzn_pricing"), "amzn_pricing", only=only)
+ if pricing:
+ ctes.append(f"pri AS ({pricing})")
+ joins.append("LEFT JOIN pri ON pri.child_asin = own.asin")
+ parents.append("pri.parent_asin")
+ sources.append("WHEN pri.parent_asin IS NOT NULL THEN 'pricing'")
+ economics = _newest_child_parent_sql(_columns_of(con, "amzn_economics"), "amzn_economics",
+ only=only)
+ if economics:
+ ctes.append(f"eco AS ({economics})")
+ joins.append("LEFT JOIN eco ON eco.child_asin = own.asin")
+ parents.append("eco.parent_asin")
+ sources.append("WHEN eco.parent_asin IS NOT NULL THEN 'economics'")
+ return ("WITH " + ", ".join(ctes) + " SELECT own.asin AS asin, "
+ f"COALESCE({', '.join([*parents, 'own.asin'])}) AS parent_asin, "
+ + (f"CASE {' '.join(sources)} ELSE 'self' END" if sources else "'self'")
+ + " AS parent_asin_source "
+ "FROM own " + " ".join(joins) + " ORDER BY own.asin")
+
+
+def asin_parents(con):
+ """`asin_parents_sql` as `{asin: (parent_asin, source)}`; {} when no own-ASIN source exists."""
+ sql = asin_parents_sql(con)
+ return {} if sql is None else {a: (p, s) for a, p, s in con.execute(sql).fetchall()}
+
+
+def sqp_limit_sentence(marker):
+ """Pure projection of saved sync facts; never asks Amazon for a catalogue."""
+ numbers = [marker[k] for k in ("maxChars", "maxPerBatch", "asinCount", "batchCount")]
+ if any(type(n) is not int or n < 0 for n in numbers) or not all(numbers[:2]):
+ raise ValueError("Invalid saved Search Query Performance limits")
+ chars, maximum, count, batches = numbers
+ return (f"Amazon accepts at most {chars} characters of ASINs per Search Query Performance "
+ f"report, about {maximum} ASINs, so {count} ASINs are requested in {batches} reports per month.")
+
+
+def sqp_refusal_message(asin, reason):
+ return f"Amazon refused ASIN {asin} for Search Query Performance: {reason}"
+
+
+def _sqp_month_key(start):
+ return entity_key("reconcile_search_query_performance_" + _utc(start).strftime("%Y_%m"))
+
+
+def _sqp_month(con, start, end):
+ state = _read_state(con, _sqp_month_key(start))
+ return json.loads(state["detail"]) if state and state.get("detail") else {
+ "start": _utc(start).date().isoformat(),
+ "end": (_utc(end) - timedelta(days=1)).date().isoformat(),
+ "batches": {}, "asins": {}, "retries": {}}
+
+
+def _sqp_save_month(con, start, marker):
+ _record_state(con, _sqp_month_key(start), "done", detail=json.dumps(marker))
+ _record_reconciliation(con, SQP_TABLE, marker)
+
+
+def _sqp_done(con, start, end, batch):
+ return batch in _sqp_month(con, start, end)["batches"]
+
+
+class SQPBudgetDeferred(RuntimeError):
+ """A batch or its single retry remains durable for the next bounded pass."""
+
+
+def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem):
+ """One month x batch using the shared report transport and outer pass budget.
+
+ A brand refusal is durable before the retry, so a budget boundary or process restart
+ resumes without the named ASINs. Only a successful document can replace mirror rows.
+ """
+ cx = _client()
+ spec = cx.REPORT_TYPES[SQP_TABLE]
+ marker = _sqp_month(con, start, end)
+ retry = marker["retries"].get(batch)
+ if retry and retry.get("failed"):
+ raise cx.AmazonError(retry["failed"])
+ requested = retry["asins"] if retry else batch.split()
+ while requested:
+ try:
+ text, meta = request(fields, spec["report"], start=_iso(start),
+ end=_iso(end - timedelta(seconds=1)),
+ options={"reportPeriod": spec["period"], "asin": " ".join(requested)})
+ break
+ except SQPBudgetDeferred:
+ raise
+ except cx.AmazonError as exc:
+ refused = cx.sqp_refused_asins(exc, requested)
+ reason = getattr(exc, "vendor_reason", str(exc))
+ reason = reason.replace("\u2014", "; ").replace("\u2013", " to ")
+ for asin in refused:
+ limit["refusedAsins"][asin] = reason
+ problem(sqp_refusal_message(asin, reason))
+ if not refused:
+ limit["errors"] = [reason]
+ _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(limit))
+ if retry:
+ retry["failed"] = str(exc)
+ _sqp_save_month(con, start, marker)
+ raise
+ if not refused:
+ raise
+ requested = [asin for asin in requested if asin not in refused]
+ retry = marker["retries"][batch] = {"asins": requested}
+ _sqp_save_month(con, start, marker)
+ else:
+ # Amazon refused every ASIN. Record the exclusion, without inventing a source summary.
+ marker["batches"][batch] = {"asins": [], "count": 0, "purchases": 0, "refused": True}
+ _sqp_save_month(con, start, marker)
+ return 0
+
+ # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success.
+ doc = json.loads(text)
+ raw = doc.get(spec["json_root"])
+ if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw):
+ raise ValueError("Search Query Performance document has no dataByAsin row array")
+ totals = {asin: {"count": 0, "purchases": 0} for asin in requested}
+ for row in raw:
+ asin = row.get("asin")
+ if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]:
+ raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch")
+ purchase = (row.get("purchaseData") or {}).get("asinPurchaseCount")
+ if type(purchase) is not int or purchase < 0:
+ raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count")
+ totals[asin]["count"] += 1
+ totals[asin]["purchases"] += purchase
+ columns, rows = parse_json_report(text, spec["json_root"],
+ document_id=meta.get("reportDocumentId"), report_type=spec["report"])
+ identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"]
+ seen = set()
+ for row in rows:
+ key = amazon_schema.row_identity_for(SQP_TABLE, row)
+ if key in seen:
+ raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query")
+ seen.add(key)
+ row["_report_period"] = spec["period"]
+ columns = list(dict.fromkeys([*columns, *identity, "_report_period",
+ "purchase_data_asin_purchase_count", *_ORIGIN_TYPES]))
+ mirror = table_name(SQP_TABLE)
+ types = _ensure(con, mirror, columns, rows, preserve_rows=True,
+ declared={"purchase_data_asin_purchase_count": "BIGINT"})
+ marker["asins"].update(totals)
+ marker["batches"][batch] = {"asins": requested, "count": len(raw),
+ "purchases": sum(t["purchases"] for t in totals.values()),
+ "reportDocumentId": meta["reportDocumentId"]}
+ con.execute("BEGIN TRANSACTION")
+ try:
+ # Empty documents remove stale rows too; other batches and months remain untouched.
+ con.execute(f"DELETE FROM {mirror} WHERE start_date=? AND end_date=? "
+ f"AND asin IN ({','.join('?' for _ in requested)})",
+ [marker["start"], marker["end"], *requested])
+ count = _replace(con, mirror, columns, types, rows)
+ _sqp_save_month(con, start, marker)
+ con.execute("COMMIT")
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+ return count
+
+
+def _validate_sqp(con):
+ state = _read_state(con, entity_key("reconcile_" + SQP_TABLE))
+ if not state or not state.get("detail"):
+ return {"state": "unproven", "reason": "Amazon Search Query Performance summary is absent"}
+ try:
+ marker = json.loads(state["detail"])
+ expected = {key: sum(t[key] for t in marker["asins"].values()) for key in ("count", "purchases")}
+ actual_rows = con.execute(
+ f"SELECT asin, COUNT(*), SUM(purchase_data_asin_purchase_count), "
+ f"COUNT(purchase_data_asin_purchase_count) FROM {table_name(SQP_TABLE)} "
+ "WHERE start_date=? AND end_date=? GROUP BY asin", [marker["start"], marker["end"]]).fetchall()
+ actual = {"count": sum(r[1] for r in actual_rows), "purchases": sum(r[2] or 0 for r in actual_rows)}
+ if not actual_rows or not marker["asins"]:
+ return {"state": "unproven", "reason": "latest Search Query Performance month has no proven rows",
+ "expected": expected, "actual": actual}
+ per_asin = {r[0]: {"count": r[1], "purchases": r[2]} for r in actual_rows}
+ matches = (expected == actual and all(r[1] == r[3] for r in actual_rows)
+ and all(per_asin.get(a, {"count": 0, "purchases": 0}) == v for a, v in marker["asins"].items())
+ and set(per_asin) <= set(marker["asins"]))
+ return {"state": "ok" if matches else "refused", "expected": expected, "actual": actual,
+ "reason": f"Amazon summary {expected}; stored detail {actual}"}
+ except Exception as exc:
+ return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"}
+
+
+def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS,
+ sqp_asins=()):
"""R18's request order: every recent month, then each table's history, oldest last.
Settlements have one discovery slot (None, None), never a createReport window.
This planner is pure; durable floors filter its fixed windows when a pass resumes.
+ SQP adds a fourth tuple member, its ASIN option, for each month x batch task.
"""
start, end = _utc(start), _utc(end)
if start >= end:
@@ -602,10 +1234,26 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY
# every new settlement document until the pass budget is spent and then ends the pass, so in
# table order it starved inventory, pricing and brand_analytics of their recent month for
# about four passes on live (52 fortnightly documents over 730 days).
- recent = ([(table, recent_start, end) for table in tables if table != "settlements"]
- + [(table, None, None) for table in tables if table == "settlements"])
- history = [(table, w0, w1) for table in tables if table != "settlements"
- for w0, w1 in reversed(_windows(start, recent_start, size))]
+ recent, history = [], []
+ for table in tables:
+ spec = _client().REPORT_TYPES[table]
+ if table == "settlements":
+ continue
+ if spec.get("snapshot"):
+ recent.append((table, None, None))
+ elif spec.get("monthly"):
+ months = list(reversed(_month_windows(start, end)))
+ tasks = ([(table, w0, w1, batch) for w0, w1 in months
+ for batch in sqp_batches(sqp_asins, spec["asin_option_max_chars"])]
+ if table == SQP_TABLE else [(table, w0, w1) for w0, w1 in months])
+ recent.extend(task for task in tasks if task[1:3] == months[0])
+ history.extend(task for task in tasks if task[1:3] != months[0])
+ else:
+ table_size = spec.get("max_window_days", size)
+ table_recent = max(start, end - timedelta(days=min(recent_days, table_size)))
+ recent.append((table, table_recent, end))
+ history.extend((table, w0, w1) for w0, w1 in reversed(_windows(start, table_recent, table_size)))
+ recent.extend((table, None, None) for table in tables if table == "settlements")
return recent + history
@@ -625,8 +1273,9 @@ def first_load_eta_cause(report_count):
"""Screen-safe explanation of the vendor ceiling and the automatic continuation."""
from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS
return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every "
- f"{REPORT_SUSTAINED_SECONDS:g} seconds. Keep the connection enabled while history "
- "loads automatically. Report processing and other syncs can add time.")
+ f"{REPORT_SUSTAINED_SECONDS:g} seconds for {report_count} remaining tasks. "
+ f"The report pacing alone needs at least {first_load_eta(report_count, burst=0)} seconds. "
+ "Keep the connection enabled. Per-report refresh floors, API pages and report processing add time.")
def _iso(dt):
@@ -662,7 +1311,20 @@ def credential(tenant_key="royal-imports", rt=None):
f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}")
try:
import core.keychain as kc
- return kc.amazon_creds(runtime)
+ fields = kc.amazon_creds(runtime)
+ if fields is None and runtime is not None and kc.first_entry_of_type(runtime, "amazon_sp") is None:
+ # Boot and resident resync reach this door even after the credential is gone.
+ # Claim the interval before work, including failures; no recursion via credential().
+ with _RETENTION_LOCK:
+ tick = time.monotonic()
+ previous = _RETENTION_CHECKED.get(key)
+ if previous is None or tick - previous >= 3600:
+ _RETENTION_CHECKED[key] = tick
+ try:
+ enforce_retention(key, runtime)
+ except Exception as exc:
+ print(f"Amazon retention check failed ({type(exc).__name__}); retry on the next check.")
+ return fields
except Exception: # noqa: BLE001
return None
@@ -786,6 +1448,7 @@ def _append_ledger(tenant_key, rt, section, row):
def append(doc):
for part in ("triggers", "checks", "audit_failures"):
doc.setdefault(part, [])
+ doc.setdefault(section, [])
# Preserve INSERT OR IGNORE's exact-event idempotency for triggers only.
if section != "triggers" or row not in doc[section]:
doc[section].append(row)
@@ -830,6 +1493,100 @@ def record_deletion_trigger(tenant_key, trigger, *, occurred_at=None, rt=None,
return retention_check(key, rt=rt)
+def _active_deletion_triggers(key, ledger, now):
+ """Historical read: a cancellation names one exact event and takes effect only later."""
+ return sorted((row for row in ledger.get("triggers", [])
+ if row["tenant"] == key and _utc(row["occurred_at"]) <= now
+ and not any(c.get("tenant") == key and c.get("kind") == "cancellation"
+ and c.get("trigger_kind") == row["kind"]
+ and c.get("trigger_occurred_at") == row["occurred_at"]
+ and _utc(row["occurred_at"]) < _utc(c["occurred_at"]) <= now
+ for c in ledger.get("cancellations", []))),
+ key=lambda row: _utc(row["occurred_at"]))
+
+
+def cancel_deletion_trigger(tenant_key, *, rt=None, occurred_at=None, principal="system"):
+ """Reconnection cancels pending credential loss during grace, never an Amazon notice.
+
+ The original trigger stays intact. A notice or termination needs its own explicit resolution;
+ storing a credential cannot overrule that independent deletion obligation.
+ """
+ key, now = _tenant_key(tenant_key), _utc(occurred_at)
+ rt = _ledger_runtime(key, rt)
+ from core import keychain as kc
+ if kc.first_entry_of_type(rt, "amazon_sp") is None:
+ return []
+ cancelled = []
+
+ def cancel(doc):
+ cancelled.clear()
+ checks = [r for r in doc.get("checks", []) if r["tenant"] == key
+ and r["kind"] == "purge" and r["success"] and _utc(r["checked_at"]) <= now]
+ for trigger in _active_deletion_triggers(key, doc, now):
+ when = _utc(trigger["occurred_at"])
+ if (trigger["kind"] not in ("revocation", "authorization_loss")
+ or not when < now < when + timedelta(days=PURGE_AFTER_DAYS)
+ or any(_utc(r["checked_at"]) >= when for r in checks)):
+ continue
+ row = {"tenant": key, "kind": "cancellation", "occurred_at": _control_stamp(now),
+ "principal": str(principal), "trigger_kind": trigger["kind"],
+ "trigger_occurred_at": trigger["occurred_at"]}
+ doc.setdefault("cancellations", []).append(row)
+ cancelled.append(row)
+
+ # Tenant-store update serializes against concurrent triggers and preserves append-only history.
+ rt.update(LIFECYCLE_KEY, cancel, flush="sync")
+ return cancelled
+
+
+def scheduled_purge(tenant_key, rt, *, now=None):
+ """Read-only policy schedule for the earliest uncancelled, unpurged mirror obligation."""
+ key, clock = _tenant_key(tenant_key), _utc(now)
+ ledger = _ledger(key, rt)
+ checks = [_utc(r["checked_at"]) for r in ledger.get("checks", [])
+ if r["tenant"] == key and r["kind"] == "purge" and r["success"]
+ and _utc(r["checked_at"]) <= clock]
+ for trigger in _active_deletion_triggers(key, ledger, clock):
+ when = _utc(trigger["occurred_at"])
+ if not any(stamp >= when for stamp in checks):
+ return {"purgeAt": _control_stamp(when + timedelta(days=PURGE_AFTER_DAYS)),
+ "deadlineAt": _control_stamp(when + timedelta(days=DELETION_DAYS)),
+ "trigger": dict(trigger)}
+ return None
+
+
+def deletion_schedule_message(plan):
+ trigger = plan["trigger"]
+ date, purge_date = trigger["occurred_at"][:10], plan["purgeAt"][:10]
+ if trigger["kind"] in ("revocation", "authorization_loss"):
+ event = "The Amazon credential was removed" if trigger["kind"] == "revocation" else "Amazon authorization was lost"
+ return (f"{event} on {date}. Amazon data in this workspace will be deleted on {purge_date} "
+ "unless the credential is added again before then.")
+ return (f"Amazon data deletion was requested on {date}. "
+ f"Amazon data in this workspace is scheduled for deletion on {purge_date}.")
+
+
+def enforce_retention(tenant_key, rt=None, *, now=None):
+ """The resident writer's policy check; a completed purge is never performed twice."""
+ key, clock = _tenant_key(tenant_key), _utc(now)
+ rt = _ledger_runtime(key, rt)
+ with _RETENTION_LOCK:
+ plan = scheduled_purge(key, rt, now=clock)
+ if plan is None:
+ return None
+ success = False
+ try:
+ from core import keychain as kc
+ if _utc(plan["purgeAt"]) <= clock and kc.first_entry_of_type(rt, "amazon_sp") is None:
+ result = purge(key, key, rt=rt, principal="retention policy", now=clock)
+ else:
+ result = plan
+ success = True
+ return result
+ finally:
+ _record_deletion_check(key, rt, "retention", success, checked_at=clock)
+
+
def retention_check(tenant_key, *, rt=None, checked_at=None, scope="all", raise_overdue=False):
"""Dated obligation check. Only a real readback can complete the covered mirror scope.
@@ -840,7 +1597,7 @@ def retention_check(tenant_key, *, rt=None, checked_at=None, scope="all", raise_
if scope not in ("all", "mirror"):
raise ValueError("deletion scope must be all or mirror")
ledger = _ledger(key, rt)
- triggers = sorted(r["occurred_at"] for r in ledger["triggers"] if r["tenant"] == key)
+ triggers = [r["occurred_at"] for r in _active_deletion_triggers(key, ledger, now)]
checks = sorted(r["checked_at"] for r in ledger["checks"]
if r["tenant"] == key and r["kind"] == "purge" and r["success"])
# A check in the future cannot prove deletion at the requested historical observation time.
@@ -865,8 +1622,8 @@ def retention_check(tenant_key, *, rt=None, checked_at=None, scope="all", raise_
return result
-def _record_deletion_check(key, rt, kind, success):
- stamp = _control_stamp()
+def _record_deletion_check(key, rt, kind, success, *, checked_at=None):
+ stamp = _control_stamp(checked_at)
_append_ledger(key, rt, "checks",
{"tenant": key, "kind": kind, "checked_at": stamp, "success": bool(success)})
return stamp
@@ -897,6 +1654,506 @@ def _sync_state_rows(con):
"FROM _sync_state ORDER BY entity").fetchall()]
+def _mirror_cache_key(tenant_key, rt):
+ key = _tenant_key(tenant_key)
+ if rt is None or _tenant_key(getattr(rt, "key", None)) != key:
+ raise ValueError("Amazon mirror requires the requested tenant's runtime")
+ path = getattr(rt, "datastore_path", None)
+ return key, str(Path(path).resolve()) if path else id(rt)
+
+
+def _mirror_json(value):
+ return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+
+def _blob_metadata(data):
+ return {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
+
+
+#: S67-T32 (D-3481): v1 names in-place keys (`
.parquet`, `sync_state.json`) and still
+#: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
+#: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
+#: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
+MANIFEST_VERSIONS = (1, 2)
+MANIFEST_VERSION = 2
+_MANIFEST_KEY = "manifest.json"
+_KEY_LEDGER = "keys.json"
+_TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
+_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
+#: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
+#: checkpoint publishes within minutes, so the grace never races an in-flight writer.
+ORPHAN_GRACE = timedelta(hours=24)
+
+
+def _checked_mirror_blob(rt, path, metadata):
+ data = rt.download_bytes(MIRROR_PREFIX + path)
+ if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
+ raise ValueError("Amazon mirror blob checksum or byte count mismatch")
+ return data
+
+
+def _content_key(stem, suffix, metadata):
+ return f"{stem}.{metadata['sha256'][:16]}{suffix}"
+
+
+def _table_blob_key(name, metadata, *, strict=True):
+ """The key a manifest entry names: its content key, or the legacy in-place key."""
+ key = str(metadata.get("key") or name + ".parquet")
+ match = _TABLE_BLOB_RE.fullmatch(key)
+ if not match or match.group(1) != name or (
+ strict and match.group(2) and not str(metadata.get("sha256") or "").startswith(match.group(2))):
+ raise ValueError("Amazon mirror manifest names an invalid table blob key")
+ return key
+
+
+def _state_blob_key(metadata, *, strict=True):
+ key = str(metadata.get("key") or "sync_state.json")
+ match = _STATE_BLOB_RE.fullmatch(key)
+ if not match or (strict and match.group(1)
+ and not str(metadata.get("sha256") or "").startswith(match.group(1))):
+ raise ValueError("Amazon mirror manifest names an invalid state blob key")
+ return key
+
+
+def _is_mirror_blob_key(key):
+ return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _STATE_BLOB_RE.fullmatch(str(key)))
+
+
+def _manifest_keys(manifest, *, strict=True):
+ """{key: sha256} for every blob a manifest names; malformed entries are skipped, never raised."""
+ out = {}
+ if not isinstance(manifest, dict):
+ return out
+ tables = manifest.get("tables")
+ for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
+ try:
+ out[_table_blob_key(str(name), metadata, strict=strict)] = metadata.get("sha256")
+ except (ValueError, AttributeError, TypeError):
+ continue
+ state = manifest.get("sync_state")
+ if isinstance(state, dict):
+ try:
+ out[_state_blob_key(state, strict=strict)] = state.get("sha256")
+ except (ValueError, AttributeError, TypeError):
+ pass
+ return out
+
+
+def _read_key_ledger(rt, tenant_key):
+ """The write-ahead key ledger. Unreadable is an error: a purge must not guess."""
+ raw = rt.download_bytes(MIRROR_PREFIX + _KEY_LEDGER)
+ key = _tenant_key(tenant_key)
+ if raw is None:
+ return {"version": 1, "tenant": key, "keys": {}}
+ doc = json.loads(raw)
+ if (not isinstance(doc, dict) or doc.get("tenant") != key or not isinstance(doc.get("keys"), dict)):
+ raise ValueError("Amazon mirror key ledger is unreadable or belongs to another tenant")
+ doc["keys"] = {str(k): v for k, v in doc["keys"].items() if _is_mirror_blob_key(k)}
+ return doc
+
+
+def _ledger_add(rt, tenant_key, keys):
+ """Record keys BEFORE their upload, so a killed checkpoint's blob is always enumerable."""
+ ledger = _read_key_ledger(rt, tenant_key)
+ stamp = _iso(datetime.now(timezone.utc))
+ fresh = [key for key in keys if key not in ledger["keys"]]
+ if not fresh:
+ return
+ for key in fresh:
+ if not _is_mirror_blob_key(key):
+ raise ValueError("Amazon mirror key is outside the naming contract")
+ ledger["keys"][key] = stamp
+ rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger))
+
+
+def _validated_snapshot(tenant_key, rt, con, raw, tmp):
+ """Every check restore makes, without writing. -> (manifest, state rows, parquet paths, rows).
+
+ Raises ValueError for anything the stored snapshot itself gets wrong; a store-access failure
+ propagates as its own type, so a caller can tell "unrestorable" from "unreachable".
+ """
+ try:
+ manifest = json.loads(raw)
+ except ValueError as exc:
+ raise ValueError("Amazon mirror manifest is not JSON") from exc
+ if (not isinstance(manifest, dict) or manifest.get("version") not in MANIFEST_VERSIONS
+ or manifest.get("tenant") != _tenant_key(tenant_key)
+ or not isinstance(manifest.get("tables"), dict) or not isinstance(manifest.get("sync_state"), dict)):
+ raise ValueError("Amazon mirror manifest has an invalid version, tenant or table map")
+ state_meta = manifest["sync_state"]
+ state_bytes = _checked_mirror_blob(rt, _state_blob_key(state_meta), state_meta)
+ rows = json.loads(state_bytes)
+ if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
+ or not rows or any(not isinstance(row, list) or len(row) != 6
+ or not isinstance(row[0], str) or not row[0].startswith(PREFIX) for row in rows)
+ or len({row[0] for row in rows}) != len(rows)):
+ raise ValueError("Amazon mirror state is incomplete or contains unrelated entities")
+ paths, total = {}, 0
+ for name, metadata in manifest["tables"].items():
+ if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
+ raise ValueError("Amazon mirror manifest contains an invalid table name")
+ data = _checked_mirror_blob(rt, _table_blob_key(name, metadata), metadata)
+ path = Path(tmp) / (name + ".parquet")
+ path.write_bytes(data)
+ try:
+ count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
+ except Exception as exc: # noqa: BLE001
+ raise ValueError("Amazon mirror parquet is unreadable") from exc
+ if type(metadata.get("rows")) is not int or count != metadata["rows"]:
+ raise ValueError("Amazon mirror parquet row count mismatch")
+ paths[name] = str(path)
+ total += count
+ return manifest, rows, paths, total
+
+
+def _snapshot_tables():
+ """Current-state tables: every report or API the client declares `snapshot`."""
+ cx = _client()
+ return tuple(t for t in TABLES if (cx.REPORT_TYPES.get(t) or {}).get("snapshot"))
+
+
+def _collapse_snapshots(con, log=None):
+ """D-3031 heal: a snapshot table holding more than one pull keeps only the latest.
+
+ -> {mirror: (rows before, rows after, copies)}. The ledger keeps its latest document's
+ dates (a DAILY document spans several), every other table its newest `_snapshot_date`;
+ AWD rows are another leg and never touched. Idempotent: one copy is left alone.
+ """
+ existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"}
+ collapsed = {}
+ for table in _snapshot_tables():
+ mirror = table_name(table)
+ if mirror not in existing:
+ continue
+ cols = _columns_of(con, mirror)
+ if "_snapshot_date" not in cols:
+ continue
+ scope = ("COALESCE(inventory_source, 'FBA') <> 'AWD'"
+ if table == "inventory" and "inventory_source" in cols else "TRUE")
+ dates = [row[0] for row in con.execute(
+ f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {mirror} WHERE {scope}").fetchall()]
+ present = sorted(d for d in dates if d is not None)
+ if len(dates) <= 1 or not present:
+ continue
+ keep = []
+ if table == "inventory":
+ saved = _read_state(con, entity_key("reconcile_" + table))
+ try:
+ marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {}
+ except (ValueError, KeyError, TypeError):
+ marked = {}
+ keep = [d for d in present if d in marked]
+ keep = keep or present[-1:]
+ stale = (f"{scope} AND (_snapshot_date IS NULL OR "
+ f"CAST(_snapshot_date AS VARCHAR) NOT IN ({','.join('?' for _ in keep)}))")
+ if not con.execute(f"SELECT count(*) FROM {mirror} WHERE {stale}", keep).fetchone()[0]:
+ continue
+ before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0]
+ con.execute("BEGIN TRANSACTION")
+ try:
+ con.execute(f"DELETE FROM {mirror} WHERE {stale}", keep)
+ con.execute("COMMIT")
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+ after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0]
+ collapsed[mirror] = (before, after, len(dates))
+ if log:
+ log(f"Amazon snapshot {mirror}: {len(dates)} copies collapsed to the latest, "
+ f"{before:,} rows to {after:,}")
+ return collapsed
+
+
+def _heal_snapshots(con, result, log):
+ """Collapse after a restore; the healed tables are dirty so the next checkpoint saves one copy."""
+ try:
+ result["dirty"].update(_collapse_snapshots(con, log))
+ except Exception as exc: # noqa: BLE001
+ message = (f"Amazon snapshot collapse failed ({type(exc).__name__}); the next resync of each "
+ "snapshot table replaces its earlier copies.")
+ result["problems"].append(message)
+ log(message)
+
+
+def restore_mirror(tenant_key, rt, log=None):
+ """Restore once per process and tenant, with all blobs validated before one atomic commit.
+
+ The tenant runtime is the only store door. A bad snapshot never partially replaces tables;
+ the caller gets a problem and the unchanged first-load decision can rebuild it from Amazon.
+ Both manifest versions restore: v1's in-place keys exactly as saved, v2's content keys.
+ """
+ log = print if log is None else log
+ cache_key = _mirror_cache_key(tenant_key, rt)
+ with _MIRROR_LOCK:
+ if cache_key in _MIRROR_CACHE:
+ return _MIRROR_CACHE[cache_key]
+ result = {"state": "local", "problems": [], "manifest": {}, "dirty": set(), "runtime": rt}
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
+ try:
+ if any(str(row[0]).startswith(PREFIX) for row in _sync_state_rows(con)):
+ # S66-T45 (lead, measured 2026-09-25): with the cache's manifest left empty here,
+ # a process whose mirror survived re-uploaded EVERY table on its first checkpoint
+ # (39 tables, 19.5 MB). The small stored manifest lets unchanged bytes be skipped.
+ try:
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
+ saved = json.loads(raw) if raw else {}
+ if (saved.get("version") in MANIFEST_VERSIONS and saved.get("tenant") == _tenant_key(tenant_key)
+ and isinstance(saved.get("tables"), dict)):
+ result["manifest"] = saved
+ except Exception: # noqa: BLE001
+ pass
+ _heal_snapshots(con, result, log)
+ _MIRROR_CACHE[cache_key] = result
+ return result
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
+ if raw is None:
+ result["state"] = "absent"
+ else:
+ obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
+ if obligation["earliestTriggerAt"]:
+ raise ValueError("Amazon deletion is outstanding; saved data cannot be restored")
+ with tempfile.TemporaryDirectory(prefix="amazon-restore-") as tmp:
+ manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
+ con.execute("BEGIN TRANSACTION")
+ try:
+ for name, path in paths.items():
+ con.execute(f"CREATE OR REPLACE TABLE {_quoted_identifier(name)} AS "
+ "SELECT * FROM read_parquet(?)", [path])
+ con.executemany("INSERT INTO _sync_state "
+ "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)", rows)
+ con.execute("COMMIT")
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+ result.update(state="restored", manifest=manifest)
+ marks = {row[0][len(PREFIX + WATERMARK_PREFIX):]: row[3] for row in rows
+ if row[0].startswith(PREFIX + WATERMARK_PREFIX)}
+ resume = ", ".join(f"{table} from {marks.get(table) or 'no watermark'}"
+ for table in sorted(set(TABLES) | set(marks)))
+ docs = [row for row in rows if row[0].startswith("amzn_settlement_doc_") and row[1] == "done"]
+ last = max(docs, key=lambda row: (row[3] or "", row[5] or "", row[0])) if docs else None
+ document = last[0][len("amzn_settlement_doc_"):] if last else "none"
+ # State is vendor-derived; keep the log to one line and enforce rule 2 here too.
+ line = (f"Amazon mirror restored from the store: {len(paths)} tables, {total:,} rows; "
+ f"resuming {resume}; settlements after document {document}")
+ log(line.replace("\u2014", "-").replace("\u2013", "-").replace("\n", " ").replace("\r", " "))
+ _heal_snapshots(con, result, log)
+ except Exception as exc:
+ result["state"] = "refused"
+ message = (f"Amazon mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
+ "row counts and state must agree. Full first load required; check tenant-store access "
+ "and the saved snapshot.")
+ result["problems"].append(message)
+ log(message)
+ finally:
+ con.close()
+ _MIRROR_CACHE[cache_key] = result
+ return result
+
+
+def _refused_may_publish(tenant_key, rt, con, cache):
+ """D-3481: a process that could not restore must not publish over a snapshot that is valid.
+
+ It publishes once its own first load is complete (its rebuild is then whole), or once a
+ fresh re-validation proves the stored snapshot unrestorable (nothing valid is lost). A
+ store-access failure during that re-check proves nothing, so it holds and asks again later.
+ """
+ if cache.get("publish_ok"):
+ return True
+ load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
+ if load and load["state"] == "done":
+ cache["publish_ok"] = True
+ return True
+ if cache.get("stored_valid") is None:
+ try:
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
+ if raw is None:
+ cache["stored_valid"] = False
+ else:
+ with tempfile.TemporaryDirectory(prefix="amazon-recheck-") as tmp:
+ _validated_snapshot(tenant_key, rt, con, raw, tmp)
+ cache["stored_valid"] = True
+ except (ValueError, KeyError, TypeError, AttributeError):
+ cache["stored_valid"] = False
+ except Exception: # noqa: BLE001
+ return False
+ if cache["stored_valid"] is False:
+ cache["publish_ok"] = True
+ return True
+ return False
+
+
+def _retire_blobs(tenant_key, rt, published, previous, log, *, orphans=True):
+ """After a publish only: delete what the previous manifest named and the new one does not,
+ plus ledger orphans older than ORPHAN_GRACE. Best effort; a failure is retried next time."""
+ try:
+ if rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) != published:
+ log("Amazon mirror cleanup skipped: the stored manifest changed after this checkpoint")
+ return 0
+ current = _manifest_keys(json.loads(published), strict=False)
+ ledger = _read_key_ledger(rt, tenant_key)
+ doomed = set(previous) - set(current)
+ if orphans:
+ cutoff = datetime.now(timezone.utc) - ORPHAN_GRACE
+ for key, stamp in ledger["keys"].items():
+ try:
+ aged = _utc(stamp) < cutoff
+ except Exception: # noqa: BLE001
+ aged = True
+ if key not in current and aged:
+ doomed.add(key)
+ removed = []
+ for key in sorted(doomed):
+ if key in current or not _is_mirror_blob_key(key):
+ continue
+ rt.delete_path(MIRROR_PREFIX + key)
+ removed.append(key)
+ if any(key in ledger["keys"] for key in removed):
+ ledger = _read_key_ledger(rt, tenant_key)
+ ledger["keys"] = {k: v for k, v in ledger["keys"].items() if k not in removed}
+ rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger))
+ return len(removed)
+ except Exception as exc: # noqa: BLE001
+ log(f"Amazon mirror cleanup deferred ({type(exc).__name__}); superseded blobs stay in the "
+ "key ledger for the next checkpoint or purge.")
+ return 0
+
+
+def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
+ """Checkpoint changed tables and all Amazon markers, publishing the manifest last.
+
+ Candidates include partially written tables. Hashing their parquet locally avoids reuploading
+ identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain
+ dirty for retry on the next pass, while landed local rows and cursors stay usable.
+
+ S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a
+ CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists
+ every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row
+ upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4)
+ leaves the previous manifest naming only complete, untouched blobs.
+ """
+ uploaded = 0
+ with _MIRROR_LOCK:
+ cache = restore_mirror(tenant_key, rt, log=log)
+ cache["dirty"].update(changed_tables)
+ try:
+ if cache.get("state") == "refused" and not _refused_may_publish(tenant_key, rt, con, cache):
+ message = ("Amazon mirror checkpoint held: this process could not restore the saved snapshot "
+ "at start, and that snapshot is still valid or unreachable. It is kept; restart to "
+ "restore it. This process saves only after its own first load completes.")
+ report["problems"].append(message)
+ log(message)
+ return
+ old = cache["manifest"]
+ named = _manifest_keys(old)
+ tables = {name for name, kind in _amazon_objects(con)
+ if name.startswith(PREFIX) and kind == "BASE TABLE"}
+ manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
+ "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}}
+ changed, pending = [], []
+ with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
+ # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
+ legacy = {name for name, meta in manifest["tables"].items() if not meta.get("key")}
+ for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
+ if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
+ raise ValueError("Amazon mirror table name is invalid")
+ path = Path(tmp) / (name + ".parquet")
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
+ data = path.read_bytes()
+ metadata = _blob_metadata(data)
+ prior = manifest["tables"].get(name, {})
+ if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
+ continue
+ key = _content_key(name, ".parquet", metadata)
+ metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
+ written_at=_iso(datetime.now(timezone.utc)), key=key)
+ manifest["tables"][name] = metadata
+ changed.append(name)
+ if key in named:
+ # The same content under its own name is already stored and published.
+ if named[key] != metadata["sha256"]:
+ raise ValueError("Amazon mirror content key collides with a published blob")
+ continue
+ pending.append((key, path))
+ rows = [row for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)]
+ state = _mirror_json(rows)
+ state_meta = {**_blob_metadata(state), "rows": len(rows)}
+ state_meta["key"] = _content_key("sync_state", ".json", state_meta)
+ published = None
+ if rows and (changed or state_meta != old.get("sync_state")
+ or manifest["tables"] != old.get("tables", {})
+ or old.get("version") != MANIFEST_VERSION):
+ state_path = None
+ if state_meta["key"] not in named:
+ state_path = Path(tmp) / "sync_state.json"
+ state_path.write_bytes(state)
+ pending.append((state_meta["key"], state_path))
+ if pending:
+ _ledger_add(rt, tenant_key, [key for key, _path in pending])
+ for key, path in pending:
+ data = path.read_bytes()
+ rt.upload_bytes(MIRROR_PREFIX + key, data)
+ uploaded += len(data)
+ manifest.update(sync_state=state_meta, written_at=_iso(datetime.now(timezone.utc)))
+ published = _mirror_json(manifest)
+ rt.upload_bytes(MIRROR_PREFIX + _MANIFEST_KEY, published)
+ uploaded += len(published)
+ cache["manifest"] = manifest
+ cache["problems"].clear()
+ if published:
+ _retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
+ if published or not changed:
+ cache["dirty"].clear()
+ log(f"Amazon mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded")
+ except Exception as exc:
+ message = (f"Amazon mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. "
+ "Local rows were kept; restore tenant-store access and resync before restarting.")
+ report["problems"].append(message)
+ log(message)
+
+
+def _purge_json(rt, name):
+ """A saved mirror JSON document for a purge: the parsed dict, or {} when absent or unreadable."""
+ try:
+ raw = rt.download_bytes(MIRROR_PREFIX + name)
+ doc = json.loads(raw) if raw else {}
+ except (TypeError, ValueError, UnicodeError, RecursionError):
+ return {}
+ return doc if isinstance(doc, dict) else {}
+
+
+def _purge_saved_mirror(tenant_key, rt, local_tables):
+ """Delete data before its inventory so a failed deletion remains retryable after restart.
+
+ Every version goes: each legacy in-place key, each key any manifest names, and each key the
+ write-ahead ledger lists, which includes blobs a killed checkpoint uploaded but never
+ published. Then the manifest, then the ledger itself, each read back.
+ """
+ with _MIRROR_LOCK:
+ key = _mirror_cache_key(tenant_key, rt)
+ cached = _MIRROR_CACHE.get(key, {})
+ # Deletion never refuses on unreadable inventory: a corrupt or foreign manifest or ledger
+ # still yields every blob name it holds, and every path below goes through THIS tenant's
+ # runtime, so a ledger labelled with another tenant cannot point a delete outside it.
+ manifest = _purge_json(rt, _MANIFEST_KEY)
+ ledger_keys = {str(k) for k in (_purge_json(rt, _KEY_LEDGER).get("keys") or {})
+ if _is_mirror_blob_key(str(k))}
+ names = ({table_name(t) for t in TABLES} | set(local_tables) | set(manifest.get("tables", {}))
+ | set(cached.get("manifest", {}).get("tables", {})) | set(cached.get("dirty", ())))
+ blobs = {name + ".parquet" for name in names if re.fullmatch(r"amzn_[a-z0-9_]+", name)}
+ blobs.add("sync_state.json")
+ blobs |= set(_manifest_keys(manifest, strict=False))
+ blobs |= set(_manifest_keys(cached.get("manifest", {}), strict=False))
+ blobs |= ledger_keys
+ paths = [MIRROR_PREFIX + blob for blob in sorted(blobs)]
+ paths.extend((MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER))
+ for path in paths:
+ if rt.delete_path(path) is False or rt.download_bytes(path) is not None:
+ raise RuntimeError("Amazon saved mirror deletion failed or left residue")
+ _MIRROR_CACHE.pop(key, None)
+
+
def _collateral_rows(con):
"""Compare complete non-Amazon row multisets, including duplicate counts, not just names."""
from collections import Counter
@@ -1030,12 +2287,12 @@ def _purge_runtime_caches(rt, tenant_key, confirmation):
return count
-def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator"):
+def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator", now=None):
"""Permanently remove one tenant's live Amazon mirror after credential revocation.
- This is an operator-only lifecycle control. It never revokes a credential itself and refuses
- to run until the tenant runtime resolves, the exact tenant slug is repeated, the mirror already
- exists, and every Amazon credential metadata row is gone.
+ Used by the operator and the retention policy. Both require the exact tenant and absent
+ credential metadata. Policy deletion also covers saved blobs after a container loses its
+ local mirror, without creating a new file as purported evidence of deletion.
"""
key = _require_purge_confirmation(tenant_key, confirmation)
rt = rt if rt is not None else _runtime_for(key)
@@ -1043,28 +2300,36 @@ def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator")
raise RuntimeError(
f"tenant {key!r} could not be resolved; refusing to guess a database target")
path = _tenant_mirror_path(key, rt)
- if not path.is_file():
+ if not path.is_file() and principal != "retention policy":
raise RuntimeError(
f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof")
_assert_amazon_credential_revoked(rt)
success, result = False, {}
+ bound_path = path
try:
- con, bound_path = _connect_tenant_mirror(key, rt)
- try:
- result = _purge_mirror(con, key, confirmation)
- finally:
- con.close()
+ if path.is_file():
+ con, bound_path = _connect_tenant_mirror(key, rt)
+ try:
+ _purge_saved_mirror(key, rt, [name for name, _kind in _amazon_objects(con)])
+ result = _purge_mirror(con, key, confirmation)
+ finally:
+ con.close()
+ else:
+ _purge_saved_mirror(key, rt, [])
+ result = {"tables": [], "rows": 0, "syncMarkers": 0,
+ "residueTables": [], "residueSyncMarkers": []}
result.update(_purge_materialized(rt, key, confirmation))
result["runtimeCacheEntries"] = _purge_runtime_caches(rt, key, confirmation)
# Keep the existing post-commit query and add a fresh, independently opened cursor.
import duckdb
- with closing(duckdb.connect(str(bound_path))) as readback:
- if _amazon_objects(readback) or any(
- str(r[0]).startswith(PREFIX) for r in _sync_state_rows(readback)):
- raise RuntimeError("independent mirror file readback found Amazon residue")
+ if bound_path.is_file():
+ with closing(duckdb.connect(str(bound_path))) as readback:
+ if _amazon_objects(readback) or any(
+ str(r[0]).startswith(PREFIX) for r in _sync_state_rows(readback)):
+ raise RuntimeError("independent mirror file readback found Amazon residue")
success = True
finally:
- stamp = _record_deletion_check(key, rt, "purge", success)
+ stamp = _record_deletion_check(key, rt, "purge", success, checked_at=now)
# Counts only. Even database-discovered table names could contain customer data, so
# never send them (or exception strings) to C2's flat, secret-refusing detail object.
_c2_event("data.purge", key, principal,
@@ -1078,7 +2343,7 @@ def purge(tenant_key, confirmation, rt=None, log=print, *, principal="operator")
"complete": False,
"unverifiedScopes": ["workbooks", "service_retained_exports", "backups",
"other_process_and_unattributed_caches"],
- "retention": retention_check(key, rt=rt)}
+ "retention": retention_check(key, rt=rt, checked_at=now)}
if not result.get("materializedVerified"):
report["unverifiedScopes"].append("materialized_store")
log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from "
@@ -1165,7 +2430,8 @@ def _record_state(con, key, phase, rows=0, detail=None):
con.execute(
"INSERT OR REPLACE INTO _sync_state "
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
- [str(key), str(phase), None, (str(detail)[:300] if detail else None), int(rows or 0),
+ [str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_"))
+ else str(detail)[:300]) if detail else None, int(rows or 0),
_iso(datetime.now(timezone.utc))])
@@ -1220,11 +2486,23 @@ def record_backfill_floor(con, table, when):
def needs_first_load(tenant_key, rt=None):
"""A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
+ rt = rt if rt is not None else _runtime_for(tenant_key)
+ restore_mirror(tenant_key, rt)
con, _path = _connect_tenant_mirror(tenant_key, rt)
try:
load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
- return bool(load and load["state"] != "done") or not any(
- read_watermark(con, table) for table in TABLES)
+ enabled = read_database_enablement(rt)["databases"]
+ added = [table for table in GROUP_A_TABLES if table in TABLES
+ and enabled.get(amazon_schema.DATABASE_PREFIX + table)]
+ # S66-T45 (lead, observed live): a table Amazon refuses outright (a 400 "not allowed",
+ # a 403 for a report it does not offer this seller, a document with no row root) never
+ # earns a watermark, so it kept this answer True and re-queued a first-load pass on every
+ # boot. Its refusal is on screen; the scheduled rotation retries it.
+ refused = {table for table in added
+ if (_read_state(con, entity_key(table)) or {}).get("state") == "refused"}
+ return (bool(load and load["state"] != "done")
+ or not any(read_watermark(con, table) for table in TABLES)
+ or any(not read_watermark(con, table) for table in added if table not in refused))
finally:
con.close()
@@ -1263,6 +2541,21 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None):
cause = f"first full load: {days} days of backfill"
spec = _client().REPORT_TYPES.get(table) or {}
rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
+ rtype = rtype or spec.get("api")
+ if table in ("economics", "order_metrics"):
+ if incremental:
+ start -= timedelta(days=3)
+ start = start.replace(hour=0, minute=0, second=0, microsecond=0)
+ end = end.replace(hour=0, minute=0, second=0, microsecond=0)
+ cause += "; completed days, with a three-day restatement overlap on incremental sync"
+ if table == "transactions":
+ end -= timedelta(minutes=2) # Finances requires postedBefore at least two minutes ago.
+ if spec.get("snapshot"):
+ cause = "one current snapshot, without a data window"
+ if spec.get("monthly"):
+ start = start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ end = end.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ cause += "; completed calendar months only"
if table == "settlements":
# The watermark filters issued reports by creation time; it never requests one.
rtype = spec.get("report_discover")
@@ -1297,15 +2590,446 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
elif isinstance(exc, AmazonOperationRefused):
state = "refused"
fix = "use an operation in the approved read-only scope; check the connector operation mapping"
+ elif "403" in str(exc):
+ state = "refused"
+ cx = _client()
+ role = (cx.REPORT_TYPES.get(table) or {}).get("role")
+ role = role or " / ".join((cx.OPERATIONS.get(operation) or {}).get("roles", ()))
+ # S66-T42 (lead, observed live): Amazon also answers 403 for a report it does not offer
+ # this seller (sold listings, RFQ bulk download) while the application HOLDS the role, so
+ # the sentence says which case applies instead of sending an admin to re-grant a held role.
+ held = all(r.strip() in cx.ROLES_TAKEN
+ for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
+ fix = (f"needs the {role} role; this application already holds it, so Amazon does not "
+ "offer this report to this seller account" if role and held else
+ f"needs the {role} role; grant it and reauthorize the application before retrying")
else:
state = "refused"
fix = "check the report and operation's required roles, credential and vendor response; retry"
message = f"{table}: {operation} {state.upper()}: {cause}; fix: {fix}"
+ message = message.replace("\u2014", "; ").replace("\u2013", " to ")
report["problems"].append(message)
log(message)
return {"state": state, "reason": message}
+def _record_reconciliation(con, table, marker):
+ """Keep the most recent landed period, even while backfill travels backwards."""
+ key = entity_key("reconcile_" + table)
+ previous = _read_state(con, key)
+ old = json.loads(previous["detail"]) if previous and previous.get("detail") else {}
+ if marker["end"] >= old.get("end", ""):
+ _record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":")))
+
+
+def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False):
+ """Atomic range/date replacement and proof; an empty restatement removes old rows too.
+
+ `snapshot=True` (S67-T32, D-3031): a current-state report is ONE copy. A successful, parsed
+ pull replaces every earlier `_snapshot_date` of the table, not only its own day, because the
+ grid pid hashes the date and a kept day doubles every row downstream. A failed pull raises
+ before this runs, so it never empties the table.
+ """
+ mirror = table_name(table)
+ columns = list(dict.fromkeys([*columns, column, *_ORIGIN_TYPES]))
+ # All JSON money columns remain fractional even when today's amounts are integers.
+ declared = {c: "DOUBLE" for c in columns if c.endswith(("_amount", "_currency_amount"))}
+ types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
+ predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
+ if column == "posted_date" and end:
+ predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND '
+ 'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)')
+ params = [start, end] if end else [start]
+ if snapshot:
+ if column != "_snapshot_date" or end:
+ raise ValueError("a snapshot replacement is keyed by one _snapshot_date")
+ predicate, params = "TRUE", []
+ con.execute("BEGIN TRANSACTION")
+ try:
+ con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params)
+ count = _replace(con, mirror, columns, types, rows)
+ if marker is not None:
+ _record_reconciliation(con, table, marker)
+ con.execute("COMMIT")
+ return count
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+
+
+def document_row_count(text, roots=()):
+ """Count source records before flattening, privacy filtering or mirror transformations."""
+ if roots:
+ doc = json.loads(text)
+ return sum(1 if isinstance(doc[root], dict) else len(doc[root]) for root in roots)
+ reader = csv.reader(io.StringIO(str(text or "").replace("\r\n", "\n").replace("\r", "\n")),
+ delimiter="\t", quoting=csv.QUOTE_NONE)
+ next(reader, None)
+ return sum(bool(raw) and not (len(raw) == 1 and not str(raw[0]).strip()) for raw in reader)
+
+
+def _ledger_date(value):
+ """The DAILY ledger uses US dates; normalize dates without inventing a report date."""
+ value = str(value or "").strip()
+ if re.fullmatch(r"\d{2}/\d{2}/\d{4}", value):
+ return datetime.strptime(value, "%m/%d/%Y").date().isoformat()
+ if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+ return datetime.strptime(value, "%Y-%m-%d").date().isoformat()
+ raise ValueError("Amazon inventory summary omitted a daily data date")
+
+
+def _snapshot_identity(table, row):
+ """Mirror identity only; legacy grid IDs and AWD's replacement keys stay unchanged."""
+ sku = row.get("seller_sku") if table == "listings" else row.get("msku") or row.get("fnsku")
+ if not sku or not row.get("_snapshot_date"):
+ raise ValueError(f"Amazon {table} snapshot is missing identity date or SKU")
+ identity = (row["_snapshot_date"], sku)
+ if table == "inventory":
+ if not row.get("disposition") or not row.get("location"):
+ raise ValueError("Amazon inventory snapshot is missing identity disposition or location")
+ identity += (row["disposition"], row["location"])
+ return identity
+
+
+def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
+ """Replace the whole FBA snapshot with one parsed document, preserving the separate AWD leg.
+
+ Listings is a catalogue as of the vendor's data/creation date. The ledger is a DAILY
+ summary: its row date wins over the report creation date, including multi-date documents.
+ Duplicate source identities refuse the whole document rather than silently dropping rows.
+ S67-T32 (D-3031): one copy. The document's dates replace every earlier date, so the table
+ holds exactly one pull, the row count a fresh first load would store.
+ """
+ stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
+ if not stamp and (table == "listings" or not rows):
+ raise ValueError("Amazon snapshot returned no data date or report creation date")
+ counts, identities = {}, set()
+ for row in rows:
+ period = _ledger_date(row.get("date")) if table == "inventory" else _utc(stamp).date().isoformat()
+ row["_snapshot_date"] = period
+ if table == "inventory":
+ row["date"] = period
+ row["inventory_source"] = "FBA"
+ identity = _snapshot_identity(table, row)
+ if identity in identities:
+ raise ValueError(f"Amazon {table} document has duplicate snapshot identities")
+ identities.add(identity)
+ counts[period] = counts.get(period, 0) + 1
+ if len(rows) != source_count:
+ raise ValueError(f"Amazon {table} snapshot row count differs from its document")
+ if not counts:
+ counts[_utc(stamp).date().isoformat()] = 0
+ columns = sorted(set(columns) | {"_snapshot_date"} | {k for row in rows for k in row})
+ if table == "inventory":
+ columns = sorted(set(columns) | {"inventory_source"})
+ mirror = table_name(table)
+ types = _ensure(con, mirror, columns, rows, preserve_rows=True)
+ dates = sorted(counts)
+ marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date",
+ "count": source_count, "dates": counts, "document": meta.get("reportDocumentId")}
+ # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg.
+ predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE"
+ con.execute("BEGIN TRANSACTION")
+ try:
+ con.execute(f"DELETE FROM {mirror} WHERE {predicate}")
+ count = _replace(con, mirror, columns, types, rows)
+ _record_reconciliation(con, table, marker)
+ con.execute("COMMIT")
+ return count
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+
+
+def _validate_snapshot(con, table):
+ """Reconcile raw document cardinality and each data date's distinct business identities."""
+ saved = _read_state(con, entity_key("reconcile_" + table))
+ if not saved or saved.get("state") != "done" or not saved.get("detail"):
+ return {"state": "unproven", "reason": "Amazon snapshot reconciliation marker is absent"}
+ try:
+ marker = json.loads(saved["detail"])
+ dates = marker["dates"]
+ if not marker["count"]:
+ return {"state": "unproven", "reason": "Amazon snapshot document is empty"}
+ predicate = "inventory_source='FBA'" if table == "inventory" else "TRUE"
+ result = con.execute(f"SELECT * FROM {table_name(table)} WHERE {predicate} AND "
+ f"_snapshot_date IN ({','.join('?' for _ in dates)})", list(dates))
+ rows = [dict(zip([c[0] for c in result.description], row)) for row in result.fetchall()]
+ actual_dates = dict.fromkeys(dates, 0)
+ identities = set()
+ for row in rows:
+ identities.add(_snapshot_identity(table, row))
+ actual_dates[row["_snapshot_date"]] += 1
+ matches = (len(rows) == len(identities) == marker["count"] and actual_dates == dates)
+ return {"state": "ok" if matches else "refused", "expected": marker["count"],
+ "actual": len(rows), "distinct": len(identities), "dates": actual_dates,
+ "reason": (f"Amazon document has {marker['count']} data rows; stored dates contain "
+ f"{len(rows)} rows and {len(identities)} distinct snapshot identities")}
+ except Exception as exc:
+ return {"state": "refused", "reason": f"Amazon snapshot reconciliation failed: {exc}"}
+
+
+def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
+ """The five money tables share the outer sync's budget, errors and completion markers."""
+ cx = _client()
+ spec = cx.REPORT_TYPES[table]
+ if table == "economics":
+ start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
+ if start >= end:
+ return 0
+ inclusive_end = (_utc(end + "T00:00:00Z") - timedelta(days=1)).date().isoformat()
+ text, meta = cx.economics_document(fields, start, inclusive_end, max_wait=max_wait)
+ records = economics_records(text)
+ rows = economics_rows(records, meta["reportDocumentId"])
+ if any(not start <= str(row.get("start_date", "")) < end for row in rows):
+ raise ValueError("Economics returned a row outside the requested date window")
+ # Every request is reconciled. This includes the newest window and ensures a resumed
+ # first-load pass cannot leave its only proof tied to an older period.
+ text, _summary_meta = cx.economics_document(
+ fields, start, inclusive_end, summary=True, max_wait=max_wait)
+ totals = economics_summary(economics_records(text))
+ marker = {"start": start, "end": end, "totals": totals}
+ return _replace_period(con, table, rows, sorted({k for r in rows for k in r}),
+ "start_date", start, end, marker)
+ if table == "transactions":
+ start, end = _iso(w0), _iso(w1)
+ transactions = cx.transactions(fields, _iso(w0 - timedelta(seconds=1)), end)
+ # Finances date endpoints can be inclusive; the mirror uses half-open UTC windows.
+ transactions = [t for t in transactions if _utc(w0) <= _utc(t["postedDate"]) < _utc(w1)]
+ rows = transaction_rows(transactions, "operation:" + uuid.uuid4().hex)
+ marker = {"start": start, "end": end, "totals": transaction_summary(transactions)}
+ return _replace_period(con, table, rows, sorted({k for r in rows for k in r}),
+ "posted_date", start, end, marker)
+ if spec.get("api"):
+ asins = _sqp_catalogue(con) if table == "catalog_items" else []
+ if table == "catalog_items":
+ _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({
+ "reason": (f"Amazon Catalog: {len(asins)} ASINs in {(len(asins) + 19) // 20} batches. "
+ "Limit: 20 ASINs per search and 2 requests per second. "
+ "Every page is followed; keep the connection enabled."),
+ }))
+ if table == "order_metrics":
+ w0 = _utc(w0).replace(hour=0, minute=0, second=0, microsecond=0)
+ w1 = _utc(w1).replace(hour=0, minute=0, second=0, microsecond=0)
+ if w0 >= w1:
+ return 0
+ start, end = (_iso(w0), _iso(w1)) if w0 is not None else (None, None)
+ records = cx.group_a_records(fields, spec["api"], start=start, end=end, asins=asins)
+ document = "operation:" + uuid.uuid4().hex
+ rows = [_origin(_flatten_row(_without_buyer_data(row)), document, i, spec["api"])
+ for i, row in enumerate(records, 1)]
+ if table == "catalog_items":
+ parents = catalog_parent_map(records, _pricing_parent_map(con),
+ fields.get("marketplace_id") or cx.US_MARKETPLACE_ID,
+ _economics_parent_map(con))
+ for row in rows:
+ asin = str(row.get("asin") or "").strip()
+ row["parent_asin"], row["parent_asin_source"] = parents.get(asin, (asin or None, "self"))
+ column = "_snapshot_date" if spec.get("snapshot") else "_period_start"
+ period = datetime.now(timezone.utc).date().isoformat() if spec.get("snapshot") else start
+ for row in rows:
+ row[column] = period
+ if end:
+ row["_period_end"] = end
+ marker = {"start": period, "end": end or period, "column": column, "count": len(records)}
+ if table == "order_metrics":
+ marker["metrics"] = _order_metric_totals(records)
+ marker["range"] = True
+ for row in rows:
+ day = _utc(str(row.get("interval") or "").split("--", 1)[0])
+ if not w0 <= day < w1:
+ raise ValueError("Amazon order metrics returned an interval outside the requested period")
+ row[column] = _iso(day)
+ return _replace_period(con, table, rows, sorted({k for r in rows for k in r}),
+ column, period, end if table == "order_metrics" else None, marker=marker,
+ snapshot=bool(spec.get("snapshot")))
+ kwargs = {} if spec.get("snapshot") else {
+ "start": _iso(w0), "end": _iso(w1 - timedelta(seconds=1))}
+ options = dict(spec.get("options") or {})
+ if spec.get("period"):
+ options["reportPeriod"] = spec["period"]
+ if spec.get("date_options"):
+ options.update(zip(spec["date_options"], (kwargs.pop("start"), kwargs.pop("end"))))
+ if options:
+ kwargs["options"] = options
+ text, meta = cx.report_document(fields, spec["report"], **kwargs, max_wait=max_wait, log=log)
+ parser_args = {"document_id": meta.get("reportDocumentId"), "report_type": spec["report"]}
+ if spec.get("json_root"):
+ columns, rows = parse_json_report(text, spec["json_root"], **parser_args)
+ if spec.get("json_extra_roots"):
+ for row in rows:
+ row["_json_root"] = spec["json_root"]
+ for root in spec["json_extra_roots"]:
+ extra_columns, extra = parse_json_report(text, root, **parser_args)
+ for row in extra:
+ row["_json_root"] = root
+ columns = merge_columns(columns, extra_columns)
+ rows.extend(extra)
+ else:
+ columns, rows = parse_flat_file(text, **parser_args)
+ if table in LEGACY_SNAPSHOT_TABLES:
+ return _replace_legacy_snapshot(con, table, columns, rows, meta, document_row_count(text))
+ if spec.get("snapshot"):
+ stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
+ if not stamp:
+ raise ValueError("Amazon snapshot returned no data date or report creation date")
+ period, column = _utc(stamp).date().isoformat(), "_snapshot_date"
+ elif table == "storage_fees":
+ period, column = _utc(w0).strftime("%Y-%m"), "month"
+ else:
+ period, column = _iso(w0), "_period_start"
+ for row in rows:
+ row[column] = period
+ if w1 is not None:
+ row["_period_end"] = _iso(w1)
+ if table == "storage_fees":
+ row["fulfillment_center"] = (row.get("fulfillment_center") or
+ row.get("fulfillment_center_id") or
+ row.get("fulfilment_center") or "all")
+ columns = sorted(set(columns) | {k for row in rows for k in row})
+ marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
+ "column": column, "count": document_row_count(
+ text, (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ())}
+ return _replace_period(con, table, rows, columns, column, period, marker=marker,
+ snapshot=bool(spec.get("snapshot")))
+
+
+def _order_metric_totals(records, *, flattened=False):
+ totals = {"orders": 0, "units": 0, "sales": {}}
+ for row in records:
+ orders = row.get("order_count" if flattened else "orderCount")
+ units = row.get("unit_count" if flattened else "unitCount")
+ money = ({"amount": row.get("total_sales_amount"), "currencyCode": row.get("total_sales_currency_code")}
+ if flattened else row.get("totalSales") or {})
+ if orders is None or units is None or money.get("amount") is None or not money.get("currencyCode"):
+ raise ValueError("Amazon order metrics omitted its order, unit or sales totals")
+ totals["orders"] += int(orders)
+ totals["units"] += int(units)
+ currency = money["currencyCode"]
+ totals["sales"][currency] = totals["sales"].get(currency, 0.0) + float(money["amount"])
+ return totals
+
+
+def _validate_orders(con):
+ """Independent Sales API order count for one completed UTC interval."""
+ saved = _read_state(con, entity_key("reconcile_order_metrics"))
+ if not saved or not saved.get("detail"):
+ return {"state": "unproven", "reason": "Sync Amazon order metrics to reconcile orders independently"}
+ try:
+ marker = json.loads(saved["detail"])
+ if not marker.get("count"):
+ return {"state": "unproven", "reason": "Amazon order metrics returned no intervals"}
+ expected = marker["metrics"]["orders"]
+ actual = con.execute(
+ "SELECT COUNT(DISTINCT amazon_order_id) FROM amzn_orders "
+ "WHERE CAST(purchase_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) "
+ "AND CAST(purchase_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ) "
+ "AND lower(COALESCE(order_status, '')) NOT IN ('cancelled', 'canceled')",
+ [marker["start"], marker["end"]]).fetchone()[0]
+ return {"state": "ok" if expected == actual else "refused",
+ "expected": expected, "actual": actual,
+ "reason": f"Amazon Sales counted {expected} orders; stored orders contain {actual} for {marker['start']} to {marker['end']}"}
+ except Exception:
+ return {"state": "unproven", "reason": "Orders lack the period or status fields needed to reconcile Amazon order metrics"}
+
+
+def _validate_margin(con, table):
+ """Compare stored detail to durable vendor evidence, per currency and latest period."""
+ state = _read_state(con, entity_key("reconcile_" + table))
+ if not state or state["state"] != "done" or not state.get("detail"):
+ return {"state": "unproven", "reason": "Amazon reconciliation marker is absent"}
+ try:
+ marker = json.loads(state["detail"])
+ column = marker.get("column") or ("start_date" if table == "economics" else "posted_date")
+ # Identifiers come only from this fixed table-kind selection, never stored JSON text.
+ if column not in ("start_date", "posted_date", "_snapshot_date", "month", "_period_start"):
+ raise ValueError("unrecognized reconciliation period column")
+ exact = "count" in marker and not marker.get("range")
+ predicate = f'"{column}" = ?' if exact else f'"{column}" >= ? AND "{column}" < ?'
+ if column == "posted_date":
+ predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND '
+ 'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)')
+ values = [marker["start"]] if exact else [marker["start"], marker["end"]]
+ result = con.execute(f"SELECT * FROM {table_name(table)} WHERE {predicate}", values)
+ names = [c[0] for c in result.description]
+ rows = [dict(zip(names, row)) for row in result.fetchall()]
+ if not rows:
+ return {"state": "unproven", "reason": "latest reconciliation period is empty"}
+ if "count" in marker:
+ expected, actual = marker["count"], len(rows)
+ matches = expected == actual
+ if table == "order_metrics":
+ expected = {"count": expected, **marker["metrics"]}
+ actual = {"count": actual, **_order_metric_totals(rows, flattened=True)}
+ matches = (expected["count"] == actual["count"] and expected["orders"] == actual["orders"]
+ and expected["units"] == actual["units"]
+ and set(expected["sales"]) == set(actual["sales"])
+ and all(abs(amount - actual["sales"][currency]) <= 0.01
+ for currency, amount in expected["sales"].items()))
+ elif table == "economics":
+ expected, actual = marker["totals"], {}
+ for row in rows:
+ group = actual.setdefault(row["currency_code"], {"sales": 0.0, "fees": 0.0, "net": 0.0})
+ pivoted_fees = sum(float(row.get(c.removesuffix("_type_name") + "_amount") or 0)
+ for c in names if c.startswith("fee_") and c.endswith("_type_name")
+ and row.get(c))
+ if abs(pivoted_fees - float(row["fees_total_amount"])) > 0.010000001:
+ raise ValueError(f"economics fee columns {pivoted_fees} vs row total {row['fees_total_amount']}")
+ for key, col in (("sales", "sales_ordered_product_sales_amount"),
+ ("fees", "fees_total_amount"), ("net", "net_proceeds_total_amount")):
+ group[key] += float(row[col])
+ matches = _totals_match(expected, actual)
+ else:
+ expected, actual, groups = marker["totals"], {}, {}
+ for row in rows:
+ groups.setdefault(row["transaction_id"], []).append(row)
+ missing_breakdowns = False
+ for transaction_id, items in groups.items():
+ heads = [r for r in items if r["item_position"] == 0]
+ if len(heads) != 1 or sorted(r["item_position"] for r in items) != list(range(len(items))):
+ raise ValueError(f"transaction {transaction_id} has missing or duplicate items")
+ head = heads[0]
+ if len(items) != head["transaction_item_count"]:
+ raise ValueError(f"transaction {transaction_id} has missing items")
+ if any(r["total_amount"] is not None for r in items if r["item_position"] != 0):
+ raise ValueError(f"transaction {transaction_id} repeats its total on an item")
+ group = actual.setdefault(head["currency_code"], {"amount": 0.0, "count": 0})
+ group["amount"] += float(head["total_amount"])
+ group["count"] += 1
+ parts = [r for r in head if r.startswith("transaction_breakdown_")
+ and r.endswith("_amount") and head[r] is not None]
+ missing_breakdowns |= not bool(parts)
+ if parts:
+ amount = sum(float(head[c]) for c in parts)
+ if abs(amount - float(head["total_amount"])) > 0.010000001:
+ return {"state": "refused", "reason": f"transaction {transaction_id}: "
+ f"breakdowns {amount} vs Amazon total {head['total_amount']}"}
+ if any(head[c + "_currency_code"] != head["currency_code"] for c in parts):
+ raise ValueError("transaction breakdown currency differs from its total")
+ for col in parts:
+ name = col.removeprefix("transaction_breakdown_").removesuffix("_amount")
+ children = [c for c in head if c.startswith(f"transaction_sub_breakdown_{name}__")
+ and c.endswith("_amount") and head[c] is not None]
+ if children and (abs(sum(float(head[c]) for c in children) - float(head[col])) > 0.010000001
+ or any(head[c + "_currency_code"] != head["currency_code"] for c in children)):
+ raise ValueError(f"transaction {transaction_id}: second-level breakdowns disagree with {col}")
+ matches = _totals_match(expected, actual)
+ if matches and missing_breakdowns:
+ return {"state": "unproven", "reason": "Amazon omitted transaction breakdowns",
+ "expected": expected, "actual": actual}
+ return {"state": "ok" if matches else "refused", "expected": expected, "actual": actual,
+ "reason": f"Amazon summary {expected}; stored detail {actual}"}
+ except Exception as exc:
+ return {"state": "refused", "reason": f"Amazon reconciliation failed: {exc}"}
+
+
+def _totals_match(expected, actual):
+ return bool(expected) and set(expected) == set(actual) and all(
+ set(expected[currency]) == set(actual[currency]) and all(
+ abs(float(value) - float(actual[currency][key])) <= (0 if key == "count" else 0.010000001)
+ for key, value in totals.items()) for currency, totals in expected.items())
+
+
def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
max_wait=900, *, scheduled=False, first_load=False, report_budget=None):
"""Pull every declared table into the tenant's mirror. -> a report dict.
@@ -1314,6 +3038,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
reason, when the tenant has stored no credential.
"""
rt = rt if rt is not None else _runtime_for(tenant_key)
+ restored = restore_mirror(tenant_key, rt, log=log)
fields = credential(tenant_key, rt)
if not fields:
raise RuntimeError(
@@ -1340,9 +3065,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
"(`connectors_amazon.py --check`) and sync only a PRODUCTION credential.")
want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
+ if scheduled and want and "settlements" not in want:
+ want.append("settlements")
end = datetime.now(timezone.utc)
report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
- "problems": [], "audit_problems": []}
+ "problems": list(restored["problems"]), "audit_problems": []}
enablement = read_database_enablement(rt)
report["available"] = enablement["available"]
for table in want:
@@ -1374,11 +3101,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
# The pinned inline producer still calls sync(scheduled=True). Do not let it start an
# unbounded history pull between the resident writer's bounded first-load passes.
for table in want:
+ if table == "settlements":
+ continue
report["tables"][table] = {"state": "deferred", "reason":
"The first load is continuing in bounded passes on the resident sync thread."}
- report["finished"] = _iso(datetime.now(timezone.utc))
- con.close()
- return report
+ want = [table for table in want if table == "settlements"]
if scheduled:
plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
for table in TABLES}, want, now=end)
@@ -1386,11 +3113,26 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
for table, reason in plan["deferred"].items():
report["tables"][table] = {"state": "deferred", "reason": reason}
log(f"{table}: DEFERRED: {reason}")
+ elif not first_load:
+ # The vendor's report floor also applies to an operator's Sync now click.
+ for table in list(want):
+ reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
+ datetime.now(timezone.utc))
+ if reason:
+ report["tables"][table] = {"state": "deferred", "reason": reason}
+ want.remove(table)
# Heads go first even when the caller listed lines first. Scheduled plans still select one.
pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items()
if head in want and sibling in want] + want))
handled, noted = set(), set()
+ # D-3031: a mirror that already holds two pulls of a snapshot heals here too, idempotently.
+ changed_tables = set()
+ try:
+ changed_tables.update(_collapse_snapshots(con, log))
+ except Exception as exc: # noqa: BLE001
+ report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
+ "resync of each snapshot table replaces its earlier copies.")
def write_window(table, cols, types, rows):
count, note = replace_window(con, table_name(table), cols, types, rows,
@@ -1404,6 +3146,36 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
work, plans, paired, contexts = [], {}, {}, {}
documents, settlement_remaining = 0, 0
failed, landed = set(), set()
+ sqp_asins, sqp_limit = [], None
+ if SQP_TABLE in pending:
+ saved = _read_state(con, SQP_LIMIT_KEY)
+ old_limit = json.loads(saved["detail"]) if saved and saved.get("detail") else {}
+ # Freeze a first-load cohort between passes. A later incremental pass refreshes it
+ # from our own catalogue, never from a separate vendor lookup.
+ sqp_asins = (old_limit.get("asins") if first_load and load and load["state"] != "done" else None)
+ sqp_asins = sqp_asins or _sqp_catalogue(con)
+ spec = cx.REPORT_TYPES[SQP_TABLE]
+ batches = sqp_batches(sqp_asins, spec["asin_option_max_chars"])
+ sqp_limit = {"asins": sqp_asins, "asinCount": len(sqp_asins), "batchCount": len(batches),
+ "maxChars": spec["asin_option_max_chars"],
+ "maxPerBatch": (spec["asin_option_max_chars"] + 1) // 11,
+ "period": spec["period"], "periods": [],
+ "refusedAsins": old_limit.get("refusedAsins", {}), "errors": old_limit.get("errors", []),
+ "partialMonthReason": "The current partial month is deferred until it closes; Search Query Performance requires a complete calendar month."}
+
+ def sqp_problem(message):
+ report["problems"].append(message)
+ log(message)
+
+ def sqp_request(*args, **kwargs):
+ nonlocal documents
+ if isinstance(report_budget, int) and documents >= max(0, report_budget):
+ raise SQPBudgetDeferred("Search Query Performance report budget reached; remaining batches continue next pass.")
+ documents += 1
+ _sync_event("amazon.sync", tenant_key, SQP_TABLE,
+ {"state": "access", "operation": "report_document"}, rt, report, log)
+ return cx.report_document(*args, **kwargs, max_wait=max_wait, log=log)
+
if first_load:
# Freeze the cohort end across passes. Recomputing it would move every window boundary.
end = _utc(load["detail"]) if load else _utc(_iso(end))
@@ -1411,31 +3183,61 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
plans = {table: sync_window(table, None, now=end) for table in pending}
floors = {table: read_backfill_floor(con, table) for table in pending}
wanted = [table for table in pending if not read_watermark(con, table)
- or read_watermark(con, table) < end]
- for task in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end):
- table, w0, w1 = task
- if w1 is None or floors[table] is None or w1 <= floors[table]:
+ or read_watermark(con, table) < plans[table]["end"]]
+ for entry in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end,
+ sqp_asins=sqp_asins):
+ task = entry if len(entry) == 4 else (*entry, None)
+ table, w0, w1, batch = task
+ if table == SQP_TABLE:
+ if not _sqp_done(con, w0, w1, batch):
+ work.append(task)
+ elif w1 is None or floors[table] is None or w1 <= floors[table]:
work.append(task)
# A failed enrichment after the final report retries completion without fetching again.
for table in wanted:
if not any(task[0] == table for task in work):
- work.append((table, None, None))
+ work.append((table, None, None, None))
# Reuse a document for head and lines only when BOTH durable cursors agree.
for head, sibling in SHARED_REPORT_SIBLINGS.items():
- head_tasks = [(w0, w1) for table, w0, w1 in work if table == head]
- line_tasks = [(w0, w1) for table, w0, w1 in work if table == sibling]
+ head_tasks = [(w0, w1) for table, w0, w1, _batch in work if table == head]
+ line_tasks = [(w0, w1) for table, w0, w1, _batch in work if table == sibling]
if head_tasks and head_tasks == line_tasks:
paired[head] = sibling
work = [task for task in work if task[0] != sibling]
else:
- work = [(table, None, None) for table in pending]
-
- for index, (table, task_start, task_end) in enumerate(work):
- if first_load and table in failed:
+ for table in pending:
+ if table == SQP_TABLE:
+ plan = plans[table] = sync_window(table, read_watermark(con, table), now=end, since=since)
+ months = _month_windows(plan["start"], plan["end"])
+ if not plan["incremental"]:
+ months.reverse()
+ tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches
+ if since is not None or not _sqp_done(con, w0, w1, batch)]
+ work.extend(tasks or [(table, None, None, None)])
+ else:
+ work.append((table, None, None, None))
+
+ if sqp_limit is not None:
+ plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since)
+ sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in _month_windows(plan["start"], plan["end"])]
+ sqp_limit["deferredReason"] = (
+ "Search Query Performance is waiting for ASINs in the listings or pricing mirror. Sync the catalogue first."
+ if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "")
+ _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
+
+ for index, (table, task_start, task_end, task_batch) in enumerate(work):
+ if (first_load or table == SQP_TABLE) and table in failed:
continue
- if (first_load and isinstance(report_budget, int) and documents >= max(0, report_budget)
- and (task_end is not None or table == "settlements")):
+ if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget)
+ and (task_end is not None or table == "settlements"
+ or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
break
+ if first_load and table in GROUP_A_TABLES:
+ reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
+ datetime.now(timezone.utc))
+ if reason:
+ report["tables"][table] = {"state": "deferred", "reason": reason}
+ continue
if table in handled:
continue
plan = plans[table] if first_load else sync_window(
@@ -1447,7 +3249,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
log(f" {table:<16} SKIPPED: no confirmed bulk report flavor")
continue
prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
- root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
+ root = cx.REPORT_TYPES[table].get("json_root")
log(f"{table}: {plan['cause']}")
sibling = SHARED_REPORT_SIBLINGS.get(table)
sibling_plan = None
@@ -1467,9 +3269,13 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
else:
sibling = None
operation = "report_document"
+ # Include attempts: a later vendor/validation failure can follow a successful row write.
+ changed_tables.add(mirror)
+ if sibling:
+ changed_tables.add(table_name(sibling))
try:
# Attempt first, so a refusal or process interruption cannot starve the other tables.
- if not first_load:
+ if not first_load and table != "settlements":
_record_state(con, entity_key("sync_tick_" + table), "attempted")
if sibling and not first_load:
_record_state(con, entity_key("sync_tick_" + sibling), "attempted")
@@ -1520,10 +3326,71 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
operations["getReports"] = {
"state": "done", "reports": len(discovered),
"downloaded": downloaded, "skipped": skipped}
+ if not discovered:
+ report["tables"][table] = {"rows": 0, "operations": operations,
+ "state": "empty", "reason": "No issued settlement reports found."}
+ landed.add((table, task_start, task_end, task_batch))
+ # No documents means no new cursor, attempt stamp or completion marker.
+ continue
if first_load and settlement_remaining:
break
+ # getReports filters createdTime. Never substitute the time we happened to pull.
+ issued_times = [_utc(item.get("createdTime") or item["dataEndTime"])
+ for item in discovered if item.get("createdTime") or item.get("dataEndTime")]
+ prior = read_watermark(con, table)
+ settlement_mark = max(issued_times + ([prior] if prior else []), default=None)
windows = ()
- if table != "settlements":
+ if table == SQP_TABLE:
+ if sqp_limit["deferredReason"]:
+ report["tables"][table] = {"state": "deferred", "reason": sqp_limit["deferredReason"]}
+ _record_state(con, entity_key("sync_tick_" + table), "done")
+ continue
+ if task_batch is not None:
+ written += _sync_sqp_batch(con, fields, task_start, task_end, task_batch,
+ sqp_limit, sqp_request, sqp_problem)
+ sqp_limit["errors"] = []
+ _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
+ elif table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES:
+ operation = cx.REPORT_TYPES[table].get("api") or "report_document"
+ spec = cx.REPORT_TYPES[table]
+ if first_load:
+ margin_windows = [(task_start, min(task_end, plan["end"]))] if task_end else (
+ [(None, None)] if spec.get("snapshot") else [])
+ elif spec.get("snapshot"):
+ margin_windows = [(None, None)]
+ elif spec.get("monthly"):
+ margin_windows = _month_windows(plan["start"], plan["end"])
+ else:
+ margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
+ if table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
+ # One due window per table tick respects the vendor floor. The watermark
+ # advances only to that window's end, so the following tick loses no history.
+ remaining_windows = len(margin_windows) - 1
+ margin_windows = [margin_windows[0]]
+ plan["end"] = margin_windows[0][1]
+ _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({
+ "reason": (f"{remaining_windows} history windows remain. Amazon allows this report at most "
+ f"once every {_readable_duration(spec['min_interval_s'])}. Keep the connection enabled; "
+ "sync resumes at its saved date. Use Sync now without a custom start date.")}))
+ elif table in GROUP_A_TABLES and table != "catalog_items":
+ _record_state(con, entity_key("limit_" + table), "done", detail="{}")
+ for w0, w1 in margin_windows:
+ # One detail window is one budget task, including its reconciliation query.
+ documents += 1
+ _sync_event("amazon.sync", tenant_key, table,
+ {"state": "access", "operation": operation}, rt, report, log)
+ written += _sync_margin_window(con, table, fields, w0, w1,
+ max_wait=max_wait, log=log)
+ if table in LEGACY_SNAPSHOT_TABLES:
+ # The dated replacement already landed; AWD must not clear that
+ # document or historical FBA/AWD rows on an explicit first load.
+ cleared = True
+ if first_load and table in GROUP_A_TABLES:
+ _record_state(con, entity_key("sync_tick_" + table), "done")
+ if first_load and w0 is not None:
+ record_backfill_floor(con, table, w0)
+ floors[table] = w0
+ elif table != "settlements":
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
else _windows(plan["start"], plan["end"]))
for w0, w1 in (() if table == "settlements" else windows):
@@ -1535,8 +3402,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
{"state": "access", "operation": "report_document"}, rt, report, log)
text, meta = cx.report_document(
fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait,
- log=lambda message: log(f"{table}: {message}"))
- if prov == "schema_artifact" and root:
+ log=lambda message: log(f"{table}: {message}"),
+ **({"options": dict(REPORT_OPTIONS[table])} if table in REPORT_OPTIONS else {}))
+ if root:
parsed_cols, rows = parse_json_report(
text, root, document_id=meta.get("reportDocumentId"), report_type=rtype)
pinned = [c for c, _t in amazon_schema.columns_for(table)]
@@ -1597,6 +3465,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
cols.append("inventory_source")
if table == "pricing":
pricing_asins.extend(_pricing_asins(rows))
+ if not cleared and _pricing_regrain(con, mirror, rows, log):
+ cleared = True
types = _ensure(con, mirror, cols, rows, declared=declared,
preserve_rows=True) if cols else {}
# Explicit windows and first loads replace the mirror; incremental passes keep history.
@@ -1611,9 +3481,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
record_backfill_floor(con, reached, w0)
floors[reached] = w0
- if first_load:
- landed.add((table, task_start, task_end))
- if any(next_table == table for next_table, _w0, _w1 in work[index + 1:]):
+ if first_load or table == SQP_TABLE:
+ landed.add((table, task_start, task_end, task_batch))
+ if any(task[0] == table for task in work[index + 1:]):
continue
if table == "pricing":
@@ -1684,7 +3554,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
_record_state(con, key, "done", total_rows)
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
- record_watermark(con, table, plan["end"], total_rows)
+ mark = settlement_mark if table == "settlements" else plan["end"]
+ if mark is not None:
+ record_watermark(con, table, mark, total_rows)
_sync_event("amazon.sync", tenant_key, table,
{"state": "done", "rows": total_rows,
"operations_ok": all(v["state"] == "done" for v in operations.values())},
@@ -1709,6 +3581,9 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
{"state": "done", "rows": sibling_total, "operations_ok": True},
rt, report, log)
log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
+ except SQPBudgetDeferred as exc:
+ report["tables"][table] = {"state": "deferred", "reason": str(exc)}
+ break
except Exception as e: # noqa: BLE001
# A failed pass retains its previous watermark so its window will be retried.
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
@@ -1716,6 +3591,17 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
problem = _sync_problem(e, table, operation, tenant_key, rt, report, log)
failed.add(table)
report["tables"][table] = {"rows": 0, **problem}
+ # S66-T42 (lead, observed live 2026-09-25): a refusal lived only in this pass's report,
+ # so Connectors > Amazon said "not loaded yet, sync it" beside a table Amazon had just
+ # refused by name (a 400 "not allowed", a FATAL). The table's own marker now carries the
+ # refusal, which `connectors_amazon.sync_state` shows verbatim, and the tick closes so the
+ # table rotates behind its peers instead of reading as syncing forever.
+ try:
+ for refused in (table, sibling) if sibling else (table,):
+ _record_state(con, entity_key(refused), problem["state"], 0, problem["reason"])
+ _record_state(con, entity_key("sync_tick_" + refused), "done", 0)
+ except Exception: # noqa: BLE001
+ pass
_sync_event("amazon.sync", tenant_key, table,
{"state": problem["state"], "rows": 0}, rt, report, log)
if sibling:
@@ -1723,6 +3609,13 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
report["tables"][sibling] = {"rows": 0, **problem}
_sync_event("amazon.sync", tenant_key, sibling,
{"state": problem["state"], "rows": 0}, rt, report, log)
+ if SQP_TABLE in pending:
+ remaining_sqp = sum(task[0] == SQP_TABLE and task not in landed for task in work)
+ if remaining_sqp and SQP_TABLE not in failed and not sqp_limit["deferredReason"]:
+ reason = f"Search Query Performance has {remaining_sqp} month and batch tasks remaining; continuing next pass."
+ sqp_limit["deferredReason"] = reason
+ report["tables"][SQP_TABLE] = {"state": "deferred", "reason": reason}
+ _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit))
if first_load:
remaining = sum(task not in landed for task in work)
if settlement_remaining:
@@ -1732,14 +3625,28 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
continuable = any(task not in landed and task[0] not in failed for task in work)
state = ("partial" if remaining and continuable else
"failed" if remaining or report["problems"] or report["audit_problems"] else "done")
+ # Requests for one report type must also respect that report's own refresh floor.
+ # Different tables can progress between those waits; use the longest table chain.
+ counts = {}
+ for task in work:
+ if task not in landed and task[0] in GROUP_A_TABLES:
+ counts[task[0]] = counts.get(task[0], 0) + 1
+ floor_eta = 0
+ for table, count in counts.items():
+ floor = cx.REPORT_TYPES[table]["min_interval_s"]
+ tick = _read_state(con, entity_key("sync_tick_" + table))
+ wait = (max(0, floor - (datetime.now(timezone.utc) - _utc(tick["updated_at"])).total_seconds())
+ if tick and tick["state"] == "done" else 0)
+ floor_eta = max(floor_eta, int(wait + max(0, count - 1) * floor + 0.999))
progress = {"state": state, "remaining": remaining,
- "etaSeconds": first_load_eta(remaining, burst=0),
+ "etaSeconds": max(first_load_eta(remaining, burst=0), floor_eta),
"cause": first_load_eta_cause(remaining), "backfillDays": DEFAULT_BACKFILL_DAYS}
report.update(progress)
report["firstLoad"] = dict(progress)
_record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end))
_record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state,
detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")}))
+ persist_mirror(tenant_key, rt, con, changed_tables, report=report, log=log)
report["finished"] = _iso(datetime.now(timezone.utc))
con.close()
return report
@@ -1797,6 +3704,21 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
out[table] = {"state": "unproven", "reason": "table is empty",
"reconciles": spec.get("validates_against")}
continue
+ if table == SQP_TABLE:
+ out[table] = {"rows": n, "reconciles": spec.get("validates_against"), **_validate_sqp(con)}
+ continue
+ if table == "listings":
+ out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
+ **_validate_snapshot(con, table)}
+ continue
+ if table in PERIOD_TABLES:
+ out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
+ **_validate_margin(con, table)}
+ continue
+ if table == "orders":
+ out[table] = {"rows": n, "reconciles": "Amazon Sales getOrderMetrics order count",
+ **_validate_orders(con)}
+ continue
if table in ENTITY_STATE_KEYS:
operation = _read_state(con, ENTITY_STATE_KEYS[table])
if not operation or operation.get("state") != "done":
@@ -1820,6 +3742,10 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
"reason": f"role operation returned {expected} but mirror carries {measured}",
"reconciles": spec.get("validates_against")}
continue
+ if table == "inventory":
+ out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
+ **_validate_snapshot(con, table)}
+ continue
out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")}
for t, r in out.items():
log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}")
@@ -1827,13 +3753,14 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
def status(tenant_key="royal-imports", rt=None):
- """What is in the mirror now. Needs no credential and no network."""
+ """What is in the mirror now, restoring once from the tenant store when needed."""
rt = rt if rt is not None else _runtime_for(tenant_key)
+ restored = restore_mirror(tenant_key, rt)
try:
con, _path = _connect_tenant_mirror(tenant_key, rt)
except Exception as e: # noqa: BLE001
return {"error": f"{type(e).__name__}: {e}"}
- out = {}
+ out = {"error": "; ".join(restored["problems"])} if restored["problems"] else {}
for table in TABLES:
mirror = table_name(table)
try:
@@ -1845,11 +3772,1487 @@ def status(tenant_key="royal-imports", rt=None):
"provenance": amazon_schema.PROVENANCE.get(table)}
if table in ENTITY_STATE_KEYS:
out[table]["operation"] = _read_state(con, ENTITY_STATE_KEYS[table])
+ con.close()
return out
# --------------------------------------------------------------------------- selftest
+class _FixtureMirrorRuntime:
+ """Dictionary blobs for this module's offline sync and purge fixtures only."""
+
+ def upload_bytes(self, path, data, message=None):
+ if not hasattr(self, "blobs"):
+ self.blobs = {}
+ self.blobs[path] = data
+
+ def download_bytes(self, path):
+ return getattr(self, "blobs", {}).get(path)
+
+ def delete_path(self, path):
+ getattr(self, "blobs", {}).pop(path, None)
+
+
+def snapshot_selftest(log=print, check=None):
+ """S66-T47: real sync with in-memory storage and fixture-only vendor transports."""
+ from types import SimpleNamespace
+ from unittest.mock import patch
+ import duckdb
+ cx = _client()
+ failures = []
+
+ def ck(name, probe):
+ try:
+ ok = bool(probe())
+ except Exception as exc:
+ ok = False
+ log(f" {name}: {type(exc).__name__}: {exc}")
+ if check:
+ check("[snapshots] " + name, ok)
+ else:
+ log(f" {'ok' if ok else 'FAIL'} {name}")
+ if not ok:
+ failures.append(name)
+
+ class Clock(datetime):
+ instant = datetime(2026, 9, 25, 12, tzinfo=timezone.utc)
+
+ @classmethod
+ def now(cls, tz=None):
+ return cls.instant.astimezone(tz)
+
+ start = Clock.instant - timedelta(days=65)
+ ck("both legacy snapshots plan one task and reduce the ETA", lambda:
+ first_load_plan(["listings", "inventory"], start, Clock.instant)
+ == [("listings", None, None), ("inventory", None, None)]
+ and first_load_eta(2, burst=0) < first_load_eta(6, burst=0))
+ listing = "seller-sku\tquantity\nA\t2\nB\t3\n"
+ ledger_header = "date\tmsku\tfnsku\tasin\tdisposition\tlocation\tending-warehouse-balance\n"
+ ledger = ledger_header + ("09/23/2026\tA\tFA\tAA\tSELLABLE\tUS\t2\n"
+ "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t3\n"
+ "09/24/2026\tA\tFA\tAA\tDAMAGED\tUS\t1\n"
+ "09/24/2026\tA\tFA\tAA\tSELLABLE\tCA\t4\n")
+ for table in ("listings", "inventory"):
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, "
+ "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ rt = SimpleNamespace(key="qa-snapshot", get=lambda *_a, **_k: None)
+ calls = []
+ payload = [listing if table == "listings" else ledger]
+ meta = {"createdTime": "2026-09-25T11:00:00Z"}
+ mirror = table_name(table)
+
+ def document(_fields, rtype, **kwargs):
+ calls.append(kwargs)
+ return payload[0], {**meta, "reportDocumentId": f"snapshot-{len(calls)}"}
+
+ def count():
+ return con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
+
+ def valid():
+ return validate(rt.key, rt=rt, log=lambda *_a: None)[table]
+
+ with patch.object(sys.modules[__name__], "_connect_tenant_mirror", side_effect=lambda *_a: (con.cursor(), None)), \
+ patch.object(sys.modules[__name__], "credential", return_value={"fixture": True}), \
+ patch.object(sys.modules[__name__], "restore_mirror", return_value={"problems": []}), \
+ patch.object(sys.modules[__name__], "persist_mirror"), \
+ patch.object(sys.modules[__name__], "retention_check", return_value={"earliestTriggerAt": None}), \
+ patch.object(sys.modules[__name__], "_sync_event"), \
+ patch.object(sys.modules[__name__], "DEFAULT_BACKFILL_DAYS", 65), \
+ patch.object(sys.modules[__name__], "datetime", Clock), \
+ patch.object(cx, "report_document", side_effect=document), \
+ patch.object(cx, "awd_inventory", return_value=[]), \
+ patch.object(cx, "access_token", side_effect=AssertionError("offline fixture")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("offline fixture")):
+ ck(table + " never synced is unproven", lambda: valid()["state"] == "unproven")
+ deferred = sync(rt.key, tables=[table], rt=rt, first_load=True, report_budget=0, log=lambda *_a: None)
+ ck(table + " a zero budget defers its one snapshot task", lambda:
+ not calls and deferred["remaining"] == 1 and deferred["etaSeconds"] == first_load_eta(1, burst=0))
+ result = sync(rt.key, tables=[table], rt=rt, first_load=True, report_budget=3, log=lambda *_a: None)
+ ck(table + " three-window first load requests once without a window", lambda:
+ len(calls) == 1 and "start" not in calls[0] and "end" not in calls[0]
+ and result["state"] == "done" and result["remaining"] == 0)
+ ck(table + " all document rows reconcile", lambda:
+ count() == (2 if table == "listings" else 4) and valid()["state"] == "ok")
+ # Prove the check is independent of the parser, without mutating any real store.
+ before = con.execute(f"SELECT * FROM {mirror}").fetchall()
+ parse = parse_flat_file
+
+ def dropping_parser(*args, **kwargs):
+ columns, records = parse(*args, **kwargs)
+ return columns, records[:-1]
+
+ with patch.object(sys.modules[__name__], "parse_flat_file", side_effect=dropping_parser):
+ dropped = sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ ck(table + " parser row loss refuses before replacing the stored snapshot", lambda:
+ dropped["tables"][table].get("state") == "refused"
+ and con.execute(f"SELECT * FROM {mirror}").fetchall() == before)
+ calls.pop() # The deliberate refused request is not part of the replay count below.
+ if table == "listings":
+ ck("listings row count equals distinct date and SKU", lambda:
+ con.execute("SELECT COUNT(*), COUNT(DISTINCT (_snapshot_date, seller_sku)) "
+ "FROM amzn_listings").fetchone() == (2, 2))
+ meta["createdTime"] = "2026-09-26T11:00:00Z"
+ sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ # S67-T32 (D-3031, reversed): a second day REPLACES the first, one load's rows.
+ ck("a second listing day replaces the first, leaving one copy", lambda:
+ con.execute("SELECT _snapshot_date, COUNT(*) FROM amzn_listings "
+ "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-26", 2)]
+ and len(calls) == 2 and all("start" not in c and "end" not in c for c in calls))
+ payload[0] = "seller-sku\tquantity\nA\t9\n"
+ sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None)
+ ck("same listing date replaces removed SKUs", lambda:
+ con.execute("SELECT _snapshot_date, seller_sku, quantity FROM amzn_listings "
+ "ORDER BY 1, 2").fetchall() == [("2026-09-26", "A", 9)]
+ and valid()["state"] == "ok")
+ else:
+ ck("ledger keeps one row per data date, SKU, disposition and location", lambda:
+ con.execute("SELECT COUNT(*), COUNT(DISTINCT (date, msku, disposition, location)), "
+ "COUNT(DISTINCT _snapshot_date) FROM amzn_inventory").fetchone() == (4, 4, 2)
+ and calls[0].get("options", {}).get("aggregateByTimePeriod") == "DAILY")
+ awd = _awd_rows([{"sku": "AWD-kept"}])
+ cols = sorted(awd[0])
+ types = _ensure(con, mirror, cols, awd, preserve_rows=True)
+ _replace(con, mirror, cols, types, awd)
+ before_awd = con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall()
+ payload[0] = ledger_header + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t9\n"
+ sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None)
+ # S67-T32 (D-3031, reversed): the next document replaces every earlier FBA date.
+ ck("ledger second document replaces every earlier data date", lambda:
+ con.execute("SELECT _snapshot_date, ending_warehouse_balance FROM amzn_inventory "
+ "WHERE inventory_source='FBA' ORDER BY 1").fetchall()
+ == [("2026-09-24", 9)])
+ ck("FBA replacement preserves AWD rows exactly", lambda:
+ con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() == before_awd)
+ # The AWD operation's existing reconciliation still checks its own rows.
+ _record_state(con, ENTITY_STATE_KEYS[table], "done", 1)
+ ck("ledger source proof excludes AWD", lambda: valid()["state"] == "ok")
+
+ # Keep the count constant but turn a second SKU into a duplicate: a count-only
+ # implementation would falsely reconcile this source document.
+ payload[0] = (listing if table == "listings" else ledger_header
+ + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t2\n"
+ "09/24/2026\tB\tFB\tAB\tSELLABLE\tUS\t3\n")
+ sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ if table == "inventory":
+ _record_state(con, ENTITY_STATE_KEYS[table], "done", 1)
+ date = "2026-09-26" if table == "listings" else "2026-09-24"
+ sku = "seller_sku" if table == "listings" else "msku"
+ try:
+ con.execute(f"UPDATE {mirror} SET {sku}='A' WHERE _snapshot_date=?", [date])
+ ck(table + " validation refuses duplicate identity at unchanged row count", lambda:
+ valid()["state"] == "refused")
+ con.execute(f"DELETE FROM {mirror} WHERE _snapshot_date=? AND _report_row_number=2", [date])
+ ck(table + " validation refuses document row loss", lambda: valid()["state"] == "refused")
+ except duckdb.Error:
+ ck(table + " validation refuses duplicate identity at unchanged row count", lambda: False)
+ ck(table + " validation refuses document row loss", lambda: False)
+ if table == "listings":
+ payload[0] = "seller-sku\tquantity\n"
+ sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ ck("empty listing restatement replaces the copy and stays unproven", lambda:
+ count() == 0 and valid()["state"] == "unproven")
+ meta["dataEndTime"] = "2026-09-27T23:00:00Z"
+ payload[0] = "seller-sku\tquantity\n" + "".join(f"S{i}\t1\n" for i in range(1205))
+ sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ ck("the report data date wins over creation time and no row cap is applied", lambda:
+ con.execute("SELECT COUNT(*) FROM amzn_listings WHERE _snapshot_date='2026-09-27'").fetchone()[0]
+ == 1205 and valid()["state"] == "ok")
+ else:
+ payload[0] = ledger_header + "09/25/2026\t\tF-fallback\tAA\tSELLABLE\tUS\t5\n"
+ sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
+ _record_state(con, ENTITY_STATE_KEYS[table], "done", 1)
+ ck("the ledger uses FNSKU when MSKU is absent", lambda:
+ valid()["state"] == "ok" and con.execute(
+ "SELECT COUNT(*) FROM amzn_inventory WHERE _snapshot_date='2026-09-25' "
+ "AND fnsku='F-fallback'").fetchone()[0] == 1)
+ con.close()
+ ck("S67-T32 saved mirror: one copy, content keys, kill-safe checkpoint, full purge",
+ lambda: mirror_selftest(log, check) == 0)
+ return len(failures)
+
+
+def mirror_selftest(log=print, check=None):
+ """S67-T32 (D-3031 + D-3481): the saved snapshot is one copy, whole, and restorable.
+
+ Offline: a DuckDB :memory: mirror and a dictionary blob store. Covers the legacy (v1)
+ manifest restoring exactly, the doubled-mirror collapse, content keys never overwriting a
+ named key, a checkpoint dying after one table upload, purge reaching the orphan, a normal
+ checkpoint round trip (NC) and a refused process never publishing over a valid snapshot.
+ """
+ from unittest.mock import patch
+ import duckdb
+ failures = []
+
+ def ck(name, probe):
+ try:
+ ok = bool(probe())
+ except Exception as exc: # noqa: BLE001
+ ok = False
+ log(f" {name}: {type(exc).__name__}: {exc}")
+ if check:
+ check("[mirror] " + name, ok)
+ else:
+ log(f" {'ok' if ok else 'FAIL'} {name}")
+ if not ok:
+ failures.append(name)
+
+ class Store:
+ key = "qa-mirror"
+
+ def __init__(self):
+ self.blobs, self.uploads, self.hook = {}, [], None
+
+ def get(self, *_a, **_k):
+ return None
+
+ def upload_bytes(self, path, data, message=None):
+ if self.hook:
+ self.hook("upload", path)
+ self.blobs[path] = bytes(data)
+ self.uploads.append(path)
+
+ def download_bytes(self, path):
+ if self.hook:
+ self.hook("download", path)
+ return self.blobs.get(path)
+
+ def delete_path(self, path):
+ self.blobs.pop(path, None)
+ return True
+
+ rt = Store()
+ manifest_path, ledger_path = MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, "
+ "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+
+ def forget():
+ _MIRROR_CACHE.pop(_mirror_cache_key(rt.key, rt), None)
+
+ def wipe():
+ """A new process whose local disk was lost."""
+ for name, _kind in _amazon_objects(con):
+ con.execute(f"DROP TABLE {_quoted_identifier(name)}")
+ con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_')")
+ forget()
+
+ def dump():
+ out = {name: sorted(con.execute(f"SELECT * FROM {_quoted_identifier(name)}").fetchall(), key=repr)
+ for name, _kind in _amazon_objects(con)}
+ out["_sync_state"] = sorted(r for r in _sync_state_rows(con) if r[0].startswith(PREFIX))
+ return out
+
+ def ledger_doc():
+ try:
+ return json.loads(rt.blobs[ledger_path])
+ except (KeyError, ValueError):
+ return {"version": 1, "tenant": rt.key, "keys": {}}
+
+ def mirror_blobs():
+ return {k for k in rt.blobs if k.startswith(MIRROR_PREFIX)}
+
+ def persist(tables=()):
+ report = {"problems": []}
+ persist_mirror(rt.key, rt, con, set(tables), report=report, log=lambda *_a: None)
+ return report
+
+ def restore():
+ return restore_mirror(rt.key, rt, log=lambda *_a: None)
+
+ def legacy_save():
+ """Today's staging layout, byte for byte: v1, in-place keys, no ledger."""
+ manifest = {"version": 1, "tenant": rt.key, "tables": {}}
+ import tempfile as _tmp
+ with _tmp.TemporaryDirectory(prefix="amazon-legacy-") as tmp:
+ for name, _kind in _amazon_objects(con):
+ path = Path(tmp) / (name + ".parquet")
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
+ data = path.read_bytes()
+ rt.blobs[MIRROR_PREFIX + name + ".parquet"] = data
+ manifest["tables"][name] = {**_blob_metadata(data), "written_at": "2026-09-27T00:00:00Z",
+ "rows": con.execute(f"SELECT count(*) FROM {name}").fetchone()[0]}
+ rows = [list(r) for r in _sync_state_rows(con) if r[0].startswith(PREFIX)]
+ state = _mirror_json(rows)
+ rt.blobs[MIRROR_PREFIX + "sync_state.json"] = state
+ manifest.update(sync_state={**_blob_metadata(state), "rows": len(rows)}, written_at="2026-09-27T00:00:00Z")
+ rt.blobs[manifest_path] = _mirror_json(manifest)
+
+ def seed():
+ con.execute("CREATE TABLE amzn_restock_recommendations AS SELECT * FROM (VALUES "
+ "('2026-09-25', 'A', 1), ('2026-09-25', 'B', 2), ('2026-09-25', 'C', 3), "
+ "('2026-09-26', 'A', 4), ('2026-09-26', 'B', 5), ('2026-09-26', 'C', 6)) "
+ "AS t(_snapshot_date, sku, qty)")
+ con.execute("CREATE TABLE amzn_inventory AS SELECT * FROM (VALUES "
+ "('2026-09-22', 'A', 'FBA', 1), ('2026-09-23', 'A', 'FBA', 2), "
+ "('2026-09-24', 'A', 'FBA', 3), (NULL, 'AWD-1', 'AWD', 9)) "
+ "AS t(_snapshot_date, msku, inventory_source, qty)")
+ con.execute("CREATE TABLE amzn_orders AS SELECT * FROM (VALUES "
+ "('2026-09-01', 'O1', 10.5), ('2026-09-02', 'O2', 11.5)) AS t(_period_start, id, amount)")
+ con.execute("CREATE TABLE amzn_listings AS SELECT * FROM (VALUES ('2026-09-26', 'S1', 1)) "
+ "AS t(_snapshot_date, seller_sku, quantity)")
+ _record_state(con, entity_key("reconcile_inventory"), "done", detail=json.dumps(
+ {"start": "2026-09-23", "end": "2026-09-24", "column": "_snapshot_date", "count": 2,
+ "dates": {"2026-09-23": 1, "2026-09-24": 1}}))
+ record_watermark(con, "orders", datetime(2026, 9, 26, tzinfo=timezone.utc), 2)
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z")
+
+ try:
+ with patch.object(sys.modules[__name__], "_connect_tenant_mirror",
+ side_effect=lambda *_a: (con.cursor(), None)):
+ # D-3031 at the write: an API snapshot pulled on a new day replaces the old day.
+ for day, qty in (("2026-09-25", 1), ("2026-09-26", 2)):
+ rows = [_origin({"sku": s, "qty": qty, "_snapshot_date": day}, "doc-" + day, i, "fixture")
+ for i, s in enumerate(("A", "B"), 1)]
+ _replace_period(con, "fba_inventory", rows, ["sku", "qty", "_snapshot_date"],
+ "_snapshot_date", day, snapshot=True)
+ ck("two snapshot days leave one copy of one load's rows", lambda: con.execute(
+ "SELECT _snapshot_date, count(*) FROM amzn_fba_inventory GROUP BY 1").fetchall()
+ == [("2026-09-26", 2)])
+ ck("a period table keeps its history", lambda: _replace_period(
+ con, "storage_fees", [_origin({"month": "2026-08", "fee": 1}, "d", 1, "f")], ["month", "fee"],
+ "month", "2026-08") == 1 and _replace_period(
+ con, "storage_fees", [_origin({"month": "2026-09", "fee": 1}, "d", 1, "f")], ["month", "fee"],
+ "month", "2026-09") == 1 and con.execute(
+ "SELECT count(DISTINCT month) FROM amzn_storage_fees").fetchone()[0] == 2)
+ con.execute("DROP TABLE amzn_fba_inventory")
+ con.execute("DROP TABLE amzn_storage_fees")
+
+ # The legacy manifest staging boots from, holding a doubled snapshot.
+ seed()
+ original = dump()
+ legacy_save()
+ legacy_blobs = dict(rt.blobs)
+ wipe()
+ result = restore()
+ ck("a legacy v1 manifest with in-place keys restores every table and marker", lambda:
+ result["state"] == "restored" and not result["problems"]
+ and dump()["amzn_orders"] == original["amzn_orders"]
+ and dump()["amzn_listings"] == original["amzn_listings"]
+ and dump()["_sync_state"] == original["_sync_state"])
+ ck("a doubled mirror collapses on restore to the latest pull and is marked dirty", lambda:
+ con.execute("SELECT _snapshot_date, count(*) FROM amzn_restock_recommendations "
+ "GROUP BY 1").fetchall() == [("2026-09-26", 3)]
+ and con.execute("SELECT _snapshot_date FROM amzn_inventory WHERE inventory_source='FBA' "
+ "ORDER BY 1").fetchall() == [("2026-09-23",), ("2026-09-24",)]
+ and con.execute("SELECT count(*) FROM amzn_inventory WHERE inventory_source='AWD'").fetchone()[0] == 1
+ and {"amzn_restock_recommendations", "amzn_inventory"} <= result["dirty"]
+ and "amzn_orders" not in result["dirty"])
+ ck("the collapse is idempotent", lambda: _collapse_snapshots(con) == {})
+
+ # First checkpoint after the legacy boot: every blob moves to a content key.
+ legacy_keys = set(_manifest_keys(json.loads(legacy_blobs[manifest_path])))
+ rt.uploads.clear()
+ report = persist()
+ saved = json.loads(rt.blobs[manifest_path])
+ named = set(_manifest_keys(saved))
+ ck("the first checkpoint migrates to content keys and retires the in-place blobs", lambda:
+ not report["problems"] and saved["version"] == MANIFEST_VERSION
+ and all(_TABLE_BLOB_RE.fullmatch(k).group(2) for k in named if k.endswith(".parquet"))
+ and not any(MIRROR_PREFIX + k in rt.blobs for k in legacy_keys)
+ and not any(MIRROR_PREFIX + k in rt.uploads for k in legacy_keys)
+ and set(ledger_doc()["keys"]) == named
+ and rt.uploads.index(ledger_path) < min(rt.uploads.index(MIRROR_PREFIX + k) for k in named)
+ and rt.uploads[-2 if rt.uploads[-1] == ledger_path else -1] == manifest_path)
+ healed = dump()
+ wipe()
+ ck("NC a normal checkpoint restores exactly, one copy each", lambda:
+ restore()["state"] == "restored" and dump() == healed
+ and con.execute("SELECT count(*) FROM amzn_restock_recommendations").fetchone()[0] == 3)
+ rt.uploads.clear()
+ before = dict(rt.blobs)
+ persist(set(original))
+ ck("an unchanged checkpoint uploads nothing", lambda: rt.uploads == [] and rt.blobs == before)
+
+ # A changed table never overwrites a key the published manifest names.
+ con.execute("UPDATE amzn_orders SET amount = amount + 1")
+ rt.uploads.clear()
+ persist({"amzn_orders"})
+ ck("content keys never overwrite a named key", lambda:
+ not any(path[len(MIRROR_PREFIX):] in named for path in rt.uploads)
+ and _manifest_keys(json.loads(rt.blobs[manifest_path]))
+ and not (MIRROR_PREFIX + saved["tables"]["amzn_orders"]["key"]) in rt.blobs)
+
+ # A checkpoint that dies after its first table blob lands.
+ published = rt.blobs[manifest_path]
+ good = dump()
+ con.execute("UPDATE amzn_orders SET amount = amount + 1")
+ con.execute("UPDATE amzn_listings SET quantity = quantity + 1")
+ parquet = []
+
+ def die_on_second(kind, path):
+ if kind == "upload" and path.endswith(".parquet"):
+ parquet.append(path)
+ if len(parquet) == 2:
+ raise OSError("fixture: the process died mid-checkpoint")
+
+ rt.hook = die_on_second
+ report = persist({"amzn_orders", "amzn_listings"})
+ rt.hook = None
+ orphan = parquet[0][len(MIRROR_PREFIX):] if parquet else "absent"
+ ck("a checkpoint that dies after one table upload leaves the manifest untouched", lambda:
+ any("persistence failed" in p for p in report["problems"])
+ and rt.blobs[manifest_path] == published and MIRROR_PREFIX + orphan in rt.blobs)
+ ck("the key ledger lists the unpublished orphan", lambda:
+ orphan in ledger_doc()["keys"]
+ and orphan not in _manifest_keys(json.loads(published)))
+ wipe()
+ ck("the next process restores the previous snapshot whole", lambda:
+ restore()["state"] == "restored" and dump() == good)
+
+ # An aged orphan is retired by the next successful publish; a fresh one is not.
+ ledger = ledger_doc()
+ ledger["keys"][orphan] = "2026-01-01T00:00:00Z"
+ rt.blobs[ledger_path] = _mirror_json(ledger)
+ con.execute("UPDATE amzn_orders SET amount = amount + 5")
+ persist({"amzn_orders"})
+ ck("an aged ledger orphan is retired after the next publish", lambda:
+ MIRROR_PREFIX + orphan not in rt.blobs
+ and orphan not in ledger_doc()["keys"]
+ and set(ledger_doc()["keys"]) >= set(
+ _manifest_keys(json.loads(rt.blobs[manifest_path]))))
+
+ # A refused restore (store unreachable at boot) must not publish its rebuild.
+ valid_manifest = rt.blobs[manifest_path]
+ wipe()
+
+ def unreachable(kind, path):
+ if kind == "download" and path == manifest_path:
+ raise OSError("fixture: store unreachable at boot")
+
+ rt.hook = unreachable
+ refused = restore()
+ rt.hook = None
+ con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-02' AS _period_start, 'O2' AS id, 1.0 AS amount")
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z")
+ rt.uploads.clear()
+ report = persist({"amzn_orders"})
+ ck("a refused restore does not publish a partial rebuild over the valid manifest", lambda:
+ refused["state"] == "refused" and rt.blobs[manifest_path] == valid_manifest
+ and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"]))
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z")
+ report = persist({"amzn_orders"})
+ ck("the refused process publishes once its own first load completes", lambda:
+ not report["problems"] and rt.blobs[manifest_path] != valid_manifest
+ and set(json.loads(rt.blobs[manifest_path])["tables"]) == {"amzn_orders"})
+
+ # A stored snapshot that is itself unrestorable names nothing valid; the rebuild may publish.
+ wipe()
+ corrupt = json.loads(rt.blobs[manifest_path])
+ corrupt["tables"]["amzn_orders"]["rows"] += 1
+ rt.blobs[manifest_path] = _mirror_json(corrupt)
+ refused = restore()
+ con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-03' AS _period_start, 'O3' AS id, 2.0 AS amount")
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z")
+ report = persist({"amzn_orders"})
+ ck("a corrupt stored snapshot refuses whole and the rebuild may publish over it", lambda:
+ refused["state"] == "refused" and not report["problems"]
+ and json.loads(rt.blobs[manifest_path])["tables"]["amzn_orders"]["rows"] == 1)
+
+ # Purge reaches every version: legacy keys, named keys and a killed checkpoint's orphan.
+ for path, data in legacy_blobs.items():
+ if path != manifest_path:
+ rt.blobs[path] = data
+ con.execute("UPDATE amzn_orders SET amount = 7")
+ parquet.clear()
+ rt.hook = lambda kind, path: (_ for _ in ()).throw(OSError("fixture death")) if (
+ kind == "upload" and path == manifest_path) else None
+ persist({"amzn_orders"})
+ rt.hook = None
+ rt.blobs["unrelated/keep.bin"] = b"keep"
+ orphans = set(ledger_doc()["keys"]) - set(
+ _manifest_keys(json.loads(rt.blobs[manifest_path])))
+ _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
+ ck("purge deletes every stored version including the orphan", lambda:
+ orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
+ finally:
+ forget()
+ con.close()
+ return len(failures)
+
+
+def margin_selftest(log=print, check=None):
+ """S66-T40 offline fixtures and red-first controls, shared with api_amazon.
+
+ Every new invariant first sees a planted defect, then the restored implementation.
+ Only in-memory DuckDB and mocked vendor transports are used.
+ """
+ from copy import deepcopy
+ from unittest.mock import patch
+ import duckdb
+ cx = _client()
+ failures = []
+
+ def ck(name, ok):
+ if check:
+ check(name, bool(ok))
+ else:
+ log(f" {'ok' if ok else 'FAIL'} {name}")
+ if not ok:
+ failures.append(name)
+
+ def rejects(call):
+ try:
+ call()
+ except (cx.AmazonError, ValueError):
+ return True
+ return False
+
+ with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")):
+ excluded = "GET_SELLER_FEEDBACK_DATA"
+ approved = cx.OPERATIONS["createReport"]
+ with patch.dict(approved, {"report_types": (*approved["report_types"], excluded)}):
+ probe = lambda: rejects(lambda: cx.require_operation("POST", cx._REPORTS_CREATE_PATH,
+ report_type=excluded))
+ with patch.object(cx, "EXCLUDED_REPORT_TYPES", frozenset()):
+ red = not probe()
+ ck("margin excluded report refusal survives a future allowlist addition", red and probe())
+ probe = lambda: not set(cx._APPROVED_REPORT_TYPES).intersection(cx.EXCLUDED_REPORT_TYPES)
+ with patch.object(cx, "_APPROVED_REPORT_TYPES", (*cx._APPROVED_REPORT_TYPES, excluded)):
+ red = not probe()
+ ck("margin every approved report is outside the compliance exclusion set", red and probe())
+
+ injected = '2026-09-01") { otherDataset { secret } }'
+ probe = lambda: rejects(lambda: cx.economics_query(injected, "2026-09-02", "MKT"))
+ with patch.object(cx, "economics_query", lambda *a, **k: a[0]):
+ red = not probe()
+ ck("margin economics dates cannot inject caller GraphQL", red and probe())
+ query = cx.economics_query("2026-09-01", "2026-09-02", "MKT")
+ ck("margin query fixes the economics dataset, DAY x MSKU and every money family",
+ all(word in query for word in ("analytics_economics_2024_03_15", "date: DAY", "productId: MSKU",
+ "fees", "ads", "netProceeds", "cost", "refundedProductSales"))
+ and "date: RANGE" in cx.economics_query("2026-09-01", "2026-09-02", "MKT", summary=True))
+
+ clock = [0.0]
+ waits = []
+ report_tokens = cx._report_pacer["tokens"]
+ def advance(seconds):
+ waits.append(seconds)
+ clock[0] += seconds
+ with patch.dict(cx._query_pacer, {"tokens": 15.0, "updated": None}):
+ for _ in range(16):
+ cx._pace_query_create(sleep=advance, now=lambda: clock[0])
+ ck("margin createQuery has an independent burst-15 one-per-minute bucket",
+ waits == [60.0] and cx._report_pacer["tokens"] == report_tokens)
+
+ pages = [{"payload": {"transactions": [{"transactionId": "p1"}], "nextToken": "next-1"}},
+ {"payload": {"transactions": [], "nextToken": "next-2"}},
+ {"payload": {"transactions": [{"transactionId": "p2"}]}}]
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_request", side_effect=pages) as request, \
+ patch.object(cx, "_pace_entity") as pace:
+ returned = cx.transactions({}, "2026-09-01T00:00:00Z", "2026-09-03T00:00:00Z")
+ ck("margin Finances follows every cursor through empty pages at 0.5 requests per second",
+ [r["transactionId"] for r in returned] == ["p1", "p2"]
+ and [c.kwargs.get("nextToken") for c in request.call_args_list] == [None, "next-1", "next-2"]
+ and all(c.args[:2] == ("transactions", 2.0) for c in pace.call_args_list))
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_create_query", return_value="query-fixture"), \
+ patch.object(cx, "_request", side_effect=[{"processingStatus": "IN_PROGRESS"},
+ {"processingStatus": "DONE", "dataDocumentId": "data-fixture"},
+ {"documentUrl": "https://fixture.invalid/document"}]), \
+ patch.object(cx, "_download_document", return_value='{"startDate":"2026-09-01"}'):
+ document, meta = cx.economics_document({}, "2026-09-01", "2026-09-02", sleep=lambda _: None)
+ ck("margin Data Kiosk polls DONE and downloads its JSONL document", "startDate" in document
+ and meta["reportDocumentId"] == "data-fixture")
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_create_query", return_value="query-fixture"), \
+ patch.object(cx, "_request", side_effect=[{"processingStatus": "FATAL", "errorDocumentId": "error-fixture"},
+ {"documentUrl": "https://fixture.invalid/error"}]), \
+ patch.object(cx, "_download_document", return_value='{"message":"Economics query refused"}'):
+ try:
+ cx.economics_document({}, "2026-09-01", "2026-09-02")
+ message = ""
+ except cx.AmazonError as exc:
+ message = str(exc)
+ ck("margin Data Kiosk surfaces the vendor error document", "Economics query refused" in message)
+
+ end, start = _utc("2026-09-03T00:00:00Z"), _utc("2026-09-01T00:00:00Z")
+ for table in ("fba_fee_estimates", "referral_fees"):
+ spec = cx.REPORT_TYPES[table]
+ probe = lambda: first_load_plan([table], end-timedelta(days=730), end) == [(table, None, None)]
+ with patch.dict(spec, {"snapshot": False}):
+ red = not probe()
+ ck(f"margin {table} plans exactly one snapshot task", red and probe())
+ ck(f"margin {table} transport refuses any snapshot data window",
+ rejects(lambda: cx._create_report("fixture", spec["report"], ["MKT"], start=_iso(start))))
+ states = {table: {"state": "done", "updated_at": _iso(end-timedelta(hours=1))}}
+ probe = lambda: table in sync_plan(states, [table], now=end)["deferred"]
+ with patch.dict(spec, {"min_interval_s": 0}):
+ red = not probe()
+ result = sync_plan(states, [table], now=end)
+ ck(f"margin {table} refresh floor defers with a reason", red and probe()
+ and "next due" in result["deferred"][table])
+ ck("margin storage windows follow complete calendar months",
+ _month_windows(_utc("2026-07-15T00:00:00Z"), end) == [(_utc("2026-07-01T00:00:00Z"), _utc("2026-08-01T00:00:00Z")),
+ (_utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z"))])
+ ck("margin economics incremental sync restates three days",
+ sync_window("economics", start, now=end)["start"] == start-timedelta(days=3))
+
+ for table in MARGIN_TABLES:
+ role = cx.REPORT_TYPES[table]["role"]
+ def probe_role():
+ report = {"problems": []}
+ got = _sync_problem(cx.AmazonError("vendor answered 403"), table,
+ cx.REPORT_TYPES[table].get("api", "report_document"),
+ "fixture", None, report, lambda *_: None)
+ return f"needs the {role} role" in got["reason"]
+ with patch.dict(cx.REPORT_TYPES[table], {"role": "wrong role"}):
+ red = not probe_role()
+ ck(f"margin {table} 403 names its required role", red and probe_role())
+
+ money = lambda value: {"amount": value, "currencyCode": "USD"}
+ economy = {"startDate": "2026-09-01", "endDate": "2026-09-01", "msku": "S1",
+ "marketplaceId": "MKT", "childAsin": "A1", "parentAsin": "P1", "fnsku": "F1",
+ "sales": {"orderedProductSales": money(100), "unitsOrdered": 2,
+ "refundedProductSales": money(0), "unitsRefunded": 0},
+ "fees": [{"feeTypeName": "FBA fulfillment fees", "charges": [
+ {"aggregatedDetail": {"totalAmount": money(3), "amount": money(3)}}]}],
+ "ads": [{"adTypeName": "Sponsored Products charge", "charge": {"totalAmount": money(1)}}],
+ "netProceeds": {"total": money(95), "perUnit": money(47.5)},
+ "cost": {"costOfGoodsSold": money(0.5)}}
+ raw = [economy, {**deepcopy(economy), "startDate": "2026-09-02", "endDate": "2026-09-02"}]
+ rows = economics_rows(raw, "economics-fixture")
+ probe = lambda values: (isinstance(values[0].get("fee_fba_fulfillment_fees_amount"), float)
+ and values[0]["fee_fba_fulfillment_fees_amount"] == 3
+ and values[0]["ads_sponsored_products_charge_amount"] == 1)
+ broken = deepcopy(rows)
+ broken[0]["fee_fba_fulfillment_fees_amount"] = "3"
+ ck("margin economics fees and ads pivot into summable numeric columns", not probe(broken) and probe(rows))
+ currency = lambda n: {"currencyAmount": n, "currencyCode": "USD"}
+ transaction = {"transactionId": "T1", "postedDate": "2026-09-01T12:00:00Z",
+ "transactionType": "Shipment", "transactionStatus": "RELEASED",
+ "totalAmount": currency(10),
+ "relatedIdentifiers": [{"relatedIdentifierName": "ORDER_ID", "relatedIdentifierValue": "O1"}],
+ "breakdowns": [{"breakdownType": "Sales", "breakdownAmount": currency(12),
+ "breakdowns": [{"breakdownType": "Principal", "breakdownAmount": currency(12)}]},
+ {"breakdownType": "Fees", "breakdownAmount": currency(-2)}],
+ "items": [{"totalAmount": currency(n), "contexts": [
+ {"contextType": "ProductContext", "sku": "S"+str(i), "asin": "A"+str(i)}]}
+ for i, n in enumerate((6, 4), 1)]}
+ transactions = [transaction, {**deepcopy(transaction), "transactionId": "T2", "items": []}]
+ items = transaction_rows(transactions, "operation:fixture")
+ probe = lambda values: (len(values) == 3 and sum(r.get("total_amount") or 0 for r in values) == 20
+ and values[1]["sku"] == "S2" and values[0]["amazon_order_id"] == "O1"
+ and values[0]["transaction_sub_breakdown_sales__principal_amount"] == 12)
+ broken = deepcopy(items)
+ broken[1]["total_amount"] = 10
+ ck("margin transactions explode items without multiplying transaction totals", not probe(broken) and probe(items))
+
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ try:
+ for table in MARGIN_TABLES:
+ ck(f"margin {table} missing marker is unproven", _validate_margin(con, table)["state"] == "unproven")
+ coarse = deepcopy(economy)
+ coarse["sales"]["orderedProductSales"] = money(200)
+ coarse["fees"][0]["charges"][0]["aggregatedDetail"]["totalAmount"] = money(6)
+ coarse["netProceeds"]["total"] = money(190)
+ def fake_economics(*args, summary=False, **kwargs):
+ values = [coarse] if summary else raw
+ return "\n".join(json.dumps(r) for r in values), {"reportDocumentId": "fixture-economics"}
+ with patch.object(cx, "economics_document", fake_economics):
+ _sync_margin_window(con, "economics", {}, start, end, max_wait=1, log=log)
+ _sync_margin_window(con, "economics", {}, start, end, max_wait=1, log=log)
+ with patch.object(cx, "transactions", return_value=transactions):
+ _sync_margin_window(con, "transactions", {}, start, end, max_wait=1, log=log)
+ _sync_margin_window(con, "transactions", {}, start, end, max_wait=1, log=log)
+ ck("margin window replacement is idempotent", con.execute("SELECT COUNT(*) FROM amzn_economics").fetchone()[0] == 2
+ and con.execute("SELECT COUNT(*) FROM amzn_transactions").fetchone()[0] == 3)
+ snapshots = []
+ def fake_report(*args, **kwargs):
+ snapshots.append(kwargs)
+ return ("sku\tasin\tfnsku\testimated-fee-total\nS1\tA1\tF1\t2.50\n",
+ {"reportDocumentId": "fixture-report", "dataEndTime": "2026-09-02T00:00:00Z"})
+ with patch.object(cx, "report_document", fake_report):
+ for table in ("fba_fee_estimates", "referral_fees"):
+ for _ in range(2):
+ _sync_margin_window(con, table, {}, None, None, max_wait=1, log=log)
+ ck("margin snapshot requests never pass start or end", all("start" not in k and "end" not in k for k in snapshots))
+ _sync_margin_window(con, "storage_fees", {}, _utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z"), max_wait=1, log=log)
+ for table in MARGIN_TABLES:
+ mirror = table_name(table)
+ # Deliberately damage a stored period before accepting its positive proof.
+ column = ("sales_ordered_product_sales_amount" if table == "economics" else
+ "total_amount" if table == "transactions" else None)
+ if column:
+ con.execute(f'UPDATE {mirror} SET "{column}"="{column}"+5 WHERE "{column}" IS NOT NULL')
+ red = _validate_margin(con, table)["state"] == "refused"
+ con.execute(f'UPDATE {mirror} SET "{column}"="{column}"-5 WHERE "{column}" IS NOT NULL')
+ else:
+ con.execute(f"INSERT INTO {mirror} SELECT * FROM {mirror}")
+ red = _validate_margin(con, table)["state"] == "refused"
+ con.execute(f"DELETE FROM {mirror} WHERE rowid IN (SELECT MAX(rowid) FROM {mirror})")
+ ck(f"margin {table} validation refuses a mismatch then accepts matching vendor evidence",
+ red and _validate_margin(con, table)["state"] == "ok")
+ with patch.object(sys.modules[__name__], "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)):
+ results = validate("fixture", rt=object(), log=lambda *_: None)
+ ck("margin public validate dispatches every new reconciliation", all(results[t]["state"] == "ok" for t in MARGIN_TABLES))
+ con.execute("UPDATE amzn_transactions SET transaction_breakdown_fees_amount=-3 WHERE item_position=0")
+ ck("margin each transaction breakdown must reconcile to its vendor total",
+ _validate_margin(con, "transactions")["state"] == "refused")
+ con.execute("UPDATE amzn_transactions SET transaction_breakdown_fees_amount=-2 WHERE item_position=0")
+ with patch.object(cx, "report_document", return_value=(
+ "sku\tasin\nS1\tA1\n", {"reportDocumentId": "next-snapshot", "dataEndTime": "2026-09-03T00:00:00Z"})):
+ _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
+ _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
+ # S67-T32 (D-3031, reversed): a new day REPLACES the earlier one; one copy, one load's rows.
+ ck("margin a second snapshot day replaces the first and same-date replay stays idempotent",
+ con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) "
+ "FROM amzn_referral_fees").fetchone() == (1, 1, "2026-09-03")
+ and _validate_margin(con, "referral_fees")["state"] == "ok")
+ kept = con.execute("SELECT * FROM amzn_referral_fees").fetchall()
+ with patch.object(cx, "report_document", side_effect=RuntimeError("fixture vendor failure")):
+ try:
+ _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
+ except RuntimeError:
+ pass
+ ck("margin a failed snapshot pull keeps the stored copy",
+ con.execute("SELECT * FROM amzn_referral_fees").fetchall() == kept and len(kept) == 1)
+ _replace_period(con, "economics", [], [c[1] for c in con.execute("PRAGMA table_info('amzn_economics')").fetchall()],
+ "start_date", "2026-09-01", "2026-09-03")
+ ck("margin empty economics restatement deletes stale detail and stays unproven",
+ _validate_margin(con, "economics")["state"] == "unproven")
+ finally:
+ con.close()
+
+ # Exercise the actual outer loop across bounded passes, not only its per-window helper.
+ class Clock(datetime):
+ @classmethod
+ def now(cls, tz=None):
+ return end.astimezone(tz)
+ class Runtime(_FixtureMirrorRuntime):
+ key = "margin-fixture"
+ def get(self, *_args, **_kwargs):
+ return None
+ def dynamic_economics(_fields, date_start, date_end, **_kwargs):
+ row = {**deepcopy(economy), "startDate": date_start, "endDate": date_start}
+ return json.dumps(row), {"reportDocumentId": "economics-" + date_start}
+ def dynamic_transactions(_fields, after, before):
+ when = _utc(after) + timedelta(seconds=2)
+ return [{**deepcopy(transaction), "transactionId": "T-" + _iso(when), "postedDate": _iso(when)}]
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ try:
+ module = sys.modules[__name__]
+ with patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 40), \
+ patch.object(module, "credential", return_value={"fixture": True}), \
+ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
+ patch.object(module, "_sync_event"), \
+ patch.object(cx, "economics_document", side_effect=dynamic_economics) as kiosk, \
+ patch.object(cx, "transactions", side_effect=dynamic_transactions) as finances, \
+ patch.object(cx, "report_document", side_effect=fake_report) as reports:
+ first = sync(Runtime.key, tables=MARGIN_TABLES, rt=Runtime(), first_load=True, report_budget=2, log=lambda *_: None)
+ ck("margin API windows consume the first-load pass budget", first["state"] == "partial"
+ and kiosk.call_count == 2 and finances.call_count == 1 and reports.call_count == 0)
+ for _ in range(3):
+ last = sync(Runtime.key, tables=MARGIN_TABLES, rt=Runtime(), first_load=True, report_budget=2, log=lambda *_: None)
+ ck("margin bounded passes resume to completion across all five table kinds",
+ last["state"] == "done" and last["remaining"] == 0
+ and all(read_watermark(con, t) for t in MARGIN_TABLES)
+ and all(_validate_margin(con, t)["state"] == "ok" for t in MARGIN_TABLES))
+ calls = reports.call_count
+ deferred = sync(Runtime.key, tables=["referral_fees", "fba_fee_estimates", "storage_fees"], rt=Runtime(), log=lambda *_: None)
+ ck("margin manual sync also honors vendor report floors", reports.call_count == calls
+ and all(r["state"] == "deferred" for r in deferred["tables"].values()))
+ finally:
+ con.close()
+ return 1 if failures else 0
+
+
+def sqp_selftest(log=print, check=None):
+ """S66-T41 fixtures, with broken inputs exercised before restored positives. No network."""
+ from unittest.mock import patch
+ import duckdb
+ cx, module = _client(), sys.modules[__name__]
+ failures = []
+
+ def ck(name, ok):
+ if check:
+ check(name, bool(ok))
+ else:
+ log(f" {'ok' if ok else 'FAIL'} {name}")
+ if not ok:
+ failures.append(name)
+
+ asins = [f"B{i:09d}" for i in range(37)]
+ valid_batches = lambda batches: (all(len(b) <= 200 for b in batches)
+ and [a for b in batches for a in b.split()] == asins)
+ ck("SQP planted 201-character batch fails before lossless stable batching passes",
+ not valid_batches(["X" * 201]) and valid_batches(sqp_batches(asins[::-1] + asins)))
+ try:
+ sqp_batches(["X" * 201])
+ oversized = False
+ except ValueError:
+ oversized = True
+ ck("SQP a single oversized identifier is refused by name, never dropped", oversized)
+ ck("SQP empty and exact-boundary batches are supported",
+ sqp_batches([]) == [] and sqp_batches(["A" * 100, "B" * 99]) == ["A" * 100 + " " + "B" * 99])
+
+ start, end = _utc("2026-07-17T00:00:00Z"), _utc("2026-09-25T00:00:00Z")
+ def calendar_probe():
+ plan = first_load_plan([SQP_TABLE], start, end, sqp_asins=asins)
+ return (len(plan) == 6 and [task[1].strftime("%Y-%m") for task in plan] == ["2026-08"] * 3 + ["2026-07"] * 3
+ and all(task[1].day == task[2].day == 1 and len(task[3]) <= 200 for task in plan))
+ with patch.object(module, "_month_windows", return_value=[(start, end)]):
+ red = not calendar_probe()
+ ck("SQP first-load plan counts every completed calendar month and batch, newest first", red and calendar_probe())
+ plan = sync_window(SQP_TABLE, _utc("2026-09-01T00:00:00Z"), now=end)
+ ck("SQP incremental window defers the partial month", plan["start"] == plan["end"]
+ and not _month_windows(plan["start"], plan["end"]))
+
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR)")
+ con.execute("INSERT INTO amzn_pricing VALUES ('B999999999')")
+ ck("SQP catalogue falls back to pricing without a listings mirror", _sqp_catalogue(con) == ["B999999999"])
+ con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
+ con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins[:19][::-1] + asins[:2]])
+ ck("SQP own listings take precedence and are sorted and deduplicated", _sqp_catalogue(con) == asins[:19])
+ foreign = asins[1]
+ reason = f"ASIN {foreign} is not part of your brand"
+ calls, problems = [], []
+
+ class Clock(datetime):
+ @classmethod
+ def now(cls, tz=None):
+ return end.astimezone(tz)
+
+ class Runtime(_FixtureMirrorRuntime):
+ key = "sqp-fixture"
+ def get(self, *_args, **_kwargs):
+ return None
+
+ def document(_fields, _report, **kwargs):
+ calls.append(kwargs)
+ requested = kwargs["options"]["asin"].split()
+ if foreign in requested:
+ raise cx.AmazonReportError("FATAL", reason, "FATAL")
+ raw = [{"asin": a, "startDate": kwargs["start"][:10], "endDate": kwargs["end"][:10],
+ "searchQueryData": {"searchQuery": "000123"}, "purchaseData": {"asinPurchaseCount": 2},
+ "impressionData": {"asinImpressionCount": 7}, "clickData": {"asinClickCount": 4},
+ "cartAddData": {"asinCartAddCount": 3}} for a in requested]
+ return json.dumps({"dataByAsin": raw}), {"reportDocumentId": "sqp-fixture-" + str(len(calls))}
+
+ try:
+ with patch.object(cx, "access_token", side_effect=AssertionError("fixture attempted LWA")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("fixture attempted network")), \
+ patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 70), \
+ patch.object(module, "credential", return_value={"fixture": True}), \
+ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
+ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document):
+ def run(**kwargs):
+ result = sync(Runtime.key, tables=[SQP_TABLE], rt=Runtime(), log=lambda *_: None, **kwargs)
+ problems.extend(result["problems"])
+ return result
+ ck("SQP no reconciliation marker is unproven", _validate_sqp(con)["state"] == "unproven")
+ first = run(first_load=True, report_budget=1)
+ ck("SQP a refused request consumes the pass budget and retains its retry task",
+ len(calls) == 1 and first["remaining"] == 4 and first["etaSeconds"] == first_load_eta(4, burst=0)
+ and read_watermark(con, SQP_TABLE) is None)
+ second = run(first_load=True, report_budget=1)
+ ck("SQP next pass retries once without the named foreign ASIN",
+ len(calls) == 2 and foreign not in calls[1]["options"]["asin"].split()
+ and len(calls[1]["options"]["asin"].split()) == 17 and second["remaining"] == 3
+ and any(sqp_refusal_message(foreign, reason) == p for p in problems))
+ for _ in range(4):
+ last = run(first_load=True, report_budget=1)
+ ck("SQP bounded passes finish all months without losing other ASINs",
+ len(calls) == 6 and last["remaining"] == 0 and last["state"] == "done"
+ and read_watermark(con, SQP_TABLE) == _utc("2026-09-01T00:00:00Z")
+ and con.execute("SELECT COUNT(*), COUNT(DISTINCT asin) FROM amzn_search_query_performance").fetchone() == (36, 18))
+ ck("SQP JSON rows keep the query as text, all nested families and report period",
+ con.execute("SELECT DISTINCT search_query_data_search_query, _report_period, "
+ "impression_data_asin_impression_count, click_data_asin_click_count, "
+ "cart_add_data_asin_cart_add_count FROM amzn_search_query_performance").fetchall()
+ == [("000123", "MONTH", 7, 4, 3)])
+ limit = json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])
+ ck("SQP durable limits retain the entire catalogue and named refusal beyond 300 characters",
+ len(_read_state(con, SQP_LIMIT_KEY)["detail"]) > 300 and limit["asins"] == asins[:19]
+ and limit["refusedAsins"] == {foreign: reason} and limit["batchCount"] == 2)
+ con.execute("UPDATE amzn_search_query_performance SET purchase_data_asin_purchase_count=9 WHERE start_date='2026-08-01'")
+ bad = _validate_sqp(con)
+ con.execute("UPDATE amzn_search_query_performance SET purchase_data_asin_purchase_count=2 WHERE start_date='2026-08-01'")
+ good = validate(Runtime.key, rt=Runtime(), log=lambda *_: None)[SQP_TABLE]
+ ck("SQP validate refuses damaged counts before reconciling the newest month",
+ bad["state"] == "refused" and bad["expected"] == {"count": 18, "purchases": 36}
+ and bad["actual"] == {"count": 18, "purchases": 162} and good["state"] == "ok")
+ before = len(calls)
+ partial = run(report_budget=1)
+ ck("SQP a current partial month makes no vendor request and states its cause",
+ len(calls) == before and partial["tables"][SQP_TABLE]["state"] == "deferred"
+ and "partial month" in partial["tables"][SQP_TABLE]["reason"])
+ # Explicit replay must replace the whole batch, including queries removed upstream.
+ con.execute("UPDATE amzn_search_query_performance SET search_query_data_search_query='stale'")
+ run(since="2026-08-01T00:00:00Z", report_budget=1)
+ ck("SQP explicit replay replaces one ASIN batch without deleting another batch or month",
+ con.execute("SELECT COUNT(*) FROM amzn_search_query_performance").fetchone()[0] == 36
+ and con.execute("SELECT COUNT(*) FROM amzn_search_query_performance WHERE search_query_data_search_query='000123'").fetchone()[0] == 17)
+ ck("SQP newest proof survives older backfill", json.loads(_read_state(
+ con, entity_key("reconcile_" + SQP_TABLE))["detail"])["start"] == "2026-08-01")
+ w0, w1 = _utc("2026-08-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z")
+ def empty_document(*_a, **_k):
+ return '{"dataByAsin":[]}', {"reportDocumentId": "empty-sqp"}
+ _sync_sqp_batch(con, {}, w0, w1, sqp_batches(asins[:18])[0], limit, empty_document, problems.append)
+ ck("SQP empty replay removes stale batch rows and cannot validate an empty month",
+ con.execute("SELECT COUNT(*) FROM amzn_search_query_performance WHERE start_date='2026-08-01'").fetchone()[0] == 1)
+ _sync_sqp_batch(con, {}, w0, w1, asins[18], limit, empty_document, problems.append)
+ ck("SQP no rows is unproven even with a summary marker", _validate_sqp(con)["state"] == "unproven")
+
+ # Exercise the real FATAL-document path. The red mutation loses the offending ASIN.
+ def fatal_probe(payload):
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_create_report", return_value="fatal-fixture"), \
+ patch.object(cx, "_request", side_effect=[{"processingStatus": "FATAL", "reportDocumentId": "error-fixture"},
+ {"url": "https://fixture.invalid/error"}]), \
+ patch.object(cx, "_download_document", return_value=payload):
+ try:
+ cx.report_document({}, cx.REPORT_TYPES[SQP_TABLE]["report"])
+ except cx.AmazonReportError as exc:
+ return cx.sqp_refused_asins(exc, asins)
+ return []
+ ck("SQP FATAL error document preserves Amazon's named foreign ASIN",
+ fatal_probe("Report failed without naming an ASIN") != [foreign] and fatal_probe(reason) == [foreign])
+ from urllib.error import HTTPError
+ with patch.object(cx.urllib.request, "urlopen", side_effect=HTTPError(
+ "https://fixture.invalid", 400, "Bad Request", {}, io.BytesIO(json.dumps(
+ {"errors": [{"message": reason}]}).encode()))), patch.object(cx, "_pace_report_create", return_value=0):
+ try:
+ cx._create_report("fixture", cx.REPORT_TYPES[SQP_TABLE]["report"], ["MKT"])
+ named = []
+ except cx.AmazonReportError as exc:
+ named = cx.sqp_refused_asins(exc, asins)
+ ck("SQP HTTP 400 preserves the same named refusal", named == [foreign])
+ # The second error entry and its details can carry the ASIN, past the old 200-char cut.
+ long_reason = "Search Query Performance brand refusal: " + "more vendor context " * 20
+ with patch.object(cx.urllib.request, "urlopen", side_effect=HTTPError(
+ "https://fixture.invalid", 400, "Bad Request", {}, io.BytesIO(json.dumps(
+ {"errors": [{"message": long_reason},
+ {"message": "Invalid ASINs", "details": foreign}]}).encode()))), \
+ patch.object(cx, "_pace_report_create", return_value=0) as pace:
+ try:
+ cx._create_report("fixture", cx.REPORT_TYPES[SQP_TABLE]["report"], ["MKT"])
+ named = []
+ except cx.AmazonReportError as exc:
+ named = cx.sqp_refused_asins(exc, asins)
+ ck("SQP 400 preserves every error and details after 200 characters and uses the shared pacer",
+ named == [foreign] and pace.call_count == 1)
+ ck("SQP unrelated failures cannot silently exclude a named valid ASIN",
+ cx.sqp_refused_asins(cx.AmazonReportError("FATAL",
+ f"Your brand owns {foreign}; service unavailable", "FATAL"), asins) == [])
+
+ # Red first: a dropped error document cannot cause the successful one-retry path.
+ def retry_probe(named):
+ month_start, month_end = _utc("2026-05-01T00:00:00Z"), _utc("2026-06-01T00:00:00Z")
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [_sqp_month_key(month_start)])
+ attempts, named_problems = [], []
+ def request(fields, rtype, **kwargs):
+ attempts.append(kwargs["options"]["asin"])
+ if len(attempts) == 1:
+ raise cx.AmazonReportError("FATAL", reason if named else "Amazon report failed", "FATAL")
+ return document(fields, rtype, **kwargs)
+ try:
+ _sync_sqp_batch(con, {}, month_start, month_end, " ".join(asins[:2]), limit, request, named_problems.append)
+ except cx.AmazonError:
+ return False
+ return attempts == [" ".join(asins[:2]), asins[0]] and named_problems == [sqp_refusal_message(foreign, reason)]
+ ck("SQP named FATAL triggers exactly one immediate retry without the foreign ASIN",
+ not retry_probe(False) and retry_probe(True))
+ month_start, month_end = _utc("2026-04-01T00:00:00Z"), _utc("2026-05-01T00:00:00Z")
+ attempts = []
+ def twice_refused(*_args, **kwargs):
+ attempts.append(kwargs["options"]["asin"])
+ raise cx.AmazonReportError("FATAL", reason, "FATAL")
+ for _ in range(2):
+ try:
+ _sync_sqp_batch(con, {}, month_start, month_end, " ".join(asins[:2]), limit, twice_refused, problems.append)
+ except cx.AmazonError:
+ pass
+ ck("SQP a failed retry is durable and never creates a third request",
+ attempts == [" ".join(asins[:2]), asins[0]])
+ requests = []
+ def unnamed(*_a, **_k):
+ requests.append(True)
+ raise cx.AmazonReportError("FATAL", "Amazon cannot generate this report", "FATAL")
+ try:
+ _sync_sqp_batch(con, {}, _utc("2026-06-01T00:00:00Z"), _utc("2026-07-01T00:00:00Z"),
+ asins[0], limit, unnamed, problems.append)
+ except cx.AmazonReportError:
+ pass
+ ck("SQP unnamed errors stay verbatim and are never retried", len(requests) == 1
+ and json.loads(_read_state(con, SQP_LIMIT_KEY)["detail"])["errors"] == ["Amazon cannot generate this report"])
+ finally:
+ con.close()
+ return 1 if failures else 0
+
+
+def group_a_selftest(log=print, check=None):
+ """S66-T42 red-first fixture controls; memory mirrors and no credentials or network."""
+ from unittest.mock import patch
+ import duckdb
+ cx, module = _client(), sys.modules[__name__]
+ failures = []
+
+ def ck(name, ok):
+ if check:
+ check("[group a] " + name, bool(ok))
+ else:
+ log(f" {'ok' if ok else 'FAIL'} group a {name}")
+ if not ok:
+ failures.append(name)
+
+ def coverage(tables, declared):
+ return (set(tables).isdisjoint(declared)
+ and set(tables) | set(declared) == set(cx.GROUP_A))
+ ck("catalogue has no omitted dataset or overlapping declaration",
+ not coverage(TABLES[1:], cx.DECLARED_DATASETS) and coverage(TABLES, cx.DECLARED_DATASETS))
+ excluded = lambda reports: not set(reports).intersection(cx.EXCLUDED_REPORT_TYPES)
+ reports = [rtype for spec in cx.GROUP_A.values() for rtype in spec.get("report_types", ())]
+ ck("no group a report is excluded", not excluded([*reports, "GET_ORDERS_DATA"]) and excluded(reports))
+ operation_ok = lambda ops: (not any(any(word in str(spec).lower() for word in
+ ("wallet", "financial_holds", "advertising-api", "amazon-ads"))
+ for spec in ops.values()))
+ ck("no wallet, deferred transaction or Ads operation is admitted",
+ not operation_ok({**cx.OPERATIONS, "planted": {"path": "sellerWallet/accounts"}})
+ and operation_ok(cx.OPERATIONS))
+ begin, end = _utc("2026-07-01T00:00:00Z"), _utc("2026-09-01T00:00:00Z")
+ for table in GROUP_A_TABLES:
+ spec = cx.REPORT_TYPES[table]
+ if spec.get("report"):
+ with patch.dict(cx.OPERATIONS["createReport"], {"report_types": ()}):
+ try:
+ cx.require_operation("POST", cx._REPORTS_CREATE_PATH, report_type=spec["report"])
+ red = False
+ except cx.AmazonOperationRefused:
+ red = True
+ admitted = cx.require_operation("POST", cx._REPORTS_CREATE_PATH, report_type=spec["report"])
+ ck(table + " report is admitted by the guarded operation", red and bool(admitted))
+ if spec.get("snapshot"):
+ def snapshot_plan():
+ return first_load_plan([table], begin, end) == [(table, None, None)]
+ with patch.dict(spec, {"snapshot": False}):
+ red = not snapshot_plan()
+ ck(table + " snapshot plans one task", red and snapshot_plan())
+ floor = {"state": "done", "updated_at": _iso(end)}
+ ck(table + " refresh floor defers then becomes due",
+ bool(_interval_reason(table, floor, end))
+ and not _interval_reason(table, floor, end + timedelta(seconds=spec["min_interval_s"])))
+ plan = first_load_plan(["long_term_storage_fees", "market_basket", "mfn_returns"], begin, end)
+ ck("monthly tasks are full months and MFN windows are at most sixty days",
+ all((w0.day == w1.day == 1) if t != "mfn_returns" else
+ (w1 - w0).total_seconds() <= 60 * 86400 for t, w0, w1 in plan)
+ and sum(t == "long_term_storage_fees" for t, *_ in plan) == 2)
+
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ calls = []
+ def document(_fields, report, **kwargs):
+ spec = next(s for s in cx.GROUP_A_TABLES.values() if s.get("report") == report)
+ calls.append((report, kwargs))
+ if spec.get("json_root"):
+ row = {"asin": "B000000001", "sku": "SKU-1", "marketplaceId": "MKT",
+ "promotionId": "promo-1", "couponId": "coupon-1", "buyer-email": "planted@example.invalid",
+ "children": [{"buyerName": "planted", "amount": 1}]}
+ body = {spec["json_root"]: [row, dict(row, sku="SKU-2")]}
+ body.update({root: [{"marketplaceId": "MKT", "status": "NORMAL"}]
+ for root in spec.get("json_extra_roots", ())})
+ text = json.dumps(body)
+ else:
+ text = ("sku\tasin\tbuyer-email\tship-address-1\tdestination-address-1\tamount\n"
+ "SKU-1\tB000000001\tplanted@example.invalid\tPrivate\tPrivate\t1.25\n"
+ "SKU-2\tB000000002\tplanted@example.invalid\tPrivate\tPrivate\t2.50\n")
+ return text, {"reportDocumentId": "fixture-" + report, "createdTime": _iso(end)}
+
+ def records(_fields, operation, **kwargs):
+ if operation == "getOrderMetrics":
+ return [{"interval": kwargs["start"] + "--" + kwargs["end"], "orderCount": 2, "unitCount": 3,
+ "totalSales": {"amount": 12.5, "currencyCode": "USD"}}]
+ return [{"asin": "B000000001", "sku": "SKU-1", "buyerEmail": "planted@example.invalid",
+ "attributes": {"recipientName": "planted", "value": "retained"}}]
+
+ try:
+ with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \
+ patch.object(cx, "report_document", side_effect=document), \
+ patch.object(cx, "group_a_records", side_effect=records):
+ for table in GROUP_A_TABLES:
+ spec = cx.REPORT_TYPES[table]
+ window = (None, None) if spec.get("snapshot") else (begin, end)
+ def ingest():
+ return _sync_margin_window(con, table, {}, *window, max_wait=1, log=lambda *_: None)
+ ck(table + " without evidence is unproven", _validate_margin(con, table)["state"] == "unproven")
+ count = ingest()
+ # A real broken mirror must fail before the matching document count passes.
+ con.execute(f"DELETE FROM {table_name(table)} WHERE _report_row_number=1")
+ red = _validate_margin(con, table)["state"] != "ok"
+ ingest()
+ actual = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
+ ck(table + " latest-period row proof rejects dropped rows and replay is idempotent",
+ red and actual == count and _validate_margin(con, table)["state"] == "ok")
+ fields = [r[1] for r in con.execute(f"PRAGMA table_info('{table_name(table)}')").fetchall()]
+ payload = str(con.execute(f"SELECT * FROM {table_name(table)}").fetchall())
+ ck(table + " buyer identity and removal addresses never reach the mirror",
+ not any(buyer_column(c) for c in fields) and "planted" not in payload and "Private" not in payload)
+ if spec.get("report"):
+ request = calls[-1][1]
+ ck(table + " request respects snapshot and report options",
+ (not spec.get("snapshot") or ("start" not in request and "end" not in request))
+ and (not spec.get("period") or request.get("options", {}).get("reportPeriod") == "MONTH")
+ and (not spec.get("date_options") or all(k in request.get("options", {}) for k in spec["date_options"])))
+ # Explicit planted bypass, then the production parser drops buyer-email.
+ sample = "sku\tbuyer-email\nSKU-1\tplanted@example.invalid\n"
+ with patch.object(module, "buyer_column", return_value=False):
+ red = "buyer_email" in parse_flat_file(sample)[0]
+ ck("buyer-column control fails when the deny rule is bypassed",
+ red and "buyer_email" not in parse_flat_file(sample)[0])
+ original_parser = parse_flat_file
+ def drop_source_row(*args, **kwargs):
+ columns, rows = original_parser(*args, **kwargs)
+ return columns, rows[1:]
+ with patch.object(module, "parse_flat_file", side_effect=drop_source_row):
+ _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None)
+ red = _validate_margin(con, "reimbursements")["state"] == "refused"
+ _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None)
+ ck("document proof counts raw source records independently of a dropping parser",
+ red and _validate_margin(con, "reimbursements")["state"] == "ok")
+ for body in ("{bad", "{}", '{"dataByAsin":null}'):
+ try:
+ parse_json_report(body, "dataByAsin")
+ refused = False
+ except ValueError:
+ refused = True
+ ck("malformed or absent JSON rows cannot masquerade as empty data " + body, refused)
+ con.execute("UPDATE amzn_order_metrics SET order_count=99")
+ red = _validate_margin(con, "order_metrics")["state"] == "refused"
+ con.execute("UPDATE amzn_order_metrics SET order_count=2")
+ ck("order metrics reconcile counts and totals, not just row count",
+ red and _validate_margin(con, "order_metrics")["state"] == "ok")
+ con.execute("CREATE TABLE amzn_orders (amazon_order_id VARCHAR, purchase_date VARCHAR, order_status VARCHAR)")
+ con.execute("INSERT INTO amzn_orders VALUES ('O1','2026-07-02T00:00:00Z','Shipped')")
+ red = _validate_orders(con)["state"] == "refused"
+ con.execute("INSERT INTO amzn_orders VALUES ('O2','2026-07-03T00:00:00Z','Shipped'), "
+ "('O3','2026-07-04T00:00:00Z','Cancelled')")
+ ck("orders independently reconcile Sales API counts and exclude cancelled orders",
+ red and _validate_orders(con)["state"] == "ok")
+ spec = cx.REPORT_TYPES["reimbursements"]
+ with patch.object(cx, "report_document", return_value=(
+ "sku\tamount\n", {"reportDocumentId": "empty", "createdTime": _iso(end)})):
+ _sync_margin_window(con, "reimbursements", {}, begin, end, max_wait=1, log=lambda *_: None)
+ ck("an empty restatement deletes stale period rows and stays unproven",
+ con.execute("SELECT COUNT(*) FROM amzn_reimbursements").fetchone()[0] == 0
+ and _validate_margin(con, "reimbursements")["state"] == "unproven")
+ finally:
+ con.close()
+
+ # Exercise the outer dispatcher and restart markers, not only the document helper.
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ clock = [end]
+ class Clock(datetime):
+ @classmethod
+ def now(cls, tz=None):
+ return clock[0].astimezone(tz)
+ selected = ("inventory_planning", "long_term_storage_fees", "order_metrics", "listing_items", "account_health")
+ class Runtime(_FixtureMirrorRuntime):
+ key = "group-a-sync-fixture"
+ def get(self, key, *args):
+ return {"databases": {amazon_schema.DATABASE_PREFIX + t: t in selected for t in TABLES}} if key == ENABLEMENT_KEY else None
+ rt = Runtime()
+ try:
+ with patch.object(module, "datetime", Clock), patch.object(module, "DEFAULT_BACKFILL_DAYS", 62), \
+ patch.object(module, "credential", return_value={"fixture": True}), \
+ patch.object(module, "_connect_tenant_mirror", side_effect=lambda *_: (con.cursor(), None)), \
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
+ patch.object(module, "_sync_event"), patch.object(cx, "report_document", side_effect=document), \
+ patch.object(cx, "group_a_records", side_effect=records), \
+ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")):
+ record_watermark(con, "orders", end)
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail=_iso(end))
+ ck("new enabled tables trigger bounded first load on an existing account", needs_first_load(rt.key, rt))
+ def run():
+ return sync(rt.key, tables=selected, first_load=True, report_budget=20, rt=rt, log=lambda *_: None)
+ first = run()
+ before = len(calls)
+ paused = run()
+ ck("first-load refresh floors preserve unfinished tasks without repeating requests",
+ first["remaining"] > 0 and paused["remaining"] == first["remaining"]
+ and len(calls) == before and paused["etaSeconds"] > first_load_eta(paused["remaining"], burst=0))
+ for _ in range(5):
+ clock[0] += timedelta(days=2)
+ last = run()
+ if not last["remaining"]:
+ break
+ checked = validate(rt.key, rt=rt, log=lambda *_: None)
+ if (last["state"] != "done" or last["remaining"]
+ or any(checked[table]["state"] != "ok" for table in selected)
+ or needs_first_load(rt.key, rt)):
+ log("group a sync fixture detail: " + json.dumps({
+ "result": last, "validation": {t: checked[t] for t in selected},
+ "watermarks": {t: str(read_watermark(con, t)) for t in selected}}))
+ ck("outer sync resumes every window and validates all new dispatch kinds",
+ last["state"] == "done" and last["remaining"] == 0
+ and all(read_watermark(con, table) for table in selected)
+ and all(checked[table]["state"] == "ok" for table in selected)
+ and not needs_first_load(rt.key, rt))
+ finally:
+ con.close()
+
+ # Client pagination: an empty middle page must not end the traversal.
+ for operation, interval in cx.ENTITY_INTERVALS.items():
+ requests, pace = [], []
+ def request(path, token, **params):
+ requests.append((path, params))
+ i = len(requests)
+ following = "page-2" if i % 3 == 1 else "page-3" if i % 3 == 2 else None
+ rows = [] if i % 3 == 2 else [{"asin": "B000000001", "sku": "SKU-" + str(i)}]
+ if operation == "getOrderMetrics":
+ return {"payload": rows}
+ if operation == "getInventorySummaries":
+ return {"payload": {"inventorySummaries": rows}, "pagination": {"nextToken": following}}
+ return {"items": rows, "pagination": {"nextToken": following}}
+ args = {"start": _iso(begin), "end": _iso(end), "asins": [f"B{i:09d}" for i in range(41)]}
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_request", side_effect=request), \
+ patch.object(cx, "_pace_entity", side_effect=lambda name, seconds, **_: pace.append((name, seconds))):
+ got = cx.group_a_records({"seller_id": "SELLER"}, operation, **args)
+ want = 9 if operation == "searchCatalogItems" else 1 if operation == "getOrderMetrics" else 3
+ ck(operation + " follows every page and uses its default request pace",
+ len(requests) == want and len(pace) == want and all(p == (operation, interval) for p in pace)
+ and len(got) == (6 if operation == "searchCatalogItems" else 1 if want == 1 else 2))
+ if operation == "searchCatalogItems":
+ sent = [a for _, params in requests[::3] for a in params["identifiers"].split(",")]
+ ck("catalog search covers all forty-one own ASINs in batches of twenty",
+ sent == args["asins"] and all(len(p["identifiers"].split(",")) <= 20 for _, p in requests))
+ ck("catalog search asks for relationships and nothing beyond the approved includes",
+ all(p["includedData"].split(",") == ["summaries", "salesRanks", "dimensions", "attributes",
+ "relationships"] for _, p in requests))
+ if operation == "getInventorySummaries":
+ ck("FBA inventory requests detailed inventory on every page",
+ all(p["details"] == "true" for _, p in requests))
+ def partition_probe(partition):
+ requests = []
+ def request(*_a, **kwargs):
+ requests.append(kwargs)
+ if len(requests) == 1:
+ return {"numberOfResults": 1001, "items": []} if partition else {"numberOfResults": 0, "items": []}
+ return {"numberOfResults": 1, "items": [{"sku": "SKU-" + str(len(requests))}]}
+ with patch.object(cx, "access_token", return_value="fixture"), \
+ patch.object(cx, "_request", side_effect=request), patch.object(cx, "_pace_entity"):
+ rows = cx.group_a_records({"seller_id": "SELLER"}, "searchListingsItems")
+ return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
+ ck("Listings search splits date ranges at the real thousand-SKU ceiling",
+ not partition_probe(False) and partition_probe(True))
+ ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
+ return 1 if failures else 0
+
+
+def catalog_parent_selftest(log=print, check=None):
+ """S67-T29 (O23) offline fixtures: the catalog Parent ASIN map covers every own ASIN.
+
+ `python -c "import sys; sys.path[:0]=['platform','platform/harness']; import amazon_store as s;
+ sys.exit(s.catalog_parent_selftest())"` from the repo root; no credentials, no network.
+ """
+ from unittest.mock import patch
+ import duckdb
+ cx = _client()
+ failures = []
+
+ def ck(name, ok):
+ if check:
+ check("[catalog parent] " + name, bool(ok))
+ else:
+ log(f" {'ok' if ok else 'FAIL'} catalog parent {name}")
+ if not ok:
+ failures.append(name)
+
+ mkt = cx.US_MARKETPLACE_ID
+ child, parent, standalone, lonely, broken = ("B0CHILD001", "B0PARENT01", "B0STAND001",
+ "B0LONELY01", "B0BROKEN01")
+ items = [
+ {"asin": child, "relationships": [
+ {"marketplaceId": "A1F83G8C2ARO7P", "relationships": [
+ {"type": "VARIATION", "parentAsins": ["B0WRONGMKT"]}]},
+ {"marketplaceId": mkt, "relationships": [
+ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]},
+ {"type": "VARIATION", "parentAsins": [parent],
+ "variationTheme": {"attributes": ["color"], "theme": "COLOR"}}]}]},
+ {"asin": parent, "relationships": [{"marketplaceId": mkt, "relationships": [
+ {"type": "VARIATION", "childAsins": [child, "B0SIBLING1"]}]}]},
+ {"asin": standalone, "relationships": [{"marketplaceId": mkt, "relationships": []}]},
+ {"asin": lonely},
+ {"asin": broken, "relationships": "not-a-list"},
+ {"asin": "B0BROKEN02", "relationships": [None, {"marketplaceId": mkt, "relationships": {"x": 1}},
+ {"marketplaceId": mkt, "relationships": [
+ {"type": "VARIATION", "parentAsins": "B0NOTALIST"},
+ {"type": "VARIATION", "parentAsins": [None, "bad"]}]}]},
+ ]
+ eco_only = "B0ECONLY01"
+ items.append({"asin": eco_only, "relationships": [{"marketplaceId": mkt, "relationships": []}]})
+ pricing = {standalone: "B0PRICEPAR", broken: "not an asin"}
+ economics = {eco_only: "B0ECOPARNT", standalone: "B0ECOLOSER", child: "B0ECOWRONG", broken: "junk"}
+ got = catalog_parent_map(items, pricing, mkt, economics)
+ ck("variation child maps to its parent from relationships", got.get(child) == (parent, "relationships"))
+ ck("variation parent maps to itself", got.get(parent) == (parent, "relationships"))
+ ck("standalone falls back to pricing parent", got.get(standalone) == ("B0PRICEPAR", "pricing"))
+ ck("no relationships and no pricing maps to itself", got.get(lonely) == (lonely, "self"))
+ ck("malformed relationships degrade to itself, never an error",
+ got.get(broken) == (broken, "self") and got.get("B0BROKEN02") == ("B0BROKEN02", "self"))
+ ck("economics child to parent is the next fallback after pricing",
+ got.get(eco_only) == ("B0ECOPARNT", "economics"))
+ ck("every item gets a parent", set(got) == {i["asin"] for i in items})
+ # Red-first: economics never beats relationships or pricing, and is absent without its map.
+ ck("economics never outranks relationships or pricing, and needs its map",
+ got.get(child) == (parent, "relationships") and got.get(standalone) == ("B0PRICEPAR", "pricing")
+ and catalog_parent_map(items, pricing, mkt).get(eco_only) == (eco_only, "self"))
+ # Red-first: another marketplace or a PACKAGE_HIERARCHY parent must never win.
+ solo_other = catalog_parent_map([dict(items[0], relationships=items[0]["relationships"][:1])], {}, mkt)
+ pack_only = catalog_parent_map([{"asin": child, "relationships": [{"marketplaceId": mkt, "relationships": [
+ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}]}]}], {}, mkt)
+ ck("another marketplace and a package parent are ignored",
+ solo_other.get(child) == (child, "self") and pack_only.get(child) == (child, "self"))
+ ck("empty and junk inputs never raise", catalog_parent_map(None) == {}
+ and catalog_parent_map([None, "x", {"asin": ""}]) == {})
+
+ con = duckdb.connect(":memory:")
+ try:
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ # Resolver before any table exists: no own-ASIN source means None, never an error.
+ ck("resolver with no listings or catalog is None and an empty map",
+ asin_parents_sql(con) is None and asin_parents(con) == {})
+ own = [child, parent, standalone, lonely, broken, eco_only]
+ # Listed ASINs Amazon's catalog returns no item for (the live 5 of 1164).
+ eco_listed, nothing = "B0ECOLIST1", "B0NOTHING1"
+ listed = own + [eco_listed, nothing, child]
+ con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
+ con.executemany("INSERT INTO amzn_listings VALUES (?)", [[a] for a in listed])
+ only_listings = asin_parents(con)
+ ck("resolver with listings only maps every listed ASIN to itself",
+ set(only_listings) == set(listed) and all(v == (k, "self") for k, v in only_listings.items()))
+ con.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR, start_date VARCHAR)")
+ con.executemany("INSERT INTO amzn_economics VALUES (?, ?, ?)", [
+ [eco_only, "B0ECOOLDPR", "2026-08-01"], [eco_only, "B0ECOPARNT", "2026-09-01"],
+ [eco_listed, "B0ECOLDPAR", "2026-07-01"], [eco_listed, "B0ECOLSTPR", "2026-09-10"],
+ [eco_listed, None, "2026-09-20"]])
+ con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, _period_start VARCHAR)")
+ con.executemany("INSERT INTO amzn_pricing VALUES (?, ?, ?)", [
+ [standalone, "B0OLDPARNT", "2026-08-01T00:00:00Z"],
+ [standalone, "B0PRICEPAR", "2026-09-01T00:00:00Z"]])
+ asked = []
+
+ def records(_fields, operation, **kwargs):
+ asked.append(list(kwargs.get("asins") or ()))
+ return [i for i in items if i["asin"] in kwargs.get("asins", ())]
+
+ # An older snapshot day with a different parent: the readback must use the LATEST day.
+ with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \
+ patch.object(cx, "group_a_records", side_effect=records):
+ count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None)
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES (?, 'B0STALEPAR', 'relationships', '2000-01-01')", [child])
+ latest = con.execute("SELECT MAX(_snapshot_date) FROM amzn_catalog_items").fetchone()[0]
+ mapped = dict((a, (p, s)) for a, p, s in con.execute(
+ "SELECT asin, parent_asin, parent_asin_source FROM amzn_catalog_items WHERE _snapshot_date=?",
+ [latest]).fetchall())
+ columns = {r[1] for r in con.execute("PRAGMA table_info('amzn_catalog_items')").fetchall()}
+ state = _read_state(con, entity_key("reconcile_catalog_items")) or {}
+ marker = json.loads(state.get("detail") or "{}")
+ ck("store readback shows the parent of a known child ASIN on the latest snapshot",
+ mapped.get(child) == (parent, "relationships"))
+ ck("catalog asks for every listed ASIN and maps every item it returns",
+ asked == [sorted(set(listed))] and set(mapped) == set(own)
+ and all(p for p, _ in mapped.values()))
+ ck("catalog sync stamps the economics fallback", mapped.get(eco_only) == ("B0ECOPARNT", "economics"))
+ # Resolver: two catalog snapshot days, an old-day-only ASIN, a latest-day catalog-only ASIN.
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES ('B0OLDONLY1', 'B0OLDONLY1', 'self', '2000-01-01')")
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES ('B0CATONLY1', 'B0CATPARNT', 'relationships', ?)", [latest])
+ resolved = asin_parents(con)
+ rows = con.execute(asin_parents_sql(con)).fetchall()
+ ck("resolver covers every own ASIN exactly once, latest catalog day only",
+ set(resolved) == set(listed) | {"B0CATONLY1"} and len(rows) == len(resolved)
+ and "B0OLDONLY1" not in resolved and all(p for p, _ in resolved.values()))
+ ck("resolver readback shows the latest parent of a known child",
+ resolved.get(child) == (parent, "relationships") and resolved.get("B0CATONLY1")
+ == ("B0CATPARNT", "relationships"))
+ ck("resolver falls back to the newest economics parent, then to itself",
+ resolved.get(eco_listed) == ("B0ECOLSTPR", "economics") and resolved.get(nothing) == (nothing, "self"))
+ # A catalog mirror synced before S67-T29 has no parent_asin column: economics and self carry it.
+ old = duckdb.connect(":memory:")
+ try:
+ old.execute("CREATE TABLE amzn_catalog_items (asin VARCHAR, _snapshot_date VARCHAR)")
+ old.execute(f"INSERT INTO amzn_catalog_items VALUES ('{child}', '2026-09-01'), ('{eco_listed}', '2026-09-01')")
+ old.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)")
+ old.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR')")
+ ck("resolver degrades on a pre-T29 catalog with no listings table",
+ asin_parents(old) == {child: (child, "self"), eco_listed: ("B0ECOLSTPR", "economics")})
+ finally:
+ old.close()
+ # O23 order in the READ-TIME resolver: catalog, then pricing, then economics, then self.
+ pri = duckdb.connect(":memory:")
+ try:
+ pri.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_listings VALUES ('{eco_listed}'), ('{nothing}')")
+ pri.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR'), "
+ f"('{nothing}', 'B0ECONOTHG')")
+ pri.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_pricing VALUES ('{eco_listed}', 'B0PRICEPA2')")
+ got = asin_parents(pri)
+ ck("resolver: a pricing parent wins over economics, economics still answers the rest",
+ got.get(eco_listed) == ("B0PRICEPA2", "pricing")
+ and got.get(nothing) == ("B0ECONOTHG", "economics"))
+ pri.execute("CREATE OR REPLACE TABLE amzn_pricing (parent_asin VARCHAR)")
+ ck("NC: a pricing mirror with no child_asin (the live mirror today) adds no pricing step",
+ asin_parents(pri).get(eco_listed) == ("B0ECOLSTPR", "economics"))
+ finally:
+ pri.close()
+ # SKU grain (S67-T29 finding): the request asks for it, and the first SKU-grain pull
+ # replaces a parent-grain mirror instead of sitting beside it.
+ grain = duckdb.connect(":memory:")
+ try:
+ quiet = lambda *_a, **_k: None # noqa: E731
+ sku_rows = [{"parent_asin": "B0PARENT01", "child_asin": "B0CHILD001", "sku": "RI-1"}]
+ ck("regrain: an absent mirror is left alone", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False)
+ grain.execute("CREATE TABLE amzn_pricing (parent_asin VARCHAR, report_date VARCHAR)")
+ grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-01'), ('B0PARENT02', '2026-09-02')")
+ ck("regrain: parent-grain rows WITHOUT child_asin never clear the mirror",
+ _pricing_regrain(grain, "amzn_pricing", [{"parent_asin": "B0PARENT03"}], quiet) is False
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 2)
+ ck("regrain: the first SKU-grain pull clears the parent-grain mirror (one grain per table)",
+ _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is True
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 0)
+ grain.execute("ALTER TABLE amzn_pricing ADD COLUMN child_asin VARCHAR")
+ grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-03', 'B0CHILD001')")
+ ck("regrain: an already SKU-grain mirror keeps its history",
+ _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 1)
+ finally:
+ grain.close()
+ with open(__file__, encoding="utf-8") as _src:
+ passes_options = _src.read().count('REPORT_OPTIONS[table]') >= 2
+ ck("the Sales and Traffic request asks for SKU grain, the same report and role",
+ REPORT_OPTIONS.get("pricing") == {"asinGranularity": "SKU"} and passes_options)
+ ck("pricing fallback reads the newest pricing period", mapped.get(standalone) == ("B0PRICEPAR", "pricing"))
+ ck("raw relationships JSON is kept beside the derived columns",
+ {"relationships", "parent_asin", "parent_asin_source"} <= columns)
+ ck("row count stays Amazon's record count", count == len(own) and marker.get("count") == len(own))
+ finally:
+ con.close()
+ return 1 if failures else 0
+
+
def selftest(log=print):
"""Prove the parsers, the typing, the naming rule and validate's refusal. No network, no store.
@@ -1966,12 +5369,13 @@ def selftest(log=print):
# NC-AZ14 — the seven tables of R4 agree with the provenance map. Two lists, one truth.
ck("NC-AZ14 TABLES agrees with the schema provenance map",
- set(TABLES) == set(amazon_schema.PROVENANCE), set(TABLES) ^ set(amazon_schema.PROVENANCE))
+ set(TABLES) | set(amazon_schema.DECLARED_DATASETS) == set(amazon_schema.PROVENANCE),
+ (set(TABLES) | set(amazon_schema.DECLARED_DATASETS)) ^ set(amazon_schema.PROVENANCE))
# NC-AZ15 — an empty parse is empty, never a phantom row.
ck("NC-AZ15 an empty document yields no columns and no rows",
parse_flat_file("", **origin) == ([], [])
- and parse_json_report("", "x", **origin) == ([], []))
+ and parse_json_report('{"x":[]}', "x", **origin) == ([], []))
# NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label.
price_input = [
@@ -2055,7 +5459,7 @@ def selftest(log=print):
ck("NC-AZ25 purge refuses when the exact tenant confirmation mismatches",
refused_confirmation)
- class _FakeRuntime:
+ class _FakeRuntime(_FixtureMirrorRuntime):
def __init__(self, entries):
self.entries = entries
@@ -2139,7 +5543,11 @@ def selftest(log=print):
ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file",
public_ok, public_detail)
- log(f"\n{'GREEN' if not fails else 'RED'}: {27 - len(fails)}/27")
+ ck("NC-AZ28 margin fixtures and red-first controls", margin_selftest(log) == 0)
+ ck("NC-AZ29 SQP fixtures and red-first controls", sqp_selftest(log) == 0)
+ ck("NC-AZ30 group a fixtures and red-first controls", group_a_selftest(log) == 0)
+ ck("NC-AZ31 legacy snapshot fixtures", snapshot_selftest(log) == 0)
+ log(f"\n{'GREEN' if not fails else 'RED'}: {30 - len(fails)}/30")
return 0 if not fails else 1
diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py
index 3f40da2c341be8040cd179a14aa40abd949ed230..2bdc93aeaa101d0f6b68ba904b060ad9da94b520 100644
--- a/platform/harness/datastore.py
+++ b/platform/harness/datastore.py
@@ -797,13 +797,63 @@ def _published_content_sha(tok, data=None):
("sha256", "contentSha", "content_sha", "sha"))
+def _seedable():
+ """Is there no usable mirror to keep? True for a missing file, or for one an early reader
+ CREATED empty (`_instance()` makes the file) before the boot seed landed: tables none but
+ `_sync_state`, and that one without a row. A file holding anything else is kept."""
+ if not DB_PATH.exists():
+ return True
+ try:
+ cur = ro_cursor()
+ try:
+ names = {r[0] for r in cur.execute(
+ "SELECT table_name FROM information_schema.tables "
+ "WHERE table_schema = 'main'").fetchall()}
+ rows = (cur.execute("SELECT count(*) FROM _sync_state").fetchone()[0]
+ if "_sync_state" in names else 0)
+ finally:
+ cur.close()
+ except Exception: # noqa: BLE001
+ return False # cannot tell: keep the file, the pre-hotfix behaviour
+ return not (names - {"_sync_state"}) and not rows
+
+
+def _install_seed(tmp):
+ """Swap the downloaded seed in, closing any instance a reader opened on the old file.
+
+ ⛔⛔ HOTFIX 2026-09-27 (D-3030). A reader that reached `_instance()` during the seed download
+ CREATED an empty royal.duckdb and kept it open; a bare `os.replace` then orphaned that handle
+ (Linux keeps the unlinked inode), so the whole process read an EMPTY mirror, 0 of 8 Odoo
+ phases, while the seed on disk had all 8 `live`. Staging's Odoo cadence is manual, so nothing
+ ever refilled it: every Odoo grid, measure column and Sheet cell refused "first sync".
+ Under `_INSTANCE_LOCK` (a plain Lock, so `use_path` is inlined, not called) no reader can
+ reopen the old file between the close and the swap; the close checkpoints its WAL, and any
+ leftover WAL is removed so it cannot be replayed onto the seed.
+ """
+ with _INSTANCE_LOCK:
+ con = _INSTANCE.get("con")
+ if con is not None:
+ try:
+ con.close()
+ except Exception: # noqa: BLE001
+ pass
+ _INSTANCE["con"] = None
+ wal = DB_PATH.with_name(DB_PATH.name + ".wal")
+ if wal.exists():
+ wal.unlink()
+ os.replace(tmp, DB_PATH)
+ _RO_TLS.__dict__.pop("cur", None)
+ _READY["ok"] = False
+ _GENERATION[0] += 1 # every other thread's cached cursor is now stale
+
+
def ensure_seed():
"""Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
from the PRIVATE dataset's seed snapshot — Analyst-ready in ~a minute instead of a 20-40 min
XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
the pilot views/routines when absent. Fail-quiet: no seed/token → normal backfill."""
import shutil
- if DB_PATH.exists():
+ if not _seedable():
return False
tok = os.environ.get("HF_TOKEN")
if not tok:
@@ -812,11 +862,10 @@ def ensure_seed():
from huggingface_hub import hf_hub_download
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
p = hf_hub_download(SEED_DATASET, SEED_FILE, repo_type="dataset", token=tok)
- # Atomic install: copy to a temp path then os.replace — a reader that guards on
- # DB_PATH.exists() must never bind a half-written file into the shared instance.
+ # Atomic install: copy to a temp path then swap it in under the instance lock.
tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
shutil.copyfile(p, tmp)
- os.replace(tmp, DB_PATH)
+ _install_seed(tmp)
# ⭐ W50-T29 — RECORD WHICH SEED THIS IS, because nothing did and the status surface could
# therefore only ever say "there is a mirror". Three fields: the dataset, the commit sha of
# the exact file downloaded, and the date the DATA was cut, which comes from the seed's own
diff --git a/platform/harness/runtime.py b/platform/harness/runtime.py
index 96990ab0f21c65573a3e107e976e8c9f6d46eaaf..cf8d988096fd0fcbd33832f64f2d5f4c57a88756 100644
--- a/platform/harness/runtime.py
+++ b/platform/harness/runtime.py
@@ -204,6 +204,14 @@ class TenantRuntime:
answer = reader([store_key for _name, store_key in keyed])
return {name: answer.get(store_key, {}) for name, store_key in keyed}
+ def get_entry(self, name, entry):
+ """One top-level entry of `name` when the bound store offers it, else None (HOTFIX
+ 2026-09-28). None also means absent; either way the caller falls back to `get`."""
+ reader = getattr(self._store(), "get_entry", None)
+ if not callable(reader):
+ return None
+ return reader(self.store_key(name), entry)
+
def revision(self, name):
"""Return the tiny durable change token for this tenant's addressed bucket."""
return self._store().revision(self.store_key(name))
diff --git a/platform/harness/telemetry.py b/platform/harness/telemetry.py
index 3925dc9828d5f3ab795d5867bb8593ba0d24bf14..60e8d35d9f36491d6e4905f7369486940723e956 100644
--- a/platform/harness/telemetry.py
+++ b/platform/harness/telemetry.py
@@ -70,8 +70,13 @@ def error(where, exc=None, **fields):
ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
with ERR_PATH.open("a", encoding="utf-8") as f:
f.write(json.dumps(rec, default=str) + "\n")
- print(f"[app-error] {rec['ts']} {rec['where']}: "
- f"{rec.get('type', '')} {rec.get('msg', '')}", file=sys.stderr, flush=True)
+ # SDLC 67 sweep: an exc-less call (routes_activity forwarding a client error) left
+ # `type`/`msg` unset, so this mirror printed an empty ": " on every one -- the JSONL line
+ # kept the real `name`/`message`/`source` fields, only the container-log mirror lost them.
+ detail = f"{rec.get('type', '')} {rec.get('msg', '')}".strip()
+ if not detail:
+ detail = " ".join(f"{k}={v}" for k, v in fields.items() if v) or "(no detail)"
+ print(f"[app-error] {rec['ts']} {rec['where']}: {detail}", file=sys.stderr, flush=True)
except Exception:
pass
diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py
index d040b9ee998c4825af1afbf2875a7255de817e08..4a0a55616efb7e6cb1da86dc6446f57c3730108e 100644
--- a/platform/modules/sheet.py
+++ b/platform/modules/sheet.py
@@ -35,7 +35,7 @@ import datetime
import re
from urllib.parse import urlencode
-from harness.filter_sql import NUMERIC_TYPES as MIRROR_NUMERIC_TYPES
+from harness.filter_sql import Compiled, NUMERIC_TYPES as MIRROR_NUMERIC_TYPES
from core.sheet_binding import (
DRAG_ACROSS,
DRAG_DOWN_NONE,
@@ -262,7 +262,7 @@ def period_window(period, today=None, *, cell_date=None):
#: field types and falls back to VARCHAR for everything else, so a `metric` or `rating` column
#: would be materialised as TEXT and then either refuse to SUM or compare as a string. Both hold a
#: number, so both are declared as one.
-MIRROR_TYPE = {"metric": "currency", "rating": "int"}
+MIRROR_TYPE = {"metric": "currency", "rating": "int", "float": "currency"}
#: The rollup folds that produce a NUMBER. The rest of `core/user_tables.py::ROLLUP_FNS`
#: (`latest`, `and`/`or`/`xor`, `concatenate`, `arrayjoin`, `arraycompact`, `arrayunique`) folds
@@ -610,6 +610,8 @@ def clean_binding(raw):
period?: {unit, start, locked?, ref?} | {unit, start, locked?, frozen: true, from, to}
| {window}, onDrag?: {across, down},
related?: {via: , filter: }}`
+
+ The legacy related bag is read for one release and rewritten into scope.filter leaves.
"""
if not isinstance(raw, dict):
raise SheetRefusal("binding_invalid")
@@ -643,6 +645,8 @@ def clean_binding(raw):
raise SheetRefusal("binding_invalid")
if scope.get("view_id"):
out_scope = {"view_id": str(scope["view_id"]).strip()}
+ if scope.get("filter") is not None:
+ out_scope["filter"] = scope["filter"]
# ⭐ THE VIEW'S NAME IS STORED AT BIND TIME, ON PURPOSE. C2's `view_missing` refusal has to
# NAME the view ("View Top accounts no longer exists"), and once the record is deleted
# there is nothing left to look a name up from. Carrying it is the only way that sentence
@@ -669,11 +673,33 @@ def clean_binding(raw):
out["onDrag"] = on_drag
if "related" in raw:
- out["related"] = clean_related(raw["related"])
+ from core.sheet_binding import BindingMetadataRefusal, rewrite_legacy_related
+ try:
+ out = rewrite_legacy_related({**out, "related": raw["related"]})
+ except BindingMetadataRefusal as exc:
+ raise SheetRefusal(exc.code, exc.subject, exc.detail) from exc
return out
+def date_formatted_cell(cell):
+ """S67-T10 (QI4, D-767): a cell is a date when its number format says so.
+
+ A Date format set by hand, the format the engine infers for DATE() / TODAY() / NOW() (written
+ into ``style.format``), or an imported Excel code carrying day or year tokens. Mirrors
+ ``aios-web/web/src/sheet/bindingRefs.ts::isDateFormattedCell``.
+ """
+ import re as _re
+ fmt = ((cell or {}).get("style") or {}).get("format") if isinstance(cell, dict) else None
+ if not isinstance(fmt, dict):
+ return False
+ if fmt.get("kind") == "date":
+ return True
+ code = fmt.get("code")
+ return (fmt.get("kind") == "code" and isinstance(code, str)
+ and _re.search(r"[dy]", _re.sub(r'"[^"]*"|\\.', "", code), _re.I) is not None)
+
+
def filter_cell_nodes(nodes):
"""Cell value slots live on leaves at any depth, never on a fixed object path."""
for node in nodes or ():
@@ -765,6 +791,31 @@ def _within_as_of_cell(node, ref, value):
# W60-T40: explicit persisted identity, separate from both flat columns and cell value slots.
LINKED_FILTER_PREFIX = "linked:"
+RELATED_FILTER_PREFIX = "related:"
+
+
+def related_field_parts(key):
+ text = str(key or "")
+ if not text.startswith(RELATED_FILTER_PREFIX):
+ return None
+ parts = text.split(":", 2)
+ if len(parts) != 3 or not all(parts[1:]):
+ raise SheetRefusal("related_invalid", text)
+ return parts[1], parts[2]
+
+
+def related_filter_ids(nodes):
+ return {key for key in filter_columns_named(nodes) if key.startswith(RELATED_FILTER_PREFIX)}
+
+
+def related_vias(nodes):
+ return {related_field_parts(key)[0] for key in related_filter_ids(nodes)}
+
+
+class _RelatedCompiled(Compiled):
+ """A normal Compiled predicate carrying far-side widening explanations."""
+
+ limits = ()
def linked_field_parts(key):
@@ -869,7 +920,7 @@ def widen_limits(nodes, columns, compiled=None):
dropped = sorted(named - used)
if not dropped:
return []
- unknown = sorted(named - set(columns or {}))
+ unknown = sorted(set(dropped) - set(columns or {}))
known_but_dropped = [c for c in dropped if c not in unknown]
out = []
if unknown:
@@ -895,7 +946,8 @@ def widen_limits(nodes, columns, compiled=None):
return out
-def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None):
+def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None, related=None,
+ _limits=None):
"""The predicate for one filter tree, or None when nothing is active.
⛔ None MEANS NO PREDICATE, NOT A FALSE ONE. `compile_filter_tree`'s own docstring says so, and
@@ -907,17 +959,41 @@ def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None):
"""
from harness.filter_sql import compile_filter_tree
import math
+ def check_rhs(items):
+ for node in items or ():
+ if not isinstance(node, dict):
+ continue
+ if isinstance(node.get("children"), list):
+ check_rhs(node["children"])
+ elif isinstance(node.get("rhs"), dict) and node["rhs"].get("colId") \
+ and not related_field_parts(node.get("colId")) \
+ and related_field_parts(node["rhs"]["colId"]):
+ # A related field is only an operand INSIDE its own related group; on an ordinary
+ # leaf `filter_sql` would ignore it and answer the unfiltered total (C2).
+ raise SheetRefusal("related_invalid", node["colId"],
+ "a related field operand requires a related condition.")
+ check_rhs(nodes)
unresolved = linked_filter_ids(nodes)
if unresolved:
raise SheetRefusal("linked_filter_unreadable", ", ".join(sorted(unresolved)),
"the linked records have not been resolved.")
+ related_ids = related_filter_ids(nodes)
+ for via in related_vias(nodes):
+ if via not in (related or {}):
+ raise SheetRefusal("related_unresolved", via)
for leaf in filter_cell_nodes(nodes):
slot = leaf.get("valueRef")
ref = slot.get("ref") if isinstance(slot, dict) else "(blank)"
if leaf.get("_cell_value_resolved") is not True:
raise SheetRefusal("filter_cell_unreadable", ref, "the source value has not been read.")
+ parts = related_field_parts(leaf.get("colId"))
+ leaf_columns = columns or {}
+ if parts:
+ via, field = parts
+ leaf_columns = related[via]["columns"]
+ leaf = {**leaf, "colId": field}
try:
- compiled = compile_filter_tree([leaf], conj="and", columns=columns or {},
+ compiled = compile_filter_tree([leaf], conj="and", columns=leaf_columns,
today=today, cohort_sets=cohort_sets)
except ValueError:
raise SheetRefusal("filter_cell_unreadable", ref,
@@ -927,6 +1003,122 @@ def compile_scope(nodes, conj, columns, *, today=None, cohort_sets=None):
for p in compiled.params)):
raise SheetRefusal("filter_cell_unreadable", ref,
"its condition cannot use this value on the selected column.")
+ if related_ids:
+ # A query also needs explanations when the entire predicate is inactive (None).
+ limits = _limits if _limits is not None else []
+
+ def only_via(node):
+ if not isinstance(node, dict):
+ return None
+ if isinstance(node.get("children"), list):
+ vias = {only_via(child) for child in node["children"]} - {""}
+ if not vias:
+ return ""
+ return next(iter(vias)) if len(vias) == 1 and None not in vias else None
+ parts = related_field_parts(node.get("colId"))
+ return parts[0] if parts else None
+
+ def strip(node, via):
+ if isinstance(node.get("children"), list):
+ return {**node, "children": [strip(child, via) for child in node["children"]]}
+ leaf = {**node, "colId": related_field_parts(node["colId"])[1]}
+ leaf.pop("_near_rhs", None)
+ rhs = node.get("rhs")
+ if isinstance(rhs, dict) and rhs.get("colId"):
+ parts = related_field_parts(rhs["colId"])
+ if not parts:
+ if node.get("op") != "eq" or not (columns.get(rhs["colId"]) or {}).get("sql") \
+ or str(columns[rhs["colId"]]["sql"]).strip().upper() == "NULL":
+ raise SheetRefusal("related_invalid", node["colId"],
+ "a correlation requires equality with a readable outer column.")
+ leaf["_near_rhs"] = rhs["colId"]
+ elif parts[0] != via:
+ raise SheetRefusal("related_invalid", node["colId"])
+ else:
+ leaf["rhs"] = {**rhs, "colId": parts[1]}
+ return leaf
+
+ def correlations(nodes, conjunction, pairs):
+ kept = []
+ for node in nodes:
+ if isinstance(node.get("children"), list):
+ nested = []
+ children = correlations(node["children"], node.get("conj", "and"), nested)
+ if nested and conjunction != "and":
+ raise SheetRefusal("related_invalid", "correlation", "use an AND relationship group.")
+ pairs.extend(nested)
+ if children:
+ kept.append({**node, "children": children})
+ elif "_near_rhs" in node:
+ if conjunction != "and":
+ raise SheetRefusal("related_invalid", node["colId"], "use an AND relationship group.")
+ pairs.append((node["_near_rhs"], node["colId"]))
+ else:
+ kept.append(node)
+ return kept
+
+ def group(children, conjunction):
+ batches, order = {}, []
+ for child in children:
+ via = only_via(child)
+ if via:
+ if via not in batches:
+ batches[via] = []
+ order.append((via, None))
+ batches[via].append(strip(child, via))
+ else:
+ order.append((None, child))
+ active = []
+ for via, child in order:
+ if via:
+ bag = related[via]
+ far_nodes = batches[via]
+ # A lone pure group is the same scope as its children. This also
+ # makes a legacy bag and its ordinary leaf emit identical SQL.
+ far_conj = conjunction
+ while len(far_nodes) == 1 and isinstance(far_nodes[0].get("children"), list):
+ far_conj = far_nodes[0].get("conj", "and")
+ far_nodes = far_nodes[0]["children"]
+ pairs = []
+ far_nodes = correlations(far_nodes, far_conj, pairs)
+ far = compile_scope(far_nodes, far_conj, bag["columns"],
+ today=today, cohort_sets=cohort_sets)
+ sql, params, dropped = _related_predicate(
+ {**bag, "scope_nodes": far_nodes, "scope_conj": far_conj,
+ "correlations": pairs}, columns,
+ today=today, cohort_sets=cohort_sets)
+ limits.extend(dropped)
+ if far is None and not pairs:
+ continue
+ used = {f"related:{via}:{key}" for key in (far.columns_used if far else ())}
+ for near_key, far_key in pairs:
+ used.update((near_key, f"related:{via}:{far_key}"))
+ active.append(Compiled(sql, params, far.uses_aggregate if far else False,
+ frozenset(used)))
+ elif isinstance(child, dict) and isinstance(child.get("children"), list):
+ part = group(child["children"], child.get("conj", "and"))
+ if part is not None:
+ active.append(part)
+ else:
+ part = compile_filter_tree([child], columns=columns or {},
+ today=today, cohort_sets=cohort_sets)
+ if part is not None:
+ active.append(part)
+ if not active:
+ return None
+ if len(active) == 1:
+ return active[0]
+ joiner = " OR " if conjunction == "or" else " AND "
+ return Compiled("(" + joiner.join(part.sql for part in active) + ")",
+ [p for part in active for p in part.params],
+ any(part.uses_aggregate for part in active),
+ frozenset().union(*(part.columns_used for part in active)))
+
+ result = group(list(nodes or ()), conj)
+ if result is not None:
+ result = _RelatedCompiled(*result)
+ result.limits = limits
+ return result
return compile_filter_tree(list(nodes or ()), conj=conj or "and", columns=columns or {},
today=today, cohort_sets=cohort_sets)
@@ -1008,7 +1200,17 @@ def _related_predicate(related, columns, *, today=None, cohort_sets=None):
if compiled is not None:
where.append(f"({compiled.sql})")
params.extend(compiled.params)
- sql = f"{local} IN (SELECT {remote} FROM {source}"
+ near_tuple, far_tuple = [local], [remote]
+ for near_key, far_key in related.get("correlations") or ():
+ near_sql = (columns.get(near_key) or {}).get("sql")
+ far_sql = (rcols.get(far_key) or {}).get("sql")
+ if any(not expr or str(expr).strip().upper() == "NULL" for expr in (near_sql, far_sql)):
+ raise SheetRefusal("related_invalid", related.get("via", "correlation"),
+ "the correlated columns must both be readable.")
+ near_tuple.append(near_sql)
+ far_tuple.append(far_sql)
+ lhs = "(" + ", ".join(near_tuple) + ")" if len(near_tuple) > 1 else local
+ sql = f"{lhs} IN (SELECT {', '.join(far_tuple)} FROM {source}"
if where:
sql += " WHERE " + " AND ".join(where)
return sql + ")", params, limits + list(related.get("limits") or ())
@@ -1048,9 +1250,11 @@ def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
where.append(f"({base})")
scope_compiled = None
+ related_limits = []
for nodes, conj, is_scope in ((wall_nodes, wall_conj, False),
(scope_nodes, scope_conj, True)):
- compiled = compile_scope(nodes, conj, columns, today=today, cohort_sets=cohort_sets)
+ compiled = compile_scope(nodes, conj, columns, today=today, cohort_sets=cohort_sets,
+ related=related, _limits=related_limits if is_scope else None)
if is_scope:
scope_compiled = compiled
if compiled is None: # nothing active: add NOTHING. See `compile_scope`.
@@ -1058,15 +1262,6 @@ def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
where.append(f"({compiled.sql})")
params.extend(list(compiled.params))
- related_limits = []
- if binding.get("related") is not None and related is None:
- raise SheetRefusal("related_unresolved", binding["related"].get("via"))
- if related is not None:
- predicate, rparams, related_limits = _related_predicate(
- related, columns, today=today, cohort_sets=cohort_sets)
- where.append(f"({predicate})")
- params.extend(rparams)
-
source = str(spec.get("from_sql") or "").strip() or str(spec.get("table") or "").strip()
if not source:
raise SheetRefusal("binding_invalid")
@@ -1075,24 +1270,23 @@ def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
sql += " WHERE " + " AND ".join(where)
# ⭐ The limits are measured against what the compiler REALLY used, so a leaf that fell out for
# any reason is reported. See `widen_limits` for the measurement that made this necessary.
- return sql, params, widen_limits(scope_nodes, columns, scope_compiled) + related_limits
-
-
-#: A drill hands `LinkGridModal` a list of record ids. It is capped because the popup renders them,
-#: and because an uncapped list is a way to pull a whole database through one cell.
-#: ⚠ THIS IS A CAP ON A DRILL POPUP, NOT ON A CONNECTOR READ. Standing rule 1 forbids capping the
-#: data pulled from a connected source; the AGGREGATE above reads the whole scope with no limit.
-#: This bounds only how many ids one popup is handed, and the true `count` is returned beside them
-#: so the caller can say "showing 500 of 24,920" rather than quietly showing 500.
-MAX_DRILL_IDS = 500
+ return sql, params, (widen_limits(scope_nodes, columns, scope_compiled)
+ + related_limits)
def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None, wall_conj="and",
- today=None, cohort_sets=None, limit=MAX_DRILL_IDS, related=None):
+ today=None, cohort_sets=None, limit=None, related=None):
"""`(sql, params, limits)` for the ROW IDS behind a bound cell (R6, R13).
Same WHERE as the aggregate, by construction: it is built by the same function so the popup
cannot disagree with the number that opened it.
+
+ ⭐⭐ S66-T48 — `limit=None` IS EVERY ID, AND THAT IS THE DEFAULT (standing rule 1). The old
+ `MAX_DRILL_IDS = 500` cap gave two reasons, that the popup rendered every id and that the list
+ travelled in a URL; the popup now renders one page and the set travels in a POST body, so both
+ are gone, and a cell over 500 records was opening as "500 matching records" with nothing on
+ screen naming the rest. A caller that must cap passes `limit` AND reports it
+ (`routes_sheet.py::drill_cell` returns `id_limit` beside the true `count`).
"""
id_expr = str(spec.get("id") or "").strip()
if not id_expr:
@@ -1103,7 +1297,9 @@ def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None,
wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, cohort_sets=cohort_sets,
related=related)
sql = sql.replace("SELECT COUNT(*) FROM", f"SELECT {id_expr} FROM", 1)
- return f"{sql} LIMIT {int(limit)}", params, limits
+ if limit is not None:
+ sql = f"{sql} LIMIT {int(limit)}"
+ return sql, params, limits
# ── W49-T10: the dependents provider (contract C3) ───────────────────────────────────────────
diff --git a/web/index.html b/web/index.html
index 97080f3cdd1f36eafb9fd0c83fe3c73691076d85..2c48f9fa5c34f5e9804602ad4ff7cb02dc46afba 100644
--- a/web/index.html
+++ b/web/index.html
@@ -5,6 +5,31 @@
Loopable
+
+