diff --git a/RELEASES.json b/RELEASES.json
index 83e5c8f99219b944f8ee1bf3747cdd4487db584c..fadef0617573e3eac0d139d643295c06771dac88 100644
--- a/RELEASES.json
+++ b/RELEASES.json
@@ -1,4 +1,4 @@
{
- "current": "3dfe966",
+ "current": "d11ecc04",
"releases": []
}
\ No newline at end of file
diff --git a/VERSION b/VERSION
index c1d9f26eabcf428103bd9d5e68e362f394886a38..e15d25d561b623035eccc2f161034ad7fbdad084 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3dfe966
+d11ecc04
diff --git a/api/ai_enrich.py b/api/ai_enrich.py
index dbeb6153ad53eaab15f1ff8a38c460c0aa2164e6..e1ac9a95a74857f14dd1b0df3ff4484a94d1f266 100644
--- a/api/ai_enrich.py
+++ b/api/ai_enrich.py
@@ -269,12 +269,15 @@ _SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: n
'exactly: UNKNOWN')
-def _ask(provider, model, prompt, max_tokens, timeout):
+def _ask(provider, model, prompt, max_tokens, timeout, system=None):
"""One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
ceiling instead of a hardcoded 300, and they read `usage` back.
+ S67-T41: `system` replaces the one-cell instruction for a caller that is not filling a cell
+ (Content Studio's drafts); every existing caller passes nothing and keeps `_SYSTEM`.
"""
+ system = system or _SYSTEM
key = (os.environ.get(provider['env']) or '').strip()
if not key:
return '', None, f"{provider['name']} has no key on this deployment"
@@ -284,7 +287,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
provider['url'], timeout=timeout,
headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
'content-type': 'application/json'},
- json={'model': model, 'max_tokens': max_tokens, 'system': _SYSTEM,
+ json={'model': model, 'max_tokens': max_tokens, 'system': system,
'messages': [{'role': 'user', 'content': prompt}]})
if r.status_code >= 400:
return '', None, f"{provider['name']} answered {r.status_code}"
@@ -300,7 +303,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
provider['url'], timeout=timeout,
headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
- 'messages': [{'role': 'system', 'content': _SYSTEM},
+ 'messages': [{'role': 'system', 'content': system},
{'role': 'user', 'content': prompt}]})
if r.status_code >= 400:
return '', None, f"{provider['name']} answered {r.status_code}"
diff --git a/api/amazon_relational.py b/api/amazon_relational.py
index 0e38b89581cb706a1d56e3e6033f2325a3783a27..141997c6c039779a438d490417a84eabfa3f6ae4 100644
--- a/api/amazon_relational.py
+++ b/api/amazon_relational.py
@@ -170,6 +170,116 @@ def mirror_columns(cur, table):
return []
+#: ⭐⭐ S67-T30 (instr 13, R29 / O23) — THE PRODUCT LOOKUPS. Every Amazon database with an ASIN
+#: shows its "Parent ASIN" and "SKU" beside it, and every one with a SKU its "ASIN" and "Parent
+#: ASIN", with no configuration. They are PLAIN TEXT fields whose value the SERVER derives, not
+#: link fields, on purpose: a link renders as a count bubble, is not groupable
+#: (`types.ts:isGroupableField`) and has no SQL binding for a filter (`routes_odoo_tables.
+#: _filter_columns` and `_group_pin` read `spec["cols"]` only). A text field with a `cols`
+#: expression groups, filters and sorts through the machinery every other column already uses.
+#: ⚠ The ASIN / SKU column of a table is the `from_col` of its declared listings link (on `asin1`
+#: / `seller_sku`), so the spec's links are the one statement of which column is which. A table
+#: that carries its own `parent_asin` (pricing, economics) gains no derived Parent ASIN; one that
+#: carries its own SKU or ASIN gains no duplicate of it; listings gains Parent ASIN only.
+#: ⛔ Parent comes from `amazon_store.asin_parents_sql` ALONE (latest catalog, else economics, else
+#: the ASIN itself), never from `amzn_catalog_items` directly: that table keeps older snapshot days
+#: (D-3031) and five own ASINs have no catalog row. SKU and ASIN come from the LATEST listings day
+#: only, because listings keeps one copy per snapshot date.
+LISTINGS = PREFIX + "listings"
+LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN = "lookup_parent_asin", "lookup_sku", "lookup_asin"
+LOOKUP_KEYS = frozenset((LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN))
+_LOOKUP_LABELS = {LOOKUP_PARENT: "Parent ASIN", LOOKUP_SKU: "SKU", LOOKUP_ASIN: "ASIN"}
+_LOOKUP_DESCRIPTIONS = {
+ LOOKUP_PARENT: ("The parent ASIN of this row's product, from Amazon's catalog, else the "
+ "economics report, else the ASIN itself. Click it to open the listing."),
+ LOOKUP_SKU: ("Your seller SKUs for this row's ASIN in the latest listings report. "
+ "Click it to open the listing."),
+ LOOKUP_ASIN: ("The ASIN of this row's SKU in the latest listings report. "
+ "Click it to open the listing."),
+}
+
+
+def _product_columns(key):
+ """`(asin_col, sku_col, asin_link_key, sku_link_key)` as the spec declares them for `key`."""
+ if key == LISTINGS:
+ return "asin1", "seller_sku", None, None # the row IS the listing: Parent ASIN only
+ asin = sku = asin_via = sku_via = None
+ for edge in connector_spec.edges_for(key):
+ if edge["target"] != LISTINGS:
+ continue
+ if edge["to_col"] == "asin1" and asin is None:
+ asin, asin_via = edge["from_col"], edge["key"]
+ elif edge["to_col"] == "seller_sku" and sku is None:
+ sku, sku_via = edge["from_col"], edge["key"]
+ return asin, sku, asin_via, sku_via
+
+
+def _latest_listings_sql(listing_names):
+ """`SELECT sku, asin` over the LATEST listings snapshot day, or None without both columns."""
+ if not {"seller_sku", "asin1"} <= set(listing_names):
+ return None
+ table = SOURCE[LISTINGS]
+ latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
+ if "_snapshot_date" in listing_names else "")
+ return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
+ f"TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM {table} "
+ f"WHERE seller_sku IS NOT NULL AND asin1 IS NOT NULL{latest}) "
+ "WHERE sku <> '' AND asin <> ''")
+
+
+def lookups(cur, key, names=None):
+ """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
+
+ `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
+ (every reader selects `FROM
` unaliased), so it binds to the outer row and never to the
+ resolver's own `asin`. DuckDB decorrelates it into a single hash join per query, which is what
+ keeps a grouped page of a read-through grid set-based rather than per-row. No match is NULL,
+ which every reader serialises as a blank cell. Nothing is capped (standing rule 1).
+ """
+ if names is None:
+ names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
+ asin, sku, asin_via, sku_via = _product_columns(key)
+ asin = asin if asin in names else None
+ sku = sku if sku in names else None
+ if not asin and not sku:
+ return []
+ listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
+ parents = None if "parent_asin" in names else amazon_store.asin_parents_sql(cur)
+ latest = _latest_listings_sql(listing_names)
+ table = SOURCE[key]
+ out = []
+
+ def add(lookup_key, after, expr, via):
+ if lookup_key in names:
+ return
+ out.append({"key": lookup_key, "label": _LOOKUP_LABELS[lookup_key], "after": after,
+ "expr": expr, "opens": {"via": via} if via else None,
+ "description": _LOOKUP_DESCRIPTIONS[lookup_key]})
+
+ if asin:
+ ref = f"TRIM(CAST({table}.{_quoted(asin)} AS VARCHAR))"
+ via = asin_via if "asin1" in listing_names else None
+ if parents:
+ add(LOOKUP_PARENT, asin,
+ f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via)
+ if latest and not sku:
+ add(LOOKUP_SKU, asin,
+ f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
+ f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via)
+ elif sku and latest:
+ ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
+ via = sku_via if "seller_sku" in listing_names else None
+ add(LOOKUP_ASIN, sku,
+ f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
+ f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via)
+ if parents:
+ add(LOOKUP_PARENT, sku,
+ f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
+ f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
+ f"WHERE _l.sku = {ref})", via)
+ return out
+
+
def _fields_for(cur, key):
"""One preset field per mirror column — the whole schema, derived. See the header."""
cols = mirror_columns(cur, SOURCE[key])
@@ -177,6 +287,12 @@ def _fields_for(cur, key):
return []
names = {c[0] for c in cols}
pinned = PINNED.get(key) if PINNED.get(key) in names else None
+ # S67-T30: each lookup lands IMMEDIATELY after the column it derives from, shown by default,
+ # and the ASIN / SKU column it sits beside is shown by default too (inventory's `msku` was not).
+ derived = {}
+ for lookup in lookups(cur, key, names):
+ derived.setdefault(lookup["after"], []).append(lookup)
+ product = {c for c in _product_columns(key)[:2] if c in names}
out = []
# The identity column goes FIRST, the rest keep the report's own order. A record reached
# through a link is titled by its first field (the record drawer's frame label), and a
@@ -186,9 +302,16 @@ def _fields_for(cur, key):
out.append(_odoo._preset({
"key": name, "label": _label(name), "type": _ftype(name, dtype),
"source": "overlay",
- "default": name == pinned or name in _DEFAULT_COLUMNS,
+ "default": name == pinned or name in _DEFAULT_COLUMNS or name in product,
"pinned": name == pinned,
}, flow="amazon_relational"))
+ for lookup in derived.get(name, ()):
+ field = {"key": lookup["key"], "label": lookup["label"], "type": "text",
+ "source": "overlay", "default": True,
+ "description": lookup["description"]}
+ if lookup["opens"]:
+ field["opens"] = dict(lookup["opens"])
+ out.append(_odoo._preset(field, flow="amazon_relational"))
# ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
# known until it syncs, so a link to a table that has never run would name a column that does
# not exist. **Absent is a normal state here, not a failure** — which is exactly what lets the
@@ -324,6 +447,10 @@ def grid_source(cur, key):
if kind in ("currency", "int"):
expr = f"TRY_CAST({expr} AS {'DOUBLE' if kind == 'currency' else 'BIGINT'})"
cols[name] = (expr, {"currency": _number, "int": _integer}.get(kind, _text))
+ # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
+ # group (count by parent) and a sort compile against them in SQL with no per-row Python.
+ for lookup in lookups(cur, key, names):
+ cols[lookup["key"]] = (lookup["expr"], _text)
return {"table": SOURCE[key], "where": "", "id": ident,
"needs_excluded": False, "cols": cols}
@@ -393,11 +520,16 @@ def _read(cur, key):
return []
quoted = ", ".join(f'"{c}"' for c in cols)
spec = grid_source(cur, key)
- rows = cur.execute(f'SELECT {spec["id"]}, {quoted} FROM {SOURCE[key]}').fetchall()
+ # S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
+ # expressions the read-through binding uses, in this one SELECT (never per row).
+ derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
+ extra = "".join(f", {expr}" for _k, expr in derived)
+ rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]}').fetchall()
+ names = cols + [k for k, _expr in derived]
out = []
for r in rows:
row = {}
- for name, val in zip(cols, r[1:]):
+ for name, val in zip(names, r[1:]):
# Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
row[name] = "" if val is None else (val if isinstance(val, (int, float))
else str(val).replace("\x00", ""))
@@ -497,6 +629,43 @@ def plan_fingerprint(cur, built):
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+def _place_lookups(stored, fields):
+ """An EXISTING table's fields with each NEW product lookup placed right after its source.
+
+ `_ensure_table_inplace` only appends, so a lookup first planned for a table stored before
+ S67-T30 would land at the far end of the grid instead of beside the ASIN or SKU it derives
+ from. A lookup already stored keeps its place (a person may have moved it) and only has its
+ `opens` bag brought up to date, which `_ensure_table_inplace` does not forward-migrate.
+ """
+ out = list(stored)
+ have = {f.get("key"): i for i, f in enumerate(out)}
+ anchor = None
+ for field in fields:
+ fkey = field.get("key")
+ if fkey not in LOOKUP_KEYS:
+ anchor = fkey
+ continue
+ if fkey in have:
+ old = out[have[fkey]]
+ if (old.get("automation") or {}).get("flowId") == "amazon_relational" \
+ and not _odoo._ut().user_edited(old):
+ if field.get("opens"):
+ old["opens"] = dict(field["opens"])
+ else:
+ old.pop("opens", None)
+ continue
+ keys = [f.get("key") for f in out]
+ if anchor not in keys:
+ continue # no source column stored yet: the append places it
+ at = keys.index(anchor) + 1
+ while at < len(keys) and keys[at] in LOOKUP_KEYS:
+ at += 1 # after any lookup already beside the same source
+ out.insert(at, dict(field))
+ have = {f.get("key"): i for i, f in enumerate(out)}
+ anchor = fkey
+ return out
+
+
def _lead_with_identity(stored, report, pinned):
"""An EXISTING table's fields with its identity column moved first, while the order is still
the report's own.
@@ -564,13 +733,16 @@ def apply_plan(rt, built, username="amazon", today=None):
# Preserve fields a person created or explicitly took over.
current = doc.get(key) or {}
wanted = {f["key"] for f in fields}
+ # S67-T30: a product lookup the mirror no longer supports retires the same way.
current["fields"] = _lead_with_identity(
[f for f in current.get("fields", [])
- if not (f.get("type") == "link"
+ if not ((f.get("type") == "link" or f.get("key") in LOOKUP_KEYS)
and (f.get("automation") or {}).get("flowId") == "amazon_relational"
and not _odoo._ut().user_edited(f)
and f.get("key") not in wanted)],
report_order.get(key) or [], PINNED.get(key))
+ if current["fields"]:
+ current["fields"] = _place_lookups(current["fields"], fields)
stub = isinstance(rows, dict) and rows.get("readThrough") is True
if not stub:
stored = current.get("rows") or {}
diff --git a/api/application_registry.py b/api/application_registry.py
index 56f880fa04026c7da219319fb4b4a816234b9492..2a4ca9c78f3cc62e670b9c680864b830f5e7f833 100644
--- a/api/application_registry.py
+++ b/api/application_registry.py
@@ -16,6 +16,11 @@ APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
FINANCE_TEMPLATE_KIND = "finance"
APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
+# S67-T40: Content Studio creates its OWN databases (content_studio.py), so it is never offered to
+# the five-step wizard either; its catalogue POST provisions References + Posts and binds Posts.
+# Appended on its own line so the wizard / Finance separation above reads exactly as S65-T10 pins it.
+CONTENT_STUDIO_TEMPLATE_KIND = "content_studio"
+APPLICATION_TEMPLATE_KINDS = APPLICATION_TEMPLATE_KINDS + (CONTENT_STUDIO_TEMPLATE_KIND,)
_KEY = "applications_v1"
_MAX = 500
@@ -65,6 +70,17 @@ _TEMPLATE_CONTRACTS = {
"fieldRoles": [],
"defaultPlacement": {"section": "app", "order": 0},
},
+ # S67-T40 (R17, R21, O3: one declarative row, no new type). S67-T41 owner correction: the App
+ # opens on its OWN surface in the App module (`content_studio.app_link`), like Catalog and
+ # Route, so the governed type only has to be a real one; its calendar is time.
+ "content_studio": {
+ "kind": "content_studio",
+ "label": "Content Studio",
+ "applicationType": "timeseries",
+ "displayMode": "timeseries",
+ "fieldRoles": [],
+ "defaultPlacement": {"section": "app", "order": 0},
+ },
}
@@ -363,6 +379,18 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
raise ApplicationError("navPlacement order must be an integer from 0 to 999")
return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
"navPlacement": {"section": "app", "order": order}}
+ if kind == CONTENT_STUDIO_TEMPLATE_KIND:
+ # Created server-side by content_studio.create right after it made the Posts database, so
+ # the binding is the one database it just created; the link can only open that database.
+ source = str(payload.get("sourceDatabase") or "").strip()[:120]
+ if not source.startswith("ut_"):
+ raise ApplicationError("Content Studio binds the Posts database it created.")
+ if allowed_databases is not None and source not in set(allowed_databases):
+ raise PermissionError("an Application cannot bind a database you cannot open")
+ # S67-T41 owner correction: no stored link. `_public` derives the App's own address from
+ # its id, so every Studio, including one created before the correction, opens in App.
+ return {"templateKind": kind, "sourceDatabase": source,
+ "navPlacement": {"section": "app", "order": 0}}
source = str(payload.get("sourceDatabase") or "").strip()[:120]
if not source:
raise ApplicationError("sourceDatabase is required")
@@ -470,12 +498,49 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
"edit", "share", "delete", "duplicate", "star", "default")})
row["starred"] = False
row["isDefault"] = False
+ if row.get("templateKind") == CONTENT_STUDIO_TEMPLATE_KIND:
+ # S67-T41 owner correction: a Studio opens on its own App surface, never in Data. The
+ # database-bound edit, share, duplicate and default doors cannot reach an App bound to a
+ # user table, so they are off; star and delete stay, routed to `/content-studio/apps`.
+ row["deepLink"] = f"/app?studio={row['id']}"
+ row["capabilities"].update({key: False for key in ("edit", "share", "duplicate", "default")})
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
"database":target_database}
return row
+#: `core.table_store.RESERVED_MEMBERS`: workspace members that are registries, not users.
+_RESERVED_WORKSPACE_MEMBERS = ("__shared__", "__assigned__", "__audit__")
+
+
+def dependent_view_counts(runtime, database):
+ """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
+ user's Views in that database's workspace. The Delete App dialog names this count before
+ confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
+ """
+ from core.view_templates import workspace_key
+ key = workspace_key(database)
+ if not key:
+ return {}
+ try:
+ doc = runtime.get(key)
+ except Exception:
+ return None
+ counts = {}
+ for member, bucket in (doc.items() if isinstance(doc, dict) else ()):
+ if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict):
+ continue
+ views = bucket.get("views")
+ for view in (views.values() if isinstance(views, dict) else ()):
+ config = view.get("config") if isinstance(view, dict) else None
+ ref = config.get("application") if isinstance(config, dict) else None
+ app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
+ if app_id:
+ counts[app_id] = counts.get(app_id, 0) + 1
+ return counts
+
+
def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
allowed_databases=None):
doc = load(runtime)
@@ -491,6 +556,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
target_database=database,
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
+ # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
+ # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
+ deletable = [row for row in rows if row["capabilities"].get("delete")]
+ if database and deletable:
+ counts = dependent_view_counts(runtime, database)
+ for row in deletable:
+ row["dependentViews"] = None if counts is None else counts.get(row["id"], 0)
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
diff --git a/api/assistant_mirror.py b/api/assistant_mirror.py
index 1054ee6c03d0f99bc454a9406d0658ff97b48776..333f119639893e4622055eff1c7b8a985a9489b1 100644
--- a/api/assistant_mirror.py
+++ b/api/assistant_mirror.py
@@ -28,6 +28,15 @@ that need them. A module-level import either way is a cycle at startup.
"""
+#: ⭐ S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what
+#: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window):
+#: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an
+#: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is
+#: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's
+#: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631).
+ASSISTANT_READ_MAX = 200_000
+
+
class MirrorRefusal(Exception):
"""A named, fail-closed refusal with the status and code `deps` should raise it as.
@@ -49,9 +58,14 @@ def _rot():
return routes_odoo_tables
-def open_cursor(session):
+def open_cursor(session, table_key=None):
"""`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
+ ⭐ S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's
+ `_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its
+ own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so
+ the assistant no longer answers 503 for Amazon while Odoo is still backfilling.
+
⛔ THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
@@ -61,7 +75,7 @@ def open_cursor(session):
"""
rot = _rot()
try:
- cur, mismatch = rot._mirror_cur(session)
+ cur, mismatch = rot._mirror_cur(session, table_key)
except RuntimeError as exc:
return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
except Exception as exc: # noqa: BLE001
@@ -118,8 +132,26 @@ def is_bound(cur, table_key):
"""
from harness import datastore
try:
- spec = _rot()._sources().get(str(table_key or ""))
- if not spec:
+ rot = _rot()
+ key = str(table_key or "")
+ spec = rot._sources().get(key)
+ if not spec and key.startswith("ut_amzn_"):
+ # S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key.
+ # `_source_for` and `_materialised_table` both learn them on demand; so must this.
+ rot._learn_amazon_sources()
+ spec = rot.GRID_SOURCES.get(key)
+ if not spec:
+ # A dynamically sized Amazon snapshot is admitted per key by the grid door, not by
+ # the provider build, so the registry can lack a key the grid serves (measured on
+ # staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the
+ # provider's own per-key resolver, exactly as `require_amazon_mirror` does, without
+ # admitting it (the read path admits it through that same door).
+ import amazon_relational as amazon
+ if key not in amazon.SOURCE:
+ return False
+ current = amazon.grid_source(cur, key) or {}
+ return bool(current) and not current.get("missing")
+ if not spec or spec.get("missing"):
return False
aliases = dict(spec.get("tables") or {})
if spec.get("table"):
@@ -129,8 +161,13 @@ def is_bound(cur, table_key):
return False
-def declared_fields(table_key):
- """`mk_fields()` for one connected grid, or None when this key is not a connected Odoo grid.
+def declared_fields(table_key, definition=None):
+ """`mk_fields()` for one connected grid, or None when this key is not a connected grid.
+
+ ⭐ S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that
+ is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid,
+ a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before
+ this, those databases were listed as answerable and then refused 409 on every read.
⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
each table's stored `fields` from this same function, so reading it directly is reading the
@@ -142,10 +179,20 @@ def declared_fields(table_key):
if key == str(table_key or ""):
return [dict(field) for field in (mk_fields() or [])
if isinstance(field, dict) and field.get("key")]
- return None
+ key = str(table_key or "")
+ if not isinstance(definition, dict) or not key.startswith("ut_"):
+ return None
+ try:
+ if _rot()._materialised_table(key):
+ return None
+ except Exception: # noqa: BLE001
+ return None
+ fields = [dict(field) for field in (definition.get("fields") or [])
+ if isinstance(field, dict) and field.get("key")]
+ return fields or None
-def answerable_fields(table_key, spec):
+def answerable_fields(table_key, spec, fields=None):
"""The declared field contract INTERSECTED with the columns the mirror can actually express.
⛔ THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
@@ -155,7 +202,8 @@ def answerable_fields(table_key, spec):
refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
for it with a loud `assistant_unknown_field`.
"""
- fields = declared_fields(table_key)
+ if fields is None:
+ fields = declared_fields(table_key)
if fields is None:
return None
cols = set((spec or {}).get("cols") or ())
@@ -197,30 +245,14 @@ def _compiled(nodes, conj, columns, *, code, subject, rot):
return pred
-def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
- filter_keys=None):
- """Every permitted row of a mirror-served grid, or a refusal that names its cause.
-
- Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
- (`{pid, **cells}`), sorted by pid.
+def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys):
+ """`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34.
- ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
- connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
- with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
- states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
- every aggregate from those same records, so a short set does not produce a smaller answer, it
- produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
- already settled this policy in the same words: REFUSE, NEVER TRIM.
-
- ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
- the field projection in Python AFTER this returns. Hand it a row missing a column the wall
- names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
- pass would quietly disagree with the pushdown it is meant to double-check.
+ Extracted verbatim from `read_rows` so the row walk, the page read and the server-side
+ aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler.
+ Three readers with one predicate cannot disagree about which rows the caller may see.
"""
- from harness import datastore
- rot = _rot()
fe = rot._fe()
-
cols = rot._filter_columns(spec, fields)
# ── the administrator's wall, compiled FIRST and pushed down ──────────────────────────────
@@ -267,23 +299,191 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
where = f"({where}) AND {pred.sql}" if where else pred.sql
params.extend(pred.params)
+ return where, params, cols
+
+
+def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
+ filter_keys=None, defn=None):
+ """Every permitted row of a mirror-served grid, or a refusal that names its cause.
+
+ Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
+ (`{pid, **cells}`), sorted by pid.
+
+ ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
+ connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
+ with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
+ states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
+ every aggregate from those same records, so a short set does not produce a smaller answer, it
+ produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
+ already settled this policy in the same words: REFUSE, NEVER TRIM.
+
+ ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
+ the field projection in Python AFTER this returns. Hand it a row missing a column the wall
+ names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
+ pass would quietly disagree with the pushdown it is meant to double-check.
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+
# ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
# spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
- win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
- where=where, params=tuple(params), order_by=spec["id"],
- limit=datastore.WINDOW_MAX, cur=cur, **frm)
- if win["total"] > len(win["rows"]):
+ # ⭐⭐ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX`
+ # pass this replaced refused any connected database past 5,000 matching rows, which is a cap
+ # on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages,
+ # KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs
+ # one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a
+ # user condition: every condition above still comes from the grid's compiler.
+ # ⛔ STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that
+ # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
+ first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
+ where=where, params=tuple(params), order_by=spec["id"],
+ limit=datastore.WINDOW_MAX, cur=cur, **frm)
+ total = int(first["total"])
+ if total > ASSISTANT_READ_MAX:
raise MirrorRefusal(
409, "assistant_source_too_large",
- f"this database matches {win['total']:,} rows for that question, and the assistant "
- f"reads a connected database in one pass of at most "
- f"{datastore.WINDOW_MAX:,} rows, so answering now would mean counting only part of "
- f"them and reporting the total as if it were whole. Narrow it with a filter and ask "
- f"again")
+ f"this database matches {total:,} rows for that question, and one assistant answer "
+ f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question "
+ f"cannot exhaust the server's memory. Answering over part of them would report a "
+ f"wrong total, so it is refused instead. Narrow it with a filter (for example a date "
+ f"range) and ask again, or ask for totals as aggregates, which the server computes "
+ f"over every row")
+ got = list(first["rows"])
+ page = list(first["rows"])
+ while len(got) < total and len(page) >= datastore.WINDOW_MAX:
+ last = page[-1][0]
+ more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
+ nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
+ where=more_where, params=tuple(params) + (last,),
+ order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm)
+ page = list(nxt["rows"])
+ if not page:
+ break
+ got.extend(page)
+ if len(got) != total:
+ raise MirrorRefusal(
+ 409, "assistant_source_changed",
+ f"this database changed while it was being read ({len(got):,} rows read against "
+ f"{total:,} counted), so the answer would not describe one consistent set of rows. "
+ f"Ask again once its connector sync has finished")
+ win = {**first, "rows": got, "total": total}
+
+ # S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does
+ # not re-read the whole tenant document (measured about 3 s a question on staging).
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
+ defn=defn if isinstance(defn, dict) else None)
+ rows = rot._pool_from_window(spec, win, stamp=stamp)
+ rows.sort(key=lambda row: row["pid"])
+ return rows, stamp, int(win["total"])
+
- stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None))
+#: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`).
+_GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"}
+
+
+def _json_value(rot, value):
+ """A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a
+ date or timestamp as its ISO text, anything else as is. Never a substituted 0."""
+ if value is None:
+ return None
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ return rot._agg_number(value)
+ iso = getattr(value, "isoformat", None)
+ if callable(iso):
+ return iso()
+ try:
+ from decimal import Decimal
+ if isinstance(value, Decimal):
+ return rot._agg_number(value)
+ except Exception: # noqa: BLE001
+ pass
+ return value
+
+
+def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
+ filter_tree=None, filter_keys=None):
+ """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
+
+ Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the
+ count of rows the permitted predicate matches, from `datastore.window`'s own count statement
+ over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract).
+
+ ⛔⛔ THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall
+ and the caller's filter with the grid's compiler, so a sum over a million-row table runs over
+ exactly the rows this caller may read and no row is pulled into Python to compute it. There
+ is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size.
+
+ ⛔ AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and
+ a field outside it has no expression here at all, so a sum over a hidden column cannot be
+ built (the caller's `_normalise_aggregates` has already refused it by name).
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+ visible = set(filter_keys) if filter_keys is not None else set(cols)
+ fs = rot._fs()
+ proj, slots = [], []
+ for item in specs or ():
+ fn, field = item.get("fn"), item.get("field")
+ if field is None:
+ slots.append(None) # count(*): the window's own total
+ continue
+ col = cols.get(field) if field in visible else None
+ if col is None:
+ raise MirrorRefusal(400, "assistant_unknown_field",
+ "every requested field must be readable in this database")
+ if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES:
+ raise MirrorRefusal(400, "query_aggregate_type",
+ "sum and avg require a readable numeric field.")
+ if fn in _GRID_AGG:
+ expr = fs.aggregate_sql(col, _GRID_AGG[fn])
+ elif fn in {"min", "max"}:
+ expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES
+ else f"{fn}({col['sql']})")
+ elif fn == "count":
+ expr = f"count({col['sql']})"
+ else:
+ raise MirrorRefusal(400, "query_aggregates_invalid",
+ "That aggregate is not available.")
+ slots.append(len(proj))
+ proj.append(f'{expr} AS "a{len(proj)}"')
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
+ win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params),
+ order_by="1", limit=1, cur=cur, **frm)
+ row = (win["rows"] or [()])[0]
+ total = int(win["total"])
+ out = []
+ for item, slot in zip(specs or (), slots):
+ value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row)
+ else None)
+ out.append({**item, "value": value})
+ return out, total
+
+
+def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
+ filter_tree=None, filter_keys=None, defn=None):
+ """ONE page of the permitted rows, paged IN SQL, in pid order. S67-T34 (rule 1).
+
+ Returns `(rows, stamp, total)` in `read_rows`'s shape. This is how a table past
+ `ASSISTANT_READ_MAX` still answers its rows: the page is `LIMIT/OFFSET` over the same
+ predicate `read_rows` runs, so only `limit` rows ever reach this process, and `total` is the
+ population that predicate matches. It carries no sort other than the pid; a caller asking for
+ one past the ceiling is refused by name upstream, because sorting a million rows in Python
+ is the memory failure the ceiling exists to prevent.
+ """
+ from harness import datastore
+ rot = _rot()
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
+ select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
+ win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
+ params=tuple(params), order_by=spec["id"], offset=int(offset),
+ limit=int(limit), cur=cur, **frm)
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
+ defn=defn if isinstance(defn, dict) else None)
rows = rot._pool_from_window(spec, win, stamp=stamp)
rows.sort(key=lambda row: row["pid"])
return rows, stamp, int(win["total"])
diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py
index b02cee0823f0731e4eae69931cbff7338ed84f37..c0b4b766dfb4082c9a4d50030bafef6afe11b3f0 100644
--- a/api/connectors_amazon.py
+++ b/api/connectors_amazon.py
@@ -49,11 +49,80 @@ class AmazonStatusSafetyError(ValueError):
"""A status response contains a forbidden credential shape."""
-def status_payload_has_secret(serialized_payload):
+class AmazonStatusRowsRefused(AmazonStatusSafetyError):
+ """S67-T31: some status ROWS were refused; the rest are carried, already guarded.
+
+ Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
+ refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
+ neither carries the refused content, so a caller may degrade those rows and keep the others.
+ """
+
+ def __init__(self, safe_status, refused):
+ super().__init__('Amazon status contains a forbidden credential shape.')
+ self.safe_status = safe_status
+ self.refused = tuple(refused)
+
+
+#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
+STATUS_SCALAR_LIMIT = 512
+STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
+
+
+def trim_status(value, limit=STATUS_SCALAR_LIMIT):
+ """Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
+
+ S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
+ characters) used to trip the length branch of the guard and take the whole connector modal
+ down with a 500. ⛔ Call this only AFTER the credential-shape scan of the UNTRIMMED value
+ (`screen_status_rows`): cutting first could slice a token shape out of a string and accept
+ the rest, which would trim a refusal into acceptance.
+ """
+ if isinstance(value, dict):
+ return {key: trim_status(item, limit) for key, item in value.items()}
+ if isinstance(value, list):
+ return [trim_status(item, limit) for item in value]
+ if isinstance(value, tuple):
+ return tuple(trim_status(item, limit) for item in value)
+ if isinstance(value, str) and len(value) > limit:
+ head = value[:limit - 3]
+ cut = head.rsplit(" ", 1)[0] if " " in head else head
+ return cut.rstrip(" ,;:") + "..."
+ return value
+
+
+def screen_status_rows(status):
+ """`(safe, refused)` for a `{table: row}` status: each row judged on its own.
+
+ A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
+ form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
+ is only too long is trimmed, then held to the full guard again.
+ """
+ import json
+
+ safe, refused = {}, []
+ for name, row in (status or {}).items():
+ try:
+ whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
+ if status_payload_has_secret(whole, max_length=None):
+ refused.append(name)
+ continue
+ trimmed = trim_status(row)
+ if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
+ refused.append(name)
+ continue
+ except (TypeError, ValueError, RecursionError):
+ refused.append(name)
+ continue
+ safe[name] = trimmed
+ return safe, refused
+
+
+def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
"""Scan serialized JSON for C2's credential keys, token shapes and oversized values.
Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
The credential key roster is the same one the connector accepts at connect time.
+ `max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
"""
import json
import re
@@ -73,10 +142,11 @@ def status_payload_has_secret(serialized_payload):
elif isinstance(value, list):
pending.extend(value)
elif isinstance(value, str):
- if (len(value) > 512 or 'Atzr|' in value
+ if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
return True
- elif value is not None and len(str(value)) > 512:
+ elif (value is not None and max_length is not None
+ and len(str(value)) > max_length):
return True
return False
@@ -162,7 +232,13 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
row["syncCause"] = cause
cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
- return assert_safe_status(status)
+ # S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
+ # `search_query_performance.unknownCause` at 935 characters); a credential shape still
+ # refuses, now naming only its rows so the caller degrades those and keeps the rest.
+ safe, refused = screen_status_rows(status)
+ if refused:
+ raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
+ return assert_safe_status(safe)
declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
for name, spec in DECLARED_DATASETS.items()}
diff --git a/api/content_studio.py b/api/content_studio.py
new file mode 100644
index 0000000000000000000000000000000000000000..10696a42b7bc4d9428277b20602836918a6ff839
--- /dev/null
+++ b/api/content_studio.py
@@ -0,0 +1,638 @@
+"""Content Studio: an App template that creates its own References and Posts databases.
+
+S67-T40 (owner instruction 1; rulings R17, R18, R21, R11). Plan-only (round 1 R5): nothing here
+publishes to any network. The template creates two TENANT-OWNED databases and binds the App to
+Posts, opening on a calendar of posts. The App is delete-safe (rule 13, ARCHITECTURE section 24):
+deleting it removes only the App record, so References and Posts and every row in them outlive it.
+
+This module is the App's own code. Shared files reach it only through a guarded import
+(`routes_templates.py`), so deleting this file removes the template and nothing else breaks.
+"""
+from __future__ import annotations
+
+TEMPLATE_KEY = "content_studio"
+LABEL = "Content Studio"
+DESCRIPTION = "Plan posts from your references. Creates References and Posts databases."
+
+#: R4: the channels a post is planned for. Channel length limits are ADVISED, never enforced (O4).
+CHANNELS = ["LinkedIn", "X", "Instagram", "TikTok", "YouTube", "Reddit"]
+NETWORKS = CHANNELS + ["Other"]
+STATUSES = ["Idea", "Draft", "Ready", "Posted", "Skipped"]
+CALENDAR_VIEW_ID = "cs_calendar"
+
+#: R15 (structure, not words): the three fields a Reference is drafted FROM are extracted from its
+#: pasted text by the per-row AI field machinery (`ai_enrich`), re-run when that text changes.
+_EXTRACT = {
+ "hook": ("Hook", "In one sentence, describe the opening move this post uses to earn attention "
+ "(for example a surprising number, a contrarian claim, a question). Describe "
+ "the technique; do not quote the post. Post: {pasted_text}"),
+ "structure": ("Structure", "List the post's structure as 3 to 6 short numbered beats, each "
+ "naming what that part does (setup, proof, turn, ask). Describe, "
+ "never quote. Post: {pasted_text}"),
+ "angle": ("Angle", "In one sentence, name the point of view or argument the post takes and "
+ "who it is for. Do not quote the post. Post: {pasted_text}"),
+}
+
+
+#: The platform ladder leads with a REASONING model, which spends a few hundred tokens thinking
+#: before it answers; at 300 the column's ceiling was reached with no answer ("the model had
+#: nothing to answer from"), so each extraction gets room for thought plus a short answer.
+EXTRACT_MAX_TOKENS = 1500
+
+
+def reference_fields():
+ """References: URL, network, author, pasted text, tags, hook, structure, angle (T40 done-when)."""
+ fields = [
+ {"key": "title", "label": "Title", "type": "text"},
+ {"key": "url", "label": "URL", "type": "url"},
+ {"key": "network", "label": "Network", "type": "select", "options": list(NETWORKS)},
+ {"key": "author", "label": "Author", "type": "text"},
+ {"key": "pasted_text", "label": "Pasted text", "type": "text"},
+ {"key": "tags", "label": "Tags", "type": "multiselect", "options": []},
+ ]
+ for key, (label, prompt) in _EXTRACT.items():
+ fields.append({"key": key, "label": label, "type": "ai_enrich",
+ "aiEnrich": {"prompt": prompt, "maxTokens": EXTRACT_MAX_TOKENS,
+ "trigger": {"mode": "on_change"}, "overwrite": "always"}})
+ return fields
+
+
+def post_fields(references_key):
+ """Posts: caption, channel, scheduled, status, media, campaign, post URL, References link,
+ attribution (T40 done-when). `media` is an `image` field: the store has no attachment type."""
+ return [
+ {"key": "caption", "label": "Caption", "type": "text"},
+ {"key": "channel", "label": "Channel", "type": "select", "options": list(CHANNELS)},
+ {"key": "scheduled", "label": "Scheduled", "type": "date"},
+ {"key": "status", "label": "Status", "type": "select", "options": list(STATUSES)},
+ {"key": "media", "label": "Media", "type": "image"},
+ {"key": "campaign", "label": "Campaign", "type": "text"},
+ {"key": "post_url", "label": "Post URL", "type": "url"},
+ {"key": "references", "label": "References", "type": "link",
+ "link": {"table": references_key, "selectionMode": "manual"}},
+ {"key": "attribution", "label": "Attribution", "type": "text"},
+ ]
+
+
+class ContentStudioError(RuntimeError):
+ """A refusal the route turns into a plain sentence; nothing half-created is left behind."""
+
+
+def _create_table(label, username, fields, st):
+ """`user_tables.create` refuses a duplicate label, so a second Studio gets "Posts 2" and so on."""
+ from core import user_tables
+ for attempt in range(1, 10):
+ name = label if attempt == 1 else f"{label} {attempt}"
+ key = user_tables.create(name, username, fields=fields, st=st)
+ if key:
+ return key
+ raise ContentStudioError(f"The {label} database could not be created. "
+ "Rename or delete an older one, then try again.")
+
+
+def create(session, name=""):
+ """Create References + Posts (tenant-owned), a calendar View on Posts, and the App bound to
+ Posts. Returns `{address, application, databases}`; on a failure part-way, the databases this
+ call created are deleted again, so a refusal leaves the workspace as it was."""
+ import application_registry
+ from core import user_tables, view_templates
+ from core import table_store
+
+ st = session.runtime
+ user = session.uname
+ app_name = " ".join(str(name or "").split())[:120] or LABEL
+ created = []
+ try:
+ references = _create_table("References", user, reference_fields(), st)
+ created.append(references)
+ posts = _create_table("Posts", user, post_fields(references), st)
+ created.append(posts)
+ ops = table_store.make(view_templates.workspace_key(posts), st=st)
+ ops.save_view(user, {"id": CALENDAR_VIEW_ID, "name": "Calendar", "createdBy": user,
+ "config": {"filters": [], "sorts": [],
+ "display": {"mode": "calendar", "dateField": "scheduled",
+ "titleField": "caption"}}})
+ application = application_registry.create(
+ st, session.tenant, user,
+ {"templateKind": TEMPLATE_KEY, "name": app_name, "sourceDatabase": posts,
+ "navPlacement": {"section": "app", "order": 0}},
+ allowed_databases=None)
+ except Exception:
+ for key in reversed(created):
+ try:
+ user_tables.delete(key, st=st)
+ except Exception:
+ pass
+ raise
+ # S67-T41 owner correction: the App opens on its own surface in the App module, not in Data.
+ return {"address": "#" + app_link(application["id"]), "application": application,
+ "databases": {"references": references, "posts": posts}}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T41 (R13, R14, R15, R19, R20, R5, O4): DRAFTING. One Draft row per chosen channel, from
+# several selected References plus a one-line topic, written from their extracted STRUCTURE
+# (hook, structure, angle), never their words, with every author credited. Plan-only: a draft
+# is a Posts row with status Draft; nothing is published anywhere.
+# ---------------------------------------------------------------------------------------------
+import re
+
+from fastapi import APIRouter, Body, Depends
+
+from deps import Session, err, require_session
+
+router = APIRouter(prefix="/content-studio")
+
+#: O4: channel limits are ADVICE in the prompt and on screen, never a truncation.
+CHANNEL_ADVICE = {
+ "LinkedIn": "LinkedIn post: up to about 3,000 characters, short paragraphs, no hashtag wall.",
+ "X": "X post: aim for 280 characters or fewer.",
+ "Instagram": "Instagram caption: up to about 2,200 characters; a strong first line.",
+ "TikTok": "TikTok caption: short, about 150 characters.",
+ "YouTube": "YouTube: a title line, then a two or three sentence description.",
+ "Reddit": "Reddit: a title line, then the body; plain, specific, no marketing tone.",
+}
+DRAFT_SYSTEM = (
+ "You draft ONE original social media post for the channel named. You are given a one-line "
+ "topic and the STRUCTURE of several reference posts: the technique of each hook, its beats "
+ "and its angle. Borrow those structural moves; write entirely new sentences. Never copy or "
+ "closely paraphrase any sentence from anywhere. Answer with the post text only: no preamble, "
+ "no quotation marks around it, no markdown heading. Never use an em dash or an en dash; use "
+ "a comma, a colon or a full stop.")
+LEDGER_SURFACE = "field_agent"
+DRAFT_MAX_TOKENS = 3000 # a reasoning model thinks first; see EXTRACT_MAX_TOKENS
+DRAFT_TIMEOUT_S = 60
+#: A shared run of this many words is a copied sentence, whatever surrounds it (R15).
+VERBATIM_WORDS = 8
+
+
+def _studio(session, app_id):
+ """(app, posts_key, references_key) for a Content Studio App this session may open."""
+ import application_registry
+ from core import user_tables
+ doc = application_registry.load(session.runtime)
+ app = doc["applications"].get(str(app_id or ""))
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KEY
+ or not application_registry.may_read(app, session.uname, session.admin)):
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
+ posts = str(app.get("sourceDatabase") or "")
+ defn = user_tables.get(posts, session.runtime) or {}
+ link = next((f.get("link") for f in defn.get("fields") or ()
+ if isinstance(f, dict) and f.get("key") == "references"), None)
+ references = str((link or {}).get("table") or "")
+ if not posts or not references:
+ raise err(409, "content_studio_incomplete",
+ "This Content Studio has lost its Posts or References database.")
+ return app, posts, references
+
+
+def _reference_rows(session, references):
+ from core import user_tables
+ defn = user_tables.get(references, session.runtime) or {}
+ rows = defn.get("rows") or {}
+ return {str(rid): (row if isinstance(row, dict) else {}) for rid, row in rows.items()}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T42 (R35, standing rule 1): THE MONTHLY DRAFTING ALLOWANCE. Drafting runs on Loopable's
+# platform keys, so each tenant gets a monthly count of drafts (one Draft row = one draft). Past
+# it, Draft pauses with a sentence naming the reset date; the platform admin raises it per tenant
+# (`PUT /content-studio/allowance`). A tenant admin cannot: that would defeat R35.
+# One per-tenant bucket, owned by this module: {"allowance": int (absent = default),
+# "months": {"YYYY-MM": drafts made}}. Month and reset are UTC.
+# ---------------------------------------------------------------------------------------------
+ALLOWANCE_KEY = "content_studio_usage"
+DEFAULT_MONTHLY_DRAFTS = 100
+ALLOWANCE_KEEP_MONTHS = 12
+_MONTH_NAMES = ("January", "February", "March", "April", "May", "June", "July", "August",
+ "September", "October", "November", "December")
+
+
+def _now():
+ from datetime import datetime, timezone
+ return datetime.now(timezone.utc)
+
+
+def _month_key(now):
+ return f"{now.year:04d}-{now.month:02d}"
+
+
+def _reset_on(now):
+ """The first day of next month (UTC): the day this month's count stops counting."""
+ from datetime import date
+ return date(now.year + 1, 1, 1) if now.month == 12 else date(now.year, now.month + 1, 1)
+
+
+def _spoken_date(day):
+ return f"{_MONTH_NAMES[day.month - 1]} {day.day}, {day.year}"
+
+
+def _limit(doc):
+ raw = (doc or {}).get("allowance")
+ if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
+ return DEFAULT_MONTHLY_DRAFTS
+ return raw
+
+
+def allowance_state(doc, now=None):
+ """{used, limit, remaining, resetsOn, paused, message}: the ONE reading of month and reset
+ that both the panel and the enforcement use."""
+ now = now or _now()
+ doc = doc if isinstance(doc, dict) else {}
+ months = doc.get("months") if isinstance(doc.get("months"), dict) else {}
+ try:
+ used = max(0, int(months.get(_month_key(now)) or 0))
+ except (TypeError, ValueError):
+ used = 0
+ limit = _limit(doc)
+ remaining = max(0, limit - used)
+ reset = _reset_on(now)
+ paused = remaining <= 0
+ message = (f"This workspace has used this month's drafting allowance. "
+ f"It resets on {_spoken_date(reset)}.") if paused else ""
+ return {"used": used, "limit": limit, "remaining": remaining,
+ "resetsOn": reset.isoformat(), "paused": paused, "message": message}
+
+
+def _short_message(remaining):
+ drafts = "1 draft" if remaining == 1 else f"{remaining} drafts"
+ channels = "1 channel" if remaining == 1 else f"up to {remaining} channels"
+ return f"This workspace has {drafts} left this month. Choose {channels}."
+
+
+def _refusal(state, wanted):
+ """The 429 a draft of `wanted` drafts earns against `state`, or None when it fits."""
+ if state["remaining"] <= 0:
+ return err(429, "drafting_allowance_used", state["message"])
+ if state["remaining"] < wanted:
+ return err(429, "drafting_allowance_short", _short_message(state["remaining"]))
+ return None
+
+
+def _read_allowance(st):
+ doc = st.get(ALLOWANCE_KEY)
+ return doc if isinstance(doc, dict) else {}
+
+
+def _count(st, month, delta):
+ """Add `delta` drafts to `month` in one read-modify-write; keeps the last 12 months."""
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
+ months[month] = max(0, int(months.get(month) or 0) + int(delta))
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
+ months.pop(stale, None)
+ doc["months"] = months
+ return doc
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+
+
+def reserve_drafts(st, wanted, now=None):
+ """Check AND reserve `wanted` drafts in ONE store update, so two concurrent drafts cannot both
+ take the last slot. Raises the 429 (no write) when it does not fit; returns the month key the
+ reservation was booked to, for `release_drafts` on a failure."""
+ now = now or _now()
+ try:
+ current = _read_allowance(st)
+ except Exception: # noqa: BLE001
+ raise err(503, "allowance_unavailable",
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
+ refusal = _refusal(allowance_state(current, now), wanted)
+ if refusal is not None: # fast path: a paused workspace writes nothing
+ raise refusal
+ month = _month_key(now)
+ verdict = {}
+
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ refused = _refusal(allowance_state(doc, now), wanted)
+ if refused is not None:
+ verdict["refusal"] = refused
+ return doc
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
+ months[month] = max(0, int(months.get(month) or 0)) + int(wanted)
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
+ months.pop(stale, None)
+ doc["months"] = months
+ return doc
+
+ try:
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+ except Exception: # noqa: BLE001
+ raise err(503, "allowance_unavailable",
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
+ if "refusal" in verdict:
+ raise verdict["refusal"]
+ return month
+
+
+def release_drafts(st, month, n):
+ """Hand back a reservation whose drafts were never written (AI failure, copy refusal, store)."""
+ try:
+ _count(st, month, -int(n))
+ except Exception: # noqa: BLE001
+ pass
+
+
+@router.put("/allowance")
+def set_allowance(body: dict = Body(default=None), session: Session = Depends(require_session)):
+ """The owner raises (or resets) one tenant's monthly drafting allowance. Platform admin only."""
+ import core.platform_admin as platform_admin
+ from harness import runtime
+ if not platform_admin.is_platform_admin(session.user):
+ raise err(403, "forbidden", "your account does not have access to this surface")
+ body = body if isinstance(body, dict) else {}
+ slug = str(body.get("tenant") or "").strip().lower()
+ monthly = body.get("monthly")
+ if monthly is not None and (isinstance(monthly, bool) or not isinstance(monthly, int)
+ or monthly < 0):
+ raise err(400, "bad_allowance",
+ "The monthly allowance must be a whole number of drafts, 0 or more.")
+ if not slug or slug not in runtime.known_tenants():
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
+ try:
+ st = runtime.get_runtime(slug)
+ except KeyError:
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
+
+ def _up(doc):
+ doc = dict(doc) if isinstance(doc, dict) else {}
+ if monthly is None:
+ doc.pop("allowance", None)
+ else:
+ doc["allowance"] = int(monthly)
+ return doc
+
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
+ return {"tenant": slug, "allowance": allowance_state(_read_allowance(st))}
+
+
+@router.get("/references")
+def list_references(app: str = "", session: Session = Depends(require_session)):
+ """The References a Draft can use: id, title, author, network, and whether its structure is in.
+ Also this workspace's drafting allowance, so the panel reports the limit before it bites."""
+ _app, _posts, references = _studio(session, app)
+ rows = _reference_rows(session, references)
+ try:
+ allowance = allowance_state(_read_allowance(session.runtime))
+ except Exception: # noqa: BLE001
+ allowance = None # unreadable: the draft route still enforces it
+ return {"references": [
+ {"id": rid, "title": str(row.get("title") or row.get("url") or "Untitled reference")[:160],
+ "author": str(row.get("author") or "")[:120], "network": str(row.get("network") or ""),
+ "extracted": bool(str(row.get("structure") or "").strip())}
+ for rid, row in rows.items()],
+ "channels": list(CHANNELS), "advice": dict(CHANNEL_ADVICE), "allowance": allowance}
+
+
+# ---------------------------------------------------------------------------------------------
+# S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an
+# App ... it should act like Catalog / Route where it just uses the data module as what its
+# building on top." So the App opens in the App module on its OWN surface (`#/app?studio=`,
+# `apps/contentStudio/ContentStudioApp.tsx`): a calendar and a board over Posts, the Draft door,
+# and the two databases underneath. Data still holds References and Posts; the App never sends
+# anyone there to use it.
+# ⚠ The shared `/applications` doors (`routes_assets.py`) admit only topic-backed databases, so
+# they refuse an App bound to a user table; the read, star and delete this App needs live here,
+# on the registry's own writers, behind the same `_studio` + `may_read` / `may_edit` walls.
+# ---------------------------------------------------------------------------------------------
+def app_link(app_id):
+ """The one address a Content Studio App opens at (App module, its own surface)."""
+ return f"/app?studio={app_id}"
+
+
+#: The Posts fields the surface draws, in the grid's own Field shape so `viewModes.CalendarView`
+#: (by Scheduled) and `KanbanView` (by Status) render them unchanged.
+SURFACE_FIELDS = ("caption", "channel", "scheduled", "status", "attribution", "campaign")
+
+
+def _surface_field(defn):
+ kind = {"date": "date", "select": "select"}.get(defn.get("type"), "text")
+ field = {"key": defn["key"], "label": str(defn.get("label") or defn["key"]), "type": kind,
+ "source": "overlay"}
+ if kind == "select":
+ field["options"] = [str(o) for o in (defn.get("options") or ())]
+ return field
+
+
+def _studio_or_missing(session, app_id):
+ """`_studio`, answering an absent App with the code the App home already reads as 'gone'."""
+ try:
+ return _studio(session, app_id)
+ except Exception as exc: # noqa: BLE001
+ if getattr(exc, "status_code", None) == 404:
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
+ raise
+
+
+@router.get("/studio")
+def studio(app: str = "", session: Session = Depends(require_session)):
+ """Everything the App surface draws: the App, Posts (surface fields + rows), References'
+ count, the statuses in board order, and the drafting allowance."""
+ from core import user_tables
+ application, posts, references = _studio(session, app)
+ if not user_tables.may_open(posts, session.uname, session.admin, st=session.runtime):
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
+ posts_def = user_tables.get(posts, session.runtime) or {}
+ refs_def = user_tables.get(references, session.runtime) or {}
+ by_key = {f.get("key"): f for f in posts_def.get("fields") or () if isinstance(f, dict)}
+ fields = [_surface_field(by_key[key]) for key in SURFACE_FIELDS if key in by_key]
+ rows = []
+ for rid, row in (posts_def.get("rows") or {}).items():
+ try:
+ pid = int(rid)
+ except (TypeError, ValueError):
+ continue
+ row = row if isinstance(row, dict) else {}
+ rows.append({"pid": pid, **{key: row.get(key) for key in SURFACE_FIELDS}})
+ rows.sort(key=lambda r: r["pid"])
+ try:
+ allowance = allowance_state(_read_allowance(session.runtime))
+ except Exception: # noqa: BLE001
+ allowance = None
+ return {"app": {"id": application["id"], "name": application.get("name") or LABEL},
+ "posts": {"database": posts, "label": str(posts_def.get("label") or "Posts"),
+ "fields": fields, "rows": rows},
+ "references": {"database": references,
+ "label": str(refs_def.get("label") or "References"),
+ "count": len(refs_def.get("rows") or {})},
+ "statuses": list(STATUSES), "allowance": allowance}
+
+
+@router.patch("/apps/{app_id}")
+def star_app(app_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Star or unstar a Content Studio on the App home (the registry's own star writer)."""
+ import application_registry
+ _app, posts, _references = _studio_or_missing(session, app_id)
+ body = body if isinstance(body, dict) else {}
+ if set(body) != {"starred"} or not isinstance(body.get("starred"), bool):
+ raise err(400, "bad_patch", "Only the star of a Content Studio changes here.")
+ try:
+ landed = application_registry.mutate(session.runtime, app_id, session.uname, session.admin,
+ {"starred": body["starred"]}, allowed_databases=[posts])
+ except PermissionError:
+ raise err(403, "forbidden", "You cannot change this App.")
+ except application_registry.ApplicationError:
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
+ return {"application": landed}
+
+
+@router.delete("/apps/{app_id}")
+def delete_app(app_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Delete the App and nothing else (rule 13): References and Posts, every row in them, and
+ every View on them outlive it. The name is confirmed inside the registry's own update."""
+ import application_registry
+ application, posts, _references = _studio_or_missing(session, app_id)
+ confirm = (body or {}).get("confirmName") if isinstance(body, dict) else None
+ if not isinstance(confirm, str) or confirm != application.get("name"):
+ raise err(400, "confirm_name", "Enter the exact App name to confirm.")
+
+ class _NameConfirmed:
+ def update(self, key, delete):
+ def confirmed(raw):
+ deleted = delete(raw)
+ if (raw.get("applications") or {}).get(app_id, {}).get("name") != confirm:
+ raise application_registry.ApplicationError("the App name changed")
+ return deleted
+ return session.runtime.update(key, confirmed)
+
+ try:
+ return application_registry.delete(_NameConfirmed(), app_id, session.uname, session.admin,
+ allowed_databases=[posts])
+ except PermissionError:
+ raise err(403, "forbidden", "You no longer have permission to delete this App.")
+ except application_registry.ApplicationError:
+ raise err(400, "confirm_name", "The App name may have changed. Close this dialog and reload to retry.")
+
+
+def _words(text):
+ return re.findall(r"[a-z0-9']+", str(text or "").lower())
+
+
+def copied_sentence(draft, sources):
+ """The first run of VERBATIM_WORDS consecutive words a draft shares with any source, or ''."""
+ grams = set()
+ for source in sources:
+ words = _words(source)
+ grams.update(" ".join(words[i:i + VERBATIM_WORDS])
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)))
+ words = _words(draft)
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)):
+ gram = " ".join(words[i:i + VERBATIM_WORDS])
+ if gram in grams:
+ return gram
+ return ""
+
+
+def attribution(rows):
+ """Every selected reference's author, credited in plain words (R14 traceability)."""
+ names = list(dict.fromkeys(
+ str(row.get("author") or row.get("title") or "an unnamed source").strip() for row in rows))
+ joined = names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
+ return f"Structure drawn from posts by {joined}."
+
+
+def draft_prompt(topic, channel, rows):
+ parts = [f"Channel: {channel}. {CHANNEL_ADVICE.get(channel, '')}", f"Topic: {topic}", ""]
+ for n, row in enumerate(rows, 1):
+ parts.append(f"Reference {n} ({row.get('network') or 'post'}):")
+ for key, label in (("hook", "Hook technique"), ("structure", "Structure"), ("angle", "Angle")):
+ value = " ".join(str(row.get(key) or "").split())[:900]
+ if value:
+ parts.append(f" {label}: {value}")
+ parts.append("")
+ parts.append("Write the post now.")
+ return "\n".join(parts)
+
+
+def draft_posts(session, app_id, reference_ids, topic, channels, *, ask=None):
+ """Generate EVERY channel first; write the Posts rows only if all succeeded (no half row)."""
+ topic = " ".join(str(topic or "").split())[:200]
+ ids = [str(x) for x in dict.fromkeys(reference_ids or ())][:50]
+ wanted = [c for c in dict.fromkeys(channels or ()) if c in CHANNELS]
+ if not ids:
+ raise err(400, "no_references", "Select at least one reference to draft from.")
+ if not topic:
+ raise err(400, "no_topic", "Type a one-line topic for the draft.")
+ if not wanted:
+ raise err(400, "no_channels", "Choose at least one channel.")
+ _app, posts, references = _studio(session, app_id)
+ rows_by_id = _reference_rows(session, references)
+ missing = [rid for rid in ids if rid not in rows_by_id]
+ if missing:
+ raise err(409, "references_missing", "A selected reference no longer exists. Reload and try again.")
+ chosen = [rows_by_id[rid] for rid in ids]
+ if not any(str(row.get("structure") or "").strip() for row in chosen):
+ raise err(409, "references_not_extracted",
+ "The selected references have no extracted structure yet. Paste their text first.")
+ # R35: the allowance is checked and reserved BEFORE any provider is resolved or called. One
+ # draft per channel; the reservation is handed back unless every row is written.
+ month = reserve_drafts(session.runtime, len(wanted))
+ try:
+ return _generate_and_save(session, posts, ids, topic, wanted, chosen, ask=ask)
+ except BaseException:
+ release_drafts(session.runtime, month, len(wanted))
+ raise
+
+
+def _generate_and_save(session, posts, ids, topic, wanted, chosen, *, ask=None):
+ import ai_enrich
+ import ai_review
+ import usage_ledger
+ from core import user_tables
+
+ live = ai_review.ladder()
+ if not live and ask is None:
+ raise err(503, "ai_unavailable", "No AI provider is configured here, so nothing was drafted.")
+ provider = (live or [{"name": "injected", "model": "injected"}])[0]
+ model = provider["model"]
+ caller = ask or ai_enrich._ask
+ sources = [row.get("pasted_text") or "" for row in chosen]
+ drafts, tokens, calls = [], 0, 0
+ try:
+ for channel in wanted:
+ prompt = draft_prompt(topic, channel, chosen)
+ text = ""
+ for attempt in range(2):
+ answer, used, problem = caller(provider, model, prompt, DRAFT_MAX_TOKENS,
+ DRAFT_TIMEOUT_S, system=DRAFT_SYSTEM)
+ calls += 1
+ tokens += int(used or 0)
+ if problem or not str(answer or "").strip():
+ raise err(502, "draft_failed", f"The {channel} draft could not be written "
+ f"({problem or 'the model returned nothing'}). Nothing was saved.")
+ text = ai_enrich.no_dashes(str(answer).strip())
+ copied = copied_sentence(text, sources)
+ if not copied:
+ break
+ prompt += f"\n\nYour last draft reused this wording from a reference: \"{copied}\". Rewrite it in new words."
+ text = ""
+ if not text:
+ raise err(502, "draft_copied", f"The {channel} draft kept repeating a reference's "
+ "wording, so it was not saved. Nothing was saved.")
+ drafts.append({"channel": channel, "caption": text})
+ finally:
+ if calls:
+ # `usage_ledger.SURFACES` names four surfaces and refuses any other, so drafts are
+ # booked as `field_agent` (AI-written cells of a user table) until a `content_studio`
+ # surface exists there (a finding for the ledger's owner; this module cannot add it).
+ usage_ledger.record(LEDGER_SURFACE, provider["name"], model, total=tokens,
+ calls=calls, st=session.runtime, user=session.uname)
+ credit = attribution(chosen)
+ new_rows = [{"caption": d["caption"], "channel": d["channel"], "status": "Draft",
+ "references": ",".join(ids), "attribution": credit} for d in drafts]
+ created = user_tables.add_rows(posts, new_rows, username=session.uname, st=session.runtime)
+ if not created:
+ raise err(503, "store_unavailable", "The drafts could not be saved. Nothing was saved.")
+ return {"created": list(created) if isinstance(created, (list, tuple)) else created,
+ "drafts": drafts, "attribution": credit, "tokens": tokens, "calls": calls,
+ "provider": provider["name"], "model": model}
+
+
+@router.post("/draft")
+def draft(body: dict = Body(default=None), session: Session = Depends(require_session)):
+ body = body if isinstance(body, dict) else {}
+ return draft_posts(session, body.get("app"), body.get("references"), body.get("topic"),
+ body.get("channels"))
diff --git a/api/deps.py b/api/deps.py
index c1c42a56dcff4efb1625e6c85c3fae99166ddb0a..bcb27545f37c62d376975e77f2fffbae144d2844 100644
--- a/api/deps.py
+++ b/api/deps.py
@@ -232,6 +232,28 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
else ("product_pool", team_id))
cache = getattr(session.runtime, "pool_cache", None)
entry = cache.get(cache_key) if isinstance(cache, dict) else None
+ if not isinstance(entry, tuple) and isinstance(cache, dict):
+ # ⭐⭐ S67-T33 / D-778: A DATABASE NEVER OPENED ANSWERS. The snapshot used to exist only
+ # once somebody opened the grid in this process, so after every restart the assistant
+ # refused a database its caller demonstrably holds. The miss is now filled by the GRID'S
+ # OWN pool builder, for the SAME scope key the grid would fill (`derive_pool_scope`
+ # above), so this is the grid's first open, not a second reader. It is read-only on Odoo
+ # (rule 4) and honours the connector pause exactly as the grid does (503 when paused
+ # with nothing cached). Permission and field checks above have already run.
+ try:
+ if database == "customer_data":
+ import routes_customers
+ routes_customers._pool_for(session.runtime, team_id, agent)
+ else:
+ import routes_products
+ routes_products._pool_for(session.runtime, team_id)
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ _assistant_refuse(409, "assistant_snapshot_unavailable",
+ f"this database's snapshot could not be built for your current "
+ f"scope ({type(exc).__name__}); try again shortly")
+ entry = cache.get(cache_key)
if (not isinstance(entry, tuple) or len(entry) < 2
or not isinstance(entry[1], (list, tuple))
or any(not isinstance(row, dict) for row in entry[1])):
@@ -247,6 +269,44 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
_ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
+def _assistant_nav_opens(session: Session, database: str, lent, definition=None) -> bool:
+ """Does the NAVIGATION offer this `ut_*` database to this caller? S67-T33 / R33.
+
+ ⭐⭐ THE AI CONNECTOR'S LIST IS THE NAV'S LIST, BY CONSTRUCTION RATHER THAN BY AGREEMENT.
+ `routes_nav.nav` admits a user table when `user_tables.nav_entries` lists it (which is
+ `may_open` plus the route-system exclusion), it is not a retired key, and
+ `perm_scope.nav_may_open` agrees (an administrator's explicit untick denies). The assistant
+ used to ask `may_open` alone, so a database the member's navigation did not show could still
+ be listed and read through the connector. This asks the nav's own predicates, in the nav's
+ order; it is not a second permission rule.
+ ⚠ `session.admin` short-circuits leg 1 of `nav_may_open` exactly as the nav's own admin
+ session does; `nav_may_open` itself returns True for an administrator.
+ """
+ import core.perm_scope as perm_scope
+ from core import route_system, user_tables
+
+ key = str(database or "")
+ try:
+ from odoo_relational import RETIRED_KEYS as retired
+ except Exception: # noqa: BLE001
+ retired = ()
+ if key in retired:
+ return False
+ if not user_tables.may_open(key, session.uname, session.admin, st=lent):
+ return False
+ if isinstance(definition, dict) and (route_system.is_managed(definition)
+ or route_system.is_managed(key)):
+ return False
+ if session.admin:
+ return True
+ user = getattr(session, "user", None)
+ if not isinstance(user, dict):
+ # No account record means no explicit per-key deny can exist, and `nav_may_open` leg 3
+ # then defers to `may_open`, which has already answered above.
+ return True
+ return bool(perm_scope.nav_may_open(user, key, st=lent))
+
+
def _assistant_table_defn(session: Session, database: str):
"""The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
@@ -276,7 +336,9 @@ def _assistant_table_defn(session: Session, database: str):
if not isinstance(definition, dict):
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
- if not user_tables.may_open(database, session.uname, session.admin, st=definition_runtime):
+ # S67-T33: the NAV's predicate, not `may_open` alone, so a database the caller's navigation
+ # does not show is refused here too (403), not merely left off the list.
+ if not _assistant_nav_opens(session, database, definition_runtime, definition):
# Deliberately no label/owner in this response: knowing a key must not become an oracle
# for somebody else's database.
_assistant_refuse(403, "assistant_forbidden",
@@ -346,7 +408,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
- *, defn=None, lent=None):
+ *, defn=None, lent=None, mode=None, extra=None):
"""Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
@@ -382,11 +444,24 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
# THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
# A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
# about it would open a cursor to learn nothing.
- if assistant_mirror.declared_fields(database) is None:
+ if database.startswith("ut_amzn_"):
+ # ⭐ S67-T33: THE GRID'S OWN ADMISSION FIRST. A dynamically sized Amazon snapshot enters
+ # the source registry only through `require_amazon_mirror`, so without it a database
+ # never opened in this process had no binding here (measured on staging for
+ # `ut_amzn_fba_inventory`). It refuses by name (503) while the Amazon mirror is unloaded.
+ import routes_odoo_tables
+ routes_odoo_tables.require_amazon_mirror(database, session, definition=definition)
+ # ⭐ S67-T33: the stored definition rides along, so an Amazon grid or a generated
+ # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
+ declared = assistant_mirror.declared_fields(database, definition)
+ if declared is None:
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database does not store its rows and it is not one of the "
"connected databases the assistant can read from the mirror")
- cur, refusal = assistant_mirror.open_cursor(session)
+ # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
+ # first sync. Passed only for an Amazon key, which is the one family it changes.
+ cur, refusal = (assistant_mirror.open_cursor(session, database)
+ if database.startswith("ut_amzn_") else assistant_mirror.open_cursor(session))
if refusal is not None:
_assistant_refuse(refusal.status, refusal.code, refusal.message)
spec = assistant_mirror.binding(cur, database)
@@ -394,7 +469,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database is served live from its connector and this deployment "
"has no read-through binding for it yet, so it holds no rows to read")
- fields = assistant_mirror.answerable_fields(database, spec)
+ fields = assistant_mirror.answerable_fields(database, spec, declared)
if not fields:
_assistant_refuse(409, "assistant_snapshot_unavailable",
"this database's connector mirror carries none of its declared columns "
@@ -422,12 +497,36 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
if grant is None:
wall_tree = None
+ filter_keys = [field["key"] for field in visible
+ if isinstance(field, dict) and field.get("key")]
+ # ⭐⭐ S67-T34: THREE READS BEHIND THE ONE PREFLIGHT ABOVE, chosen by `mode`, all compiling the
+ # SAME wall and filter (`assistant_mirror._predicate`). "schema" reads and counts no row
+ # (D-2630: describe answers the field list of a million-row table without counting it);
+ # ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
+ # ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
try:
- rows, stamp, total = assistant_mirror.read_rows(
- session, database, cur, spec, fields,
- wall_tree=wall_tree, filter_tree=filter_tree,
- filter_keys=[field["key"] for field in visible
- if isinstance(field, dict) and field.get("key")])
+ if mode == "schema":
+ return [], fields, {"read": "connector mirror"}, definition.get("label") or database
+ if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
+ values, total = assistant_mirror.aggregate(
+ session, database, cur, spec, fields, mode[1],
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
+ if isinstance(extra, dict):
+ extra["aggregates"], extra["total"] = values, total
+ return ([], fields, {"read": "connector mirror", "rows": total},
+ definition.get("label") or database)
+ if isinstance(mode, tuple) and mode and mode[0] == "page":
+ rows, stamp, total = assistant_mirror.read_page(
+ session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
+ defn=definition)
+ if isinstance(extra, dict):
+ extra["total"] = total
+ else:
+ rows, stamp, total = assistant_mirror.read_rows(
+ session, database, cur, spec, fields,
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
+ defn=definition)
except assistant_mirror.MirrorRefusal as exc:
_assistant_refuse(exc.status, exc.code, exc.message)
raise # unreachable; see above
@@ -459,7 +558,33 @@ def _assistant_requested_fields(fields, visible_fields):
return out
-def assistant_read_scope(session: Session, database, fields=None, filters=None):
+def _assistant_oversize_reason(database: str, definition, lent) -> str:
+ """The named cause an oversize stored table carries, or "". S67-T33 / rule 1.
+
+ ⛔ AN EVICTED TABLE KEEPS ITS DEFINITION AND LOSES ITS ROWS, so the materialised reader used
+ to answer it as a complete table of 0 rows. `user_tables.limit_report` is the one evaluator
+ of that stamp (the grid serves the same sentence), so its cause and recommendation are read
+ from there rather than re-worded here.
+ """
+ from core import user_tables
+
+ if not user_tables.oversize_of(database, defn=definition if isinstance(definition, dict)
+ else {}):
+ return ""
+ report = user_tables.limit_report(database, st=lent) or {}
+ label = str((definition or {}).get("label") or database)
+ return (f"{label} is not stored: {report.get('cause') or 'its rows were not stored'}. "
+ f"{str(report.get('recommendation') or '').rstrip('.')}. The assistant refuses "
+ f"rather than answering 0 rows").replace("\u2014", ",").replace("\u2013", "-")
+
+
+def _assistant_refuse_oversize(database: str, definition, lent) -> None:
+ reason = _assistant_oversize_reason(database, definition, lent)
+ if reason:
+ _assistant_refuse(409, "assistant_source_too_large", reason)
+
+
+def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None):
"""Return a detached, permissioned snapshot for one Assistant tool call.
Signature (the C8 contract for E):
@@ -468,6 +593,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
permission_scope_applied, fields, filters, records
}
+ ⭐ S67-T34: `mode` changes only HOW a mirror-served database is read, never who may read it
+ (every wall below runs unchanged): "schema" returns the field contract with no row read,
+ ("aggregate", specs) adds `server_aggregates` and `total_rows` computed in SQL over every
+ permitted row, ("page", offset, limit) returns one SQL page plus `total_rows`. Every other
+ source kind ignores it and answers as before.
+
`database` is exactly one source. `fields` is an explicit list of visible field keys (or
``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
`core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
@@ -478,6 +609,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
from harness import filter_eval
key = str(database or "").strip()
+ extra = {}
if key in _ASSISTANT_BUILT_INS:
rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
label = _ASSISTANT_BUILT_INS[key]["label"]
@@ -494,12 +626,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
from core import user_tables
_defn, _lent = _assistant_table_defn(session, key)
if user_tables.materialises(key, st=_lent, defn=_defn):
+ # S67-T33: a stored table whose rows were evicted refuses by name, never answers 0.
+ # Only on the STORED reader: a read-through table's rows come from the mirror.
+ _assistant_refuse_oversize(key, _defn, _lent)
rows, canonical_fields, source_version, label = _assistant_user_table(
session, key, fields, filters, defn=_defn, lent=_lent)
source_kind = "materialised_user_table"
else:
rows, canonical_fields, source_version, label = _assistant_mirror_table(
- session, key, fields, filters, defn=_defn, lent=_lent)
+ session, key, fields, filters, defn=_defn, lent=_lent, mode=mode, extra=extra)
source_kind = "mirror_user_table"
else:
_assistant_refuse(404, "assistant_unknown_database", "that database is not available")
@@ -556,7 +691,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
records.append({"pid": copy.deepcopy(row["pid"]),
**{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
- return {
+ out = {
"database": key,
"label": str(label),
"source_kind": source_kind,
@@ -568,6 +703,11 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
"filters": copy.deepcopy(requested_filter),
"records": records,
}
+ if "aggregates" in extra:
+ out["server_aggregates"] = copy.deepcopy(extra["aggregates"])
+ if "total" in extra:
+ out["total_rows"] = int(extra["total"])
+ return out
def assistant_permitted_sources(session: Session, databases):
@@ -672,6 +812,19 @@ def assistant_source_status(session: Session, databases=None):
mirror_cur = None
return mirror_cur
+ amazon_cur = {}
+
+ def _amazon_mirror(key):
+ """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
+ if "cur" not in amazon_cur:
+ try:
+ import assistant_mirror
+ cur, refusal = assistant_mirror.open_cursor(session, key)
+ amazon_cur["cur"] = None if refusal is not None else cur
+ except Exception: # noqa: BLE001
+ amazon_cur["cur"] = None
+ return amazon_cur["cur"]
+
def _definitions():
nonlocal definitions, read_definitions
if not read_definitions:
@@ -752,6 +905,10 @@ def assistant_source_status(session: Session, databases=None):
entry = cache.get(cache_key) if isinstance(cache, dict) else None
if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
row["answerable"] = True
+ elif isinstance(cache, dict):
+ # ⭐ S67-T33 / D-778: a miss is no longer a refusal; the read builds the snapshot
+ # through the grid's own pool builder, so this source is answerable now.
+ row["answerable"] = True
else:
row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
continue
@@ -762,10 +919,16 @@ def assistant_source_status(session: Session, databases=None):
if not isinstance(definition, dict):
continue
lent = user_tables.lend(session.runtime, user_tables=defs)
- if not user_tables.may_open(key, session.uname, session.admin, st=lent):
+ # ⭐ S67-T33 / R33: the NAV's predicate, so this list and the navigation are one set.
+ if not _assistant_nav_opens(session, key, lent, definition):
continue
row["permitted"] = True
if user_tables.materialises(key, st=lent, defn=definition):
+ oversize = _assistant_oversize_reason(key, definition, lent)
+ if oversize:
+ # Rule 1's second sentence: listed, with its cause, never answered as 0 rows.
+ row["reason"] = oversize
+ continue
row["answerable"] = True
continue
# ⭐⭐ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
@@ -778,7 +941,9 @@ def assistant_source_status(session: Session, databases=None):
# is transient and retrying works, while a grid this deployment has no read-through
# binding for will not fix itself by waiting. One sentence for both would make the first
# look permanent, which is the failure mode the ruling exists to prevent.
- cur = _mirror()
+ # ⭐ S67-T33: an Amazon grid asks ITS OWN mirror (`_table_cursor`), never the Odoo
+ # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
+ cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
if cur is None:
row["reason"] = ("this database is read live from its connector mirror, and that "
"mirror is not open on this deployment right now; try again shortly")
@@ -801,6 +966,49 @@ def assistant_source_status(session: Session, databases=None):
return out
+#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
+#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
+#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
+#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
+#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
+#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
+#: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
+#: answers 503 when the store is unreadable); a cached registry is never served in its place.
+#: The key carries the identity of the reader and the store it reads, so a swapped store or a
+#: patched `users.registry` / `store.get` misses instead of replaying another registry.
+_SESSION_REGISTRY = {"key": None, "reg": None}
+_SESSION_REGISTRY_LOCK = __import__("threading").Lock()
+
+
+def _session_registry():
+ """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed.
+
+ ⛔ POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`),
+ and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
+ fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
+ memo on HF), and its `get()` already holds the bucket for the life of the process, so the
+ re-read this memo removes does not exist there: it reads straight through.
+ """
+ key = None
+ try:
+ backend = store.backend()
+ token = (store.revision("users") or {}).get("token") if backend == "pg" else None
+ if token not in (None, ""):
+ where = os.environ.get("AIOS_TENANT") or "royal-imports"
+ key = (backend, where, id(users.registry), id(store.get), str(token))
+ except Exception: # noqa: BLE001 -- fail-closed below
+ key = None
+ if key is not None:
+ with _SESSION_REGISTRY_LOCK:
+ if _SESSION_REGISTRY["key"] == key:
+ return _SESSION_REGISTRY["reg"]
+ reg = users.registry() or {}
+ if key is not None:
+ with _SESSION_REGISTRY_LOCK:
+ _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
+ return reg
+
+
def _user_for(claims):
"""The account named by a verified cookie, or None — with the epoch check that makes a
stateless cookie revocable.
@@ -833,12 +1041,12 @@ def _user_for(claims):
# `Exception` into `{}`, so a store `PoolTimeout` on staging answered a valid cookie with 401
# `invalid_session` and the client signed the owner out. Still fail-closed: nobody is admitted.
try:
- reg = users.registry() or {}
+ reg = _session_registry()
except Exception as exc:
print(f"[aios-api] session account store unavailable: {type(exc).__name__}: {exc}")
raise err(503, "session_store_unavailable",
"your account could not be checked just now. Try again in a moment")
- rec = reg.get(uname)
+ rec = copy.deepcopy(reg.get(uname)) # the registry may be the held copy: never mutate it
if (rec and rec.get("active", True)
and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
pub = users._public(uname, rec)
diff --git a/api/formula_runtime.py b/api/formula_runtime.py
index 65c76eb99f8fc3ec8fe305392c07db0c98049733..7f12e7df4885a3d442cb6a74dfc47c66d9a3b4d6 100644
--- a/api/formula_runtime.py
+++ b/api/formula_runtime.py
@@ -83,6 +83,8 @@ FN_ARITY = {
"ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
"FILTER": (2, 3), "SORT": (1, 4),
+ # S67-T24: the reference workbooks' `IF(ISNUMBER(SEARCH(...)))` and `ISFORMULA(...)`.
+ "SEARCH": (2, 3), "SEARCHB": (2, 3), "ISFORMULA": (1, 1),
}
# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
@@ -457,6 +459,15 @@ FN_CATALOGUE = {
"description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
"ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
"description": "Returns TRUE when a value is text.", "contexts": _BOTH},
+ "ISFORMULA": {"category": "Information", "signature": "ISFORMULA(reference)",
+ "description": "Returns TRUE when the cell it names holds a formula.",
+ "contexts": _SHEET},
+ "SEARCH": {"category": "Text", "signature": "SEARCH(find_text, within_text, [start_num])",
+ "description": "Returns the position of one text value inside another, ignoring "
+ "case; ? matches one character and * any run.", "contexts": _BOTH},
+ "SEARCHB": {"category": "Text", "signature": "SEARCHB(find_text, within_text, [start_num])",
+ "description": "Counts like SEARCH; each character is one byte in this workbook.",
+ "contexts": _BOTH},
"ISERROR": {"category": "Information", "signature": "ISERROR(value)",
"description": "Returns TRUE when a formula would refuse to produce a value.",
"contexts": _BOTH},
@@ -625,6 +636,9 @@ PARITY_FIXTURES = {
"EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
"ISNUMBER": ("ISNUMBER(1)", "TRUE"),
"ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
+ "SEARCH": ("SEARCH(\"B\", \"abc\")", 2),
+ "SEARCHB": ("SEARCHB(\"c?\", \"abcd\")", 3),
+ "ISFORMULA": ("ISFORMULA(B2)", "FALSE"),
"ISERROR": ("ISERROR(1 / 0)", "TRUE"),
}
@@ -917,7 +931,7 @@ RANGE_ARGS = {
#: ⭐ THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
#: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
-ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,)}
+ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,), "ISFORMULA": (0,)}
def _is_range_position(fn, index):
@@ -1863,6 +1877,17 @@ def _eval_value(node, row, today, diagnostics=None):
for code in local:
_diagnose(diagnostics, code)
return "" if first is None and isinstance(row, _SheetInputs) and not local else first
+ if fn in ("ISNUMBER", "ISTEXT") and len(node.get("args") or ()) == 1:
+ # ⭐ S67-T24: AN IS-FUNCTION NEVER PASSES AN ERROR ON. Excel answers FALSE for
+ # `ISNUMBER(#VALUE!)`, which is what makes `IF(ISNUMBER(SEARCH(x, y)), ...)` (2,512 cells
+ # in the Inventory System's Catalog) take its FALSE branch instead of failing whole.
+ local = []
+ try:
+ value = _eval(node["args"][0], row, today, local)
+ except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error,
+ FormulaError, _ReferenceFault):
+ return False
+ return False if local else _call(fn, [value], today, diagnostics)
if fn == "ISERROR":
# The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
local = []
@@ -1893,6 +1918,11 @@ def _eval_value(node, row, today, diagnostics=None):
_diagnose(diagnostics, "bad_value")
return None
args.append(here)
+ if fn == "ISFORMULA":
+ # ⭐ S67-T24: ASKED OF THE CELL, NOT OF ITS VALUE. Only a Sheet document knows which
+ # cells hold formulas (`SheetCalculation.specs`); a grid row has no formula cells.
+ spec = row.engine.specs.get(args[0]) if isinstance(row, _SheetInputs) and args[0] else None
+ return bool(isinstance(spec, dict) and spec.get("formula"))
return _call(fn, args, today, diagnostics)
# ⭐ RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
# `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
@@ -1928,6 +1958,27 @@ def _eval_value(node, row, today, diagnostics=None):
raise FormulaError("formula AST node is not supported")
+def _display_text(value):
+ """A calculated value as the grid shows it: Excel's TRUE / FALSE for a boolean (S67-T24; a
+ Python `str(True)` painted "True"), blank for nothing, else the value's own text."""
+ if value is None:
+ return ""
+ if isinstance(value, bool):
+ return "TRUE" if value else "FALSE"
+ return str(value)
+
+
+def _search_pattern(text):
+ """SEARCH's wildcard text as a regex: `?` one character, `*` any run, `~x` a literal x."""
+ out, index = [], 0
+ while index < len(text):
+ ch = text[index]
+ if ch == "~" and index + 1 < len(text) and text[index + 1] in "?*~":
+ out.append(re.escape(text[index + 1])); index += 2; continue
+ out.append(".*?" if ch == "*" else "." if ch == "?" else re.escape(ch)); index += 1
+ return "".join(out)
+
+
def _addr_arg(node, diagnostics):
"""The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
if isinstance(node, dict) and node.get("t") == "a1":
@@ -2324,6 +2375,19 @@ def _call(fn, a, today, diagnostics=None):
if fn == "FIND":
start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
return pos+1 if pos >= 0 else None
+ if fn in ("SEARCH", "SEARCHB"):
+ # Excel's SEARCH: case-insensitive, `?` one character, `*` any run, `~` escapes the next;
+ # absent text or a start outside the text is #VALUE!. SEARCHB counts bytes only in a
+ # double-byte locale, so in this one it is SEARCH.
+ within = _text(a[1])
+ start = _strict_num(a[2], diagnostics) if len(a) > 2 else 1
+ if start is None or not 1 <= int(start) <= len(within) + 1:
+ _diagnose(diagnostics, "bad_value"); return None
+ found = re.compile(_search_pattern(_text(a[0])), re.IGNORECASE | re.DOTALL).search(
+ within, int(start) - 1)
+ if not found:
+ _diagnose(diagnostics, "bad_value"); return None
+ return found.start() + 1
if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
if fn == "REGEX_EXTRACT": return m.group(0) if m else None
@@ -3727,7 +3791,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
if excel: cell["error"]["excelCode"] = excel
else:
- cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"]))
+ cell.update(state="fresh", display=_display_text(cell["value"]))
cell.pop("refusal", None)
cell.pop("error", None)
cell.pop("spill", None)
@@ -3770,7 +3834,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
base = engine.specs.get(child, {})
style = {**(owner.get("style") or {}), **(base.get("style") or {})}
record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
- "display": "" if engine.values.get(child) is None else str(engine.values[child]),
+ "display": _display_text(engine.values.get(child)),
"spillParent": origin, "style": style,
"formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
**({"base": base} if base else {})}}}
diff --git a/api/main.py b/api/main.py
index 672626ff911d7595404b6caaf81fbf9928b18ef2..be7592d9ef565fde1ca8b11d69dfd4f7edcbf103 100644
--- a/api/main.py
+++ b/api/main.py
@@ -92,6 +92,7 @@ from core import store # noqa: E402
# 409: a view write whose caller was looking at an older revision. See its handler below.
from core import view_audit # noqa: E402
from deps import Session, module_gate # noqa: E402
+from deps import require_session as deps_require_session # noqa: E402 (S67-T20 freshness door)
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
@@ -351,6 +352,21 @@ app.include_router(routes_uploads.router)
# control is that the enumeration must NAME every router in this file.**
app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated
app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory
+
+
+@app.get("/api/v1/connectors/odoo/freshness")
+def odoo_sync_freshness(session: Session = Depends(deps_require_session)):
+ """⭐⭐ S67-T20 / C4 — IS THIS WORKSPACE'S ODOO DATA STALE, AND DID THE LAST SYNC FAIL.
+
+ `{stale, lastGoodSyncAt, schedule, staleAfterSeconds, lastRunAt, phase, frozen, failure}`
+ from `odoo_relational.sync_freshness`. The Connectors card and Connectors > Odoo word it.
+ ⚠ A SIBLING OF `GET /connectors/odoo/status`, not a field on it, because that route is another
+ lane's file this wave; the integrator may fold these fields into it (COMBINE-AMEND, S67-T20).
+ Same administrator door as that route.
+ """
+ routes_connectors._odoo_admin(session)
+ import odoo_relational as _rel_fresh
+ return _rel_fresh.sync_freshness(session.runtime)
# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so
# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
@@ -1359,11 +1375,17 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
cid = str(cid or default_cid)
report = report if isinstance(report, dict) else {}
failed = report.get("failed")
+ # S67-T19: a read-through `limits` notice is the design working, not a failed
+ # database; health reads only the entries that withheld rows (`_rel.health_view`).
database_failures, unmatched_failures = (
- routes_connectors._odoo_result_database_failures(_rt, cid, report))
+ routes_connectors._odoo_result_database_failures(
+ _rt, cid, _rel.health_view(report)))
failure = (routes_connectors._odoo_failure(failed, "materialization")
if failed else None)
- if not failure and (database_failures or unmatched_failures):
+ # S67-T19: count drift within tolerance is recorded per database (below) and does not
+ # fail the connection; a real failure or a stale mirror's gap still does.
+ if not failure and (_rel.blocking_failures(database_failures, report)
+ or unmatched_failures):
failure = routes_connectors._odoo_failure(
"one or more databases failed", "materialization")
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
@@ -1440,6 +1462,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
_ut.record_odoo_sync_status(
phase="error" if safe_errors else "ok", errors=safe_errors,
st=_rt, connection_id=cid)
+ # S67-T20 / C4: the tenant-wide freshness stamp follows the DEFAULT connection, the
+ # one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
+ if cid == str(default_cid):
+ try:
+ _rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at)
+ except Exception: # noqa: BLE001
+ pass
meta_state = _rebuild_meta_relational(
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
if isinstance(meta_state_out, dict):
@@ -1469,6 +1498,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
_ut.record_odoo_sync_status(
phase="error", errors=[f"{failure['code']}: {failure['message']}"],
st=_rt, connection_id=cid)
+ # S67-T20 / C4: a failed rebuild is a sync that did NOT end well.
+ import odoo_relational as _rel_failed
+ _rel_failed.record_sync_outcome(_rt, ok=False)
except Exception:
pass
return False
diff --git a/api/odoo_relational.py b/api/odoo_relational.py
index 7df59114292c98d38c2e7f3201ec10d444b2e614..de4fc7159627e489e0a2c628da3cd6822d6ab9af 100644
--- a/api/odoo_relational.py
+++ b/api/odoo_relational.py
@@ -3619,6 +3619,112 @@ def sync_seconds(rt):
return max(int(secs), SYNC_FLOOR_SECONDS)
+#: ⭐⭐ S67-T20 / C4 — how far past its cadence a sync may run before its data is called stale.
+#: Two hours of grace on top of the interval: a daily sync is stale after 26 hours (C4's number),
+#: long enough that one slow pass does not raise a warning and short enough that a dead sync is
+#: named on the next day, not three weeks later as staging's was.
+STALE_GRACE_SECONDS = 2 * 3600
+
+
+def _utc_now():
+ return _dt.datetime.now(_dt.timezone.utc)
+
+
+def _parse_iso(value):
+ """An ISO timestamp as an aware UTC datetime, or None when it is not one."""
+ if not value:
+ return None
+ try:
+ at = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ return at if at.tzinfo else at.replace(tzinfo=_dt.timezone.utc)
+
+
+def record_sync_outcome(rt, ok, at=None):
+ """Stamp this tenant's last sync, and its last GOOD sync when `ok`, on the connector's config.
+
+ ⛔ ON THE CONFIG DOCUMENT, NOT ON `odoo_sync_status`, because that document is REPLACED whole
+ on every phase write (`core.user_tables.record_odoo_sync_status`), so a stamp kept there would
+ be erased by the next `syncing`. Every writer of this config document updates it in place.
+ """
+ stamp = at or _utc_now().isoformat()
+
+ def _up(cur):
+ cur = dict(cur) if isinstance(cur, dict) else {}
+ cur["lastSyncAt"] = stamp
+ if ok:
+ cur["lastGoodSyncAt"] = stamp
+ return cur
+
+ rt.update(CONFIG_KEY, _up, flush="sync")
+ return stamp
+
+
+def _effective_sync_seconds(preset):
+ """The interval a sync can actually keep: the tenant's preset, floored, and never faster than
+ the deployment's own resync floor (`AIOS_STORE_RESYNC_SECONDS`), which `main.py` applies with
+ `max()`. None for `manual`: a person who asked not to be synced has no schedule to fall
+ behind."""
+ secs = SYNC_PRESETS.get(preset)
+ if secs is None:
+ return None
+ secs = max(int(secs), SYNC_FLOOR_SECONDS)
+ import os
+ try:
+ env = int(str(os.environ.get("AIOS_STORE_RESYNC_SECONDS") or "").strip() or 0)
+ except ValueError:
+ env = 0
+ return max(secs, env)
+
+
+def sync_freshness(rt, now=None):
+ """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
+ fail. PURE over the store reads.
+
+ `stale` is true when the last good sync is older than the effective cadence plus
+ `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
+ (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
+ the last run's own reason, its code prefix removed, when that run ended in error.
+ """
+ now = now or _utc_now()
+ try:
+ raw = rt.get(CONFIG_KEY) if rt is not None else None
+ except Exception: # noqa: BLE001
+ raw = None
+ raw = raw if isinstance(raw, dict) else {}
+ cfg = read_config(rt)
+ try:
+ status = rt.get(_ut().ODOO_SYNC_STATUS_KEY) if rt is not None else None
+ except Exception: # noqa: BLE001
+ status = None
+ status = status if isinstance(status, dict) else {}
+ phase = str(status.get("phase") or "unknown")
+ last_good = raw.get("lastGoodSyncAt") or None
+ # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
+ if not last_good and phase == "ok":
+ last_good = status.get("lastRun") or None
+ every = _effective_sync_seconds(cfg["syncEvery"])
+ allowed = None if every is None else every + STALE_GRACE_SECONDS
+ stale = False
+ if allowed is not None and not cfg["frozen"]:
+ good_at = _parse_iso(last_good)
+ stale = good_at is None or (now - good_at).total_seconds() > allowed
+ failure = None
+ errors = status.get("errors") if isinstance(status.get("errors"), list) else []
+ if phase == "error" and errors:
+ text = str(errors[0])
+ code, _sep, message = text.partition(": ")
+ if not _sep:
+ code, message = "sync_failed", text
+ failure = {"code": code.strip()[:80], "message": message.strip()[:300],
+ "at": status.get("lastRun") or None}
+ return {"stale": bool(stale), "lastGoodSyncAt": last_good,
+ "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
+ "lastRunAt": status.get("lastRun") or None, "phase": phase,
+ "frozen": cfg["frozen"], "failure": failure}
+
+
def frozen(rt):
"""Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential."""
@@ -4558,6 +4664,93 @@ def available(rt=None, tenant_key=None, connection_id=None):
return False
+#: ⭐ S67-T19 — the `limits` effects that REPORT a serving choice rather than withheld rows.
+#: `read_through` says a database is served THROUGH the mirror with no ceiling at all
+#: (`core.user_tables.limit_report`), which is the design working, not a failure.
+INFORMATIONAL_LIMIT_EFFECTS = frozenset({"read_through"})
+
+
+def failing_limits(limits):
+ """Only the `limits` entries that describe rows a pass did NOT serve, PURE.
+
+ ⛔⛔ STAGING'S `materialization_failed` WAS THIS, NAMED 2026-09-28 FROM ITS OWN RECEIPTS. The
+ rebuild handed its whole `limits` map to the health classifier, which reads EVERY entry as a
+ materialization failure, so the two read-through grains (`ut_odoo_order_lines`,
+ `ut_odoo_gl_lines`, "no ceiling applies to them") failed the connection on every single sync
+ and `odoo_sync_status` could never read `ok`. The report keeps them; only health drops them.
+ """
+ if not isinstance(limits, dict):
+ return limits
+ return {key: entry for key, entry in limits.items()
+ if not (isinstance(entry, dict)
+ and entry.get("effect") in INFORMATIONAL_LIMIT_EFFECTS)}
+
+
+#: ⭐ S67-T19 — the reconciliation DRIFT a good sync may carry: Odoo keeps taking invoices while a
+#: rebuild runs, and a hard delete leaves a mirror row until `reconcile_deletes` (every fourth
+#: pass). Measured on a mirror pulled 25 minutes before its rebuild: invoices 32,980 vs 32,978,
+#: orders 34,059 vs 34,058, agents 17 vs 18, GL lines 1,024,102 vs 1,024,178, customers 3,715 vs
+#: 3,716. A stale mirror is nothing like that: staging's unpulled seed was 626 orders and 732
+#: invoices short. The receipt keeps its exact counts either way; this only decides whether the
+#: CONNECTION's sync counts as good.
+RECONCILE_DRIFT_ROWS = 10
+RECONCILE_DRIFT_RATIO = 0.002
+
+
+def reconciliation_drift(receipt):
+ """`(difference, tolerance)` for a count-reconciliation receipt, or None when it has no counts."""
+ if not isinstance(receipt, dict):
+ return None
+ source = receipt.get("sourceCount")
+ if isinstance(receipt.get("unexplainedCount"), int):
+ diff = int(receipt["unexplainedCount"])
+ else:
+ mirror = receipt.get("mirrorCount")
+ if not isinstance(source, int) or not isinstance(mirror, int):
+ return None
+ diff = abs(source - mirror)
+ base = source if isinstance(source, int) and source > 0 else 0
+ return diff, max(RECONCILE_DRIFT_ROWS, int(base * RECONCILE_DRIFT_RATIO))
+
+
+def blocking_failures(database_failures, report):
+ """The database failures that make a sync NOT good, PURE.
+
+ A reconciliation failure whose count drift is within `reconciliation_drift`'s tolerance is
+ recorded against its database and does not fail the connection; every other failure does.
+ """
+ receipts = {}
+ for name in ("databases", "canonicalDatabases"):
+ group = report.get(name) if isinstance(report, dict) else None
+ if isinstance(group, dict):
+ receipts.update(group)
+ out = {}
+ for key, failure in (database_failures or {}).items():
+ if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
+ drift = reconciliation_drift(receipts.get(key))
+ if drift is not None and drift[0] <= drift[1]:
+ continue
+ out[key] = failure
+ return out
+
+
+def health_view(report):
+ """The refresh report as the sync-health classifier must read it, PURE.
+
+ Identical to `report` except that `limits` carries only `failing_limits`; the returned report
+ itself (the log line, the Sync report) still names every read-through database.
+ """
+ if not isinstance(report, dict) or not isinstance(report.get("limits"), dict):
+ return report
+ view = dict(report)
+ remaining = failing_limits(report["limits"])
+ if remaining:
+ view["limits"] = remaining
+ else:
+ view.pop("limits", None)
+ return view
+
+
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
today=None, retire=True, on_apply=None, source_client=None):
"""Refresh each enabled database independently; retain failed databases unchanged.
@@ -4844,6 +5037,16 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
return out
+#: ⭐ S67-T19 — the store errors a connection refresh retries ONCE. Staging's rebuild at
+#: 2026-09-28 16:03Z failed whole with `ProtocolViolation` (SQLSTATE 08P01), which is how the
+#: DigitalOcean pool (PgBouncer) reports its own timeouts while every lane container shares it;
+#: nothing was written and the next scheduled pass was a day away. `refresh` is idempotent, so a
+#: second attempt after a pause is safe; a refusal or any other exception is never retried.
+TRANSIENT_STORE_ERRORS = frozenset({"ProtocolViolation", "OperationalError", "PoolTimeout",
+ "ConnectionTimeout", "AdminShutdown"})
+REFRESH_RETRY_SECONDS = 20
+
+
def refresh_all(rt, tenant, username="automation", cur=None, today=None):
"""Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
@@ -4880,12 +5083,20 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
primary = None
for entry_cid in order:
named = None if (entry_cid is None or entry_cid == default) else entry_cid
- try:
- one = refresh(rt, tenant, username=username,
- cur=(cur if named is None else None),
- today=today, connection_id=named)
- except Exception as exc: # noqa: BLE001
- one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
+ for attempt in (1, 2):
+ try:
+ one = refresh(rt, tenant, username=username,
+ cur=(cur if named is None else None),
+ today=today, connection_id=named)
+ break
+ except Exception as exc: # noqa: BLE001
+ # S67-T19: one retry for a TRANSIENT store error only; every other failure is
+ # recorded exactly as before.
+ if attempt == 1 and type(exc).__name__ in TRANSIENT_STORE_ERRORS:
+ _time.sleep(REFRESH_RETRY_SECONDS)
+ continue
+ one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
+ break
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
if named is None and primary is None:
primary = one
diff --git a/api/routes_connectors.py b/api/routes_connectors.py
index 7937e644e71765cb0aae0b66314b127668fc1810..832f240d6239efc43f1fb7502106b1035ffb5213 100644
--- a/api/routes_connectors.py
+++ b/api/routes_connectors.py
@@ -31,7 +31,7 @@ from datetime import datetime, timezone
from collections import OrderedDict
from typing import Annotated, Literal
-from fastapi import APIRouter, Body, Depends, Query
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
import oauth_connect
from deps import Session, err, require_session, running_main
@@ -326,9 +326,11 @@ TOKEN_CONNECTORS = [
"hint": "Create a Production app client in the Solution Provider Portal, self authorize "
"it, then paste client_id, client_secret, refresh_token, seller_id and "
"marketplace_id.",
- "note": "The credential is verified live against your marketplace participations before "
- "it is stored. Bulk reads use the Reports API, so a backfill is one document "
- "rather than a page loop."},
+ # S66-T27: storing is offline (`keychain._test_amazon_entry` probes only on an explicit
+ # Test), so this no longer claims the key was verified before it was stored.
+ "note": "Choose Test in the Keychain tab to verify the credential live against your "
+ "marketplace participations. Bulk reads use the Reports API, so a backfill is one "
+ "document rather than a page loop."},
]
#: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
@@ -446,6 +448,11 @@ _register_probers()
# W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace,
# never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
# `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
+#: The directory states that mean "this workspace holds this connection": connected, stored but
+#: asking to be reconnected, or stored past its rotation deadline (`ConnectorsPage`'s `held`).
+_GRAPH_HELD_STATES = frozenset(("connected", "reconnect", "expired"))
+
+
def _graph_refusal(connection_key, reason, status="refused"):
return {"connectionKey": str(connection_key or ""), "status": status,
"reason": str(reason or ""), "tables": [], "relationships": []}
@@ -598,7 +605,12 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False):
graphs = {}
connected = set()
for connection_key, row in row_by_key.items():
- if row.get("state") == "connected":
+ # ⭐ S67-T31 / D-718 — A HELD CONNECTION HAS A GRAPH. Staging's Amazon card reads
+ # `reconnect` (its stored key has no passing test on record) while 38 `ut_amzn_*` tables
+ # are materialised, and the old `== "connected"` test refused the graph as "Connect this
+ # provider" over a connection that is plainly there. The nodes are still only tables that
+ # exist and that this reader may open, so admitting the held states draws no template.
+ if row.get("state") in _GRAPH_HELD_STATES:
connected.add(connection_key)
graphs[connection_key] = _graph_refusal(
connection_key,
@@ -978,9 +990,21 @@ def directory(rt, uname, is_admin=False, incomplete=None):
else "keychain_personal"),
# Said once, on the row, rather than in a paragraph somewhere (R13): storing the
# key is real and useful, and it is not the same as the data being on screen.
- "note": "Storing the key connects the account; reading its data lands in a later "
- "release.",
+ # ⭐ S66-T27 / S67-T31 — EXCEPT AMAZON, whose data DOES land (38 `ut_amzn_*`
+ # tables on staging), so the generic sentence told an administrator the opposite
+ # of the truth. Amazon carries its own authored note.
+ # S67 lane D: so does Meta Ads, whose authored note this line used to overwrite
+ # with the generic one; every card now shows its own (Stripe's and Shopify's own
+ # note IS the generic sentence, so theirs is unchanged).
+ "note": t["note"],
})
+ # S66-T27 — WHAT `reconnect` MEANS, said on the card. On Amazon it is "a key is stored
+ # but no passing Test is on record for it" (`keychain.entry_verified`), measured on
+ # staging's admin view with the data still syncing. The bare word read as a broken
+ # connection with no way to tell what to do.
+ if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
+ rows[-1]["needs"] = ("Amazon access for the stored key is not confirmed yet. "
+ "Choose Test in the Keychain tab, or replace the key.")
contribute("token", _tokens,
"the token-connector registry could not be composed",
@@ -3791,7 +3815,17 @@ def _amazon_databases(rt, is_admin, spec=None):
enabled = enablement.get("databases") or {}
cat = amazon_schema.catalogue()
tables = cat.get("tables") or {}
- states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
+ # ⭐ S67-T31 — A STATUS THE GUARD REFUSES COSTS ITS OWN ROW, NEVER THE MODAL. `sync_state`
+ # trims an over-long cause itself; a credential shape still refuses, and the refusal carries
+ # the guarded rows that passed, so only the refused tables read "Sync status is unavailable
+ # for this database." (a missing status is exactly what `_amazon_database_sync_health`
+ # degrades). Any other failure reading status degrades every row, never the whole page.
+ try:
+ states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
+ except cx.AmazonStatusSafetyError as refused:
+ states = dict(getattr(refused, "safe_status", None) or {})
+ except Exception: # noqa: BLE001
+ states = {}
# Which OTHER declared tables are filled from the SAME report document. Derived from the
# report map so a table added to `REPORT_TYPES` gets this disclosure for free.
@@ -4289,6 +4323,8 @@ def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offs
# question about a PERSON and a table, not about an administrator flag alone.
databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""),
page=page, fields=fields)
+ if key == "amazon_sp":
+ databases = _screen_amazon_database_rows(databases)
budget.check()
overview = _detail_overview(row, databases, graph)
if page is not None and page.total:
@@ -4325,6 +4361,36 @@ def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offs
"relationships": graph})
+def _screen_amazon_database_rows(rows):
+ """S67-T31: each Amazon database row answers the status guard ON ITS OWN.
+
+ `_safe_connector_status` still guards the whole detail below, and it used to be the first and
+ only judge, so one oversized scalar in one row aborted every row, the Overview and the
+ Keychain with a bare 500. Here an over-long string is trimmed, and a row that still fails (a
+ credential shape anywhere in it) keeps only its key, its shipped name and its switch, with
+ "Sync status is unavailable for this database." as its status. The refused value is dropped,
+ never echoed.
+ """
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns the guard
+ listed = list(rows or [])
+ safe, _refused = cx.screen_status_rows(dict(enumerate(listed)))
+ out = []
+ for index, row in enumerate(listed):
+ if index in safe:
+ out.append(safe[index])
+ continue
+ key = str((row or {}).get("key") or "")
+ enabled = (row or {}).get("enabled") is True
+ label = LABELS_AMAZON.get(key) or key
+ out.append({"key": key, "label": label, "defaultLabel": label, "native": "",
+ "enabled": enabled,
+ **({"sync": None} if "sync" in (row or {}) else {}),
+ "syncHealth": _amazon_database_sync_health(None, enabled),
+ "fields": [], "excluded": [], "sharesFieldsWith": [],
+ **({"fieldsDeferred": True} if (row or {}).get("fieldsDeferred") else {})})
+ return out
+
+
def _set_amazon_database_enablement(database_key, body, session):
"""The testable core of Amazon's tenant-scoped administrator enablement door."""
if not session.admin:
@@ -4406,16 +4472,46 @@ def connectors_detail(connector_key: str, session: Session = Depends(require_ses
workspace is connected to. What a member does NOT get is the tenant-wide credential door
(`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
"""
- detail = connector_detail(
- session.runtime, session.user, session.uname,
- bool(session.admin), connector_key, databases_offset=databases_offset,
- databases_limit=databases_limit, fields=fields)
+ try:
+ detail = connector_detail(
+ session.runtime, session.user, session.uname,
+ bool(session.admin), connector_key, databases_offset=databases_offset,
+ databases_limit=databases_limit, fields=fields)
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_detail_failed",
+ "The connector details could not be read.")
if detail is None:
raise err(404, "unknown_connector",
"this workspace has no connector with that key")
return detail
+def _plain_read_failure(exc, code, sentence):
+ """S67-T31: an unexpected failure on a connector read answers the app's JSON error shape.
+
+ The owner's modal read "The server answered 500." because a guard refusal escaped as a bare
+ `Internal Server Error` with no body. Every read the modal makes now answers
+ `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
+ and no exception text (which can echo a payload) reaches the wire.
+ """
+ import logging # noqa: PLC0415
+ logging.getLogger(__name__).exception("connector read failed: %s", code)
+ name = type(exc).__name__
+ if name in ("StoreUnavailable", "PoolTimeout"):
+ return err(503, "store_unavailable",
+ "The workspace store did not answer in time, so nothing could be read.")
+ try:
+ import connectors_amazon as cx # noqa: PLC0415
+ if isinstance(exc, cx.AmazonStatusSafetyError):
+ return err(500, "connector_status_withheld",
+ "This connector's status could not be shown safely, so it was withheld.")
+ except Exception: # noqa: BLE001
+ pass
+ return err(500, code, sentence)
+
+
def _detail_row(rt, uname, is_admin, connector_key):
row = next((row for row in directory(rt, uname, is_admin)
if row.get("key") == connector_key), None)
@@ -4429,22 +4525,35 @@ def connector_database_fields(connector_key: str, database_key: str,
session: Session = Depends(require_session)):
"""The same field disclosure and database access as detail, for one database only."""
rt = _ConnectorDetailReads(session.runtime)
- _detail_row(rt, session.uname, bool(session.admin), connector_key)
- rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname,
- page=_DatabasePage(database_key=database_key))
- if not rows:
- raise err(404, "unknown_database", "This connector has no visible database with that key.")
- return _safe_connector_status(connector_key, {
- "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]})
+ try:
+ _detail_row(rt, session.uname, bool(session.admin), connector_key)
+ rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname,
+ page=_DatabasePage(database_key=database_key))
+ if not rows:
+ raise err(404, "unknown_database",
+ "This connector has no visible database with that key.")
+ return _safe_connector_status(connector_key, {
+ "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]})
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_fields_failed",
+ "The fields of this database could not be read.")
@router.get("/connectors/{connector_key}/relationships")
def connector_relationships(connector_key: str, session: Session = Depends(require_session)):
"""Load the permitted graph when the summary client's Relationships tab opens."""
rt = _ConnectorDetailReads(session.runtime)
- row = _detail_row(rt, session.uname, bool(session.admin), connector_key)
- return _safe_connector_status(connector_key, relationship_graphs(
- [row], session.user, rt, session.uname, bool(session.admin))[connector_key])
+ try:
+ row = _detail_row(rt, session.uname, bool(session.admin), connector_key)
+ return _safe_connector_status(connector_key, relationship_graphs(
+ [row], session.user, rt, session.uname, bool(session.admin))[connector_key])
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ raise _plain_read_failure(exc, "connector_relationships_failed",
+ "The relationship graph could not be read.")
class _ReplacementConflict(Exception):
diff --git a/api/routes_external.py b/api/routes_external.py
index 6eb5bea833832b925031d7aea92fc7e2756e8f1a..f479438ef786347f27b90e520fba1c7084bdaefa 100644
--- a/api/routes_external.py
+++ b/api/routes_external.py
@@ -728,8 +728,12 @@ def _resolve_external_scope(
if include_schema:
if row["answerable"] or strict_schema:
try:
+ # ⭐ S67-T34 (D-2630): THE FIELD CONTRACT, NOT THE ROWS. Describing a
+ # database used to read (and count) every row first, so the largest
+ # connected table refused its own schema. "schema" runs every wall and
+ # reads no row; sources that are not mirror-served ignore it.
scoped = deps.assistant_read_scope(
- session, database, fields=None, filters=None,
+ session, database, fields=None, filters=None, mode="schema",
)
except HTTPException as exc:
# HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
@@ -1821,8 +1825,11 @@ def _query_database(session: Session, database: str, payload: dict):
# This is the same permissioned record reader the in-app Assistant uses. It resolves the
# database, field and row walls on every call before this adapter sorts, caps, or aggregates.
+ # ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
+ # its aggregates run in SQL over every permitted row and its rows are read only as needed.
scoped = deps.assistant_read_scope(
- session, database, fields=None, filters=payload.get("filters"))
+ session, database, fields=None, filters=payload.get("filters"), mode="schema")
+ mirror = scoped.get("source_kind") == "mirror_user_table"
visible_fields = list(scoped.get("fields") or [])
visible_keys = {
field.get("key") for field in visible_fields
@@ -1851,6 +1858,104 @@ def _query_database(session: Session, database: str, payload: dict):
sorts.append({"field": rule["field"], "direction": rule["direction"]})
aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
+ server_aggregates = None
+ paged_total = None
+ if mirror:
+ # ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
+ # are computed by the mirror in SQL over exactly the rows this caller may read, never
+ # over a fetched slice, and never by pulling a million rows into this process.
+ if aggregates:
+ summary = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"),
+ mode=("aggregate", aggregates))
+ server_aggregates = summary.get("server_aggregates")
+ import assistant_mirror
+ # The summary already counted the permitted population, so a table past the one-pass
+ # ceiling goes straight to SQL paging instead of pulling a first 5,000-row page only to
+ # refuse it.
+ known = (summary.get("total_rows") if aggregates else None)
+ try:
+ if known is not None and int(known) > assistant_mirror.ASSISTANT_READ_MAX:
+ raise err(409, "assistant_source_too_large",
+ f"this database matches {int(known):,} rows for that question, "
+ f"past the {assistant_mirror.ASSISTANT_READ_MAX:,} rows one answer "
+ f"holds in memory")
+ scoped = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"))
+ except HTTPException as exc:
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
+ if info.get("code") != "assistant_source_too_large":
+ raise
+ # ⭐ RULE 1: PAST THE ONE-PASS CEILING THE ROWS ARE PAGED IN SQL, NOT REFUSED. Only
+ # this page reaches the process. A sort cannot be honoured at this size without
+ # holding every row, so it is refused by name with the fix, never silently dropped.
+ if sorts:
+ raise err(409, "query_sort_too_large",
+ f"{info.get('message') or 'This database is too large to sort here.'} "
+ f"Sorting is refused at this size because it would need every row in "
+ f"memory. Ask without sorts to page it in record order, or narrow it "
+ f"with a filter and sort again.") from exc
+ paged_total = True
+ if paged_total:
+ offset = (0 if payload.get("cursor") is None else None)
+ page_digest = _json_digest({
+ "database": database, "fields": selected_keys, "filters": payload.get("filters"),
+ "sorts": [], "aggregates": aggregates, "limit": limit, "paged": True,
+ })
+ first = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"), mode=("page", 0, 1))
+ snap = _json_digest({
+ "source_version": first.get("source_version"),
+ "visible_fields": [field["key"] for field in visible_fields
+ if isinstance(field, dict) and field.get("key") in visible_keys],
+ "total_rows": first.get("total_rows"),
+ })
+ if offset is None:
+ offset = _decode_query_cursor(session, payload.get("cursor"), page_digest, snap)
+ total_rows = int(first.get("total_rows") or 0)
+ if offset > total_rows:
+ raise err(409, "query_cursor_stale",
+ "The permitted result set changed. Start again without a cursor.")
+ page = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"),
+ mode=("page", offset, limit))
+ page_rows = list(page.get("records") or [])
+ for row in page_rows:
+ _assert_finite_query_value({"pid": row.get("pid"),
+ **{key: row.get(key) for key in selected_keys}})
+ records = [{"pid": copy.deepcopy(row.get("pid")),
+ **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
+ for row in page_rows]
+ next_offset = offset + limit
+ next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset)
+ if next_offset < total_rows else None)
+ result = {
+ "database": page.get("database"), "label": page.get("label"),
+ "source_kind": page.get("source_kind"),
+ "source_version": copy.deepcopy(page.get("source_version")),
+ "retrieved_at": page.get("retrieved_at"), "permission_scope_applied": True,
+ "fields": [copy.deepcopy(field) for field in visible_fields
+ if field.get("key") in selected_keys],
+ "filters": copy.deepcopy(page.get("filters")), "sorts": [],
+ "records": records, "aggregates": copy.deepcopy(server_aggregates or []),
+ "page": {"limit": limit, "returned": len(records), "offset": offset,
+ "has_more": next_cursor is not None, "next_cursor": next_cursor},
+ "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
+ "max_aggregates": _QUERY_MAX_AGGREGATES,
+ "max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
+ "max_filter_nodes": _QUERY_MAX_FILTER_NODES,
+ "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
+ }
+ _assert_finite_query_value(result)
+ audit.record("external.query", session.tenant, session.uname, {
+ "surface": "ai_api",
+ "authentication": session.claims.get("external_auth_type", "api_key"),
+ "field_count": len(selected_keys), "filter_applied": page.get("filters") is not None,
+ "sort_count": 0, "aggregate_count": len(aggregates), "page_size": limit,
+ "returned_count": len(records), "cursor_used": offset > 0,
+ })
+ return result
+
rows = list(scoped.get("records") or [])
inspected_keys = (set(selected_keys)
| {rule["field"] for rule in sorts}
@@ -1901,7 +2006,8 @@ def _query_database(session: Session, database: str, payload: dict):
"filters": copy.deepcopy(scoped.get("filters")),
"sorts": copy.deepcopy(sorts),
"records": records,
- "aggregates": _aggregate_records(rows, aggregates),
+ "aggregates": (copy.deepcopy(server_aggregates) if server_aggregates is not None
+ else _aggregate_records(rows, aggregates)),
"page": {"limit": limit, "returned": len(records), "offset": offset,
"has_more": next_cursor is not None, "next_cursor": next_cursor},
"caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
diff --git a/api/routes_module_index.py b/api/routes_module_index.py
index d776470f9b5f717ab9123e8df539d38543a0492f..b9d246e393da257a09298d657a1e2c06566172dc 100644
--- a/api/routes_module_index.py
+++ b/api/routes_module_index.py
@@ -5,8 +5,10 @@ from fastapi import APIRouter, Body, Depends, HTTPException
from core.sheets import SheetFolderRefusal, validate_folder_tree
from deps import Session, err, module_gate, require_session
+import application_registry
import routes_assets
import routes_automation
+from routes_applications import _may_open
from routes_nav import _placeable_top_keys
import voice_control_plane as voice
@@ -18,10 +20,16 @@ def _sentence(value):
return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
+# D-3282 (standing rule 1): a limit the server cannot lift is reported with its cause and the fix.
+_REFUSAL_SENTENCES = {
+ "too_many_folders": "Folders hold at most 200 per rail. Delete or merge a folder, then try again.",
+}
+
+
def _refuse(exc):
status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
"module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
- return err(status, exc.code, _sentence(exc.message))
+ return err(status, exc.code, _sentence(_REFUSAL_SENTENCES.get(exc.code, exc.message)))
def _access(module: str, session: Session = Depends(require_session)):
@@ -57,7 +65,19 @@ def _readable_ids(module, session):
# only listed items, so an unverifiable placement renders no tile. Use local voice
# records, never audience-option assembly, just to verify placement ids.
if module == "app":
- return {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
+ # S67-T38 (O24): the App home also renders the seeded Finance projections, so the wall
+ # admits their ids through the SAME grant filter /nav/applications uses. Never ungated:
+ # a Finance card the session cannot open is not placeable.
+ stored = {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
+ finance = application_registry.finance_application_records(
+ may_open=lambda permission: _may_open(session, permission))
+ # S67-T41 owner correction: a Content Studio App is bound to a USER table, which the
+ # topic-only `/applications` index refuses, so its tile arrives from /nav/applications.
+ # Its id is admitted through the registry's own `may_read` (the same read that lists it).
+ studios = {row["id"] for row in application_registry.list_apps(
+ session.runtime, session.uname, session.admin)
+ if row.get("templateKind") == application_registry.CONTENT_STUDIO_TEMPLATE_KIND}
+ return stored | {row["id"] for row in finance} | studios
# ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
# *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
# it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py
index af8e30f2acd99543dfd536b1b39f7692a3c25b32..8a0457587d279e52a80df39d2b5be4b84911b74e 100644
--- a/api/routes_odoo_tables.py
+++ b/api/routes_odoo_tables.py
@@ -23,6 +23,7 @@ import hashlib
import hmac
import json
import math
+import threading
import time
from contextlib import contextmanager
@@ -622,10 +623,27 @@ GRID_SOURCES = {}
_SOURCE_PROVIDERS = []
_SOURCE_ANALYSES = {}
+#: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
+#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
+#: keys to the shared dict as it went, and returned the dict to any caller that found it
+#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
+#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
+#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
+#: this lock and publishes it in one `dict.update`; a reader waits on the lock until
+#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
+#: Amazon fallback) happens only after a build has been published, so it can never stand in
+#: for one either.
+_SOURCES_LOCK = threading.RLock()
+_SOURCES_BUILT = False
+_SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
+
def invalidate_sources():
"""Forget measured bindings after a connector publishes a new mirror schema."""
- GRID_SOURCES.clear()
+ global _SOURCES_BUILT
+ with _SOURCES_LOCK:
+ _SOURCES_BUILT = False
+ GRID_SOURCES.clear()
def register_source_provider(fn):
@@ -672,20 +690,60 @@ def _sources():
⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
importing that module at file-import time would drag the Odoo layer into every process that
mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
+
+ ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
+ lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
+ and so does one that holds only admissions made since the last invalidation.
"""
- if GRID_SOURCES:
+ global _SOURCES_BUILT, _SOURCES_BUILDER
+ if _SOURCES_BUILT and GRID_SOURCES:
return GRID_SOURCES
- for build in list(_SOURCE_PROVIDERS):
+ with _SOURCES_LOCK:
+ if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
+ return GRID_SOURCES
+ _SOURCES_BUILDER = threading.get_ident()
try:
- extra = build() or {}
- except Exception: # noqa: BLE001
- # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must
- # not go dark because a newer connector's module is unhappy — and vice versa.
- continue
- if extra:
+ built = {}
+ for build in list(_SOURCE_PROVIDERS):
+ try:
+ extra = build() or {}
+ except Exception: # noqa: BLE001
+ # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
+ # grids must not go dark because a newer connector's module is unhappy, and
+ # vice versa.
+ continue
+ if extra:
+ built.update(extra)
+ _ut().register_connected(*extra)
+ # A key admitted since the last invalidation was measured at least as recently as
+ # this build, so it stands; everything else comes from the build. ONE update, so a
+ # lock-free reader of the dict sees the old entry or the new one, never a gap.
+ GRID_SOURCES.update({**built, **GRID_SOURCES})
+ _SOURCES_BUILT = True
+ finally:
+ _SOURCES_BUILDER = None
+ return GRID_SOURCES
+
+
+def _learn_amazon_sources():
+ """Publish the Amazon provider's keys into a registry that was built without any of them.
+
+ HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot
+ BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates
+ it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through
+ Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more
+ than one window. A registry that already knows one Amazon key is not re-asked.
+ """
+ _sources()
+ if any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
+ return
+ extra = _amazon_sources()
+ if not extra:
+ return
+ with _SOURCES_LOCK:
+ if not any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
GRID_SOURCES.update(extra)
_ut().register_connected(*extra)
- return GRID_SOURCES
def _odoo_sources():
@@ -1055,11 +1113,13 @@ def _amazon_sources():
return {}
-def require_amazon_mirror(table_key, session, definition=None):
+def require_amazon_mirror(table_key, session, definition=None, st=None):
"""Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
The provider contributes no keys on an Odoo-only mirror. A persisted read-through
definition still must refuse there instead of falling through to its empty stored rows.
+ S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
+ window route stops paying a second definitions projection for this one check.
"""
if not str(table_key).startswith("ut_amzn_"):
return
@@ -1069,7 +1129,8 @@ def require_amazon_mirror(table_key, session, definition=None):
if table_key not in amazon.SOURCE:
return
if definition is None:
- definition = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
+ definition = _defn_or_refuse(session, table_key, st=st, defs_only=True,
+ scope_applied=True)
try:
cur, mismatch = _mirror_cur(session, table_key)
except RuntimeError as exc:
@@ -1086,8 +1147,17 @@ def require_amazon_mirror(table_key, session, definition=None):
raise err(503, "amazon_mirror_not_loaded", current["missing"])
# A dynamically oversized snapshot may have a durable stub before this process
# has measured it. Admit its binding without changing another source's registry.
+ # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
+ # key. Admitted into an empty or Amazon-less registry, this one key used to make
+ # `_sources()` skip its build, so the grid resolved while its link targets
+ # (orders, read-through) were dispatched as stored tables: a first-page 500.
if spec is None or spec.get("missing"):
- GRID_SOURCES[table_key] = current
+ _learn_amazon_sources()
+ with _SOURCES_LOCK:
+ _sources()
+ held = GRID_SOURCES.get(table_key)
+ if held is None or held.get("missing"):
+ GRID_SOURCES[table_key] = current
finally:
cur.close()
@@ -1322,18 +1392,14 @@ def _source_for(cur, table_key):
if module_cache_source(key) is not None:
return None
spec = _sources().get(key)
- if not spec and key.startswith("ut_amzn_") and not any(
- k.startswith("ut_amzn_") for k in GRID_SOURCES):
+ if not spec and key.startswith("ut_amzn_"):
# HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
# Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
# runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
# read-through Amazon grid answered "no read-through binding". Ask the Amazon provider
# alone, once it can answer; a registry that already knows one Amazon key is not re-asked.
- extra = _amazon_sources()
- if extra:
- GRID_SOURCES.update(extra)
- _ut().register_connected(*extra)
- spec = GRID_SOURCES.get(key)
+ _learn_amazon_sources()
+ spec = GRID_SOURCES.get(key)
if not spec:
return None
if spec.get("missing"):
@@ -1721,11 +1787,133 @@ def _related_source_value(session, entry, record):
def _materialised_table(table_key):
- """Storage dispatch, independent of the browser's fallible status roster."""
- return module_cache_source(table_key) is not None or table_key not in _sources()
+ """Storage dispatch, independent of the browser's fallible status roster.
+
+ ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
+ (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
+ the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
+ """
+ if module_cache_source(table_key) is not None:
+ return True
+ if table_key in _sources():
+ return False
+ if str(table_key).startswith("ut_amzn_"):
+ _learn_amazon_sources()
+ return table_key not in GRID_SOURCES
+ return True
+
+
+#: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's
+#: ROWS, each held against the durable `user_tables` revision it was read at. A grouped Amazon page
+#: used to re-read the definitions projection once for its own wall and once more for EVERY link
+#: target, plus the stored listings entry, on every page; on staging each of those reads is one
+#: large JSONB projection (measured 33 to 186 s a page under load, 2026-09-28). Every `put` and
+#: `update` bumps the revision column in the same statement, so an unchanged token proves the
+#: bytes unchanged; the token is read BEFORE the value, so a write racing the read can only cache
+#: newer bytes under the older token (a miss next time, never a stale hit). ⚠ Nothing cached here
+#: is a permission: the wall (`may_read`, `apply_row_scope`, the compiled predicate) is evaluated
+#: per request against the session's own user, only the bytes it evaluates over are reused.
+_REV_MEMO: dict = {}
+_REV_MEMO_MAX = 64
+# Route handlers run on a thread pool: the lock makes the evict (`next(iter(...))`) and the
+# insert one step, so a concurrent insert can never raise "dictionary changed size".
+_REV_MEMO_LOCK = threading.Lock()
+#: "No token was handed in, read one." Distinct from None, which means "unreadable, hold nothing".
+#: It is the `...` literal (a singleton) so a signature can spell its default `...` and still
+#: define when `verify_grid_ux.py` execs `_relation_rows` on its own, without this module name.
+_UNREAD = ...
+
+
+def _user_tables_token(rt):
+ """The durable change token of this tenant's `user_tables` bucket, or None when unreadable."""
+ try:
+ return rt.revision(_ut().STORE_KEY)["token"]
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
+ """`read()`, reused while this tenant's `user_tables` revision is unchanged.
+
+ ⛔ `clone` runs on the way in AND on every hit, so a caller that edits what it was handed
+ cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
+ answer never is (it is the shape a failed read takes, and caching it would make one outage
+ outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
+ `token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
+ trip, not one per memo. It was read before any value this call reads, which is the only
+ ordering the staleness argument above needs.
+ """
+ if token is _UNREAD:
+ token = _user_tables_token(rt)
+ tenant = getattr(rt, "key", None)
+ slot = (kind, tenant, name)
+ if token is not None and tenant:
+ with _REV_MEMO_LOCK:
+ hit = _REV_MEMO.get(slot)
+ if hit is not None and hit[0] == token:
+ return clone(hit[1])
+ value = read()
+ if token is not None and tenant and keep(value):
+ held = (token, clone(value))
+ with _REV_MEMO_LOCK:
+ if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
+ _REV_MEMO.pop(next(iter(_REV_MEMO)), None)
+ _REV_MEMO[slot] = held
+ return value
+
+
+def _page_defs(session):
+ """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's.
+
+ Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked
+ against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link
+ target does not pay a second revision round trip.
+ The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
+ `user_tables` revision instead of once per question. Kept only when every definition really is
+ a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
+ 28 MB document on a failed read and that must never be held, let alone copied per request.
+ ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
+ lends lazily live for this one request and are never carried to the next.
+ """
+ import copy
+ import core.store as store
+
+ ut = _ut()
+ rt = session.runtime
+ token = _user_tables_token(rt)
+
+ def keep(v):
+ tables = [d for d in (v or {}).values() if isinstance(d, dict)]
+ return bool(tables) and all(store.is_projected(d) for d in tables)
+
+ defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt),
+ keep=keep, clone=copy.deepcopy, token=token)
+ return ut.lend(rt, **{ut.STORE_KEY: defs}), token
+
+
+def _stored_link_rows(session, table, token=_UNREAD):
+ """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
+
+ ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
+ cache, an app reader) or a read-through binding answers from somewhere this token does not
+ watch, so those read exactly as before. The wall is NOT applied here; the caller applies
+ `apply_row_scope` for its own user on every request. Holding it across users is sound only
+ because `perm_scope._read_user_table` never consults `user` for a `ut_*` key: every
+ user-dependent step (hidden fields, row wall, BU scope) sits above it, in the caller.
+ """
+ import core.perm_scope as perm_scope
+
+ rt = session.runtime
+ if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
+ or module_cache_source(table) is not None or not _materialised_table(table)):
+ return perm_scope._read(table, session.user, rt)
+ return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
+ keep=lambda v: bool(v[1]),
+ clone=lambda v: ([dict(f) for f in v[0]], [dict(r) for r in v[1]]),
+ token=token)
-def _traversal_defn_or_refuse(session, table_key):
+def _traversal_defn_or_refuse(session, table_key, st=None):
"""R50 / C4: admit an intermediate after the caller has established both end grants.
The session may already open both ends of a declared two-hop link. Its intermediate
@@ -1733,7 +1921,7 @@ def _traversal_defn_or_refuse(session, table_key):
read still applies the row wall, hidden-field closure, BU scope and mirror guard.
"""
ut = _ut()
- st = ut.lend_defs(session.runtime)
+ st = st if st is not None else ut.lend_defs(session.runtime)
defn = ut.get(table_key, st=st)
if not defn:
raise err(404, "unknown_table", "that database does not exist")
@@ -1741,21 +1929,26 @@ def _traversal_defn_or_refuse(session, table_key):
def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
- traversal_of=None):
+ traversal_of=None, st=None, rev=...): # `...` IS `_UNREAD`
"""Internal join values, after the ordinary row wall, with no row-count ceiling.
Join columns may be hidden on the wire. Only identities leave the resolver; the native
payload builders still apply their complete hidden-field closure before returning cells.
With traversal_of, the caller must already have established both ends of the declared
edge named by that key. Only join values leave this intermediate read, never a cell.
+ S67-T28: `st` is a lent DEFINITIONS snapshot the caller already holds (`_page_defs`); the wall
+ is asked of it instead of re-reading every definition per target. A stored target's rows come
+ through `_stored_link_rows`, reused per `user_tables` revision; its wall still runs here.
+ `rev` is the revision token `_page_defs` already read for this request; absent, the stored
+ target is read directly, as before S67-T28.
"""
import core.perm_scope as perm_scope
import core.relations as relations
columns = set(columns) | {relations.IDENTITY_COL}
if not _materialised_table(table):
- read = (_compiled_read(table, session, traversal_intermediate=True)
- if traversal_of is not None else _compiled_read(table, session))
+ read = (_compiled_read(table, session, traversal_intermediate=True, st=st)
+ if traversal_of is not None else _compiled_read(table, session, st=st))
spec = read["spec"]
bindings = dict(spec["cols"], pid=(spec["id"], int))
if columns - set(bindings):
@@ -1778,11 +1971,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
# This is an internal join, never a cell response. Apply the same permanent wall on the
# complete contract before taking any join value, including a join on a hidden column.
- if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime):
+ if traversal_of is None and not perm_scope.may_read(
+ session.user, table, st=st if st is not None else session.runtime):
raise err(403, "forbidden_table", "This account cannot open the linked database.")
if module_cache_source(table) is not None:
session.require(table)
- fields, rows = perm_scope._read(table, session.user, session.runtime)
+ # The revision memo serves a caller that handed this request's token (the page's link fills,
+ # S67-T28); any other caller reads exactly as before, which is also what lets
+ # `verify_grid_ux.py` exec this function alone with a stubbed `perm_scope`.
+ fields, rows = (perm_scope._read(table, session.user, session.runtime) if rev is ...
+ else _stored_link_rows(session, table, token=rev))
rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
if where:
raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
@@ -2570,7 +2768,8 @@ def _may_see_view(session, vid, store):
def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
- search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False):
+ search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
+ st=None):
"""EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
@@ -2630,10 +2829,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
# tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
# filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
# used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
+ # S67-T28: `st` lends a definitions snapshot the caller already holds (`_page_defs`), the
+ # same `st=` `_defn_or_refuse` has always accepted. None keeps its own `lend_defs` read.
if traversal_intermediate:
- defn = _traversal_defn_or_refuse(session, table_key)
+ defn = _traversal_defn_or_refuse(session, table_key, st=st)
else:
- defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
+ defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
fields = None
for _bucket, key, _label, mk_fields in rel.TABLES:
if key == table_key:
@@ -3105,7 +3306,12 @@ def odoo_table_rows(table_key: str,
"limit, filters, sorts and search in the query string.")
ids = _record_ids(body)
- require_amazon_mirror(table_key, session)
+ # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
+ # asks it instead of reading its own definitions projection (one store round trip fewer on
+ # every Amazon page). Not reordered past it: the check may register this grid's binding,
+ # which is what `_materialised_table` below then reads.
+ page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
+ require_amazon_mirror(table_key, session, st=page[0] if page else None)
if _materialised_table(table_key):
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
sorts, search, related, groupBy, groupKey, ids=ids)
@@ -3116,8 +3322,11 @@ def odoo_table_rows(table_key: str,
# is a page of* - and that is provable only while there is one of it. What stays here is what
# a WINDOW alone needs: the total order, the projection, the overlay merge and the two
# sentences a page earns (`window_clamped`, the deep-page walk).
+ # S67-T28 (D-3480 point 1): ONE definitions snapshot answers this page's own wall and every
+ # link target's below, re-read only when `user_tables` has moved (`_page_defs`).
+ page_st, page_rev = page if page is not None else _page_defs(session)
read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
- sorts=sorts, search=search, related=related)
+ sorts=sorts, search=search, related=related, st=page_st)
read = _restrict_row_ids(read, ids)
defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
cols, limits, frm = read["cols"], read["limits"], read["frm"]
@@ -3184,14 +3393,16 @@ def odoo_table_rows(table_key: str,
def read_amazon_targets(target, column, values):
if target not in _opened:
- _opened[target] = _may_open(session, target)
+ _opened[target] = _may_open(session, target, st=page_st)
if not _opened[target]:
return []
if not _materialised_table(target):
- return _relation_rows(session, target, [column], match=(column, values))
+ return _relation_rows(session, target, [column], match=(column, values),
+ st=page_st, rev=page_rev)
if target not in _stored:
_stored[target] = _relation_rows(
- session, target, sorted(_stored_cols.get(target, set()) | {column}))
+ session, target, sorted(_stored_cols.get(target, set()) | {column}),
+ st=page_st, rev=page_rev)
wanted = {str(value) for value in values}
# The same string comparison `_relation_rows`' own `match=` applies to a stored table.
return [row for row in _stored[target] if str(row.get(column)) in wanted]
@@ -3214,11 +3425,14 @@ def odoo_table_rows(table_key: str,
# the wall and the predicate, so the argument it demands is the list we just fetched.
# ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
# answered "may this session open this surface".
- shared_defs = _so().fields(table_key, st=session.runtime)
+ # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request
+ # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
+ # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
+ shared_defs = _so().fields(table_key, st=page_st)
if shared_defs:
fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
- st=session.runtime).items():
+ st=page_st).items():
overlays.setdefault(pid, {}).update(cells)
# RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
# ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
@@ -3549,9 +3763,11 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
# who assumed both were bodies would have wired the rows and left the summaries answering over
# the entire database under a tab that names one record. `_compiled_read` accepts either shape,
# so the two doors cannot pin differently without this argument being deleted.
+ # S67-T28: the same page snapshot `/rows` takes, so opening a grouped view (its group counts
+ # come from here) no longer re-reads every definition while `user_tables` is unchanged.
read = _compiled_read(table_key, session, tree=query["filters"],
filterConj=query["filterConj"], search=query["search"],
- related=payload.get("related"))
+ related=payload.get("related"), st=_page_defs(session)[0])
read = _restrict_row_ids(read, ids)
out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
read["params"], read["frm"], aggs, group_by=group_by,
diff --git a/api/routes_sheet.py b/api/routes_sheet.py
index ff9cd53376982d3c2af708194321571c510b6b67..fc5e75933931ad8188feb9605f663a8d31c72ad2 100644
--- a/api/routes_sheet.py
+++ b/api/routes_sheet.py
@@ -2565,8 +2565,13 @@ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()
def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
- code="ending_cell_unreadable"):
- """Shared cell read for Ending and Filter, including scoped bound and formula inputs."""
+ code="ending_cell_unreadable", as_date=False):
+ """Shared cell read for Ending and Filter, including scoped bound and formula inputs.
+
+ S67-T10: with ``as_date`` a date-formatted number (typed, or a formula's result such as
+ 46293) is returned as its ISO day, so a Filter condition compares it as that date for every
+ operator; a plain number with no date format is returned unchanged.
+ """
from modules import sheet as sheet_mod
import formula_runtime as fr
@@ -2648,6 +2653,13 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
if cell.get("state") == "stale":
refuse("the cell needs to be refreshed.")
value = cell.get("value", cell.get("text"))
+ if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
+ and sheet_mod.date_formatted_cell(cell)):
+ from harness import windows as W
+ day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
+ if not day:
+ refuse("its value is not a real date.")
+ return day
return value
@@ -2655,7 +2667,8 @@ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen
from modules import sheet as sheet_mod
try:
return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
- session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"))
+ session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable",
+ as_date=True))
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
diff --git a/api/routes_templates.py b/api/routes_templates.py
index 6b499da97ce889b05feb8192ed10bf612a23ba7c..0c7ada55c0c120ee79cbf30734cade841497e0a4 100644
--- a/api/routes_templates.py
+++ b/api/routes_templates.py
@@ -137,6 +137,16 @@ def apply_template(key: str, body: dict = Body(default=None),
import core.table_store as table_store
ops = table_store.make(ws_key, st=session.runtime)
applied = []
+ # S67-T39 (R12, D-2233): a template View bound to an App that no longer exists instantiates as
+ # a plain view of its database; the live App ids are read once, and an unreadable registry
+ # keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
+ try:
+ import application_registry
+ raw_registry = session.runtime.get(application_registry._KEY)
+ live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
+ and isinstance(raw_registry.get("applications"), dict) else None)
+ except Exception:
+ live_apps = None
for view in tpl.get('views') or ():
# ⛔ A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
# — mutating one would write this caller's username into the template every other tenant
@@ -145,6 +155,10 @@ def apply_template(key: str, body: dict = Body(default=None),
# side.
payload = dict(view)
payload['config'] = dict(view.get('config') or {})
+ dead_ref = payload['config'].get('application')
+ if (live_apps is not None and isinstance(dead_ref, dict)
+ and str(dead_ref.get('id') or '') not in live_apps):
+ payload['config'].pop('application')
payload['createdBy'] = session.uname
saved = ops.save_view(session.uname, payload)
applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
@@ -316,8 +330,17 @@ def _agent_templates():
]
+def _content_studio():
+ """The Content Studio App's own module, or None when it is absent (rule 13: delete-safe)."""
+ try:
+ import content_studio
+ except ImportError:
+ return None
+ return content_studio
+
+
def _module_templates(session):
- from core import sheet_templates, user_tables
+ from core import registry, sheet_templates, user_tables
from routes_connectors import _entry_types
from routes_keychain import env_odoo_available
from routes_nav import application_templates
@@ -332,7 +355,9 @@ def _module_templates(session):
saved = _saved_sheet_templates(session.runtime)
data_templates = [(row, _data_template_database(row, definitions))
for row in _templates().TEMPLATES]
- agent_templates = _agent_templates() if _may_open(session, "automation") else []
+ # S67-T36 (R25): an archived Agents module offers no agent template; the create route stays.
+ agent_templates = (_agent_templates() if _may_open(session, "automation")
+ and not registry.is_archived("automation") else [])
databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
databases.update(db for row in saved for db in row["databases"])
databases.update(database for _row, database in data_templates if database)
@@ -361,6 +386,16 @@ def _module_templates(session):
sections = [{"module": "sheet", "templates": sheet},
{"module": "data", "templates": data},
{"module": "app", "templates": app}]
+ # S67-T40 (R18): every tenant's App catalogue offers Content Studio, which creates its own
+ # databases. Guarded (rule 13): without the module the catalogue simply omits the row.
+ # ⚠ Appended AFTER `sections` (it is the App section's own list, so the order stays Finance,
+ # then Studio): `verify_shell_rail.py` scopes its S65-T10 needles to the span from the Finance
+ # offer to `sections = [`, and its finance-offer-wizard control mutates the FIRST
+ # `creation_mode="post"` in this file. A Studio row inside that span, or above it, masks it.
+ studio = _content_studio()
+ if studio is not None:
+ app.append(_catalog_row(studio.TEMPLATE_KEY, studio.LABEL, studio.DESCRIPTION, [],
+ creation_mode="post"))
if agent_templates:
sections.append({"module": "agents", "templates": [
_catalog_row(row["key"], row["name"], row["description"],
@@ -509,6 +544,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
raise err(404, "unknown_template", "That template does not exist.")
if not template["creatable"]:
raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
+ studio = _content_studio() if module == "app" else None
+ if studio is not None and key == studio.TEMPLATE_KEY:
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was created.")
+ try:
+ return studio.create(session, body.get("name") or "")
+ except studio.ContentStudioError as exc:
+ raise err(409, "content_studio_refused", str(exc))
+ except PermissionError as exc:
+ raise err(403, "forbidden", str(exc))
if module == "sheet":
database = template["requires"][0]["database"] if template["requires"] else ""
result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
@@ -529,3 +574,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
app = result["application"]
return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
**result}
+
+# S67-T41: Content Studio's own routes (`/api/v1/content-studio/*`) ride this router because the
+# App wires no mount of its own into main.py; GUARDED, so deleting content_studio.py drops the
+# routes and nothing else (rule 13).
+# ⚠ An inline guarded import, NOT a call to `_content_studio()`: this tail belongs to the
+# dispatcher block, which `verify_home.py`'s W61-T23 declaration-order control moves ABOVE that
+# helper's definition, where a call is a NameError that crashes the whole gate.
+try:
+ import content_studio as _studio_module
+except ImportError:
+ _studio_module = None
+if _studio_module is not None and getattr(_studio_module, "router", None) is not None:
+ router.include_router(_studio_module.router)
diff --git a/platform/core/daily_sales_repair.py b/platform/core/daily_sales_repair.py
index d57257b99ace950452b7130711e8bcf22512d8b4..c069974c44d057c66e32b6bf98bd7fd7d8fdaa56 100644
--- a/platform/core/daily_sales_repair.py
+++ b/platform/core/daily_sales_repair.py
@@ -498,7 +498,8 @@ def _checked_projection(projected, edits, formula_check):
!= {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
raise RepairRefusal(
"repair_unplanned_formula_change",
- "Formula validation would change a cell outside the supplied source diff.")
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
if row is None:
sheet = sheet_names[slot[0]]
@@ -515,7 +516,8 @@ def _checked_projection(projected, edits, formula_check):
or row["after"].get("kind") != "formula":
raise RepairRefusal(
"repair_unplanned_formula_change",
- "Formula validation would change a cell outside the supplied source diff.")
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
recalculated = after.get(slot)
if not isinstance(recalculated, dict) \
or (row["kind"] == "source"
@@ -1333,6 +1335,404 @@ def _build_daily_sales_bindings_plan(current, request, resolve_binding, formula_
"helper": helper, "ending_row": ending, "observed": observed}
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ S67-T21 / R34 — EVERY METRIC COMPUTED; ONLY THE OWNER'S OWN INPUTS STAY TYPED
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# The owner's weekly scorecard read pasted numbers: the weekly block and the middle of a dozen
+# rows were values copied once and never recomputed. This mode applies an explicit, fully named
+# cell plan (each cell a formula, a Cell Database binding or a blank), resolved and recalculated
+# BEFORE a byte moves, and REFUSES unless the projected workbook keeps no hardcoded number but the
+# owner's inputs. The plan itself is built by the operator from the Excel reference; this module is
+# what makes it safe to apply: owner inputs and labels are untouchable, every binding resolves to
+# a fresh value with no limits, the formula census stays empty, and the backup lands first.
+
+SR_TAB = "Summary (SR)"
+
+#: R34 (owner, 2026-09-28), verbatim cells. Summary (All)'s six are the owner's inputs; Summary
+#: (SR)'s are the only numbers the Excel file itself types on that tab (its targets and goals).
+OWNER_INPUTS = {
+ SUMMARY_TAB: frozenset({"Q25", "T20", "T37", "T38", "T51", "T55"}),
+ SR_TAB: frozenset({"T24", "T25", "T34", "T67", "T71", "T105", "T112"}),
+}
+_PLANNED_KINDS = ("formula", "bound", "empty")
+#: The tab-level visual keys a plan may DECLARE (O42: match the Excel file's freeze). Anything else
+#: visual still refuses, exactly as every content-only repair above does.
+_PLANNED_TAB_VISUAL_KEYS = ("freeze",)
+
+
+def hardcoded_numbers(book):
+ """`{tab name: {address: number}}` for every literal numeric cell, PURE. R34's census."""
+ out = {}
+ for tab in book.get("sheets") or ():
+ found = {}
+ for address, cell in (tab.get("cells") or {}).items():
+ value = (cell or {}).get("value")
+ if (cell or {}).get("kind") == "literal" and type(value) in (int, float):
+ found[address] = value
+ out[str(tab.get("name") or "")] = found
+ return out
+
+
+def _bound_value(value, fn):
+ """A resolved binding's value as the document stores it, or a refusal.
+
+ ⭐ A `min`/`max` over a date column answers a DATE ("Latest data" is `max(invoice_date)`), which
+ is stored as its ISO text, the shape every formula date in this workbook already has.
+ """
+ import datetime as _dtm
+ if isinstance(value, (_dtm.date, _dtm.datetime)):
+ if fn not in ("min", "max"):
+ raise RepairRefusal("repair_binding_unreadable", "Only min or max may answer a date.")
+ return value.isoformat()[:10]
+ if isinstance(value, str) and fn in ("min", "max") and re.fullmatch(r"\d{4}-\d{2}-\d{2}.*", value):
+ return value[:10]
+ if value is None and fn in ("avg", "min", "max"):
+ return None
+ if type(value) not in (int, float) or not math.isfinite(value):
+ raise RepairRefusal("repair_binding_unreadable", "A binding must resolve a finite number.")
+ return value
+
+
+def _visual_without(book, planned_notes, keys=("note",)):
+ """`visual_metadata` with each planned cell's declared visual `keys` removed (a note for
+ T21, a sparkline for T23), so a declared change is the only visual change a plan may make."""
+ shaped = copy.deepcopy(book)
+ for tab in shaped.get("sheets") or ():
+ name = str(tab.get("name") or "").casefold()
+ for address, cell in (tab.get("cells") or {}).items():
+ if (name, address) in planned_notes and isinstance(cell, dict):
+ for key in keys:
+ cell.pop(key, None)
+ return visual_metadata(shaped)
+
+
+def _build_computed_metrics_plan(current, request, resolve_binding, formula_check):
+ """R34: validate and resolve an explicit per-cell plan; refuse unless it leaves no hardcode."""
+ spec = request["computed_metrics"]
+ if not isinstance(spec, dict) or any(key in request for key in (
+ "formula_repair", "related_migration", "summary_block", "source_cells",
+ "binding_cells", "daily_sales_bindings")):
+ raise RepairRefusal("repair_metrics_plan_conflict",
+ "Run the computed metrics plan as its own repair.")
+ if not callable(formula_check) or not callable(resolve_binding):
+ raise RepairRefusal("repair_binding_resolver_required",
+ "Computed metrics require the resolver and the formula engine.")
+ refreshed_at = str(request.get("refreshed_at") or "").strip()
+ if not refreshed_at:
+ raise RepairRefusal("repair_timestamp_required", "Computed metrics require a refresh timestamp.")
+ rows = spec.get("cells")
+ if not isinstance(rows, list) or not rows:
+ raise RepairRefusal("repair_metrics_plan_empty", "The computed metrics plan names no cell.")
+ edits, observed, seen, planned_notes = [], [], set(), set()
+ # ⭐ A binding may read a cell this same plan writes (a month's helper start/end dates, a
+ # year header), so bindings resolve against the workbook WITH the planned formulas and blanks
+ # applied and recalculated, never against the stale current one.
+ staged = []
+ for row in rows:
+ want = row.get("cell") if isinstance(row, dict) else None
+ if isinstance(want, dict) and want.get("kind") in ("formula", "empty"):
+ tab = _tab(current, str(row.get("sheet") or ""))
+ address = _addr(row.get("address"))
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ cell = ({"kind": "formula", "state": "stale", "formula": str(want.get("formula") or "")}
+ if want["kind"] == "formula" else {"kind": "empty", "state": "typed"})
+ staged.append({"sheet": str(tab.get("name")), "address": address, "before": before,
+ "after": sheets.clean_cell(_preserve_visual(before, cell) or cell, address)})
+ def recalculated(extra):
+ projected = _project(current, staged + extra)
+ checked = formula_check(copy.deepcopy(projected))
+ return checked["book"] if isinstance(checked, dict) and isinstance(
+ checked.get("book"), dict) else projected
+
+ def resolve_row(row, book):
+ sheet = str(_tab(current, str(row.get("sheet") or "")).get("name"))
+ address = _addr(row.get("address"))
+ binding = sheets.clean_cell({"kind": "bound", "binding": row["cell"].get("binding")},
+ address)["binding"]
+ try:
+ result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(book),
+ copy.deepcopy(_tab(book, sheet)))
+ except Exception as exc: # noqa: BLE001
+ detail = getattr(exc, "detail", None)
+ info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
+ code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
+ return None, RepairRefusal(code, sheet + "!" + address + " binding resolution refused.")
+ return (binding, result), None
+
+ # ⭐ ROUNDS, because a binding may read a date another binding produces through formulas
+ # (Summary (SR)'s week axis runs E12 `max(invoice_date)` -> E15 -> R8 -> F8 -> F31's window).
+ # Each round resolves what it can read, the next projection carries those values, and only a
+ # cell whose filter input was not yet readable waits; no progress in a round refuses by name.
+ resolved, pending = {}, [row for row in rows if isinstance(row, dict)
+ and isinstance(row.get("cell"), dict)
+ and row["cell"].get("kind") == "bound"]
+ for _round in range(6):
+ if not pending:
+ break
+ extra = []
+ for (sheet, address), (binding, result) in resolved.items():
+ value = result.get("value") if isinstance(result, dict) else None
+ try:
+ value = _bound_value(value, (binding.get("aggregate") or {}).get("fn"))
+ except RepairRefusal:
+ value = None
+ extra.append({"sheet": sheet, "address": address, "before": None,
+ "after": sheets.clean_cell({"kind": "bound", "state": "fresh",
+ "binding": binding, "value": value,
+ "display": "" if value is None else str(value)},
+ address)})
+ book = recalculated(extra)
+ waiting, first_error = [], None
+ for row in pending:
+ got, error = resolve_row(row, book)
+ if got is not None:
+ resolved[(str(_tab(current, str(row["sheet"])).get("name")),
+ _addr(row.get("address")))] = got
+ elif error.code == "filter_cell_unreadable":
+ waiting.append(row)
+ first_error = first_error or error
+ else:
+ raise error
+ if len(waiting) == len(pending):
+ raise first_error
+ pending = waiting
+ if pending:
+ raise RepairRefusal("filter_cell_unreadable", "A binding's filter cell never became readable.")
+ for row in rows:
+ if not isinstance(row, dict):
+ raise RepairRefusal("repair_metrics_cell_invalid", "Every planned cell must be an object.")
+ sheet_name = str(row.get("sheet") or "").strip()
+ address = _addr(row.get("address"))
+ tab = _tab(current, sheet_name)
+ sheet_name = str(tab.get("name") or sheet_name)
+ slot = (sheet_name.casefold(), address)
+ if slot in seen:
+ raise RepairRefusal("repair_edit_duplicate", "A planned cell may be named only once.")
+ seen.add(slot)
+ if address in OWNER_INPUTS.get(sheet_name, ()):
+ raise RepairRefusal("repair_owner_input", sheet_name + "!" + address
+ + " is one of the owner's typed inputs and stays typed.")
+ cells = tab.get("cells") or {}
+ before = copy.deepcopy(cells.get(address))
+ prior_value = (before or {}).get("value")
+ want = row.get("cell")
+ if not isinstance(want, dict) or want.get("kind") not in (*_PLANNED_KINDS, "note_only"):
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
+ + " must be planned as a formula, a binding, a blank or a note.")
+ kind = want["kind"]
+ # A typed ISO date is a value (Summary (SR)'s frozen E13/E15), never a label.
+ if (before or {}).get("kind") == "literal" and isinstance(prior_value, str) \
+ and prior_value.strip() and kind != "note_only" \
+ and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", prior_value.strip()):
+ raise RepairRefusal("repair_label_protected", sheet_name + "!" + address
+ + " holds a label, which a metrics plan never replaces.")
+ if kind == "note_only":
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
+ elif kind == "formula":
+ text = str(want.get("formula") or "")
+ if not text.startswith("="):
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
+ + " needs a formula that starts with =.")
+ desired = _preserve_visual(before, {"kind": "formula", "state": "stale", "formula": text})
+ elif kind == "empty":
+ desired = _preserve_visual(before, {"kind": "empty", "state": "typed"}) \
+ or {"kind": "empty", "state": "typed"}
+ else:
+ binding, result = resolved[(sheet_name, address)]
+ fn = (binding.get("aggregate") or {}).get("fn")
+ if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
+ or result.get("error") or result.get("state") not in (None, "fresh"):
+ raise RepairRefusal("repair_binding_unreadable", sheet_name + "!" + address
+ + " did not resolve a fresh bound value.")
+ if result.get("limits"):
+ raise RepairRefusal("repair_binding_limits", sheet_name + "!" + address
+ + " has unapplied or limited binding conditions.")
+ value = _bound_value(result["value"], fn)
+ if (before or {}).get("kind") == "bound" and before.get("binding") == binding \
+ and type(prior_value) in (int, float) and type(value) in (int, float) \
+ and math.isclose(prior_value, value, rel_tol=1e-9, abs_tol=1e-9):
+ value = prior_value # the same number; a rerun changes nothing
+ desired = _preserve_visual(before, {
+ "kind": "bound", "state": "fresh", "binding": binding, "value": value,
+ "display": "" if value is None else str(value), "refreshed_at": refreshed_at})
+ observed.append({"sheet": sheet_name, "address": address, "value": value})
+ if "note" in row:
+ note = str(row.get("note") or "").strip()
+ if note:
+ desired["note"] = note
+ else:
+ desired.pop("note", None)
+ planned_notes.add(slot)
+ after = sheets.clean_cell(desired, address)
+ # A planned formula is `derived`: the recalculation below is what fills its value, and
+ # `_checked_projection` accepts a recalculated planned cell only under that kind.
+ edit_kind = {"formula": "derived", "bound": "binding"}.get(kind, "metric")
+ edits.append({"kind": edit_kind, "name": sheet_name + "!" + address, "sheet": sheet_name,
+ "address": address, "before": before, "after": after})
+ tab_visual = spec.get("tab_visual") or {}
+ if not isinstance(tab_visual, dict):
+ raise RepairRefusal("repair_metrics_visual_invalid", "Tab visual changes must be an object.")
+ for sheet_name, keys in tab_visual.items():
+ _tab(current, sheet_name)
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_PLANNED_TAB_VISUAL_KEYS):
+ raise RepairRefusal("repair_metrics_visual_invalid",
+ "A metrics plan may declare only a tab's freeze.")
+ projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
+ _daily_sales_invariants(projected, formula_check)
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
+ for name, found in hardcoded_numbers(projected).items()
+ if name in OWNER_INPUTS}
+ if any(remaining.values()):
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers would remain: "
+ + json.dumps(remaining, sort_keys=True))
+ return [edit for edit in edits if edit["before"] != edit["after"]], {
+ "observed": observed, "planned_notes": sorted(f"{s}!{a}" for s, a in planned_notes),
+ "tab_visual": copy.deepcopy(tab_visual),
+ "hardcoded_before": hardcoded_numbers(current), "hardcoded_after": hardcoded_numbers(projected)}
+
+
+def _apply_tab_visual(st, workbook_id, tab_visual, *, actor="", actor_name=""):
+ """Write the DECLARED tab-level visual keys through the ordinary document door, then prove
+ nothing else moved. Returns the readback snapshot."""
+ doc = sheets.workbook_snapshot(st, workbook_id)
+ before_cells = {str(t.get("name")): copy.deepcopy(t.get("cells") or {}) for t in doc["sheets"]}
+ before_visual = visual_metadata(doc)
+ for sheet_name, keys in tab_visual.items():
+ tab = _tab(doc, sheet_name)
+ for key, value in keys.items():
+ tab[key] = copy.deepcopy(value)
+ sheets.put_workbook(st, workbook_id, doc, expect_rev=int(doc.get("rev") or 0),
+ actor=actor, actor_name=actor_name, lossless=True)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ for sheet_name, keys in tab_visual.items():
+ tab = _tab(readback, sheet_name)
+ for key, value in keys.items():
+ if _canonical(tab.get(key)) != _canonical(value):
+ raise RepairRefusal("repair_visual_readback", sheet_name + " " + key
+ + " did not read back as declared.")
+ if {str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} != before_cells:
+ raise RepairRefusal("repair_visual_readback", "A declared visual change moved cell content.")
+ expected = copy.deepcopy(before_visual)
+ for item in expected:
+ for key, value in (tab_visual.get(item.get("name")) or {}).items():
+ item[key] = copy.deepcopy(value)
+ if visual_metadata(readback) != _canonical(expected):
+ raise RepairRefusal("repair_visual_readback", "An undeclared visual property changed.")
+ return readback
+
+
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+# ⭐⭐ S67-T22 / S67-T23 — DECLARED VISUAL REPAIRS: Excel's rule colours and its 3M sparklines
+# ═════════════════════════════════════════════════════════════════════════════════════════════
+#
+# Every repair above REFUSES a visual change, on purpose. These two tickets ARE visual changes, so
+# they get their own mode that names exactly what may move: a tab's `rules` (T22, O31) and a cell's
+# `sparkline` (T23, O32). Cell content, every other visual key and every other cell must read back
+# unchanged; the backup lands before anything is written.
+
+_VISUAL_REPAIR_TAB_KEYS = ("rules",)
+
+
+def _build_visual_repair_plan(current, request):
+ spec = request["visual_repair"]
+ if not isinstance(spec, dict) or any(key in request for key in (
+ "formula_repair", "related_migration", "summary_block", "source_cells",
+ "binding_cells", "daily_sales_bindings", "computed_metrics")):
+ raise RepairRefusal("repair_visual_plan_conflict", "Run the visual repair as its own repair.")
+ tab_visual = copy.deepcopy(spec.get("tab_visual") or {})
+ for sheet_name, keys in tab_visual.items():
+ _tab(current, sheet_name)
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_VISUAL_REPAIR_TAB_KEYS):
+ raise RepairRefusal("repair_visual_invalid", "A visual repair may declare only a tab's rules.")
+ keys["rules"] = sheets._clean_rules(keys["rules"], lossless=True)
+ edits, seen = [], set()
+ for row in spec.get("cells") or ():
+ if not isinstance(row, dict) or "sparkline" not in row:
+ raise RepairRefusal("repair_visual_invalid", "A visual repair cell names only its sparkline.")
+ tab = _tab(current, str(row.get("sheet") or ""))
+ sheet_name, address = str(tab.get("name")), _addr(row.get("address"))
+ if (sheet_name.casefold(), address) in seen:
+ raise RepairRefusal("repair_edit_duplicate", "A repaired cell may be named only once.")
+ seen.add((sheet_name.casefold(), address))
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
+ if row["sparkline"] is None:
+ desired.pop("sparkline", None)
+ else:
+ clean = sheets._clean_sparkline(row["sparkline"])
+ if clean is None:
+ raise RepairRefusal("repair_visual_invalid", sheet_name + "!" + address
+ + " names a sparkline that cannot be drawn.")
+ desired["sparkline"] = clean
+ after = sheets.clean_cell(desired, address)
+ content = ("kind", "state", "value", "display", "formula", "binding", "text")
+ if any((before or {}).get(key) != after.get(key) for key in content if key != "state") \
+ and not (before is None and after.get("kind") == "empty"):
+ raise RepairRefusal("repair_visual_content_changed", sheet_name + "!" + address
+ + " would change content, which a visual repair never does.")
+ edits.append({"kind": "visual", "name": sheet_name + "!" + address, "sheet": sheet_name,
+ "address": address, "before": before, "after": after})
+ return [e for e in edits if e["before"] != e["after"]], {
+ "tab_visual": tab_visual, "sparkline_cells": sorted(e["name"] for e in edits)}
+
+def _execute_visual_repair(st, backup_store, request, current, revision, digest, workbook_id,
+ *, actor="", actor_name=""):
+ """Back up, then write ONLY the declared sparklines (exact cell edits) and tab rules."""
+ edits, evidence = _build_visual_repair_plan(current, request)
+ if not edits and not evidence["tab_visual"]:
+ return {"workbook_id": workbook_id, "revision_before": revision, "revision_after": revision,
+ "applied": 0, "noop": True, "visual_repair": evidence}
+ planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
+ projected = _project(current, edits)
+ if _visual_without(projected, planned, ("sparkline",)) != \
+ _visual_without(current, planned, ("sparkline",)):
+ raise RepairRefusal("repair_visual_metadata_changed",
+ "The projected repair changes a visual property it did not declare.")
+ exact = [{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
+ for row in edits]
+ diff_digest = sheets.workbook_digest({"edits": exact, "tab_visual": evidence["tab_visual"]})
+ backup_key = _write_backup(backup_store, current, digest, diff_digest, workbook_id)
+ revision_after = revision
+ if exact:
+ sheets.apply_exact_cell_edits(st, workbook_id, exact, revision, digest,
+ actor=actor, actor_name=actor_name)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ if int(readback.get("rev") or 0) != revision + 1:
+ raise RepairRefusal("repair_readback_revision",
+ "The repaired workbook revision did not read back as expected.")
+ for row in edits:
+ actual = (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"])
+ if actual != sheets.clean_cell(row["after"], row["address"]):
+ raise RepairRefusal("repair_readback_cell",
+ row["sheet"] + "!" + row["address"] + " did not read back exactly.")
+ revision_after = int(readback["rev"])
+ if evidence["tab_visual"]:
+ revision_after = int(_apply_tab_visual(st, workbook_id, evidence["tab_visual"],
+ actor=actor, actor_name=actor_name)["rev"])
+ return {"workbook_id": workbook_id, "revision_before": revision,
+ "revision_after": revision_after, "applied": len(exact), "backup_key": backup_key,
+ "backup_digest": digest, "diff_digest": diff_digest, "visual_repair": evidence}
+
+
+def restore_backup(st, backup_store, backup_key, *, workbook_id, actor="", actor_name=""):
+ """NC for every mode here: put a backup's workbook back and prove it round-trips."""
+ record = backup_store.get(backup_key)
+ if not isinstance(record, dict) or record.get("workbook_id") != workbook_id:
+ raise RepairRefusal("repair_backup_missing", "That backup does not exist for this workbook.")
+ saved = copy.deepcopy(record["workbook"])
+ current = sheets.workbook_snapshot(st, workbook_id)
+ sheets.put_workbook(st, workbook_id, saved, expect_rev=int(current.get("rev") or 0),
+ actor=actor, actor_name=actor_name, lossless=True)
+ readback = sheets.workbook_snapshot(st, workbook_id)
+ same = ({str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]}
+ == {str(t.get("name")): t.get("cells") or {} for t in saved["sheets"]}
+ and visual_metadata(readback) == visual_metadata(saved))
+ if not same:
+ raise RepairRefusal("repair_restore_unverified", "The restored workbook did not round-trip.")
+ return readback
+
+
def _build_plan(current, request, resolve_binding):
if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
raise RepairRefusal(
@@ -1525,12 +1925,24 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
raise RepairRefusal("repair_revision_mismatch",
"The named workbook is not at the expected repair revision.")
digest = sheets.workbook_digest(current)
+ if "visual_repair" in request:
+ return _execute_visual_repair(st, backup_store, request, current, revision, digest,
+ target_workbook_id, actor=actor, actor_name=actor_name)
summary_evidence = None
summary_errors_before = None
migrated = None
formula_evidence = None
daily_evidence = None
- if "daily_sales_bindings" in request:
+ metrics_evidence = None
+ if "computed_metrics" in request:
+ formula_check = _daily_sales_formula_checker(formula_check)
+ edits, metrics_evidence = _build_computed_metrics_plan(
+ current, request, resolve_binding, formula_check)
+ if not edits and not metrics_evidence["tab_visual"]:
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
+ "revision_after": revision, "applied": 0, "noop": True,
+ "computed_metrics": metrics_evidence}
+ elif "daily_sales_bindings" in request:
formula_check = _daily_sales_formula_checker(formula_check)
edits, daily_evidence = _build_daily_sales_bindings_plan(
current, request, resolve_binding, formula_check)
@@ -1579,7 +1991,11 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
"The projected repair introduces a formula error or leaves one in its repair scope, "
"so no repair was applied.")
visual_before = visual_metadata(current)
- if visual_metadata(projected) != visual_before:
+ planned_notes = set()
+ if metrics_evidence is not None:
+ planned_notes = {tuple(item.split("!", 1)) for item in metrics_evidence["planned_notes"]}
+ planned_notes = {(sheet.casefold(), address) for sheet, address in planned_notes}
+ if _visual_without(projected, planned_notes) != _visual_without(current, planned_notes):
raise RepairRefusal("repair_visual_metadata_changed",
"The projected repair changes Daily Sales visual metadata.")
@@ -1611,7 +2027,7 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
or _cell_map(checked_readback) != _cell_map(readback):
raise RepairRefusal("repair_readback_formula_state",
"The repaired workbook did not read back calculation-current.")
- if visual_metadata(readback) != visual_before:
+ if _visual_without(readback, planned_notes) != _visual_without(current, planned_notes):
raise RepairRefusal("repair_readback_visual_metadata",
"Daily Sales visual metadata changed during repair.")
if formula_evidence is not None:
@@ -1667,4 +2083,19 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
result["daily_sales_bindings"] = daily_evidence
result["formula_error_census"] = {
"before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
+ if metrics_evidence is not None:
+ _daily_sales_invariants(readback, formula_check)
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
+ for name, found in hardcoded_numbers(readback).items() if name in OWNER_INPUTS}
+ if any(remaining.values()):
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers remain after the write: "
+ + json.dumps(remaining, sort_keys=True))
+ if metrics_evidence["tab_visual"]:
+ visual_readback = _apply_tab_visual(st, target_workbook_id, metrics_evidence["tab_visual"],
+ actor=actor, actor_name=actor_name)
+ result["revision_after"] = visual_readback["rev"]
+ metrics_evidence["hardcoded_readback"] = hardcoded_numbers(readback)
+ result["computed_metrics"] = metrics_evidence
+ result["formula_error_census"] = {
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
return result
diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py
index 2ecd0e110759998d3811c07e4c79f7e7368afbc6..ac5c6cb060582da8833d5e98c6df5d987724ae52 100644
--- a/platform/core/external_tools.py
+++ b/platform/core/external_tools.py
@@ -42,12 +42,19 @@ _STRING = {"type": "string"}
_ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
_FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
"currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
+# S67-T14 / contract C2: a border side is the legacy `true` (read as a thin black line) or
+# `{style, color}` with one of Excel's seven line styles and a six digit hex colour; `false` or an
+# absent side is no side. `sheets.py::_clean_border_side` refuses anything else by code, and this
+# publishes the same shape so an AI client can draw a styled border at all.
+_BORDER_SIDE = {"oneOf": [{"type": "boolean"}, _object({
+ "style": {"enum": list(sheets.CELL_BORDER_STYLES)},
+ "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"}}, ("style", "color"))]}
_STYLE = _object({
**{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
**{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
"size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
"textRotation": {"type": "number"}, "format": _FORMAT,
- "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}),
+ "border": _object({side: _BORDER_SIDE for side in ("top", "right", "bottom", "left")}),
})
_VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
_FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
@@ -62,15 +69,25 @@ _BINDING = _object({
"period": _object({"unit": _STRING, "start": _STRING}),
"ending": {"type": "object"},
}, ("database", "aggregate", "scope"))
-_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
- "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
- "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
_RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
for key in ("col0", "row0", "col1", "row1")},
("col0", "row0", "col1", "row1"))]}
+# S67-T18: an in-cell sparkline, `{kind, range}` exactly as `sheets.py::_clean_sparkline` stores
+# it; the range is a rectangle on the SAME tab (an A1 range such as "B2:M2", or the dict).
+_SPARKLINE = _object({"kind": {"enum": list(sheets.SPARKLINE_KINDS)}, "range": _RANGE},
+ ("kind", "range"))
+_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
+ "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
+ "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING,
+ "sparkline": _SPARKLINE}, ("kind",))
+# S67-T13: a colour scale's two ends and what each end means. `sheets.py::_clean_rules` refuses
+# a type outside `SHEET_RULE_SCALE_TYPES`; the enum publishes the same list to the caller.
+_SCALE_TYPE = {"enum": list(sheets.SHEET_RULE_SCALE_TYPES)}
_RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
"value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
- "scale": {"type": "object"}}, ("range", "op"))
+ "scale": _object({"low": _STRING, "high": _STRING,
+ "minType": _SCALE_TYPE, "maxType": _SCALE_TYPE})},
+ ("range", "op"))
_CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
"stacking": _STRING, "source": {"type": "object"},
"series": {"type": "array", "items": {"type": "object"}},
diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py
index f2c308477c631434959a219e36f24acfdf963bda..496b8c568257451ec0014edbbae423f1b7a163ae 100644
--- a/platform/core/grid_events.py
+++ b/platform/core/grid_events.py
@@ -632,6 +632,57 @@ def _grant_marker(ctx, ws):
return _has_grants
+#: S67-T39 / R36: the Application registry's store key. READ here, never imported:
+#: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE).
+_APPLICATIONS_KEY = 'applications_v1'
+
+
+def _live_application_ids(ctx):
+ """The set of App ids the tenant registry holds, or None when that is not KNOWN.
+
+ None (read failed, or no registry document with an `applications` map) means "strip
+ nothing": an unknown answer must never turn a live App binding into a plain view.
+ """
+ try:
+ doc = _store_of(ctx).get(_APPLICATIONS_KEY)
+ except Exception as exc:
+ _tel.error('applications:live', exc)
+ return None
+ apps = doc.get('applications') if isinstance(doc, dict) else None
+ return {str(key) for key in apps} if isinstance(apps, dict) else None
+
+
+def _application_ref_id(view):
+ config = view.get('config') if isinstance(view, dict) else None
+ ref = config.get('application') if isinstance(config, dict) else None
+ return str(ref.get('id') or '') if isinstance(ref, dict) else None
+
+
+def _drop_dead_application_refs(ctx, views):
+ """S67-T39 / R36 / D-2233: a View bound to a DELETED App is served as a plain view of its
+ database (the `application` member dropped, everything else kept), so it opens with no
+ error. Projection only, and it never raises: any failure serves the views unchanged.
+ The registry is read only when some View actually carries a reference.
+ """
+ try:
+ if not any(_application_ref_id(v) is not None for v in (views or {}).values()):
+ return views
+ live = _live_application_ids(ctx)
+ if live is None:
+ return views
+ out = {}
+ for vid, view in views.items():
+ ref_id = _application_ref_id(view)
+ if ref_id is not None and ref_id not in live:
+ config = {k: v for k, v in view['config'].items() if k != 'application'}
+ view = {**view, 'config': config}
+ out[vid] = view
+ return out
+ except Exception as exc:
+ _tel.error('applications:dead_refs', exc)
+ return views
+
+
def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
@@ -707,6 +758,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
merged.update(ws.get('views') or {}) # own views win the id collision
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
for vid, v in merged.items()}
+ if ws.get('views'):
+ ws['views'] = _drop_dead_application_refs(ctx, ws['views'])
return ws
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
# API caller has no durable dict and must be told, not quietly served memory.
@@ -1584,7 +1637,13 @@ def handle_one(event, ctx):
_refuse(ctx, 'invalid_application_ref',
'that Application reference is not a valid typed identity', view_id)
return False
- config['application'] = application
+ # S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View
+ # saves as a plain view of its database. Unknown registry state keeps the ref.
+ _live = _live_application_ids(ctx)
+ if _live is not None and application['id'] not in _live:
+ application = None
+ else:
+ config['application'] = application
# Wave-6 item 10: a direct non-grid display mode rides the view config, validated
# structurally (mode whitelist, refs must be this table's fields). A bound Application's
# config remains in the registry and is resolved on read, so an offered clone is dropped.
diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py
index efeb99085ec3818f71338190ddd7819a040f4b7e..bb82e8790330ec3d9fa76a02f3ef0b4770d0ac51 100644
--- a/platform/core/perm_scope.py
+++ b/platform/core/perm_scope.py
@@ -221,12 +221,39 @@ def assistant_entry(user, module):
that reader must be able to name the migrated grant that admitted a database. In particular,
a store-outage admin identity with no ``perms`` document is not an unresolved permission that
may be widened into data access.
+
+ ⭐ S67-T34 / D-276: ONE NAMED EXCEPTION, the grant the NAVIGATION uses. For the two registry
+ topics in ``ASSISTANT_NAV_TOPICS`` an administrator account is admitted unfiltered (unless
+ explicitly unticked), because ``nav_may_open`` leg 1 opens them to it; the AI list must equal
+ the nav (R33). See the comment at that leg for why the outage identity no longer reaches it.
"""
e = entry(user, module)
- if (not is_migrated(user) or not isinstance(e, dict)
- or not bool(e.get('access', True)) or not may_access(user, module)):
- return None
- return e
+ if (is_migrated(user) and isinstance(e, dict)
+ and bool(e.get('access', True)) and may_access(user, module)):
+ return e
+ # ⭐⭐ S67-T34 / D-276 / R26: THE ADMINISTRATOR THE NAVIGATION ADMITS. `nav_may_open` leg 1
+ # opens every registry topic to an administrator, and `_clean_perms` writes no per-topic
+ # entry for one, so an administrator saw `customer_data` and `product_data` (Odoo products,
+ # R26's Inventory source) in the nav and had them refused here. MEASURED on staging
+ # 2026-09-28: aaron (admin, no `perms_v`) had both in the nav and neither in the AI list. The
+ # answer is the grant the nav already uses, admitted for exactly these topics: an account
+ # whose ROLE is admin, minus an explicit `access: false` (narrower than the nav, never wider).
+ # Unfiltered, as the nav's break-glass is.
+ # ⚠ THE STORE-OUTAGE IDENTITY the docstring names is no longer built by `deps._user_for`
+ # (it admits a stored record or the QA runner, nothing else), so the role is read off a
+ # stored account here. If a synthetic admin identity is ever reintroduced, it must be kept
+ # out of this leg.
+ # ⛔ NOT A MEMBER. A non-administrator still needs the explicit migrated grant above, so a
+ # user without it does not see the database (the NC).
+ if (module in ASSISTANT_NAV_TOPICS and perms.is_admin(user)
+ and not (isinstance(e, dict) and e.get('access') is False)):
+ return dict(e) if isinstance(e, dict) else {'access': True, 'filter': None}
+ return None
+
+
+#: S67-T34: the registry topics the navigation opens to an administrator by break-glass, which
+#: `assistant_entry` therefore admits for a migrated administrator without a per-topic entry.
+ASSISTANT_NAV_TOPICS = frozenset({'customer_data', 'product_data'})
# ── FIELDS ───────────────────────────────────────────────────────────────────────────────────
diff --git a/platform/core/registry.py b/platform/core/registry.py
index ae3c90ed139055dca5823c8d1b6c4b4b95af3310..ad4901aeb538a3ddb2752d5f474c8cf52e5bc8e6 100644
--- a/platform/core/registry.py
+++ b/platform/core/registry.py
@@ -203,7 +203,12 @@ REGISTRY = [
'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
'them") made that literal: `validate_only` takes it out of the account menu too, '
'which is the last place it was still visible. The proof survives; the door does not.'},
+ # S67-T36 (owner instr 24, R25, 2026-09-28): Agents is TABLED. `archived` hides the UI only:
+ # no rail row, no Settings module toggle, no Home section, no agent templates. The
+ # `/automations` APIs, stored automations and their schedules are untouched and keep running.
+ # ⭐ RESTORING IS THIS ONE LINE: delete `'archived': True` below and every door comes back.
{'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
+ 'archived': True,
'nav': True,
'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
'(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
@@ -280,6 +285,11 @@ def brand_filterable(key) -> bool:
return bool(BY_KEY.get(key, {}).get('brand'))
+def is_archived(key) -> bool:
+ """True when the registry row `key` is archived (tabled for everyone). Unknown keys: False."""
+ return bool(BY_KEY.get(key, {}).get('archived'))
+
+
# ═════════════════════════════════════════════════════════════════════════════════════════════
# ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE
# ═════════════════════════════════════════════════════════════════════════════════════════════
diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py
index 1bc4d6ce5726f9cf67e7db286b78602b3944eeb1..ec057f106cf47a08b0f589b77aee9b29ce004ce6 100644
--- a/platform/core/sheet_import.py
+++ b/platform/core/sheet_import.py
@@ -43,9 +43,11 @@ from __future__ import annotations
import re
import zipfile
import hashlib
+import io
import json
import math
import os
+import posixpath
import tempfile
import xml.etree.ElementTree as ET
from datetime import date, datetime, time, timezone
@@ -392,12 +394,11 @@ def style_of(cell, notes):
if border is not None:
sides = {}
for side in sheets._BORDER_SIDES:
- edge = getattr(border, side, None)
- if edge is not None and getattr(edge, "style", None):
- sides[side] = True
+ line = border_line(getattr(border, side, None), notes)
+ if line is not None:
+ sides[side] = line
if sides:
out["border"] = sides
- notes.add("a border keeps its side but not its weight, style or colour")
fmt, note = classify_format(getattr(cell, "number_format", ""))
if fmt:
@@ -407,6 +408,47 @@ def style_of(cell, notes):
return out
+#: S67-T14: the OOXML line styles that are not one of C2's seven, each drawn as its nearest.
+#: The dash and dot variants keep their PATTERN over their weight, because a patterned line drawn
+#: solid reads as a different border while a medium dash drawn at 1px still reads as dashed.
+_BORDER_NEAREST = {
+ "mediumDashed": "dashed", "dashDot": "dashed", "dashDotDot": "dashed",
+ "mediumDashDot": "dashed", "mediumDashDotDot": "dashed", "slantDashDot": "dashed",
+}
+
+
+def border_line(edge, notes):
+ """One openpyxl border side as C2's `{style, color}`, or None when the side draws nothing.
+
+ The seven Excel names C2 shares pass through unchanged; the dash dot variants become their
+ nearest with a note naming the approximation. The colour is the explicit RGB when there is one;
+ an automatic colour is black, which is what Excel draws; a theme or indexed colour is drawn in
+ black too, with a note, because a colour nobody chose is never guessed (see `argb_of`).
+ """
+ style = str(getattr(edge, "style", None) or "") if edge is not None else ""
+ if not style or style == "none":
+ return None
+ if style not in sheets.CELL_BORDER_STYLES:
+ nearest = _BORDER_NEAREST.get(style, "thin")
+ notes.add("a " + style + " border is drawn as " + nearest)
+ style = nearest
+ colour = getattr(edge, "color", None)
+ argb = argb_of(colour)
+ if argb:
+ hex_colour = "#" + argb.lower()
+ else:
+ hex_colour = "#000000"
+ kind = getattr(colour, "type", "") if colour is not None else ""
+ # ⚠ `type`, never `.auto`: openpyxl answers an unset descriptor with a (truthy) error
+ # STRING, so `colour.auto` reads as set on every theme colour.
+ automatic = (colour is None or kind == "auto"
+ or (kind == "indexed" and getattr(colour, "indexed", None) == 64)
+ or (kind == "rgb" and str(getattr(colour, "rgb", "")).upper() == "00000000"))
+ if not automatic:
+ notes.add("a theme or indexed border colour was drawn in black")
+ return {"style": style, "color": hex_colour}
+
+
# ── values ───────────────────────────────────────────────────────────────────────────────────
def json_value(value):
@@ -1037,6 +1079,134 @@ def guard_import_archive(path):
"That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
+# ── sparklines (S67-T18) ─────────────────────────────────────────────────────────────────────
+
+#: ⭐ S67-T18: Excel keeps sparkline groups in the worksheet's x14 extension, which openpyxl drops,
+#: so they are read from the worksheet PART itself. A group with no `type` draws lines, `column`
+#: draws columns and `stacked` draws win/loss bars (Excel's name for `sheets.SPARKLINE_KINDS`'s
+#: `winloss`). The part was already admitted by `guard_import_archive`, so this second read adds
+#: no new exposure.
+_X14 = "{http://schemas.microsoft.com/office/spreadsheetml/2009/9/main}"
+_XM = "{http://schemas.microsoft.com/office/excel/2006/main}"
+_MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+_DOC_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
+_PKG_REL_NS = "{http://schemas.openxmlformats.org/package/2006/relationships}"
+_SPARKLINE_KIND_OF = {"line": "line", "column": "column", "stacked": "winloss"}
+
+
+def _worksheet_parts(archive):
+ """`{tab name: worksheet part}` through `xl/workbook.xml` and its relationships, never by index."""
+ names = set(archive.namelist())
+ if "xl/workbook.xml" not in names or "xl/_rels/workbook.xml.rels" not in names:
+ return {}
+ targets = {}
+ for rel in ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")).iter(_PKG_REL_NS + "Relationship"):
+ target = str(rel.get("Target") or "")
+ targets[rel.get("Id")] = (target.lstrip("/") if target.startswith("/")
+ else posixpath.normpath("xl/" + target))
+ out = {}
+ for sheet in ET.fromstring(archive.read("xl/workbook.xml")).iter(_MAIN_NS + "sheet"):
+ part = targets.get(sheet.get(_DOC_REL_NS + "id"))
+ if part in names:
+ out[str(sheet.get("name") or "")] = part
+ return out
+
+
+def _sparkline_groups(stream):
+ """`[(type, [(formula, sqref), ...]), ...]` for one worksheet part's x14 sparkline groups."""
+ groups = []
+ inside = 0 # a group's subtree is kept until the group itself ends
+ for event, element in ET.iterparse(stream, events=("start", "end")):
+ if element.tag == _X14 + "sparklineGroup":
+ if event == "start":
+ inside += 1
+ continue
+ inside -= 1
+ entries = [((spark.findtext(_XM + "f") or "").strip(),
+ (spark.findtext(_XM + "sqref") or "").strip())
+ for spark in element.iter(_X14 + "sparkline")]
+ groups.append((str(element.get("type") or "line"), entries))
+ element.clear()
+ elif event == "end" and not inside:
+ element.clear()
+ return groups
+
+
+def _sparklines_of(path, notes):
+ """`{tab name: groups}` for every worksheet that carries x14 sparklines; `{}` for the rest.
+
+ A part whose bytes never mention a sparkline group is not parsed at all, so a workbook with
+ no sparklines imports exactly as it did before S67-T18.
+ """
+ out = {}
+ try:
+ with zipfile.ZipFile(path) as archive:
+ for name, part in _worksheet_parts(archive).items():
+ raw = archive.read(part)
+ if b"sparklineGroup" not in raw:
+ continue
+ groups = _sparkline_groups(io.BytesIO(raw))
+ if groups:
+ out[name] = groups
+ except (KeyError, OSError, ET.ParseError, zipfile.BadZipFile):
+ notes.add("the workbook's sparklines could not be read, so none were imported")
+ return {}
+ return out
+
+
+def _count_phrase(count, one, many):
+ return (one if count == 1 else str(count) + " " + many)
+
+
+def _apply_sparklines(tab_name, cells, groups, blank_styles, notes):
+ """Write `cell.sparkline {kind, range}` on each location cell of this tab's groups.
+
+ `sheets._clean_sparkline` stores a rectangle LOCAL to its tab, so a sparkline drawn from
+ another tab's data cannot be carried: it is left out and NAMED, never dropped in silence
+ (the module note). A location that is not one cell, or data that is not a range, is named too.
+ A blank location becomes an `empty` cell holding only the sparkline, which is exactly what
+ `clean_cell`'s empty branch keeps; its compact blank-style reference moves onto it.
+ """
+ other_tab = unreadable = 0
+ for kind_name, entries in groups:
+ kind = _SPARKLINE_KIND_OF.get(kind_name)
+ if kind is None:
+ unreadable += len(entries)
+ continue
+ for formula, sqref in entries:
+ source = sheet_of_ref(formula)
+ if source and source.casefold() != tab_name.casefold():
+ other_tab += 1
+ continue
+ rect = rect_of(formula)
+ places = sqref.replace("$", "").upper().split()
+ if (rect is None or len(places) != 1 or ":" in places[0]
+ or not _address_in_document(places[0])):
+ unreadable += 1
+ continue
+ addr = places[0]
+ spark = {"kind": kind, "range": rect}
+ cell = cells.get(addr)
+ if cell is None:
+ if len(cells) >= sheets._MAX_CELLS_PER_SHEET:
+ unreadable += 1
+ continue
+ cell = {"kind": "empty", "sparkline": spark}
+ if addr in blank_styles:
+ cell["excel"] = {"styleId": blank_styles.pop(addr)}
+ cells[addr] = cell
+ else:
+ cell["sparkline"] = spark
+ if other_tab:
+ notes.add(_count_phrase(other_tab, "a sparkline", "sparklines")
+ + " drawn from another tab's data " + ("was" if other_tab == 1 else "were")
+ + " left out")
+ if unreadable:
+ notes.add(_count_phrase(unreadable, "a sparkline", "sparklines")
+ + " with a location or data range that could not be read "
+ + ("was" if unreadable == 1 else "were") + " left out")
+
+
def _print_of(ws):
"""Store print setup and inclusive print areas; export executes these settings later."""
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
@@ -1148,6 +1318,7 @@ def import_workbook(path, *, source="", progress=None):
counts = {"cells": 0, "formulas": 0}
book_notes = set()
+ sparklines = _sparklines_of(path, book_notes) # S67-T18
authored = _authored_chart_document(path, book_notes)
chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
if authored and len(chart_tabs) != len(authored["sheets"]):
@@ -1209,6 +1380,8 @@ def import_workbook(path, *, source="", progress=None):
cells = _cells_of(ws, cached, notes, counts, excel)
_say("formats", 0.7 + 0.25 * (index / total))
_validations_of(ws, cells, notes)
+ if name in sparklines: # S67-T18, before usedExtent
+ _apply_sparklines(name, cells, sparklines[name], excel["blankStyles"], notes)
cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
"cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
diff --git a/platform/core/sheets.py b/platform/core/sheets.py
index 8d09b6673d7255f99a61de847113b3a3deeae803..7c8c11cfd433fcae1387ad729085acff30d4134f 100644
--- a/platform/core/sheets.py
+++ b/platform/core/sheets.py
@@ -152,6 +152,14 @@ MAX_FONT_PT = 96
_STYLE_MARKS = ("bold", "italic", "underline")
_BORDER_SIDES = ("top", "right", "bottom", "left")
+#: ⭐⭐ S67-T14 / CONTRACT C2: Excel's seven border line styles, name for name. Mirrored in
+#: `aios-web/web/src/sheet/types.ts::CELL_BORDER_STYLES`, and `verify_sheet.py` compares the two
+#: lists. A side is either the legacy `True` (read as a thin black line and stored unchanged, so a
+#: document written before S67 reads back byte for byte) or `{"style": ,
+#: "color": "#rrggbb"}`.
+CELL_BORDER_STYLES = ("thin", "medium", "thick", "dashed", "dotted", "double", "hair")
+_BORDER_COLOUR = re.compile(r"^#[0-9a-f]{6}$")
+
#: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
#: outlives every layout change, exactly as a stored hex outlives a palette.
_MAX_INDENT = 8
@@ -204,6 +212,12 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
#: every rule written before this ticket already is, so writing the word onto them would make a
#: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
+#: ⭐ S67-T13: Excel's colour scale point types, one list for both ends. `lowest` / `highest` are
+#: measured from the range at paint, so the scale re-spreads as the data changes; the other three
+#: read their number from the rule's `value` (low end) or `value2` (high end). A type outside this
+#: list is REFUSED by name, never dropped: a dropped type reads back as a different scale. Mirrored
+#: in `web/src/sheet/types.ts::SheetRuleScaleType`.
+SHEET_RULE_SCALE_TYPES = ("lowest", "highest", "number", "percent", "percentile")
# The client compiler accepts 500 characters per condition.
MAX_RULE_FORMULA_LENGTH = 500
@@ -652,13 +666,44 @@ def _clean_style(raw):
border = raw.get("border")
if isinstance(border, dict):
- sides = {side: True for side in _BORDER_SIDES if border.get(side) is True}
+ sides = {}
+ for side in _BORDER_SIDES:
+ line = _clean_border_side(border.get(side))
+ if line is not None:
+ sides[side] = line
if sides:
out["border"] = sides
return out or None
+def _clean_border_side(raw):
+ """One C2 border side: `True`, a `{style, color}` dict, or None when the side is absent.
+
+ ⛔⛔ THE SECOND DOCUMENTED EXCEPTION TO "UNKNOWN KEYS ARE DROPPED IN SILENCE". A border side
+ that names a line style outside the seven, or a colour that is not a six digit hex, is
+ REFUSED by name. Dropping it would store a cell with no line where the person drew one, and
+ the caller could not learn its request went nowhere: `_clean_format`'s reason, one key over.
+ A side that is `False`, None or absent is simply no side. A legacy `True` stays `True`.
+ """
+ if raw is None or raw is False:
+ return None
+ if raw is True:
+ return True
+ if not isinstance(raw, dict):
+ return None
+ style = raw.get("style")
+ colour = raw.get("color")
+ if isinstance(colour, str):
+ colour = colour.strip().lower()
+ if (not isinstance(style, str) or style not in CELL_BORDER_STYLES
+ or not isinstance(colour, str) or not _BORDER_COLOUR.match(colour)):
+ raise SheetRefusal(
+ "cell_border_invalid",
+ "A border needs a line style such as thin and a six digit hex colour such as #000000.")
+ return {"style": style, "color": colour}
+
+
# ── the cell ─────────────────────────────────────────────────────────────────────────────────
def _clean_validation(raw):
@@ -1131,15 +1176,31 @@ def _clean_rule_scale(raw):
⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
`_clean_rules` already persists for every op that takes operands, so a colour scale needed no
new numeric slot at all.
+
+ ⭐ S67-T13: `minType` / `maxType` say what each end MEANS (`SHEET_RULE_SCALE_TYPES`). Each is
+ kept only when present and never injected, so a stored boundless scale reads back byte for
+ byte, and input with no types returns exactly the `{low, high}` it always did. An unknown type
+ is refused by name.
"""
if not isinstance(raw, dict):
return None
+ types = {}
+ for key in ("minType", "maxType"):
+ kind = raw.get(key)
+ if kind is None or kind == "":
+ continue
+ if kind not in SHEET_RULE_SCALE_TYPES:
+ raise SheetRefusal(
+ "sheet_rule_scale_type_unknown",
+ "A colour scale end must be one of " + ", ".join(SHEET_RULE_SCALE_TYPES) + ".")
+ types[key] = kind
out = {}
for key in ("low", "high"):
token = _text(raw.get(key), 40)
if not token or not _COLOUR.match(token):
return None
out[key] = token
+ out.update(types)
return out
diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py
index 77ab07c24ac0521a36ddfd604fbc8b463d676293..39707a197c8e0955cf7783c7dd2b3c2b833033de 100644
--- a/platform/harness/amazon_client.py
+++ b/platform/harness/amazon_client.py
@@ -1457,7 +1457,10 @@ def group_a_records(fields, operation, *, start=None, end=None, asins=(),
params.update(includedData="summaries,issues,offers,fulfillmentAvailability,attributes", pageSize=20)
root, payload_key, next_key = "items", None, "pageToken"
else:
- params.update(identifiersType="ASIN", includedData="summaries,salesRanks,dimensions,attributes",
+ # `relationships` (S67-T29, O23): same Catalog Items endpoint and Product Listing role;
+ # it carries VARIATION parentAsins so the mirror can map every own ASIN to its parent.
+ params.update(identifiersType="ASIN",
+ includedData="summaries,salesRanks,dimensions,attributes,relationships",
pageSize=20)
root, payload_key, next_key = "items", None, "pageToken"
batches = [None]
diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py
index 8785dbdcaf2370fb320547eddae97961c4bb3c92..2bc94de4e1557e3a35c099519802bfad0b349c80 100644
--- a/platform/harness/amazon_store.py
+++ b/platform/harness/amazon_store.py
@@ -853,6 +853,206 @@ def _sqp_catalogue(con):
return []
+_ASIN_RE = re.compile(r"[A-Z0-9]{10}")
+
+
+def _columns_of(con, table):
+ return {r[0] for r in con.execute(
+ "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()}
+
+
+_NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number")
+
+
+def _newest_child_parent_sql(columns, table, only=None):
+ """SELECT child_asin, parent_asin: one row per child, the newest period's parent; None if absent.
+
+ `only` is an optional SQL SELECT of the child ASINs wanted. It filters BEFORE the window, which
+ keeps or drops whole `child_asin` partitions, so every kept child's answer is unchanged. Once
+ the catalog carries parents (its first sync after S67-T29), `asin_parents_sql` passes only the
+ few own ASINs the catalog leaves without one, so S67-T30's per-page lookup stops ranking all of
+ economics (572,320 rows on staging, about 100 ms a query) to answer them.
+ """
+ if not {"child_asin", "parent_asin"} <= columns:
+ return None
+ order = ", ".join(f'"{c}" DESC NULLS LAST' for c in _NEWEST_ORDER if c in columns) or "1"
+ wanted = f" AND TRIM(CAST(child_asin AS VARCHAR)) IN ({only})" if only else ""
+ return (f'SELECT child_asin, parent_asin FROM (SELECT TRIM(CAST(child_asin AS VARCHAR)) AS child_asin, '
+ f'TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, ROW_NUMBER() OVER ('
+ f'PARTITION BY TRIM(CAST(child_asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM "{table}" '
+ f"WHERE child_asin IS NOT NULL AND parent_asin IS NOT NULL "
+ f"AND TRIM(CAST(parent_asin AS VARCHAR)) <> ''{wanted}) WHERE _rn = 1")
+
+
+def _child_parent_map(con, table):
+ """child ASIN -> parent ASIN from the newest rows of `table`; {} when it has no child_asin."""
+ sql = _newest_child_parent_sql(_columns_of(con, table), table)
+ if sql is None:
+ return {}
+ return {str(c): str(p) for c, p in con.execute(sql).fetchall()
+ if _ASIN_RE.fullmatch(str(c)) and _ASIN_RE.fullmatch(str(p))}
+
+
+def _pricing_parent_map(con):
+ """O23 fallback. Used only when pricing HAS child_asin: the live Sales and Traffic mirror is
+ stored at parent granularity (parent_asin only), so this is {} there and economics carries on."""
+ return _child_parent_map(con, "amzn_pricing")
+
+
+def _economics_parent_map(con):
+ """Data Kiosk economics child_asin -> parent_asin, newest start_date wins."""
+ return _child_parent_map(con, "amzn_economics")
+
+
+def _catalog_parent(item, asin, pricing_map, marketplace, economics_map=None):
+ """One item's (parent_asin, source); never raises on a missing or malformed `relationships`."""
+ parents, is_parent = [], False
+ groups = item.get("relationships")
+ for group in groups if isinstance(groups, list) else ():
+ if not isinstance(group, dict):
+ continue
+ if marketplace and group.get("marketplaceId") not in (None, marketplace):
+ continue
+ entries = group.get("relationships")
+ for rel in entries if isinstance(entries, list) else ():
+ # Only VARIATION defines a parent; PACKAGE_HIERARCHY is a pack, not a family.
+ if not isinstance(rel, dict) or rel.get("type") != "VARIATION":
+ continue
+ found = rel.get("parentAsins")
+ parents += [str(a).strip() for a in (found if isinstance(found, list) else ())
+ if _ASIN_RE.fullmatch(str(a).strip())]
+ if isinstance(rel.get("childAsins"), list) and rel["childAsins"]:
+ is_parent = True
+ if parents:
+ return parents[0], "relationships"
+ if is_parent:
+ return asin, "relationships"
+ for source, mapping in (("pricing", pricing_map), ("economics", economics_map)):
+ fallback = str((mapping or {}).get(asin) or "").strip()
+ if _ASIN_RE.fullmatch(fallback):
+ return fallback, source
+ return asin, "self"
+
+
+def catalog_parent_map(items, pricing_map=None, marketplace=None, economics_map=None):
+ """S67-T29 (O23): own ASIN -> (parent ASIN, source) for every catalog item.
+
+ Source order: the catalog VARIATION `parentAsins` for the requested marketplace; an item that
+ is itself a variation parent maps to itself; else pricing's `parent_asin` for that ASIN as
+ `child_asin` (O23); else economics child_asin -> parent_asin; else itself. A standalone ASIN
+ therefore maps to itself, never an error.
+ """
+ out = {}
+ for item in items or ():
+ if not isinstance(item, dict):
+ continue
+ asin = str(item.get("asin") or "").strip()
+ if asin:
+ out[asin] = _catalog_parent(item, asin, pricing_map, marketplace, economics_map)
+ return out
+
+
+#: S67 lane D (S67-T29 finding): Sales and Traffic defaults `asinGranularity` to PARENT, so the
+#: saved `amzn_pricing` mirror carried `parent_asin` with no `child_asin` and no `sku`, and
+#: `connector_spec`'s pricing identity `(child_asin, parent_asin)` and its two listings links never
+#: formed. SKU grain returns parentAsin, childAsin and sku per row: the SAME report type and role
+#: (the PRD's Amazon compliance line: no new endpoint or role).
+REPORT_OPTIONS = {"pricing": {"asinGranularity": "SKU"}}
+
+
+def _pricing_regrain(con, mirror, rows, log=print):
+ """Clear a PARENT-grain pricing mirror before the first SKU-grain rows land; True if cleared.
+
+ `amzn_pricing` keeps history across incremental passes, so without this the old parent-grain
+ days would sit beside SKU-grain days and any sum across days would count twice. A mirror that
+ already holds any `child_asin`, an empty or absent mirror, or rows without `child_asin` are
+ left alone.
+ """
+ if not any((row or {}).get("child_asin") for row in rows):
+ return False
+ have = _columns_of(con, mirror) # empty for an absent table (first load), never raises
+ if not have:
+ return False
+ if "child_asin" in have and con.execute(
+ f'SELECT COUNT(*) FROM {mirror} WHERE "child_asin" IS NOT NULL').fetchone()[0]:
+ return False
+ stale = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
+ if not stale:
+ return False
+ con.execute(f"DELETE FROM {mirror} WHERE 1=1")
+ log(f" pricing: replaced {stale} parent-grain rows with SKU-grain rows (one grain per table)")
+ return True
+
+
+def asin_parents_sql(con):
+ """S67-T29: ONE read-time resolver, one row per own ASIN: `(asin, parent_asin, parent_asin_source)`.
+
+ Own ASINs = DISTINCT listings `asin1` UNION the LATEST catalog `_snapshot_date`'s `asin` (the
+ snapshot tables keep one copy per sync day, D-3031, so only the newest day counts). Parent =
+ that latest catalog row's `parent_asin` (source as stamped at sync), else pricing child_asin ->
+ parent_asin when the pricing mirror carries child_asin (source `pricing`, O23), else economics
+ child_asin -> parent_asin (newest start_date, source `economics`), else the ASIN itself
+ (`self`). Covers own ASINs Amazon's catalog search returns no item for, without stub catalog
+ rows. Degrades with absent tables or columns; None only when no own-ASIN source exists.
+ Reusable verbatim by a SQL join (S67-T30): wrap it as a subquery.
+ """
+ listings, catalog = _columns_of(con, "amzn_listings"), _columns_of(con, "amzn_catalog_items")
+ latest = ('(SELECT MAX("_snapshot_date") FROM amzn_catalog_items)')
+ own = []
+ if "asin1" in listings:
+ own.append("SELECT TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM amzn_listings WHERE asin1 IS NOT NULL")
+ has_catalog = {"asin", "_snapshot_date"} <= catalog
+ if has_catalog:
+ own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin FROM amzn_catalog_items "
+ f'WHERE asin IS NOT NULL AND "_snapshot_date" = {latest}')
+ if not own:
+ return None
+ ctes = ["own AS (SELECT DISTINCT asin FROM (" + " UNION ".join(own) + ") WHERE asin <> '')"]
+ joins, parents, sources = [], [], []
+ if has_catalog and "parent_asin" in catalog:
+ source = ("CAST(parent_asin_source AS VARCHAR)" if "parent_asin_source" in catalog
+ else "'relationships'")
+ order = '"_report_row_number"' if "_report_row_number" in catalog else "1"
+ ctes.append(
+ "cat AS (SELECT asin, parent_asin, parent_asin_source FROM (SELECT TRIM(CAST(asin AS VARCHAR)) AS asin, "
+ "TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, "
+ f"COALESCE({source}, 'relationships') AS parent_asin_source, ROW_NUMBER() OVER ("
+ f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM amzn_catalog_items "
+ f"WHERE \"_snapshot_date\" = {latest} AND parent_asin IS NOT NULL "
+ "AND TRIM(CAST(parent_asin AS VARCHAR)) <> '') WHERE _rn = 1)")
+ joins.append("LEFT JOIN cat ON cat.asin = own.asin")
+ parents.append("cat.parent_asin")
+ sources.append("WHEN cat.parent_asin IS NOT NULL THEN cat.parent_asin_source")
+ # Economics is only the fallback, so rank only the own ASINs the catalog left without a parent.
+ only = "SELECT asin FROM own" + (" WHERE asin NOT IN (SELECT asin FROM cat)" if joins else "")
+ # O23: pricing is the fallback BEFORE economics. None (no step) while the pricing mirror has no
+ # `child_asin`, which is the live Sales and Traffic mirror today (S67-T29 finding).
+ pricing = _newest_child_parent_sql(_columns_of(con, "amzn_pricing"), "amzn_pricing", only=only)
+ if pricing:
+ ctes.append(f"pri AS ({pricing})")
+ joins.append("LEFT JOIN pri ON pri.child_asin = own.asin")
+ parents.append("pri.parent_asin")
+ sources.append("WHEN pri.parent_asin IS NOT NULL THEN 'pricing'")
+ economics = _newest_child_parent_sql(_columns_of(con, "amzn_economics"), "amzn_economics",
+ only=only)
+ if economics:
+ ctes.append(f"eco AS ({economics})")
+ joins.append("LEFT JOIN eco ON eco.child_asin = own.asin")
+ parents.append("eco.parent_asin")
+ sources.append("WHEN eco.parent_asin IS NOT NULL THEN 'economics'")
+ return ("WITH " + ", ".join(ctes) + " SELECT own.asin AS asin, "
+ f"COALESCE({', '.join([*parents, 'own.asin'])}) AS parent_asin, "
+ + (f"CASE {' '.join(sources)} ELSE 'self' END" if sources else "'self'")
+ + " AS parent_asin_source "
+ "FROM own " + " ".join(joins) + " ORDER BY own.asin")
+
+
+def asin_parents(con):
+ """`asin_parents_sql` as `{asin: (parent_asin, source)}`; {} when no own-ASIN source exists."""
+ sql = asin_parents_sql(con)
+ return {} if sql is None else {a: (p, s) for a, p, s in con.execute(sql).fetchall()}
+
+
def sqp_limit_sentence(marker):
"""Pure projection of saved sync facts; never asks Amazon for a catalogue."""
numbers = [marker[k] for k in ("maxChars", "maxPerBatch", "asinCount", "batchCount")]
@@ -1470,6 +1670,21 @@ def _blob_metadata(data):
return {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
+#: S67-T32 (D-3481): v1 names in-place keys (`
.parquet`, `sync_state.json`) and still
+#: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a
+#: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever
+#: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable.
+MANIFEST_VERSIONS = (1, 2)
+MANIFEST_VERSION = 2
+_MANIFEST_KEY = "manifest.json"
+_KEY_LEDGER = "keys.json"
+_TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet")
+_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json")
+#: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live
+#: checkpoint publishes within minutes, so the grace never races an in-flight writer.
+ORPHAN_GRACE = timedelta(hours=24)
+
+
def _checked_mirror_blob(rt, path, metadata):
data = rt.download_bytes(MIRROR_PREFIX + path)
if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}:
@@ -1477,11 +1692,195 @@ def _checked_mirror_blob(rt, path, metadata):
return data
+def _content_key(stem, suffix, metadata):
+ return f"{stem}.{metadata['sha256'][:16]}{suffix}"
+
+
+def _table_blob_key(name, metadata, *, strict=True):
+ """The key a manifest entry names: its content key, or the legacy in-place key."""
+ key = str(metadata.get("key") or name + ".parquet")
+ match = _TABLE_BLOB_RE.fullmatch(key)
+ if not match or match.group(1) != name or (
+ strict and match.group(2) and not str(metadata.get("sha256") or "").startswith(match.group(2))):
+ raise ValueError("Amazon mirror manifest names an invalid table blob key")
+ return key
+
+
+def _state_blob_key(metadata, *, strict=True):
+ key = str(metadata.get("key") or "sync_state.json")
+ match = _STATE_BLOB_RE.fullmatch(key)
+ if not match or (strict and match.group(1)
+ and not str(metadata.get("sha256") or "").startswith(match.group(1))):
+ raise ValueError("Amazon mirror manifest names an invalid state blob key")
+ return key
+
+
+def _is_mirror_blob_key(key):
+ return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _STATE_BLOB_RE.fullmatch(str(key)))
+
+
+def _manifest_keys(manifest, *, strict=True):
+ """{key: sha256} for every blob a manifest names; malformed entries are skipped, never raised."""
+ out = {}
+ if not isinstance(manifest, dict):
+ return out
+ tables = manifest.get("tables")
+ for name, metadata in (tables.items() if isinstance(tables, dict) else ()):
+ try:
+ out[_table_blob_key(str(name), metadata, strict=strict)] = metadata.get("sha256")
+ except (ValueError, AttributeError, TypeError):
+ continue
+ state = manifest.get("sync_state")
+ if isinstance(state, dict):
+ try:
+ out[_state_blob_key(state, strict=strict)] = state.get("sha256")
+ except (ValueError, AttributeError, TypeError):
+ pass
+ return out
+
+
+def _read_key_ledger(rt, tenant_key):
+ """The write-ahead key ledger. Unreadable is an error: a purge must not guess."""
+ raw = rt.download_bytes(MIRROR_PREFIX + _KEY_LEDGER)
+ key = _tenant_key(tenant_key)
+ if raw is None:
+ return {"version": 1, "tenant": key, "keys": {}}
+ doc = json.loads(raw)
+ if (not isinstance(doc, dict) or doc.get("tenant") != key or not isinstance(doc.get("keys"), dict)):
+ raise ValueError("Amazon mirror key ledger is unreadable or belongs to another tenant")
+ doc["keys"] = {str(k): v for k, v in doc["keys"].items() if _is_mirror_blob_key(k)}
+ return doc
+
+
+def _ledger_add(rt, tenant_key, keys):
+ """Record keys BEFORE their upload, so a killed checkpoint's blob is always enumerable."""
+ ledger = _read_key_ledger(rt, tenant_key)
+ stamp = _iso(datetime.now(timezone.utc))
+ fresh = [key for key in keys if key not in ledger["keys"]]
+ if not fresh:
+ return
+ for key in fresh:
+ if not _is_mirror_blob_key(key):
+ raise ValueError("Amazon mirror key is outside the naming contract")
+ ledger["keys"][key] = stamp
+ rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger))
+
+
+def _validated_snapshot(tenant_key, rt, con, raw, tmp):
+ """Every check restore makes, without writing. -> (manifest, state rows, parquet paths, rows).
+
+ Raises ValueError for anything the stored snapshot itself gets wrong; a store-access failure
+ propagates as its own type, so a caller can tell "unrestorable" from "unreachable".
+ """
+ try:
+ manifest = json.loads(raw)
+ except ValueError as exc:
+ raise ValueError("Amazon mirror manifest is not JSON") from exc
+ if (not isinstance(manifest, dict) or manifest.get("version") not in MANIFEST_VERSIONS
+ or manifest.get("tenant") != _tenant_key(tenant_key)
+ or not isinstance(manifest.get("tables"), dict) or not isinstance(manifest.get("sync_state"), dict)):
+ raise ValueError("Amazon mirror manifest has an invalid version, tenant or table map")
+ state_meta = manifest["sync_state"]
+ state_bytes = _checked_mirror_blob(rt, _state_blob_key(state_meta), state_meta)
+ rows = json.loads(state_bytes)
+ if (not isinstance(rows, list) or len(rows) != state_meta.get("rows")
+ or not rows or any(not isinstance(row, list) or len(row) != 6
+ or not isinstance(row[0], str) or not row[0].startswith(PREFIX) for row in rows)
+ or len({row[0] for row in rows}) != len(rows)):
+ raise ValueError("Amazon mirror state is incomplete or contains unrelated entities")
+ paths, total = {}, 0
+ for name, metadata in manifest["tables"].items():
+ if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict):
+ raise ValueError("Amazon mirror manifest contains an invalid table name")
+ data = _checked_mirror_blob(rt, _table_blob_key(name, metadata), metadata)
+ path = Path(tmp) / (name + ".parquet")
+ path.write_bytes(data)
+ try:
+ count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
+ except Exception as exc: # noqa: BLE001
+ raise ValueError("Amazon mirror parquet is unreadable") from exc
+ if type(metadata.get("rows")) is not int or count != metadata["rows"]:
+ raise ValueError("Amazon mirror parquet row count mismatch")
+ paths[name] = str(path)
+ total += count
+ return manifest, rows, paths, total
+
+
+def _snapshot_tables():
+ """Current-state tables: every report or API the client declares `snapshot`."""
+ cx = _client()
+ return tuple(t for t in TABLES if (cx.REPORT_TYPES.get(t) or {}).get("snapshot"))
+
+
+def _collapse_snapshots(con, log=None):
+ """D-3031 heal: a snapshot table holding more than one pull keeps only the latest.
+
+ -> {mirror: (rows before, rows after, copies)}. The ledger keeps its latest document's
+ dates (a DAILY document spans several), every other table its newest `_snapshot_date`;
+ AWD rows are another leg and never touched. Idempotent: one copy is left alone.
+ """
+ existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"}
+ collapsed = {}
+ for table in _snapshot_tables():
+ mirror = table_name(table)
+ if mirror not in existing:
+ continue
+ cols = _columns_of(con, mirror)
+ if "_snapshot_date" not in cols:
+ continue
+ scope = ("COALESCE(inventory_source, 'FBA') <> 'AWD'"
+ if table == "inventory" and "inventory_source" in cols else "TRUE")
+ dates = [row[0] for row in con.execute(
+ f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {mirror} WHERE {scope}").fetchall()]
+ present = sorted(d for d in dates if d is not None)
+ if len(dates) <= 1 or not present:
+ continue
+ keep = []
+ if table == "inventory":
+ saved = _read_state(con, entity_key("reconcile_" + table))
+ try:
+ marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {}
+ except (ValueError, KeyError, TypeError):
+ marked = {}
+ keep = [d for d in present if d in marked]
+ keep = keep or present[-1:]
+ stale = (f"{scope} AND (_snapshot_date IS NULL OR "
+ f"CAST(_snapshot_date AS VARCHAR) NOT IN ({','.join('?' for _ in keep)}))")
+ if not con.execute(f"SELECT count(*) FROM {mirror} WHERE {stale}", keep).fetchone()[0]:
+ continue
+ before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0]
+ con.execute("BEGIN TRANSACTION")
+ try:
+ con.execute(f"DELETE FROM {mirror} WHERE {stale}", keep)
+ con.execute("COMMIT")
+ except Exception:
+ con.execute("ROLLBACK")
+ raise
+ after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0]
+ collapsed[mirror] = (before, after, len(dates))
+ if log:
+ log(f"Amazon snapshot {mirror}: {len(dates)} copies collapsed to the latest, "
+ f"{before:,} rows to {after:,}")
+ return collapsed
+
+
+def _heal_snapshots(con, result, log):
+ """Collapse after a restore; the healed tables are dirty so the next checkpoint saves one copy."""
+ try:
+ result["dirty"].update(_collapse_snapshots(con, log))
+ except Exception as exc: # noqa: BLE001
+ message = (f"Amazon snapshot collapse failed ({type(exc).__name__}); the next resync of each "
+ "snapshot table replaces its earlier copies.")
+ result["problems"].append(message)
+ log(message)
+
+
def restore_mirror(tenant_key, rt, log=None):
"""Restore once per process and tenant, with all blobs validated before one atomic commit.
The tenant runtime is the only store door. A bad snapshot never partially replaces tables;
the caller gets a problem and the unchanged first-load decision can rebuild it from Amazon.
+ Both manifest versions restore: v1's in-place keys exactly as saved, v2's content keys.
"""
log = print if log is None else log
cache_key = _mirror_cache_key(tenant_key, rt)
@@ -1496,46 +1895,25 @@ def restore_mirror(tenant_key, rt, log=None):
# a process whose mirror survived re-uploaded EVERY table on its first checkpoint
# (39 tables, 19.5 MB). The small stored manifest lets unchanged bytes be skipped.
try:
- raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json")
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
saved = json.loads(raw) if raw else {}
- if (saved.get("version") == 1 and saved.get("tenant") == _tenant_key(tenant_key)
+ if (saved.get("version") in MANIFEST_VERSIONS and saved.get("tenant") == _tenant_key(tenant_key)
and isinstance(saved.get("tables"), dict)):
result["manifest"] = saved
except Exception: # noqa: BLE001
pass
+ _heal_snapshots(con, result, log)
_MIRROR_CACHE[cache_key] = result
return result
- raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json")
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
if raw is None:
result["state"] = "absent"
else:
obligation = retention_check(tenant_key, rt=rt, raise_overdue=True)
if obligation["earliestTriggerAt"]:
raise ValueError("Amazon deletion is outstanding; saved data cannot be restored")
- manifest = json.loads(raw)
- if (manifest.get("version") != 1 or manifest.get("tenant") != _tenant_key(tenant_key)
- or not isinstance(manifest.get("tables"), dict)):
- raise ValueError("Amazon mirror manifest has an invalid version, tenant or table map")
- state_bytes = _checked_mirror_blob(rt, "sync_state.json", manifest["sync_state"])
- rows = json.loads(state_bytes)
- if (not isinstance(rows, list) or len(rows) != manifest["sync_state"]["rows"]
- or not rows or any(not isinstance(row, list) or len(row) != 6
- or not isinstance(row[0], str) or not row[0].startswith(PREFIX) for row in rows)
- or len({row[0] for row in rows}) != len(rows)):
- raise ValueError("Amazon mirror state is incomplete or contains unrelated entities")
with tempfile.TemporaryDirectory(prefix="amazon-restore-") as tmp:
- paths, total = {}, 0
- for name, metadata in manifest["tables"].items():
- if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
- raise ValueError("Amazon mirror manifest contains an invalid table name")
- data = _checked_mirror_blob(rt, name + ".parquet", metadata)
- path = Path(tmp) / (name + ".parquet")
- path.write_bytes(data)
- count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0]
- if type(metadata.get("rows")) is not int or count != metadata["rows"]:
- raise ValueError("Amazon mirror parquet row count mismatch")
- paths[name] = str(path)
- total += count
+ manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp)
con.execute("BEGIN TRANSACTION")
try:
for name, path in paths.items():
@@ -1559,6 +1937,7 @@ def restore_mirror(tenant_key, rt, log=None):
line = (f"Amazon mirror restored from the store: {len(paths)} tables, {total:,} rows; "
f"resuming {resume}; settlements after document {document}")
log(line.replace("\u2014", "-").replace("\u2013", "-").replace("\n", " ").replace("\r", " "))
+ _heal_snapshots(con, result, log)
except Exception as exc:
result["state"] = "refused"
message = (f"Amazon mirror restore refused ({type(exc).__name__}); saved manifest, checksums, "
@@ -1572,26 +1951,110 @@ def restore_mirror(tenant_key, rt, log=None):
return result
+def _refused_may_publish(tenant_key, rt, con, cache):
+ """D-3481: a process that could not restore must not publish over a snapshot that is valid.
+
+ It publishes once its own first load is complete (its rebuild is then whole), or once a
+ fresh re-validation proves the stored snapshot unrestorable (nothing valid is lost). A
+ store-access failure during that re-check proves nothing, so it holds and asks again later.
+ """
+ if cache.get("publish_ok"):
+ return True
+ load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
+ if load and load["state"] == "done":
+ cache["publish_ok"] = True
+ return True
+ if cache.get("stored_valid") is None:
+ try:
+ raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY)
+ if raw is None:
+ cache["stored_valid"] = False
+ else:
+ with tempfile.TemporaryDirectory(prefix="amazon-recheck-") as tmp:
+ _validated_snapshot(tenant_key, rt, con, raw, tmp)
+ cache["stored_valid"] = True
+ except (ValueError, KeyError, TypeError, AttributeError):
+ cache["stored_valid"] = False
+ except Exception: # noqa: BLE001
+ return False
+ if cache["stored_valid"] is False:
+ cache["publish_ok"] = True
+ return True
+ return False
+
+
+def _retire_blobs(tenant_key, rt, published, previous, log, *, orphans=True):
+ """After a publish only: delete what the previous manifest named and the new one does not,
+ plus ledger orphans older than ORPHAN_GRACE. Best effort; a failure is retried next time."""
+ try:
+ if rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) != published:
+ log("Amazon mirror cleanup skipped: the stored manifest changed after this checkpoint")
+ return 0
+ current = _manifest_keys(json.loads(published), strict=False)
+ ledger = _read_key_ledger(rt, tenant_key)
+ doomed = set(previous) - set(current)
+ if orphans:
+ cutoff = datetime.now(timezone.utc) - ORPHAN_GRACE
+ for key, stamp in ledger["keys"].items():
+ try:
+ aged = _utc(stamp) < cutoff
+ except Exception: # noqa: BLE001
+ aged = True
+ if key not in current and aged:
+ doomed.add(key)
+ removed = []
+ for key in sorted(doomed):
+ if key in current or not _is_mirror_blob_key(key):
+ continue
+ rt.delete_path(MIRROR_PREFIX + key)
+ removed.append(key)
+ if any(key in ledger["keys"] for key in removed):
+ ledger = _read_key_ledger(rt, tenant_key)
+ ledger["keys"] = {k: v for k, v in ledger["keys"].items() if k not in removed}
+ rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger))
+ return len(removed)
+ except Exception as exc: # noqa: BLE001
+ log(f"Amazon mirror cleanup deferred ({type(exc).__name__}); superseded blobs stay in the "
+ "key ledger for the next checkpoint or purge.")
+ return 0
+
+
def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
"""Checkpoint changed tables and all Amazon markers, publishing the manifest last.
Candidates include partially written tables. Hashing their parquet locally avoids reuploading
identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain
dirty for retry on the next pass, while landed local rows and cursors stay usable.
+
+ S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a
+ CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists
+ every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row
+ upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4)
+ leaves the previous manifest naming only complete, untouched blobs.
"""
uploaded = 0
with _MIRROR_LOCK:
cache = restore_mirror(tenant_key, rt, log=log)
cache["dirty"].update(changed_tables)
try:
+ if cache.get("state") == "refused" and not _refused_may_publish(tenant_key, rt, con, cache):
+ message = ("Amazon mirror checkpoint held: this process could not restore the saved snapshot "
+ "at start, and that snapshot is still valid or unreachable. It is kept; restart to "
+ "restore it. This process saves only after its own first load completes.")
+ report["problems"].append(message)
+ log(message)
+ return
old = cache["manifest"]
+ named = _manifest_keys(old)
tables = {name for name, kind in _amazon_objects(con)
if name.startswith(PREFIX) and kind == "BASE TABLE"}
- manifest = {"version": 1, "tenant": _tenant_key(tenant_key),
+ manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
"tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}}
- changed = []
+ changed, pending = [], []
with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp:
- for name in sorted((cache["dirty"] | (tables - set(manifest["tables"]))) & tables):
+ # A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
+ legacy = {name for name, meta in manifest["tables"].items() if not meta.get("key")}
+ for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
if not re.fullmatch(r"amzn_[a-z0-9_]+", name):
raise ValueError("Amazon mirror table name is invalid")
path = Path(tmp) / (name + ".parquet")
@@ -1599,28 +2062,49 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
"(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
data = path.read_bytes()
metadata = _blob_metadata(data)
- if metadata["sha256"] == manifest["tables"].get(name, {}).get("sha256"):
+ prior = manifest["tables"].get(name, {})
+ if metadata["sha256"] == prior.get("sha256") and prior.get("key"):
continue
+ key = _content_key(name, ".parquet", metadata)
metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
- written_at=_iso(datetime.now(timezone.utc)))
- rt.upload_bytes(MIRROR_PREFIX + name + ".parquet", data)
- uploaded += len(data)
+ written_at=_iso(datetime.now(timezone.utc)), key=key)
manifest["tables"][name] = metadata
changed.append(name)
- rows = [row for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)]
- state = _mirror_json(rows)
- state_meta = {**_blob_metadata(state), "rows": len(rows)}
- if rows and (changed or state_meta != old.get("sync_state")
- or manifest["tables"] != old.get("tables", {})):
- rt.upload_bytes(MIRROR_PREFIX + "sync_state.json", state)
- uploaded += len(state)
- manifest.update(sync_state=state_meta, written_at=_iso(datetime.now(timezone.utc)))
- data = _mirror_json(manifest)
- rt.upload_bytes(MIRROR_PREFIX + "manifest.json", data)
- uploaded += len(data)
- cache["manifest"] = manifest
- cache["problems"].clear()
- cache["dirty"].clear()
+ if key in named:
+ # The same content under its own name is already stored and published.
+ if named[key] != metadata["sha256"]:
+ raise ValueError("Amazon mirror content key collides with a published blob")
+ continue
+ pending.append((key, path))
+ rows = [row for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)]
+ state = _mirror_json(rows)
+ state_meta = {**_blob_metadata(state), "rows": len(rows)}
+ state_meta["key"] = _content_key("sync_state", ".json", state_meta)
+ published = None
+ if rows and (changed or state_meta != old.get("sync_state")
+ or manifest["tables"] != old.get("tables", {})
+ or old.get("version") != MANIFEST_VERSION):
+ state_path = None
+ if state_meta["key"] not in named:
+ state_path = Path(tmp) / "sync_state.json"
+ state_path.write_bytes(state)
+ pending.append((state_meta["key"], state_path))
+ if pending:
+ _ledger_add(rt, tenant_key, [key for key, _path in pending])
+ for key, path in pending:
+ data = path.read_bytes()
+ rt.upload_bytes(MIRROR_PREFIX + key, data)
+ uploaded += len(data)
+ manifest.update(sync_state=state_meta, written_at=_iso(datetime.now(timezone.utc)))
+ published = _mirror_json(manifest)
+ rt.upload_bytes(MIRROR_PREFIX + _MANIFEST_KEY, published)
+ uploaded += len(published)
+ cache["manifest"] = manifest
+ cache["problems"].clear()
+ if published:
+ _retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
+ if published or not changed:
+ cache["dirty"].clear()
log(f"Amazon mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded")
except Exception as exc:
message = (f"Amazon mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. "
@@ -1629,20 +2113,41 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print):
log(message)
+def _purge_json(rt, name):
+ """A saved mirror JSON document for a purge: the parsed dict, or {} when absent or unreadable."""
+ try:
+ raw = rt.download_bytes(MIRROR_PREFIX + name)
+ doc = json.loads(raw) if raw else {}
+ except (TypeError, ValueError, UnicodeError, RecursionError):
+ return {}
+ return doc if isinstance(doc, dict) else {}
+
+
def _purge_saved_mirror(tenant_key, rt, local_tables):
- """Delete data before its inventory so a failed deletion remains retryable after restart."""
+ """Delete data before its inventory so a failed deletion remains retryable after restart.
+
+ Every version goes: each legacy in-place key, each key any manifest names, and each key the
+ write-ahead ledger lists, which includes blobs a killed checkpoint uploaded but never
+ published. Then the manifest, then the ledger itself, each read back.
+ """
with _MIRROR_LOCK:
key = _mirror_cache_key(tenant_key, rt)
cached = _MIRROR_CACHE.get(key, {})
- raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json")
- manifest = json.loads(raw) if raw else {}
+ # Deletion never refuses on unreadable inventory: a corrupt or foreign manifest or ledger
+ # still yields every blob name it holds, and every path below goes through THIS tenant's
+ # runtime, so a ledger labelled with another tenant cannot point a delete outside it.
+ manifest = _purge_json(rt, _MANIFEST_KEY)
+ ledger_keys = {str(k) for k in (_purge_json(rt, _KEY_LEDGER).get("keys") or {})
+ if _is_mirror_blob_key(str(k))}
names = ({table_name(t) for t in TABLES} | set(local_tables) | set(manifest.get("tables", {}))
| set(cached.get("manifest", {}).get("tables", {})) | set(cached.get("dirty", ())))
- paths = []
- for name in sorted(names):
- if re.fullmatch(r"amzn_[a-z0-9_]+", name):
- paths.append(MIRROR_PREFIX + name + ".parquet")
- paths.extend((MIRROR_PREFIX + "sync_state.json", MIRROR_PREFIX + "manifest.json"))
+ blobs = {name + ".parquet" for name in names if re.fullmatch(r"amzn_[a-z0-9_]+", name)}
+ blobs.add("sync_state.json")
+ blobs |= set(_manifest_keys(manifest, strict=False))
+ blobs |= set(_manifest_keys(cached.get("manifest", {}), strict=False))
+ blobs |= ledger_keys
+ paths = [MIRROR_PREFIX + blob for blob in sorted(blobs)]
+ paths.extend((MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER))
for path in paths:
if rt.delete_path(path) is False or rt.download_bytes(path) is not None:
raise RuntimeError("Amazon saved mirror deletion failed or left residue")
@@ -2117,8 +2622,14 @@ def _record_reconciliation(con, table, marker):
_record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":")))
-def _replace_period(con, table, rows, columns, column, start, end=None, marker=None):
- """Atomic range/date replacement and proof; an empty restatement removes old rows too."""
+def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False):
+ """Atomic range/date replacement and proof; an empty restatement removes old rows too.
+
+ `snapshot=True` (S67-T32, D-3031): a current-state report is ONE copy. A successful, parsed
+ pull replaces every earlier `_snapshot_date` of the table, not only its own day, because the
+ grid pid hashes the date and a kept day doubles every row downstream. A failed pull raises
+ before this runs, so it never empties the table.
+ """
mirror = table_name(table)
columns = list(dict.fromkeys([*columns, column, *_ORIGIN_TYPES]))
# All JSON money columns remain fractional even when today's amounts are integers.
@@ -2129,6 +2640,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND '
'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)')
params = [start, end] if end else [start]
+ if snapshot:
+ if column != "_snapshot_date" or end:
+ raise ValueError("a snapshot replacement is keyed by one _snapshot_date")
+ predicate, params = "TRUE", []
con.execute("BEGIN TRANSACTION")
try:
con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params)
@@ -2177,11 +2692,13 @@ def _snapshot_identity(table, row):
def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
- """Replace complete returned data dates, preserving history and the separate AWD leg.
+ """Replace the whole FBA snapshot with one parsed document, preserving the separate AWD leg.
Listings is a catalogue as of the vendor's data/creation date. The ledger is a DAILY
summary: its row date wins over the report creation date, including multi-date documents.
Duplicate source identities refuse the whole document rather than silently dropping rows.
+ S67-T32 (D-3031): one copy. The document's dates replace every earlier date, so the table
+ holds exactly one pull, the row count a fresh first load would store.
"""
stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime")
if not stamp and (table == "listings" or not rows):
@@ -2210,16 +2727,11 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
dates = sorted(counts)
marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date",
"count": source_count, "dates": counts, "document": meta.get("reportDocumentId")}
- placeholders = ",".join("?" for _ in dates)
- predicate = f"_snapshot_date IN ({placeholders})"
- # Undated pre-migration copies cannot be attributed to a day. Replace them once with
- # dated source evidence; never fabricate a date from the time we happened to fetch.
- predicate = f"({predicate} OR _snapshot_date IS NULL)"
- if table == "inventory":
- predicate += " AND COALESCE(inventory_source, 'FBA') <> 'AWD'"
+ # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg.
+ predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE"
con.execute("BEGIN TRANSACTION")
try:
- con.execute(f"DELETE FROM {mirror} WHERE {predicate}", dates)
+ con.execute(f"DELETE FROM {mirror} WHERE {predicate}")
count = _replace(con, mirror, columns, types, rows)
_record_reconciliation(con, table, marker)
con.execute("COMMIT")
@@ -2306,6 +2818,13 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
document = "operation:" + uuid.uuid4().hex
rows = [_origin(_flatten_row(_without_buyer_data(row)), document, i, spec["api"])
for i, row in enumerate(records, 1)]
+ if table == "catalog_items":
+ parents = catalog_parent_map(records, _pricing_parent_map(con),
+ fields.get("marketplace_id") or cx.US_MARKETPLACE_ID,
+ _economics_parent_map(con))
+ for row in rows:
+ asin = str(row.get("asin") or "").strip()
+ row["parent_asin"], row["parent_asin_source"] = parents.get(asin, (asin or None, "self"))
column = "_snapshot_date" if spec.get("snapshot") else "_period_start"
period = datetime.now(timezone.utc).date().isoformat() if spec.get("snapshot") else start
for row in rows:
@@ -2322,7 +2841,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
raise ValueError("Amazon order metrics returned an interval outside the requested period")
row[column] = _iso(day)
return _replace_period(con, table, rows, sorted({k for r in rows for k in r}),
- column, period, end if table == "order_metrics" else None, marker=marker)
+ column, period, end if table == "order_metrics" else None, marker=marker,
+ snapshot=bool(spec.get("snapshot")))
kwargs = {} if spec.get("snapshot") else {
"start": _iso(w0), "end": _iso(w1 - timedelta(seconds=1))}
options = dict(spec.get("options") or {})
@@ -2370,7 +2890,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log):
marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period,
"column": column, "count": document_row_count(
text, (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ())}
- return _replace_period(con, table, rows, columns, column, period, marker=marker)
+ return _replace_period(con, table, rows, columns, column, period, marker=marker,
+ snapshot=bool(spec.get("snapshot")))
def _order_metric_totals(records, *, flattened=False):
@@ -2605,7 +3126,13 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items()
if head in want and sibling in want] + want))
handled, noted = set(), set()
+ # D-3031: a mirror that already holds two pulls of a snapshot heals here too, idempotently.
changed_tables = set()
+ try:
+ changed_tables.update(_collapse_snapshots(con, log))
+ except Exception as exc: # noqa: BLE001
+ report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
+ "resync of each snapshot table replaces its earlier copies.")
def write_window(table, cols, types, rows):
count, note = replace_window(con, table_name(table), cols, types, rows,
@@ -2875,7 +3402,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
{"state": "access", "operation": "report_document"}, rt, report, log)
text, meta = cx.report_document(
fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait,
- log=lambda message: log(f"{table}: {message}"))
+ log=lambda message: log(f"{table}: {message}"),
+ **({"options": dict(REPORT_OPTIONS[table])} if table in REPORT_OPTIONS else {}))
if root:
parsed_cols, rows = parse_json_report(
text, root, document_id=meta.get("reportDocumentId"), report_type=rtype)
@@ -2937,6 +3465,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
cols.append("inventory_source")
if table == "pricing":
pricing_asins.extend(_pricing_asins(rows))
+ if not cleared and _pricing_regrain(con, mirror, rows, log):
+ cleared = True
types = _ensure(con, mirror, cols, rows, declared=declared,
preserve_rows=True) if cols else {}
# Explicit windows and first loads replace the mirror; incremental passes keep history.
@@ -3364,16 +3894,16 @@ def snapshot_selftest(log=print, check=None):
"FROM amzn_listings").fetchone() == (2, 2))
meta["createdTime"] = "2026-09-26T11:00:00Z"
sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
- ck("NC two listing dates keep two copies", lambda:
+ # S67-T32 (D-3031, reversed): a second day REPLACES the first, one load's rows.
+ ck("a second listing day replaces the first, leaving one copy", lambda:
con.execute("SELECT _snapshot_date, COUNT(*) FROM amzn_listings "
- "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-25", 2), ("2026-09-26", 2)]
+ "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-26", 2)]
and len(calls) == 2 and all("start" not in c and "end" not in c for c in calls))
payload[0] = "seller-sku\tquantity\nA\t9\n"
sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None)
- ck("same listing date replaces removed SKUs and retains the previous date", lambda:
+ ck("same listing date replaces removed SKUs", lambda:
con.execute("SELECT _snapshot_date, seller_sku, quantity FROM amzn_listings "
- "ORDER BY 1, 2").fetchall()
- == [("2026-09-25", "A", 2), ("2026-09-25", "B", 3), ("2026-09-26", "A", 9)]
+ "ORDER BY 1, 2").fetchall() == [("2026-09-26", "A", 9)]
and valid()["state"] == "ok")
else:
ck("ledger keeps one row per data date, SKU, disposition and location", lambda:
@@ -3387,10 +3917,11 @@ def snapshot_selftest(log=print, check=None):
before_awd = con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall()
payload[0] = ledger_header + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t9\n"
sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None)
- ck("ledger same data date replaces while older dates survive", lambda:
+ # S67-T32 (D-3031, reversed): the next document replaces every earlier FBA date.
+ ck("ledger second document replaces every earlier data date", lambda:
con.execute("SELECT _snapshot_date, ending_warehouse_balance FROM amzn_inventory "
"WHERE inventory_source='FBA' ORDER BY 1").fetchall()
- == [("2026-09-23", 2), ("2026-09-24", 9)])
+ == [("2026-09-24", 9)])
ck("FBA replacement preserves AWD rows exactly", lambda:
con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() == before_awd)
# The AWD operation's existing reconciliation still checks its own rows.
@@ -3419,8 +3950,8 @@ def snapshot_selftest(log=print, check=None):
if table == "listings":
payload[0] = "seller-sku\tquantity\n"
sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
- ck("empty listing restatement clears only that date and stays unproven", lambda:
- count() == 2 and valid()["state"] == "unproven")
+ ck("empty listing restatement replaces the copy and stays unproven", lambda:
+ count() == 0 and valid()["state"] == "unproven")
meta["dataEndTime"] = "2026-09-27T23:00:00Z"
payload[0] = "seller-sku\tquantity\n" + "".join(f"S{i}\t1\n" for i in range(1205))
sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None)
@@ -3436,6 +3967,308 @@ def snapshot_selftest(log=print, check=None):
"SELECT COUNT(*) FROM amzn_inventory WHERE _snapshot_date='2026-09-25' "
"AND fnsku='F-fallback'").fetchone()[0] == 1)
con.close()
+ ck("S67-T32 saved mirror: one copy, content keys, kill-safe checkpoint, full purge",
+ lambda: mirror_selftest(log, check) == 0)
+ return len(failures)
+
+
+def mirror_selftest(log=print, check=None):
+ """S67-T32 (D-3031 + D-3481): the saved snapshot is one copy, whole, and restorable.
+
+ Offline: a DuckDB :memory: mirror and a dictionary blob store. Covers the legacy (v1)
+ manifest restoring exactly, the doubled-mirror collapse, content keys never overwriting a
+ named key, a checkpoint dying after one table upload, purge reaching the orphan, a normal
+ checkpoint round trip (NC) and a refused process never publishing over a valid snapshot.
+ """
+ from unittest.mock import patch
+ import duckdb
+ failures = []
+
+ def ck(name, probe):
+ try:
+ ok = bool(probe())
+ except Exception as exc: # noqa: BLE001
+ ok = False
+ log(f" {name}: {type(exc).__name__}: {exc}")
+ if check:
+ check("[mirror] " + name, ok)
+ else:
+ log(f" {'ok' if ok else 'FAIL'} {name}")
+ if not ok:
+ failures.append(name)
+
+ class Store:
+ key = "qa-mirror"
+
+ def __init__(self):
+ self.blobs, self.uploads, self.hook = {}, [], None
+
+ def get(self, *_a, **_k):
+ return None
+
+ def upload_bytes(self, path, data, message=None):
+ if self.hook:
+ self.hook("upload", path)
+ self.blobs[path] = bytes(data)
+ self.uploads.append(path)
+
+ def download_bytes(self, path):
+ if self.hook:
+ self.hook("download", path)
+ return self.blobs.get(path)
+
+ def delete_path(self, path):
+ self.blobs.pop(path, None)
+ return True
+
+ rt = Store()
+ manifest_path, ledger_path = MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER
+ con = duckdb.connect(":memory:")
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, "
+ "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+
+ def forget():
+ _MIRROR_CACHE.pop(_mirror_cache_key(rt.key, rt), None)
+
+ def wipe():
+ """A new process whose local disk was lost."""
+ for name, _kind in _amazon_objects(con):
+ con.execute(f"DROP TABLE {_quoted_identifier(name)}")
+ con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_')")
+ forget()
+
+ def dump():
+ out = {name: sorted(con.execute(f"SELECT * FROM {_quoted_identifier(name)}").fetchall(), key=repr)
+ for name, _kind in _amazon_objects(con)}
+ out["_sync_state"] = sorted(r for r in _sync_state_rows(con) if r[0].startswith(PREFIX))
+ return out
+
+ def ledger_doc():
+ try:
+ return json.loads(rt.blobs[ledger_path])
+ except (KeyError, ValueError):
+ return {"version": 1, "tenant": rt.key, "keys": {}}
+
+ def mirror_blobs():
+ return {k for k in rt.blobs if k.startswith(MIRROR_PREFIX)}
+
+ def persist(tables=()):
+ report = {"problems": []}
+ persist_mirror(rt.key, rt, con, set(tables), report=report, log=lambda *_a: None)
+ return report
+
+ def restore():
+ return restore_mirror(rt.key, rt, log=lambda *_a: None)
+
+ def legacy_save():
+ """Today's staging layout, byte for byte: v1, in-place keys, no ledger."""
+ manifest = {"version": 1, "tenant": rt.key, "tables": {}}
+ import tempfile as _tmp
+ with _tmp.TemporaryDirectory(prefix="amazon-legacy-") as tmp:
+ for name, _kind in _amazon_objects(con):
+ path = Path(tmp) / (name + ".parquet")
+ con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
+ "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
+ data = path.read_bytes()
+ rt.blobs[MIRROR_PREFIX + name + ".parquet"] = data
+ manifest["tables"][name] = {**_blob_metadata(data), "written_at": "2026-09-27T00:00:00Z",
+ "rows": con.execute(f"SELECT count(*) FROM {name}").fetchone()[0]}
+ rows = [list(r) for r in _sync_state_rows(con) if r[0].startswith(PREFIX)]
+ state = _mirror_json(rows)
+ rt.blobs[MIRROR_PREFIX + "sync_state.json"] = state
+ manifest.update(sync_state={**_blob_metadata(state), "rows": len(rows)}, written_at="2026-09-27T00:00:00Z")
+ rt.blobs[manifest_path] = _mirror_json(manifest)
+
+ def seed():
+ con.execute("CREATE TABLE amzn_restock_recommendations AS SELECT * FROM (VALUES "
+ "('2026-09-25', 'A', 1), ('2026-09-25', 'B', 2), ('2026-09-25', 'C', 3), "
+ "('2026-09-26', 'A', 4), ('2026-09-26', 'B', 5), ('2026-09-26', 'C', 6)) "
+ "AS t(_snapshot_date, sku, qty)")
+ con.execute("CREATE TABLE amzn_inventory AS SELECT * FROM (VALUES "
+ "('2026-09-22', 'A', 'FBA', 1), ('2026-09-23', 'A', 'FBA', 2), "
+ "('2026-09-24', 'A', 'FBA', 3), (NULL, 'AWD-1', 'AWD', 9)) "
+ "AS t(_snapshot_date, msku, inventory_source, qty)")
+ con.execute("CREATE TABLE amzn_orders AS SELECT * FROM (VALUES "
+ "('2026-09-01', 'O1', 10.5), ('2026-09-02', 'O2', 11.5)) AS t(_period_start, id, amount)")
+ con.execute("CREATE TABLE amzn_listings AS SELECT * FROM (VALUES ('2026-09-26', 'S1', 1)) "
+ "AS t(_snapshot_date, seller_sku, quantity)")
+ _record_state(con, entity_key("reconcile_inventory"), "done", detail=json.dumps(
+ {"start": "2026-09-23", "end": "2026-09-24", "column": "_snapshot_date", "count": 2,
+ "dates": {"2026-09-23": 1, "2026-09-24": 1}}))
+ record_watermark(con, "orders", datetime(2026, 9, 26, tzinfo=timezone.utc), 2)
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z")
+
+ try:
+ with patch.object(sys.modules[__name__], "_connect_tenant_mirror",
+ side_effect=lambda *_a: (con.cursor(), None)):
+ # D-3031 at the write: an API snapshot pulled on a new day replaces the old day.
+ for day, qty in (("2026-09-25", 1), ("2026-09-26", 2)):
+ rows = [_origin({"sku": s, "qty": qty, "_snapshot_date": day}, "doc-" + day, i, "fixture")
+ for i, s in enumerate(("A", "B"), 1)]
+ _replace_period(con, "fba_inventory", rows, ["sku", "qty", "_snapshot_date"],
+ "_snapshot_date", day, snapshot=True)
+ ck("two snapshot days leave one copy of one load's rows", lambda: con.execute(
+ "SELECT _snapshot_date, count(*) FROM amzn_fba_inventory GROUP BY 1").fetchall()
+ == [("2026-09-26", 2)])
+ ck("a period table keeps its history", lambda: _replace_period(
+ con, "storage_fees", [_origin({"month": "2026-08", "fee": 1}, "d", 1, "f")], ["month", "fee"],
+ "month", "2026-08") == 1 and _replace_period(
+ con, "storage_fees", [_origin({"month": "2026-09", "fee": 1}, "d", 1, "f")], ["month", "fee"],
+ "month", "2026-09") == 1 and con.execute(
+ "SELECT count(DISTINCT month) FROM amzn_storage_fees").fetchone()[0] == 2)
+ con.execute("DROP TABLE amzn_fba_inventory")
+ con.execute("DROP TABLE amzn_storage_fees")
+
+ # The legacy manifest staging boots from, holding a doubled snapshot.
+ seed()
+ original = dump()
+ legacy_save()
+ legacy_blobs = dict(rt.blobs)
+ wipe()
+ result = restore()
+ ck("a legacy v1 manifest with in-place keys restores every table and marker", lambda:
+ result["state"] == "restored" and not result["problems"]
+ and dump()["amzn_orders"] == original["amzn_orders"]
+ and dump()["amzn_listings"] == original["amzn_listings"]
+ and dump()["_sync_state"] == original["_sync_state"])
+ ck("a doubled mirror collapses on restore to the latest pull and is marked dirty", lambda:
+ con.execute("SELECT _snapshot_date, count(*) FROM amzn_restock_recommendations "
+ "GROUP BY 1").fetchall() == [("2026-09-26", 3)]
+ and con.execute("SELECT _snapshot_date FROM amzn_inventory WHERE inventory_source='FBA' "
+ "ORDER BY 1").fetchall() == [("2026-09-23",), ("2026-09-24",)]
+ and con.execute("SELECT count(*) FROM amzn_inventory WHERE inventory_source='AWD'").fetchone()[0] == 1
+ and {"amzn_restock_recommendations", "amzn_inventory"} <= result["dirty"]
+ and "amzn_orders" not in result["dirty"])
+ ck("the collapse is idempotent", lambda: _collapse_snapshots(con) == {})
+
+ # First checkpoint after the legacy boot: every blob moves to a content key.
+ legacy_keys = set(_manifest_keys(json.loads(legacy_blobs[manifest_path])))
+ rt.uploads.clear()
+ report = persist()
+ saved = json.loads(rt.blobs[manifest_path])
+ named = set(_manifest_keys(saved))
+ ck("the first checkpoint migrates to content keys and retires the in-place blobs", lambda:
+ not report["problems"] and saved["version"] == MANIFEST_VERSION
+ and all(_TABLE_BLOB_RE.fullmatch(k).group(2) for k in named if k.endswith(".parquet"))
+ and not any(MIRROR_PREFIX + k in rt.blobs for k in legacy_keys)
+ and not any(MIRROR_PREFIX + k in rt.uploads for k in legacy_keys)
+ and set(ledger_doc()["keys"]) == named
+ and rt.uploads.index(ledger_path) < min(rt.uploads.index(MIRROR_PREFIX + k) for k in named)
+ and rt.uploads[-2 if rt.uploads[-1] == ledger_path else -1] == manifest_path)
+ healed = dump()
+ wipe()
+ ck("NC a normal checkpoint restores exactly, one copy each", lambda:
+ restore()["state"] == "restored" and dump() == healed
+ and con.execute("SELECT count(*) FROM amzn_restock_recommendations").fetchone()[0] == 3)
+ rt.uploads.clear()
+ before = dict(rt.blobs)
+ persist(set(original))
+ ck("an unchanged checkpoint uploads nothing", lambda: rt.uploads == [] and rt.blobs == before)
+
+ # A changed table never overwrites a key the published manifest names.
+ con.execute("UPDATE amzn_orders SET amount = amount + 1")
+ rt.uploads.clear()
+ persist({"amzn_orders"})
+ ck("content keys never overwrite a named key", lambda:
+ not any(path[len(MIRROR_PREFIX):] in named for path in rt.uploads)
+ and _manifest_keys(json.loads(rt.blobs[manifest_path]))
+ and not (MIRROR_PREFIX + saved["tables"]["amzn_orders"]["key"]) in rt.blobs)
+
+ # A checkpoint that dies after its first table blob lands.
+ published = rt.blobs[manifest_path]
+ good = dump()
+ con.execute("UPDATE amzn_orders SET amount = amount + 1")
+ con.execute("UPDATE amzn_listings SET quantity = quantity + 1")
+ parquet = []
+
+ def die_on_second(kind, path):
+ if kind == "upload" and path.endswith(".parquet"):
+ parquet.append(path)
+ if len(parquet) == 2:
+ raise OSError("fixture: the process died mid-checkpoint")
+
+ rt.hook = die_on_second
+ report = persist({"amzn_orders", "amzn_listings"})
+ rt.hook = None
+ orphan = parquet[0][len(MIRROR_PREFIX):] if parquet else "absent"
+ ck("a checkpoint that dies after one table upload leaves the manifest untouched", lambda:
+ any("persistence failed" in p for p in report["problems"])
+ and rt.blobs[manifest_path] == published and MIRROR_PREFIX + orphan in rt.blobs)
+ ck("the key ledger lists the unpublished orphan", lambda:
+ orphan in ledger_doc()["keys"]
+ and orphan not in _manifest_keys(json.loads(published)))
+ wipe()
+ ck("the next process restores the previous snapshot whole", lambda:
+ restore()["state"] == "restored" and dump() == good)
+
+ # An aged orphan is retired by the next successful publish; a fresh one is not.
+ ledger = ledger_doc()
+ ledger["keys"][orphan] = "2026-01-01T00:00:00Z"
+ rt.blobs[ledger_path] = _mirror_json(ledger)
+ con.execute("UPDATE amzn_orders SET amount = amount + 5")
+ persist({"amzn_orders"})
+ ck("an aged ledger orphan is retired after the next publish", lambda:
+ MIRROR_PREFIX + orphan not in rt.blobs
+ and orphan not in ledger_doc()["keys"]
+ and set(ledger_doc()["keys"]) >= set(
+ _manifest_keys(json.loads(rt.blobs[manifest_path]))))
+
+ # A refused restore (store unreachable at boot) must not publish its rebuild.
+ valid_manifest = rt.blobs[manifest_path]
+ wipe()
+
+ def unreachable(kind, path):
+ if kind == "download" and path == manifest_path:
+ raise OSError("fixture: store unreachable at boot")
+
+ rt.hook = unreachable
+ refused = restore()
+ rt.hook = None
+ con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-02' AS _period_start, 'O2' AS id, 1.0 AS amount")
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z")
+ rt.uploads.clear()
+ report = persist({"amzn_orders"})
+ ck("a refused restore does not publish a partial rebuild over the valid manifest", lambda:
+ refused["state"] == "refused" and rt.blobs[manifest_path] == valid_manifest
+ and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"]))
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z")
+ report = persist({"amzn_orders"})
+ ck("the refused process publishes once its own first load completes", lambda:
+ not report["problems"] and rt.blobs[manifest_path] != valid_manifest
+ and set(json.loads(rt.blobs[manifest_path])["tables"]) == {"amzn_orders"})
+
+ # A stored snapshot that is itself unrestorable names nothing valid; the rebuild may publish.
+ wipe()
+ corrupt = json.loads(rt.blobs[manifest_path])
+ corrupt["tables"]["amzn_orders"]["rows"] += 1
+ rt.blobs[manifest_path] = _mirror_json(corrupt)
+ refused = restore()
+ con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-03' AS _period_start, 'O3' AS id, 2.0 AS amount")
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z")
+ report = persist({"amzn_orders"})
+ ck("a corrupt stored snapshot refuses whole and the rebuild may publish over it", lambda:
+ refused["state"] == "refused" and not report["problems"]
+ and json.loads(rt.blobs[manifest_path])["tables"]["amzn_orders"]["rows"] == 1)
+
+ # Purge reaches every version: legacy keys, named keys and a killed checkpoint's orphan.
+ for path, data in legacy_blobs.items():
+ if path != manifest_path:
+ rt.blobs[path] = data
+ con.execute("UPDATE amzn_orders SET amount = 7")
+ parquet.clear()
+ rt.hook = lambda kind, path: (_ for _ in ()).throw(OSError("fixture death")) if (
+ kind == "upload" and path == manifest_path) else None
+ persist({"amzn_orders"})
+ rt.hook = None
+ rt.blobs["unrelated/keep.bin"] = b"keep"
+ orphans = set(ledger_doc()["keys"]) - set(
+ _manifest_keys(json.loads(rt.blobs[manifest_path])))
+ _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)])
+ ck("purge deletes every stored version including the orphan", lambda:
+ orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"})
+ finally:
+ forget()
+ con.close()
return len(failures)
@@ -3666,8 +4499,19 @@ def margin_selftest(log=print, check=None):
"sku\tasin\nS1\tA1\n", {"reportDocumentId": "next-snapshot", "dataEndTime": "2026-09-03T00:00:00Z"})):
_sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
_sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
- ck("margin two snapshot dates survive while same-date replay stays idempotent",
- con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date) FROM amzn_referral_fees").fetchone() == (2, 2))
+ # S67-T32 (D-3031, reversed): a new day REPLACES the earlier one; one copy, one load's rows.
+ ck("margin a second snapshot day replaces the first and same-date replay stays idempotent",
+ con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) "
+ "FROM amzn_referral_fees").fetchone() == (1, 1, "2026-09-03")
+ and _validate_margin(con, "referral_fees")["state"] == "ok")
+ kept = con.execute("SELECT * FROM amzn_referral_fees").fetchall()
+ with patch.object(cx, "report_document", side_effect=RuntimeError("fixture vendor failure")):
+ try:
+ _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log)
+ except RuntimeError:
+ pass
+ ck("margin a failed snapshot pull keeps the stored copy",
+ con.execute("SELECT * FROM amzn_referral_fees").fetchall() == kept and len(kept) == 1)
_replace_period(con, "economics", [], [c[1] for c in con.execute("PRAGMA table_info('amzn_economics')").fetchall()],
"start_date", "2026-09-01", "2026-09-03")
ck("margin empty economics restatement deletes stale detail and stays unproven",
@@ -4188,6 +5032,9 @@ def group_a_selftest(log=print, check=None):
sent = [a for _, params in requests[::3] for a in params["identifiers"].split(",")]
ck("catalog search covers all forty-one own ASINs in batches of twenty",
sent == args["asins"] and all(len(p["identifiers"].split(",")) <= 20 for _, p in requests))
+ ck("catalog search asks for relationships and nothing beyond the approved includes",
+ all(p["includedData"].split(",") == ["summaries", "salesRanks", "dimensions", "attributes",
+ "relationships"] for _, p in requests))
if operation == "getInventorySummaries":
ck("FBA inventory requests detailed inventory on every page",
all(p["details"] == "true" for _, p in requests))
@@ -4204,6 +5051,205 @@ def group_a_selftest(log=print, check=None):
return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"]
ck("Listings search splits date ranges at the real thousand-SKU ceiling",
not partition_probe(False) and partition_probe(True))
+ ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0)
+ return 1 if failures else 0
+
+
+def catalog_parent_selftest(log=print, check=None):
+ """S67-T29 (O23) offline fixtures: the catalog Parent ASIN map covers every own ASIN.
+
+ `python -c "import sys; sys.path[:0]=['platform','platform/harness']; import amazon_store as s;
+ sys.exit(s.catalog_parent_selftest())"` from the repo root; no credentials, no network.
+ """
+ from unittest.mock import patch
+ import duckdb
+ cx = _client()
+ failures = []
+
+ def ck(name, ok):
+ if check:
+ check("[catalog parent] " + name, bool(ok))
+ else:
+ log(f" {'ok' if ok else 'FAIL'} catalog parent {name}")
+ if not ok:
+ failures.append(name)
+
+ mkt = cx.US_MARKETPLACE_ID
+ child, parent, standalone, lonely, broken = ("B0CHILD001", "B0PARENT01", "B0STAND001",
+ "B0LONELY01", "B0BROKEN01")
+ items = [
+ {"asin": child, "relationships": [
+ {"marketplaceId": "A1F83G8C2ARO7P", "relationships": [
+ {"type": "VARIATION", "parentAsins": ["B0WRONGMKT"]}]},
+ {"marketplaceId": mkt, "relationships": [
+ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]},
+ {"type": "VARIATION", "parentAsins": [parent],
+ "variationTheme": {"attributes": ["color"], "theme": "COLOR"}}]}]},
+ {"asin": parent, "relationships": [{"marketplaceId": mkt, "relationships": [
+ {"type": "VARIATION", "childAsins": [child, "B0SIBLING1"]}]}]},
+ {"asin": standalone, "relationships": [{"marketplaceId": mkt, "relationships": []}]},
+ {"asin": lonely},
+ {"asin": broken, "relationships": "not-a-list"},
+ {"asin": "B0BROKEN02", "relationships": [None, {"marketplaceId": mkt, "relationships": {"x": 1}},
+ {"marketplaceId": mkt, "relationships": [
+ {"type": "VARIATION", "parentAsins": "B0NOTALIST"},
+ {"type": "VARIATION", "parentAsins": [None, "bad"]}]}]},
+ ]
+ eco_only = "B0ECONLY01"
+ items.append({"asin": eco_only, "relationships": [{"marketplaceId": mkt, "relationships": []}]})
+ pricing = {standalone: "B0PRICEPAR", broken: "not an asin"}
+ economics = {eco_only: "B0ECOPARNT", standalone: "B0ECOLOSER", child: "B0ECOWRONG", broken: "junk"}
+ got = catalog_parent_map(items, pricing, mkt, economics)
+ ck("variation child maps to its parent from relationships", got.get(child) == (parent, "relationships"))
+ ck("variation parent maps to itself", got.get(parent) == (parent, "relationships"))
+ ck("standalone falls back to pricing parent", got.get(standalone) == ("B0PRICEPAR", "pricing"))
+ ck("no relationships and no pricing maps to itself", got.get(lonely) == (lonely, "self"))
+ ck("malformed relationships degrade to itself, never an error",
+ got.get(broken) == (broken, "self") and got.get("B0BROKEN02") == ("B0BROKEN02", "self"))
+ ck("economics child to parent is the next fallback after pricing",
+ got.get(eco_only) == ("B0ECOPARNT", "economics"))
+ ck("every item gets a parent", set(got) == {i["asin"] for i in items})
+ # Red-first: economics never beats relationships or pricing, and is absent without its map.
+ ck("economics never outranks relationships or pricing, and needs its map",
+ got.get(child) == (parent, "relationships") and got.get(standalone) == ("B0PRICEPAR", "pricing")
+ and catalog_parent_map(items, pricing, mkt).get(eco_only) == (eco_only, "self"))
+ # Red-first: another marketplace or a PACKAGE_HIERARCHY parent must never win.
+ solo_other = catalog_parent_map([dict(items[0], relationships=items[0]["relationships"][:1])], {}, mkt)
+ pack_only = catalog_parent_map([{"asin": child, "relationships": [{"marketplaceId": mkt, "relationships": [
+ {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}]}]}], {}, mkt)
+ ck("another marketplace and a package parent are ignored",
+ solo_other.get(child) == (child, "self") and pack_only.get(child) == (child, "self"))
+ ck("empty and junk inputs never raise", catalog_parent_map(None) == {}
+ and catalog_parent_map([None, "x", {"asin": ""}]) == {})
+
+ con = duckdb.connect(":memory:")
+ try:
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
+ # Resolver before any table exists: no own-ASIN source means None, never an error.
+ ck("resolver with no listings or catalog is None and an empty map",
+ asin_parents_sql(con) is None and asin_parents(con) == {})
+ own = [child, parent, standalone, lonely, broken, eco_only]
+ # Listed ASINs Amazon's catalog returns no item for (the live 5 of 1164).
+ eco_listed, nothing = "B0ECOLIST1", "B0NOTHING1"
+ listed = own + [eco_listed, nothing, child]
+ con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
+ con.executemany("INSERT INTO amzn_listings VALUES (?)", [[a] for a in listed])
+ only_listings = asin_parents(con)
+ ck("resolver with listings only maps every listed ASIN to itself",
+ set(only_listings) == set(listed) and all(v == (k, "self") for k, v in only_listings.items()))
+ con.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR, start_date VARCHAR)")
+ con.executemany("INSERT INTO amzn_economics VALUES (?, ?, ?)", [
+ [eco_only, "B0ECOOLDPR", "2026-08-01"], [eco_only, "B0ECOPARNT", "2026-09-01"],
+ [eco_listed, "B0ECOLDPAR", "2026-07-01"], [eco_listed, "B0ECOLSTPR", "2026-09-10"],
+ [eco_listed, None, "2026-09-20"]])
+ con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, _period_start VARCHAR)")
+ con.executemany("INSERT INTO amzn_pricing VALUES (?, ?, ?)", [
+ [standalone, "B0OLDPARNT", "2026-08-01T00:00:00Z"],
+ [standalone, "B0PRICEPAR", "2026-09-01T00:00:00Z"]])
+ asked = []
+
+ def records(_fields, operation, **kwargs):
+ asked.append(list(kwargs.get("asins") or ()))
+ return [i for i in items if i["asin"] in kwargs.get("asins", ())]
+
+ # An older snapshot day with a different parent: the readback must use the LATEST day.
+ with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \
+ patch.object(cx, "group_a_records", side_effect=records):
+ count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None)
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES (?, 'B0STALEPAR', 'relationships', '2000-01-01')", [child])
+ latest = con.execute("SELECT MAX(_snapshot_date) FROM amzn_catalog_items").fetchone()[0]
+ mapped = dict((a, (p, s)) for a, p, s in con.execute(
+ "SELECT asin, parent_asin, parent_asin_source FROM amzn_catalog_items WHERE _snapshot_date=?",
+ [latest]).fetchall())
+ columns = {r[1] for r in con.execute("PRAGMA table_info('amzn_catalog_items')").fetchall()}
+ state = _read_state(con, entity_key("reconcile_catalog_items")) or {}
+ marker = json.loads(state.get("detail") or "{}")
+ ck("store readback shows the parent of a known child ASIN on the latest snapshot",
+ mapped.get(child) == (parent, "relationships"))
+ ck("catalog asks for every listed ASIN and maps every item it returns",
+ asked == [sorted(set(listed))] and set(mapped) == set(own)
+ and all(p for p, _ in mapped.values()))
+ ck("catalog sync stamps the economics fallback", mapped.get(eco_only) == ("B0ECOPARNT", "economics"))
+ # Resolver: two catalog snapshot days, an old-day-only ASIN, a latest-day catalog-only ASIN.
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES ('B0OLDONLY1', 'B0OLDONLY1', 'self', '2000-01-01')")
+ con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) "
+ "VALUES ('B0CATONLY1', 'B0CATPARNT', 'relationships', ?)", [latest])
+ resolved = asin_parents(con)
+ rows = con.execute(asin_parents_sql(con)).fetchall()
+ ck("resolver covers every own ASIN exactly once, latest catalog day only",
+ set(resolved) == set(listed) | {"B0CATONLY1"} and len(rows) == len(resolved)
+ and "B0OLDONLY1" not in resolved and all(p for p, _ in resolved.values()))
+ ck("resolver readback shows the latest parent of a known child",
+ resolved.get(child) == (parent, "relationships") and resolved.get("B0CATONLY1")
+ == ("B0CATPARNT", "relationships"))
+ ck("resolver falls back to the newest economics parent, then to itself",
+ resolved.get(eco_listed) == ("B0ECOLSTPR", "economics") and resolved.get(nothing) == (nothing, "self"))
+ # A catalog mirror synced before S67-T29 has no parent_asin column: economics and self carry it.
+ old = duckdb.connect(":memory:")
+ try:
+ old.execute("CREATE TABLE amzn_catalog_items (asin VARCHAR, _snapshot_date VARCHAR)")
+ old.execute(f"INSERT INTO amzn_catalog_items VALUES ('{child}', '2026-09-01'), ('{eco_listed}', '2026-09-01')")
+ old.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)")
+ old.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR')")
+ ck("resolver degrades on a pre-T29 catalog with no listings table",
+ asin_parents(old) == {child: (child, "self"), eco_listed: ("B0ECOLSTPR", "economics")})
+ finally:
+ old.close()
+ # O23 order in the READ-TIME resolver: catalog, then pricing, then economics, then self.
+ pri = duckdb.connect(":memory:")
+ try:
+ pri.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_listings VALUES ('{eco_listed}'), ('{nothing}')")
+ pri.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR'), "
+ f"('{nothing}', 'B0ECONOTHG')")
+ pri.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
+ pri.execute(f"INSERT INTO amzn_pricing VALUES ('{eco_listed}', 'B0PRICEPA2')")
+ got = asin_parents(pri)
+ ck("resolver: a pricing parent wins over economics, economics still answers the rest",
+ got.get(eco_listed) == ("B0PRICEPA2", "pricing")
+ and got.get(nothing) == ("B0ECONOTHG", "economics"))
+ pri.execute("CREATE OR REPLACE TABLE amzn_pricing (parent_asin VARCHAR)")
+ ck("NC: a pricing mirror with no child_asin (the live mirror today) adds no pricing step",
+ asin_parents(pri).get(eco_listed) == ("B0ECOLSTPR", "economics"))
+ finally:
+ pri.close()
+ # SKU grain (S67-T29 finding): the request asks for it, and the first SKU-grain pull
+ # replaces a parent-grain mirror instead of sitting beside it.
+ grain = duckdb.connect(":memory:")
+ try:
+ quiet = lambda *_a, **_k: None # noqa: E731
+ sku_rows = [{"parent_asin": "B0PARENT01", "child_asin": "B0CHILD001", "sku": "RI-1"}]
+ ck("regrain: an absent mirror is left alone", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False)
+ grain.execute("CREATE TABLE amzn_pricing (parent_asin VARCHAR, report_date VARCHAR)")
+ grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-01'), ('B0PARENT02', '2026-09-02')")
+ ck("regrain: parent-grain rows WITHOUT child_asin never clear the mirror",
+ _pricing_regrain(grain, "amzn_pricing", [{"parent_asin": "B0PARENT03"}], quiet) is False
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 2)
+ ck("regrain: the first SKU-grain pull clears the parent-grain mirror (one grain per table)",
+ _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is True
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 0)
+ grain.execute("ALTER TABLE amzn_pricing ADD COLUMN child_asin VARCHAR")
+ grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-03', 'B0CHILD001')")
+ ck("regrain: an already SKU-grain mirror keeps its history",
+ _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False
+ and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 1)
+ finally:
+ grain.close()
+ with open(__file__, encoding="utf-8") as _src:
+ passes_options = _src.read().count('REPORT_OPTIONS[table]') >= 2
+ ck("the Sales and Traffic request asks for SKU grain, the same report and role",
+ REPORT_OPTIONS.get("pricing") == {"asinGranularity": "SKU"} and passes_options)
+ ck("pricing fallback reads the newest pricing period", mapped.get(standalone) == ("B0PRICEPAR", "pricing"))
+ ck("raw relationships JSON is kept beside the derived columns",
+ {"relationships", "parent_asin", "parent_asin_source"} <= columns)
+ ck("row count stays Amazon's record count", count == len(own) and marker.get("count") == len(own))
+ finally:
+ con.close()
return 1 if failures else 0
diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py
index 6b6280534c8874f3f78e28a3e1603d4f2cddcee3..4a0a55616efb7e6cb1da86dc6446f57c3730108e 100644
--- a/platform/modules/sheet.py
+++ b/platform/modules/sheet.py
@@ -682,6 +682,24 @@ def clean_binding(raw):
return out
+def date_formatted_cell(cell):
+ """S67-T10 (QI4, D-767): a cell is a date when its number format says so.
+
+ A Date format set by hand, the format the engine infers for DATE() / TODAY() / NOW() (written
+ into ``style.format``), or an imported Excel code carrying day or year tokens. Mirrors
+ ``aios-web/web/src/sheet/bindingRefs.ts::isDateFormattedCell``.
+ """
+ import re as _re
+ fmt = ((cell or {}).get("style") or {}).get("format") if isinstance(cell, dict) else None
+ if not isinstance(fmt, dict):
+ return False
+ if fmt.get("kind") == "date":
+ return True
+ code = fmt.get("code")
+ return (fmt.get("kind") == "code" and isinstance(code, str)
+ and _re.search(r"[dy]", _re.sub(r'"[^"]*"|\\.', "", code), _re.I) is not None)
+
+
def filter_cell_nodes(nodes):
"""Cell value slots live on leaves at any depth, never on a fixed object path."""
for node in nodes or ():
diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT
index 4f2f8cf804bc0cc2970bbc2881886994a31203ad..4b9f44b361da6a2d3e4419803daeb6a33e374c18 100644
--- a/web/public/AIOS_DEPLOY_RECEIPT
+++ b/web/public/AIOS_DEPLOY_RECEIPT
@@ -1 +1 @@
-4bddbdac0a66f19f85828e1077bbbc16266db0d5f1e63fa1448e0a2b41a57313
+eb084c3f9d52c9da87adb2168567d2ecf4bb936d483cfea8bcdcfe15160dedde
diff --git a/web/src/apps/contentStudio/ContentStudioApp.tsx b/web/src/apps/contentStudio/ContentStudioApp.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..5281bbfcdcbc6e72564751b49413a78df60b61a5
--- /dev/null
+++ b/web/src/apps/contentStudio/ContentStudioApp.tsx
@@ -0,0 +1,271 @@
+import { Suspense, lazy, useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
+import { API_V1, CREDENTIALS } from "../../apiContract";
+import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface";
+import { dataPathHash } from "../../customer-grid/dataPath";
+import type { Field, Row } from "../../customer-grid/types";
+import { CalendarView, KanbanView } from "../../customer-grid/viewModes";
+import "../../styles/wave50/grid.css";
+
+/** Drafting is its own drawer (S67-T41/T42); it loads only when the owner asks to draft. */
+const DraftPanel = lazy(() => import("./DraftPanel"));
+
+interface StudioPost { pid: number; caption?: string | null; channel?: string | null; scheduled?: string | null;
+ status?: string | null; attribution?: string | null; campaign?: string | null }
+interface Studio {
+ app: { id: string; name: string };
+ posts: { database: string; label: string; fields: Field[]; rows: StudioPost[] };
+ references: { database: string; label: string; count: number };
+ statuses: string[];
+}
+type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; studio: Studio };
+type Mode = "calendar" | "board";
+
+/** The server's own sentence first (top-level `error.message`), then the legacy nesting. */
+const sentence = (body: unknown, fallback: string): string => {
+ const row = body as { error?: { message?: string }; detail?: { error?: { message?: string } } } | null;
+ return row?.error?.message || row?.detail?.error?.message || fallback;
+};
+
+const plural = (n: number, one: string, many: string) => `${n} ${n === 1 ? one : many}`;
+
+/** A post's one-line name on a calendar chip or a board card: its channel, then its opening words. */
+const titleOf = (post: StudioPost): string => {
+ const words = String(post.caption ?? "").replace(/\s+/g, " ").trim();
+ const head = words.length > 70 ? `${words.slice(0, 70).trimEnd()}…` : words || "Untitled post";
+ return post.channel ? `${post.channel}: ${head}` : head;
+};
+
+const TITLE_FIELD: Field = { key: "title", label: "Post", type: "text", source: "overlay" };
+
+/**
+ * S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an
+ * App ... it should act like Catalog / Route where it just uses the data module as what its
+ * building on top." So this is the App: it opens in the App module on its own surface and works
+ * the Posts and References databases underneath, which stay in Data. A calendar of posts by
+ * Scheduled beside the posts not yet scheduled, a board of posts by Status (a drag moves one),
+ * a post drawer to schedule it, and the Draft door. Plan-only (round 1 R5): nothing publishes.
+ * The calendar and the board are the grid's own `CalendarView` and `KanbanView`, unchanged.
+ */
+export default function ContentStudioApp({ appId }: { appId: string }) {
+ const [load, setLoad] = useState({ phase: "loading" });
+ const [epoch, setEpoch] = useState(0);
+ const [mode, setMode] = useState("calendar");
+ const [draftOpen, setDraftOpen] = useState(false);
+ const [openPid, setOpenPid] = useState(null);
+ const [notice, setNotice] = useState("");
+ const [moved, setMoved] = useState>({});
+
+ useEffect(() => {
+ const controller = new AbortController();
+ void fetch(`${API_V1}/content-studio/studio?app=${encodeURIComponent(appId)}`,
+ { credentials: CREDENTIALS, signal: controller.signal })
+ .then(async (response) => {
+ const body = await response.json().catch(() => null);
+ if (!response.ok) throw new Error(sentence(body, "This Content Studio could not be loaded. Reload to retry."));
+ setLoad({ phase: "ready", studio: body as Studio });
+ setMoved({});
+ })
+ .catch((failure: unknown) => {
+ if (controller.signal.aborted) return;
+ setLoad({ phase: "error",
+ message: failure instanceof Error ? failure.message : "This Content Studio could not be loaded. Reload to retry." });
+ });
+ return () => controller.abort();
+ }, [appId, epoch]);
+
+ // A draft written from the drawer lands in Posts; read the App again so it appears here.
+ useEffect(() => {
+ const reload = () => setEpoch((value) => value + 1);
+ window.addEventListener("aios:grid-refresh", reload);
+ return () => window.removeEventListener("aios:grid-refresh", reload);
+ }, []);
+
+ const studio = load.phase === "ready" ? load.studio : null;
+ const fieldByKey = useMemo(() => {
+ const map = new Map((studio?.posts.fields ?? []).map((field) => [field.key, field]));
+ map.set(TITLE_FIELD.key, TITLE_FIELD);
+ return map;
+ }, [studio]);
+ const rows: Row[] = useMemo(() => (studio?.posts.rows ?? []).map((post) => ({
+ ...post, status: moved[post.pid] ?? post.status ?? null, title: titleOf(post),
+ }) as Row), [studio, moved]);
+ const scheduledField = fieldByKey.get("scheduled");
+ const statusField = fieldByKey.get("status");
+ const unscheduled = rows.filter((row) => !String(row.scheduled ?? "").trim());
+ const openPost = studio?.posts.rows.find((post) => post.pid === openPid) ?? null;
+
+ const postsDatabase = studio?.posts.database ?? "";
+ // KanbanView is memo'd and keeps its props stable on purpose, so both handlers are too.
+ const openCard = useCallback((pid: number) => setOpenPid(pid), []);
+ const move = useCallback((pid: number, value: string) => {
+ if (!postsDatabase) return;
+ setNotice("");
+ setMoved((current) => ({ ...current, [pid]: value }));
+ void fetch(`${API_V1}/tables/${encodeURIComponent(postsDatabase)}/rows/${pid}`, {
+ method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ status: value }),
+ }).then(async (response) => {
+ if (!response.ok) throw new Error(sentence(await response.json().catch(() => null), "The post could not be moved. Try again."));
+ setEpoch((current) => current + 1);
+ }).catch((failure: unknown) => {
+ setMoved((current) => { const next = { ...current }; delete next[pid]; return next; });
+ setNotice(failure instanceof Error ? failure.message : "The post could not be moved. Try again.");
+ });
+ }, [postsDatabase]);
+
+ if (load.phase === "loading") {
+ return
+ {allowance.used} of {allowance.limit} drafts used this month
+
: null}
+
+ ;
+}
diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx
index d56705aad476a3516984e864d5af98e8373f3b21..bd1090083c7851cccab804f03e8483d7f29c0173 100644
--- a/web/src/connectors/ConnectorModal.tsx
+++ b/web/src/connectors/ConnectorModal.tsx
@@ -890,6 +890,17 @@ const CONNECTOR_DETAIL_DEADLINE_MS = 20_000;
// Closing the modal unmounts it; keep the last successful read for the next open.
const connectorDetailCache = new Map();
+// S67-T31: a refused read shows the server's own plain cause, or this component's sentence when
+// the server sent none (a bare 500 has no JSON body). A status code never reaches the screen:
+// "The server answered 500." was the owner's whole report, and it told him nothing he could act on.
+function plainReadFailure(
+ body: { error?: { message?: string }; detail?: { error?: { message?: string } } } | null,
+ fallback: string,
+): string {
+ const said = (body?.error?.message || body?.detail?.error?.message || "").trim();
+ return said || fallback;
+}
+
// Shell.tsx::signOut emits SIGNED_OUT_EVENT; an unloaded module has no cache to clear.
export function clearConnectorDetailCache(): void {
connectorDetailCache.clear();
@@ -1120,9 +1131,7 @@ export default function ConnectorModal({
if (dead) throw new Error("closed");
if (timedOut) throw new Error(timeoutMessage);
if (!res.ok) {
- throw new Error(
- body?.error?.message || body?.detail?.error?.message || `The server answered ${res.status}.`
- );
+ throw new Error(plainReadFailure(body, `The details for ${row.label} could not be read.`));
}
const page = parseConnectorDetail(body, connectorKey);
if (!page) {
@@ -1230,9 +1239,7 @@ export default function ConnectorModal({
| null;
if (controller.signal.aborted) return;
if (!res.ok) {
- throw new Error(
- body?.error?.message || body?.detail?.error?.message || `The server answered ${res.status}.`
- );
+ throw new Error(plainReadFailure(body, "The relationships for this connection could not be read."));
}
// The same reader, and the same selected-key guard, as the inline graph on a full read.
setGraphRead({ reads: forReads, graph: parseRelationshipGraph(body, connectorKey) });
diff --git a/web/src/connectors/ConnectorsPage.tsx b/web/src/connectors/ConnectorsPage.tsx
index b4ec1307b6004662d51ff0b40851ff7ef2dc2cd6..108d578277f6e883906c961ba8c37bd89da4a232 100644
--- a/web/src/connectors/ConnectorsPage.tsx
+++ b/web/src/connectors/ConnectorsPage.tsx
@@ -67,6 +67,15 @@ import {
type OdooSchema,
type OdooStatus,
} from "../automation/automationApi";
+// ⭐ S67-T20 / C4: the freshness READER and its words, never the settings panel. The tombstone
+// above still holds: `OdooConfigPanel` stays out of this file.
+import {
+ freshnessWarnings,
+ getOdooFreshness,
+ OdooFreshnessNote,
+ syncDate,
+ type OdooFreshness,
+} from "./OdooConfig";
import "../styles/wave45/agent-operator.css";
// ⭐ W49-T12 — the card's own open control and the relationship canvas rules live here now. The
// `CONNECTOR_GRAPH_CSS` string this file used to inject through `