diff --git a/RELEASES.json b/RELEASES.json index 83e5c8f99219b944f8ee1bf3747cdd4487db584c..fadef0617573e3eac0d139d643295c06771dac88 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "3dfe966", + "current": "d11ecc04", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index c1d9f26eabcf428103bd9d5e68e362f394886a38..e15d25d561b623035eccc2f161034ad7fbdad084 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3dfe966 +d11ecc04 diff --git a/api/ai_enrich.py b/api/ai_enrich.py index dbeb6153ad53eaab15f1ff8a38c460c0aa2164e6..e1ac9a95a74857f14dd1b0df3ff4484a94d1f266 100644 --- a/api/ai_enrich.py +++ b/api/ai_enrich.py @@ -269,12 +269,15 @@ _SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: n 'exactly: UNKNOWN') -def _ask(provider, model, prompt, max_tokens, timeout): +def _ask(provider, model, prompt, max_tokens, timeout, system=None): """One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text. ⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's ceiling instead of a hardcoded 300, and they read `usage` back. + S67-T41: `system` replaces the one-cell instruction for a caller that is not filling a cell + (Content Studio's drafts); every existing caller passes nothing and keeps `_SYSTEM`. """ + system = system or _SYSTEM key = (os.environ.get(provider['env']) or '').strip() if not key: return '', None, f"{provider['name']} has no key on this deployment" @@ -284,7 +287,7 @@ def _ask(provider, model, prompt, max_tokens, timeout): provider['url'], timeout=timeout, headers={'x-api-key': key, 'anthropic-version': '2023-06-01', 'content-type': 'application/json'}, - json={'model': model, 'max_tokens': max_tokens, 'system': _SYSTEM, + json={'model': model, 'max_tokens': max_tokens, 'system': system, 'messages': [{'role': 'user', 'content': prompt}]}) if r.status_code >= 400: return '', None, f"{provider['name']} answered {r.status_code}" @@ -300,7 +303,7 @@ def _ask(provider, model, prompt, max_tokens, timeout): provider['url'], timeout=timeout, headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'}, json={'model': model, 'max_tokens': max_tokens, 'temperature': 0, - 'messages': [{'role': 'system', 'content': _SYSTEM}, + 'messages': [{'role': 'system', 'content': system}, {'role': 'user', 'content': prompt}]}) if r.status_code >= 400: return '', None, f"{provider['name']} answered {r.status_code}" diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 0e38b89581cb706a1d56e3e6033f2325a3783a27..141997c6c039779a438d490417a84eabfa3f6ae4 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -170,6 +170,116 @@ def mirror_columns(cur, table): return [] +#: ⭐⭐ S67-T30 (instr 13, R29 / O23) — THE PRODUCT LOOKUPS. Every Amazon database with an ASIN +#: shows its "Parent ASIN" and "SKU" beside it, and every one with a SKU its "ASIN" and "Parent +#: ASIN", with no configuration. They are PLAIN TEXT fields whose value the SERVER derives, not +#: link fields, on purpose: a link renders as a count bubble, is not groupable +#: (`types.ts:isGroupableField`) and has no SQL binding for a filter (`routes_odoo_tables. +#: _filter_columns` and `_group_pin` read `spec["cols"]` only). A text field with a `cols` +#: expression groups, filters and sorts through the machinery every other column already uses. +#: ⚠ The ASIN / SKU column of a table is the `from_col` of its declared listings link (on `asin1` +#: / `seller_sku`), so the spec's links are the one statement of which column is which. A table +#: that carries its own `parent_asin` (pricing, economics) gains no derived Parent ASIN; one that +#: carries its own SKU or ASIN gains no duplicate of it; listings gains Parent ASIN only. +#: ⛔ Parent comes from `amazon_store.asin_parents_sql` ALONE (latest catalog, else economics, else +#: the ASIN itself), never from `amzn_catalog_items` directly: that table keeps older snapshot days +#: (D-3031) and five own ASINs have no catalog row. SKU and ASIN come from the LATEST listings day +#: only, because listings keeps one copy per snapshot date. +LISTINGS = PREFIX + "listings" +LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN = "lookup_parent_asin", "lookup_sku", "lookup_asin" +LOOKUP_KEYS = frozenset((LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN)) +_LOOKUP_LABELS = {LOOKUP_PARENT: "Parent ASIN", LOOKUP_SKU: "SKU", LOOKUP_ASIN: "ASIN"} +_LOOKUP_DESCRIPTIONS = { + LOOKUP_PARENT: ("The parent ASIN of this row's product, from Amazon's catalog, else the " + "economics report, else the ASIN itself. Click it to open the listing."), + LOOKUP_SKU: ("Your seller SKUs for this row's ASIN in the latest listings report. " + "Click it to open the listing."), + LOOKUP_ASIN: ("The ASIN of this row's SKU in the latest listings report. " + "Click it to open the listing."), +} + + +def _product_columns(key): + """`(asin_col, sku_col, asin_link_key, sku_link_key)` as the spec declares them for `key`.""" + if key == LISTINGS: + return "asin1", "seller_sku", None, None # the row IS the listing: Parent ASIN only + asin = sku = asin_via = sku_via = None + for edge in connector_spec.edges_for(key): + if edge["target"] != LISTINGS: + continue + if edge["to_col"] == "asin1" and asin is None: + asin, asin_via = edge["from_col"], edge["key"] + elif edge["to_col"] == "seller_sku" and sku is None: + sku, sku_via = edge["from_col"], edge["key"] + return asin, sku, asin_via, sku_via + + +def _latest_listings_sql(listing_names): + """`SELECT sku, asin` over the LATEST listings snapshot day, or None without both columns.""" + if not {"seller_sku", "asin1"} <= set(listing_names): + return None + table = SOURCE[LISTINGS] + latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})' + if "_snapshot_date" in listing_names else "") + return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, " + f"TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM {table} " + f"WHERE seller_sku IS NOT NULL AND asin1 IS NOT NULL{latest}) " + "WHERE sku <> '' AND asin <> ''") + + +def lookups(cur, key, names=None): + """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains. + + `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name + (every reader selects `FROM ` unaliased), so it binds to the outer row and never to the + resolver's own `asin`. DuckDB decorrelates it into a single hash join per query, which is what + keeps a grouped page of a read-through grid set-based rather than per-row. No match is NULL, + which every reader serialises as a blank cell. Nothing is capped (standing rule 1). + """ + if names is None: + names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])} + asin, sku, asin_via, sku_via = _product_columns(key) + asin = asin if asin in names else None + sku = sku if sku in names else None + if not asin and not sku: + return [] + listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])} + parents = None if "parent_asin" in names else amazon_store.asin_parents_sql(cur) + latest = _latest_listings_sql(listing_names) + table = SOURCE[key] + out = [] + + def add(lookup_key, after, expr, via): + if lookup_key in names: + return + out.append({"key": lookup_key, "label": _LOOKUP_LABELS[lookup_key], "after": after, + "expr": expr, "opens": {"via": via} if via else None, + "description": _LOOKUP_DESCRIPTIONS[lookup_key]}) + + if asin: + ref = f"TRIM(CAST({table}.{_quoted(asin)} AS VARCHAR))" + via = asin_via if "asin1" in listing_names else None + if parents: + add(LOOKUP_PARENT, asin, + f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via) + if latest and not sku: + add(LOOKUP_SKU, asin, + f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) " + f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via) + elif sku and latest: + ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))" + via = sku_via if "seller_sku" in listing_names else None + add(LOOKUP_ASIN, sku, + f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) " + f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via) + if parents: + add(LOOKUP_PARENT, sku, + f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) " + f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin " + f"WHERE _l.sku = {ref})", via) + return out + + def _fields_for(cur, key): """One preset field per mirror column — the whole schema, derived. See the header.""" cols = mirror_columns(cur, SOURCE[key]) @@ -177,6 +287,12 @@ def _fields_for(cur, key): return [] names = {c[0] for c in cols} pinned = PINNED.get(key) if PINNED.get(key) in names else None + # S67-T30: each lookup lands IMMEDIATELY after the column it derives from, shown by default, + # and the ASIN / SKU column it sits beside is shown by default too (inventory's `msku` was not). + derived = {} + for lookup in lookups(cur, key, names): + derived.setdefault(lookup["after"], []).append(lookup) + product = {c for c in _product_columns(key)[:2] if c in names} out = [] # The identity column goes FIRST, the rest keep the report's own order. A record reached # through a link is titled by its first field (the record drawer's frame label), and a @@ -186,9 +302,16 @@ def _fields_for(cur, key): out.append(_odoo._preset({ "key": name, "label": _label(name), "type": _ftype(name, dtype), "source": "overlay", - "default": name == pinned or name in _DEFAULT_COLUMNS, + "default": name == pinned or name in _DEFAULT_COLUMNS or name in product, "pinned": name == pinned, }, flow="amazon_relational")) + for lookup in derived.get(name, ()): + field = {"key": lookup["key"], "label": lookup["label"], "type": "text", + "source": "overlay", "default": True, + "description": lookup["description"]} + if lookup["opens"]: + field["opens"] = dict(lookup["opens"]) + out.append(_odoo._preset(field, flow="amazon_relational")) # ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not # known until it syncs, so a link to a table that has never run would name a column that does # not exist. **Absent is a normal state here, not a failure** — which is exactly what lets the @@ -324,6 +447,10 @@ def grid_source(cur, key): if kind in ("currency", "int"): expr = f"TRY_CAST({expr} AS {'DOUBLE' if kind == 'currency' else 'BIGINT'})" cols[name] = (expr, {"currency": _number, "int": _integer}.get(kind, _text)) + # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a + # group (count by parent) and a sort compile against them in SQL with no per-row Python. + for lookup in lookups(cur, key, names): + cols[lookup["key"]] = (lookup["expr"], _text) return {"table": SOURCE[key], "where": "", "id": ident, "needs_excluded": False, "cols": cols} @@ -393,11 +520,16 @@ def _read(cur, key): return [] quoted = ", ".join(f'"{c}"' for c in cols) spec = grid_source(cur, key) - rows = cur.execute(f'SELECT {spec["id"]}, {quoted} FROM {SOURCE[key]}').fetchall() + # S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL + # expressions the read-through binding uses, in this one SELECT (never per row). + derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)] + extra = "".join(f", {expr}" for _k, expr in derived) + rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]}').fetchall() + names = cols + [k for k, _expr in derived] out = [] for r in rows: row = {} - for name, val in zip(cols, r[1:]): + for name, val in zip(names, r[1:]): # Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write. row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val).replace("\x00", "")) @@ -497,6 +629,43 @@ def plan_fingerprint(cur, built): return hashlib.sha256(payload.encode("utf-8")).hexdigest() +def _place_lookups(stored, fields): + """An EXISTING table's fields with each NEW product lookup placed right after its source. + + `_ensure_table_inplace` only appends, so a lookup first planned for a table stored before + S67-T30 would land at the far end of the grid instead of beside the ASIN or SKU it derives + from. A lookup already stored keeps its place (a person may have moved it) and only has its + `opens` bag brought up to date, which `_ensure_table_inplace` does not forward-migrate. + """ + out = list(stored) + have = {f.get("key"): i for i, f in enumerate(out)} + anchor = None + for field in fields: + fkey = field.get("key") + if fkey not in LOOKUP_KEYS: + anchor = fkey + continue + if fkey in have: + old = out[have[fkey]] + if (old.get("automation") or {}).get("flowId") == "amazon_relational" \ + and not _odoo._ut().user_edited(old): + if field.get("opens"): + old["opens"] = dict(field["opens"]) + else: + old.pop("opens", None) + continue + keys = [f.get("key") for f in out] + if anchor not in keys: + continue # no source column stored yet: the append places it + at = keys.index(anchor) + 1 + while at < len(keys) and keys[at] in LOOKUP_KEYS: + at += 1 # after any lookup already beside the same source + out.insert(at, dict(field)) + have = {f.get("key"): i for i, f in enumerate(out)} + anchor = fkey + return out + + def _lead_with_identity(stored, report, pinned): """An EXISTING table's fields with its identity column moved first, while the order is still the report's own. @@ -564,13 +733,16 @@ def apply_plan(rt, built, username="amazon", today=None): # Preserve fields a person created or explicitly took over. current = doc.get(key) or {} wanted = {f["key"] for f in fields} + # S67-T30: a product lookup the mirror no longer supports retires the same way. current["fields"] = _lead_with_identity( [f for f in current.get("fields", []) - if not (f.get("type") == "link" + if not ((f.get("type") == "link" or f.get("key") in LOOKUP_KEYS) and (f.get("automation") or {}).get("flowId") == "amazon_relational" and not _odoo._ut().user_edited(f) and f.get("key") not in wanted)], report_order.get(key) or [], PINNED.get(key)) + if current["fields"]: + current["fields"] = _place_lookups(current["fields"], fields) stub = isinstance(rows, dict) and rows.get("readThrough") is True if not stub: stored = current.get("rows") or {} diff --git a/api/application_registry.py b/api/application_registry.py index 56f880fa04026c7da219319fb4b4a816234b9492..2a4ca9c78f3cc62e670b9c680864b830f5e7f833 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -16,6 +16,11 @@ APPLICATION_TYPES = ("chart", "route", "map", "timeseries") APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog") FINANCE_TEMPLATE_KIND = "finance" APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,) +# S67-T40: Content Studio creates its OWN databases (content_studio.py), so it is never offered to +# the five-step wizard either; its catalogue POST provisions References + Posts and binds Posts. +# Appended on its own line so the wizard / Finance separation above reads exactly as S65-T10 pins it. +CONTENT_STUDIO_TEMPLATE_KIND = "content_studio" +APPLICATION_TEMPLATE_KINDS = APPLICATION_TEMPLATE_KINDS + (CONTENT_STUDIO_TEMPLATE_KIND,) _KEY = "applications_v1" _MAX = 500 @@ -65,6 +70,17 @@ _TEMPLATE_CONTRACTS = { "fieldRoles": [], "defaultPlacement": {"section": "app", "order": 0}, }, + # S67-T40 (R17, R21, O3: one declarative row, no new type). S67-T41 owner correction: the App + # opens on its OWN surface in the App module (`content_studio.app_link`), like Catalog and + # Route, so the governed type only has to be a real one; its calendar is time. + "content_studio": { + "kind": "content_studio", + "label": "Content Studio", + "applicationType": "timeseries", + "displayMode": "timeseries", + "fieldRoles": [], + "defaultPlacement": {"section": "app", "order": 0}, + }, } @@ -363,6 +379,18 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract raise ApplicationError("navPlacement order must be an integer from 0 to 999") return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"], "navPlacement": {"section": "app", "order": order}} + if kind == CONTENT_STUDIO_TEMPLATE_KIND: + # Created server-side by content_studio.create right after it made the Posts database, so + # the binding is the one database it just created; the link can only open that database. + source = str(payload.get("sourceDatabase") or "").strip()[:120] + if not source.startswith("ut_"): + raise ApplicationError("Content Studio binds the Posts database it created.") + if allowed_databases is not None and source not in set(allowed_databases): + raise PermissionError("an Application cannot bind a database you cannot open") + # S67-T41 owner correction: no stored link. `_public` derives the App's own address from + # its id, so every Studio, including one created before the correction, opens in App. + return {"templateKind": kind, "sourceDatabase": source, + "navPlacement": {"section": "app", "order": 0}} source = str(payload.get("sourceDatabase") or "").strip()[:120] if not source: raise ApplicationError("sourceDatabase is required") @@ -470,12 +498,49 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas "edit", "share", "delete", "duplicate", "star", "default")}) row["starred"] = False row["isDefault"] = False + if row.get("templateKind") == CONTENT_STUDIO_TEMPLATE_KIND: + # S67-T41 owner correction: a Studio opens on its own App surface, never in Data. The + # database-bound edit, share, duplicate and default doors cannot reach an App bound to a + # user table, so they are off; star and delete stay, routed to `/content-studio/apps`. + row["deepLink"] = f"/app?studio={row['id']}" + row["capabilities"].update({key: False for key in ("edit", "share", "duplicate", "default")}) target_database = target_database if target_database in compatible else (compatible[0] if compatible else None) row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"], "database":target_database} return row +#: `core.table_store.RESERVED_MEMBERS`: workspace members that are registries, not users. +_RESERVED_WORKSPACE_MEMBERS = ("__shared__", "__assigned__", "__audit__") + + +def dependent_view_counts(runtime, database): + """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY + user's Views in that database's workspace. The Delete App dialog names this count before + confirm. None when the workspace cannot be read, so the dialog never states a guessed 0. + """ + from core.view_templates import workspace_key + key = workspace_key(database) + if not key: + return {} + try: + doc = runtime.get(key) + except Exception: + return None + counts = {} + for member, bucket in (doc.items() if isinstance(doc, dict) else ()): + if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict): + continue + views = bucket.get("views") + for view in (views.values() if isinstance(views, dict) else ()): + config = view.get("config") if isinstance(view, dict) else None + ref = config.get("application") if isinstance(config, dict) else None + app_id = str(ref.get("id") or "") if isinstance(ref, dict) else "" + if app_id: + counts[app_id] = counts.get(app_id, 0) + 1 + return counts + + def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None, allowed_databases=None): doc = load(runtime) @@ -491,6 +556,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases, target_database=database, is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id")))) + # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows + # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it. + deletable = [row for row in rows if row["capabilities"].get("delete")] + if database and deletable: + counts = dependent_view_counts(runtime, database) + for row in deletable: + row["dependentViews"] = None if counts is None else counts.get(row["id"], 0) return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True) diff --git a/api/assistant_mirror.py b/api/assistant_mirror.py index 1054ee6c03d0f99bc454a9406d0658ff97b48776..333f119639893e4622055eff1c7b8a985a9489b1 100644 --- a/api/assistant_mirror.py +++ b/api/assistant_mirror.py @@ -28,6 +28,15 @@ that need them. A module-level import either way is a cycle at startup. """ +#: ⭐ S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what +#: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window): +#: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an +#: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is +#: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's +#: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631). +ASSISTANT_READ_MAX = 200_000 + + class MirrorRefusal(Exception): """A named, fail-closed refusal with the status and code `deps` should raise it as. @@ -49,9 +58,14 @@ def _rot(): return routes_odoo_tables -def open_cursor(session): +def open_cursor(session, table_key=None): """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None. + ⭐ S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's + `_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its + own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so + the assistant no longer answers 503 for Amazon while Odoo is still backfilling. + ⛔ THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync @@ -61,7 +75,7 @@ def open_cursor(session): """ rot = _rot() try: - cur, mismatch = rot._mirror_cur(session) + cur, mismatch = rot._mirror_cur(session, table_key) except RuntimeError as exc: return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc)) except Exception as exc: # noqa: BLE001 @@ -118,8 +132,26 @@ def is_bound(cur, table_key): """ from harness import datastore try: - spec = _rot()._sources().get(str(table_key or "")) - if not spec: + rot = _rot() + key = str(table_key or "") + spec = rot._sources().get(key) + if not spec and key.startswith("ut_amzn_"): + # S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key. + # `_source_for` and `_materialised_table` both learn them on demand; so must this. + rot._learn_amazon_sources() + spec = rot.GRID_SOURCES.get(key) + if not spec: + # A dynamically sized Amazon snapshot is admitted per key by the grid door, not by + # the provider build, so the registry can lack a key the grid serves (measured on + # staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the + # provider's own per-key resolver, exactly as `require_amazon_mirror` does, without + # admitting it (the read path admits it through that same door). + import amazon_relational as amazon + if key not in amazon.SOURCE: + return False + current = amazon.grid_source(cur, key) or {} + return bool(current) and not current.get("missing") + if not spec or spec.get("missing"): return False aliases = dict(spec.get("tables") or {}) if spec.get("table"): @@ -129,8 +161,13 @@ def is_bound(cur, table_key): return False -def declared_fields(table_key): - """`mk_fields()` for one connected grid, or None when this key is not a connected Odoo grid. +def declared_fields(table_key, definition=None): + """`mk_fields()` for one connected grid, or None when this key is not a connected grid. + + ⭐ S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that + is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid, + a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before + this, those databases were listed as answerable and then refused 409 on every read. ⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built each table's stored `fields` from this same function, so reading it directly is reading the @@ -142,10 +179,20 @@ def declared_fields(table_key): if key == str(table_key or ""): return [dict(field) for field in (mk_fields() or []) if isinstance(field, dict) and field.get("key")] - return None + key = str(table_key or "") + if not isinstance(definition, dict) or not key.startswith("ut_"): + return None + try: + if _rot()._materialised_table(key): + return None + except Exception: # noqa: BLE001 + return None + fields = [dict(field) for field in (definition.get("fields") or []) + if isinstance(field, dict) and field.get("key")] + return fields or None -def answerable_fields(table_key, spec): +def answerable_fields(table_key, spec, fields=None): """The declared field contract INTERSECTED with the columns the mirror can actually express. ⛔ THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field @@ -155,7 +202,8 @@ def answerable_fields(table_key, spec): refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request for it with a loud `assistant_unknown_field`. """ - fields = declared_fields(table_key) + if fields is None: + fields = declared_fields(table_key) if fields is None: return None cols = set((spec or {}).get("cols") or ()) @@ -197,30 +245,14 @@ def _compiled(nodes, conj, columns, *, code, subject, rot): return pred -def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None, - filter_keys=None): - """Every permitted row of a mirror-served grid, or a refusal that names its cause. - - Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape - (`{pid, **cells}`), sorted by pid. +def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys): + """`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34. - ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a - connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED - with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt` - states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes - every aggregate from those same records, so a short set does not produce a smaller answer, it - produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over - already settled this policy in the same words: REFUSE, NEVER TRIM. - - ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and - the field projection in Python AFTER this returns. Hand it a row missing a column the wall - names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces - pass would quietly disagree with the pushdown it is meant to double-check. + Extracted verbatim from `read_rows` so the row walk, the page read and the server-side + aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler. + Three readers with one predicate cannot disagree about which rows the caller may see. """ - from harness import datastore - rot = _rot() fe = rot._fe() - cols = rot._filter_columns(spec, fields) # ── the administrator's wall, compiled FIRST and pushed down ────────────────────────────── @@ -267,23 +299,191 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t where = f"({where}) AND {pred.sql}" if where else pred.sql params.extend(pred.params) + return where, params, cols + + +def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None, + filter_keys=None, defn=None): + """Every permitted row of a mirror-served grid, or a refusal that names its cause. + + Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape + (`{pid, **cells}`), sorted by pid. + + ⛔ NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a + connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED + with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt` + states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes + every aggregate from those same records, so a short set does not produce a smaller answer, it + produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over + already settled this policy in the same words: REFUSE, NEVER TRIM. + + ⛔ FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and + the field projection in Python AFTER this returns. Hand it a row missing a column the wall + names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces + pass would quietly disagree with the pushdown it is meant to double-check. + """ + from harness import datastore + rot = _rot() + where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys) + # ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', - where=where, params=tuple(params), order_by=spec["id"], - limit=datastore.WINDOW_MAX, cur=cur, **frm) - if win["total"] > len(win["rows"]): + # ⭐⭐ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX` + # pass this replaced refused any connected database past 5,000 matching rows, which is a cap + # on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages, + # KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs + # one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a + # user condition: every condition above still comes from the grid's compiler. + # ⛔ STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that + # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short. + first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', + where=where, params=tuple(params), order_by=spec["id"], + limit=datastore.WINDOW_MAX, cur=cur, **frm) + total = int(first["total"]) + if total > ASSISTANT_READ_MAX: raise MirrorRefusal( 409, "assistant_source_too_large", - f"this database matches {win['total']:,} rows for that question, and the assistant " - f"reads a connected database in one pass of at most " - f"{datastore.WINDOW_MAX:,} rows, so answering now would mean counting only part of " - f"them and reporting the total as if it were whole. Narrow it with a filter and ask " - f"again") + f"this database matches {total:,} rows for that question, and one assistant answer " + f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question " + f"cannot exhaust the server's memory. Answering over part of them would report a " + f"wrong total, so it is refused instead. Narrow it with a filter (for example a date " + f"range) and ask again, or ask for totals as aggregates, which the server computes " + f"over every row") + got = list(first["rows"]) + page = list(first["rows"]) + while len(got) < total and len(page) >= datastore.WINDOW_MAX: + last = page[-1][0] + more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?" + nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', + where=more_where, params=tuple(params) + (last,), + order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm) + page = list(nxt["rows"]) + if not page: + break + got.extend(page) + if len(got) != total: + raise MirrorRefusal( + 409, "assistant_source_changed", + f"this database changed while it was being read ({len(got):,} rows read against " + f"{total:,} counted), so the answer would not describe one consistent set of rows. " + f"Ask again once its connector sync has finished") + win = {**first, "rows": got, "total": total} + + # S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does + # not re-read the whole tenant document (measured about 3 s a question on staging). + stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), + defn=defn if isinstance(defn, dict) else None) + rows = rot._pool_from_window(spec, win, stamp=stamp) + rows.sort(key=lambda row: row["pid"]) + return rows, stamp, int(win["total"]) + - stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None)) +#: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`). +_GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"} + + +def _json_value(rot, value): + """A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a + date or timestamp as its ISO text, anything else as is. Never a substituted 0.""" + if value is None: + return None + if isinstance(value, (int, float)) and not isinstance(value, bool): + return rot._agg_number(value) + iso = getattr(value, "isoformat", None) + if callable(iso): + return iso() + try: + from decimal import Decimal + if isinstance(value, Decimal): + return rot._agg_number(value) + except Exception: # noqa: BLE001 + pass + return value + + +def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None, + filter_tree=None, filter_keys=None): + """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631). + + Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the + count of rows the permitted predicate matches, from `datastore.window`'s own count statement + over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract). + + ⛔⛔ THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall + and the caller's filter with the grid's compiler, so a sum over a million-row table runs over + exactly the rows this caller may read and no row is pulled into Python to compute it. There + is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size. + + ⛔ AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and + a field outside it has no expression here at all, so a sum over a hidden column cannot be + built (the caller's `_normalise_aggregates` has already refused it by name). + """ + from harness import datastore + rot = _rot() + where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys) + visible = set(filter_keys) if filter_keys is not None else set(cols) + fs = rot._fs() + proj, slots = [], [] + for item in specs or (): + fn, field = item.get("fn"), item.get("field") + if field is None: + slots.append(None) # count(*): the window's own total + continue + col = cols.get(field) if field in visible else None + if col is None: + raise MirrorRefusal(400, "assistant_unknown_field", + "every requested field must be readable in this database") + if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES: + raise MirrorRefusal(400, "query_aggregate_type", + "sum and avg require a readable numeric field.") + if fn in _GRID_AGG: + expr = fs.aggregate_sql(col, _GRID_AGG[fn]) + elif fn in {"min", "max"}: + expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES + else f"{fn}({col['sql']})") + elif fn == "count": + expr = f"count({col['sql']})" + else: + raise MirrorRefusal(400, "query_aggregates_invalid", + "That aggregate is not available.") + slots.append(len(proj)) + proj.append(f'{expr} AS "a{len(proj)}"') + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params), + order_by="1", limit=1, cur=cur, **frm) + row = (win["rows"] or [()])[0] + total = int(win["total"]) + out = [] + for item, slot in zip(specs or (), slots): + value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row) + else None) + out.append({**item, "value": value}) + return out, total + + +def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None, + filter_tree=None, filter_keys=None, defn=None): + """ONE page of the permitted rows, paged IN SQL, in pid order. S67-T34 (rule 1). + + Returns `(rows, stamp, total)` in `read_rows`'s shape. This is how a table past + `ASSISTANT_READ_MAX` still answers its rows: the page is `LIMIT/OFFSET` over the same + predicate `read_rows` runs, so only `limit` rows ever reach this process, and `total` is the + population that predicate matches. It carries no sort other than the pid; a caller asking for + one past the ceiling is refused by name upstream, because sorting a million rows in Python + is the memory failure the ceiling exists to prevent. + """ + from harness import datastore + rot = _rot() + where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys) + select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where, + params=tuple(params), order_by=spec["id"], offset=int(offset), + limit=int(limit), cur=cur, **frm) + stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None), + defn=defn if isinstance(defn, dict) else None) rows = rot._pool_from_window(spec, win, stamp=stamp) rows.sort(key=lambda row: row["pid"]) return rows, stamp, int(win["total"]) diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index b02cee0823f0731e4eae69931cbff7338ed84f37..c0b4b766dfb4082c9a4d50030bafef6afe11b3f0 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -49,11 +49,80 @@ class AmazonStatusSafetyError(ValueError): """A status response contains a forbidden credential shape.""" -def status_payload_has_secret(serialized_payload): +class AmazonStatusRowsRefused(AmazonStatusSafetyError): + """S67-T31: some status ROWS were refused; the rest are carried, already guarded. + + Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps + refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES: + neither carries the refused content, so a caller may degrade those rows and keep the others. + """ + + def __init__(self, safe_status, refused): + super().__init__('Amazon status contains a forbidden credential shape.') + self.safe_status = safe_status + self.refused = tuple(refused) + + +#: The status guard's scalar ceiling, and the sentence a row wears when its status is refused. +STATUS_SCALAR_LIMIT = 512 +STATUS_UNAVAILABLE = "Sync status is unavailable for this database." + + +def trim_status(value, limit=STATUS_SCALAR_LIMIT): + """Cut every string over `limit` to a word-bounded excerpt that fits, recursively. + + S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935 + characters) used to trip the length branch of the guard and take the whole connector modal + down with a 500. ⛔ Call this only AFTER the credential-shape scan of the UNTRIMMED value + (`screen_status_rows`): cutting first could slice a token shape out of a string and accept + the rest, which would trim a refusal into acceptance. + """ + if isinstance(value, dict): + return {key: trim_status(item, limit) for key, item in value.items()} + if isinstance(value, list): + return [trim_status(item, limit) for item in value] + if isinstance(value, tuple): + return tuple(trim_status(item, limit) for item in value) + if isinstance(value, str) and len(value) > limit: + head = value[:limit - 3] + cut = head.rsplit(" ", 1)[0] if " " in head else head + return cut.rstrip(" ,;:") + "..." + return value + + +def screen_status_rows(status): + """`(safe, refused)` for a `{table: row}` status: each row judged on its own. + + A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed + form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that + is only too long is trimmed, then held to the full guard again. + """ + import json + + safe, refused = {}, [] + for name, row in (status or {}).items(): + try: + whole = json.dumps(row, ensure_ascii=False, allow_nan=False) + if status_payload_has_secret(whole, max_length=None): + refused.append(name) + continue + trimmed = trim_status(row) + if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)): + refused.append(name) + continue + except (TypeError, ValueError, RecursionError): + refused.append(name) + continue + safe[name] = trimmed + return safe, refused + + +def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT): """Scan serialized JSON for C2's credential keys, token shapes and oversized values. Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content. The credential key roster is the same one the connector accepts at connect time. + `max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`). """ import json import re @@ -73,10 +142,11 @@ def status_payload_has_secret(serialized_payload): elif isinstance(value, list): pending.extend(value) elif isinstance(value, str): - if (len(value) > 512 or 'Atzr|' in value + if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)): return True - elif value is not None and len(str(value)) > 512: + elif (value is not None and max_length is not None + and len(str(value)) > max_length): return True return False @@ -162,7 +232,13 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): row["syncCause"] = cause cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..." row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part) - return assert_safe_status(status) + # S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was + # `search_query_performance.unknownCause` at 935 characters); a credential shape still + # refuses, now naming only its rows so the caller degrades those and keeps the rest. + safe, refused = screen_status_rows(status) + if refused: + raise AmazonStatusRowsRefused(assert_safe_status(safe), refused) + return assert_safe_status(safe) declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]} for name, spec in DECLARED_DATASETS.items()} diff --git a/api/content_studio.py b/api/content_studio.py new file mode 100644 index 0000000000000000000000000000000000000000..10696a42b7bc4d9428277b20602836918a6ff839 --- /dev/null +++ b/api/content_studio.py @@ -0,0 +1,638 @@ +"""Content Studio: an App template that creates its own References and Posts databases. + +S67-T40 (owner instruction 1; rulings R17, R18, R21, R11). Plan-only (round 1 R5): nothing here +publishes to any network. The template creates two TENANT-OWNED databases and binds the App to +Posts, opening on a calendar of posts. The App is delete-safe (rule 13, ARCHITECTURE section 24): +deleting it removes only the App record, so References and Posts and every row in them outlive it. + +This module is the App's own code. Shared files reach it only through a guarded import +(`routes_templates.py`), so deleting this file removes the template and nothing else breaks. +""" +from __future__ import annotations + +TEMPLATE_KEY = "content_studio" +LABEL = "Content Studio" +DESCRIPTION = "Plan posts from your references. Creates References and Posts databases." + +#: R4: the channels a post is planned for. Channel length limits are ADVISED, never enforced (O4). +CHANNELS = ["LinkedIn", "X", "Instagram", "TikTok", "YouTube", "Reddit"] +NETWORKS = CHANNELS + ["Other"] +STATUSES = ["Idea", "Draft", "Ready", "Posted", "Skipped"] +CALENDAR_VIEW_ID = "cs_calendar" + +#: R15 (structure, not words): the three fields a Reference is drafted FROM are extracted from its +#: pasted text by the per-row AI field machinery (`ai_enrich`), re-run when that text changes. +_EXTRACT = { + "hook": ("Hook", "In one sentence, describe the opening move this post uses to earn attention " + "(for example a surprising number, a contrarian claim, a question). Describe " + "the technique; do not quote the post. Post: {pasted_text}"), + "structure": ("Structure", "List the post's structure as 3 to 6 short numbered beats, each " + "naming what that part does (setup, proof, turn, ask). Describe, " + "never quote. Post: {pasted_text}"), + "angle": ("Angle", "In one sentence, name the point of view or argument the post takes and " + "who it is for. Do not quote the post. Post: {pasted_text}"), +} + + +#: The platform ladder leads with a REASONING model, which spends a few hundred tokens thinking +#: before it answers; at 300 the column's ceiling was reached with no answer ("the model had +#: nothing to answer from"), so each extraction gets room for thought plus a short answer. +EXTRACT_MAX_TOKENS = 1500 + + +def reference_fields(): + """References: URL, network, author, pasted text, tags, hook, structure, angle (T40 done-when).""" + fields = [ + {"key": "title", "label": "Title", "type": "text"}, + {"key": "url", "label": "URL", "type": "url"}, + {"key": "network", "label": "Network", "type": "select", "options": list(NETWORKS)}, + {"key": "author", "label": "Author", "type": "text"}, + {"key": "pasted_text", "label": "Pasted text", "type": "text"}, + {"key": "tags", "label": "Tags", "type": "multiselect", "options": []}, + ] + for key, (label, prompt) in _EXTRACT.items(): + fields.append({"key": key, "label": label, "type": "ai_enrich", + "aiEnrich": {"prompt": prompt, "maxTokens": EXTRACT_MAX_TOKENS, + "trigger": {"mode": "on_change"}, "overwrite": "always"}}) + return fields + + +def post_fields(references_key): + """Posts: caption, channel, scheduled, status, media, campaign, post URL, References link, + attribution (T40 done-when). `media` is an `image` field: the store has no attachment type.""" + return [ + {"key": "caption", "label": "Caption", "type": "text"}, + {"key": "channel", "label": "Channel", "type": "select", "options": list(CHANNELS)}, + {"key": "scheduled", "label": "Scheduled", "type": "date"}, + {"key": "status", "label": "Status", "type": "select", "options": list(STATUSES)}, + {"key": "media", "label": "Media", "type": "image"}, + {"key": "campaign", "label": "Campaign", "type": "text"}, + {"key": "post_url", "label": "Post URL", "type": "url"}, + {"key": "references", "label": "References", "type": "link", + "link": {"table": references_key, "selectionMode": "manual"}}, + {"key": "attribution", "label": "Attribution", "type": "text"}, + ] + + +class ContentStudioError(RuntimeError): + """A refusal the route turns into a plain sentence; nothing half-created is left behind.""" + + +def _create_table(label, username, fields, st): + """`user_tables.create` refuses a duplicate label, so a second Studio gets "Posts 2" and so on.""" + from core import user_tables + for attempt in range(1, 10): + name = label if attempt == 1 else f"{label} {attempt}" + key = user_tables.create(name, username, fields=fields, st=st) + if key: + return key + raise ContentStudioError(f"The {label} database could not be created. " + "Rename or delete an older one, then try again.") + + +def create(session, name=""): + """Create References + Posts (tenant-owned), a calendar View on Posts, and the App bound to + Posts. Returns `{address, application, databases}`; on a failure part-way, the databases this + call created are deleted again, so a refusal leaves the workspace as it was.""" + import application_registry + from core import user_tables, view_templates + from core import table_store + + st = session.runtime + user = session.uname + app_name = " ".join(str(name or "").split())[:120] or LABEL + created = [] + try: + references = _create_table("References", user, reference_fields(), st) + created.append(references) + posts = _create_table("Posts", user, post_fields(references), st) + created.append(posts) + ops = table_store.make(view_templates.workspace_key(posts), st=st) + ops.save_view(user, {"id": CALENDAR_VIEW_ID, "name": "Calendar", "createdBy": user, + "config": {"filters": [], "sorts": [], + "display": {"mode": "calendar", "dateField": "scheduled", + "titleField": "caption"}}}) + application = application_registry.create( + st, session.tenant, user, + {"templateKind": TEMPLATE_KEY, "name": app_name, "sourceDatabase": posts, + "navPlacement": {"section": "app", "order": 0}}, + allowed_databases=None) + except Exception: + for key in reversed(created): + try: + user_tables.delete(key, st=st) + except Exception: + pass + raise + # S67-T41 owner correction: the App opens on its own surface in the App module, not in Data. + return {"address": "#" + app_link(application["id"]), "application": application, + "databases": {"references": references, "posts": posts}} + + +# --------------------------------------------------------------------------------------------- +# S67-T41 (R13, R14, R15, R19, R20, R5, O4): DRAFTING. One Draft row per chosen channel, from +# several selected References plus a one-line topic, written from their extracted STRUCTURE +# (hook, structure, angle), never their words, with every author credited. Plan-only: a draft +# is a Posts row with status Draft; nothing is published anywhere. +# --------------------------------------------------------------------------------------------- +import re + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/content-studio") + +#: O4: channel limits are ADVICE in the prompt and on screen, never a truncation. +CHANNEL_ADVICE = { + "LinkedIn": "LinkedIn post: up to about 3,000 characters, short paragraphs, no hashtag wall.", + "X": "X post: aim for 280 characters or fewer.", + "Instagram": "Instagram caption: up to about 2,200 characters; a strong first line.", + "TikTok": "TikTok caption: short, about 150 characters.", + "YouTube": "YouTube: a title line, then a two or three sentence description.", + "Reddit": "Reddit: a title line, then the body; plain, specific, no marketing tone.", +} +DRAFT_SYSTEM = ( + "You draft ONE original social media post for the channel named. You are given a one-line " + "topic and the STRUCTURE of several reference posts: the technique of each hook, its beats " + "and its angle. Borrow those structural moves; write entirely new sentences. Never copy or " + "closely paraphrase any sentence from anywhere. Answer with the post text only: no preamble, " + "no quotation marks around it, no markdown heading. Never use an em dash or an en dash; use " + "a comma, a colon or a full stop.") +LEDGER_SURFACE = "field_agent" +DRAFT_MAX_TOKENS = 3000 # a reasoning model thinks first; see EXTRACT_MAX_TOKENS +DRAFT_TIMEOUT_S = 60 +#: A shared run of this many words is a copied sentence, whatever surrounds it (R15). +VERBATIM_WORDS = 8 + + +def _studio(session, app_id): + """(app, posts_key, references_key) for a Content Studio App this session may open.""" + import application_registry + from core import user_tables + doc = application_registry.load(session.runtime) + app = doc["applications"].get(str(app_id or "")) + if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KEY + or not application_registry.may_read(app, session.uname, session.admin)): + raise err(404, "content_studio_not_found", "That Content Studio does not exist.") + posts = str(app.get("sourceDatabase") or "") + defn = user_tables.get(posts, session.runtime) or {} + link = next((f.get("link") for f in defn.get("fields") or () + if isinstance(f, dict) and f.get("key") == "references"), None) + references = str((link or {}).get("table") or "") + if not posts or not references: + raise err(409, "content_studio_incomplete", + "This Content Studio has lost its Posts or References database.") + return app, posts, references + + +def _reference_rows(session, references): + from core import user_tables + defn = user_tables.get(references, session.runtime) or {} + rows = defn.get("rows") or {} + return {str(rid): (row if isinstance(row, dict) else {}) for rid, row in rows.items()} + + +# --------------------------------------------------------------------------------------------- +# S67-T42 (R35, standing rule 1): THE MONTHLY DRAFTING ALLOWANCE. Drafting runs on Loopable's +# platform keys, so each tenant gets a monthly count of drafts (one Draft row = one draft). Past +# it, Draft pauses with a sentence naming the reset date; the platform admin raises it per tenant +# (`PUT /content-studio/allowance`). A tenant admin cannot: that would defeat R35. +# One per-tenant bucket, owned by this module: {"allowance": int (absent = default), +# "months": {"YYYY-MM": drafts made}}. Month and reset are UTC. +# --------------------------------------------------------------------------------------------- +ALLOWANCE_KEY = "content_studio_usage" +DEFAULT_MONTHLY_DRAFTS = 100 +ALLOWANCE_KEEP_MONTHS = 12 +_MONTH_NAMES = ("January", "February", "March", "April", "May", "June", "July", "August", + "September", "October", "November", "December") + + +def _now(): + from datetime import datetime, timezone + return datetime.now(timezone.utc) + + +def _month_key(now): + return f"{now.year:04d}-{now.month:02d}" + + +def _reset_on(now): + """The first day of next month (UTC): the day this month's count stops counting.""" + from datetime import date + return date(now.year + 1, 1, 1) if now.month == 12 else date(now.year, now.month + 1, 1) + + +def _spoken_date(day): + return f"{_MONTH_NAMES[day.month - 1]} {day.day}, {day.year}" + + +def _limit(doc): + raw = (doc or {}).get("allowance") + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0: + return DEFAULT_MONTHLY_DRAFTS + return raw + + +def allowance_state(doc, now=None): + """{used, limit, remaining, resetsOn, paused, message}: the ONE reading of month and reset + that both the panel and the enforcement use.""" + now = now or _now() + doc = doc if isinstance(doc, dict) else {} + months = doc.get("months") if isinstance(doc.get("months"), dict) else {} + try: + used = max(0, int(months.get(_month_key(now)) or 0)) + except (TypeError, ValueError): + used = 0 + limit = _limit(doc) + remaining = max(0, limit - used) + reset = _reset_on(now) + paused = remaining <= 0 + message = (f"This workspace has used this month's drafting allowance. " + f"It resets on {_spoken_date(reset)}.") if paused else "" + return {"used": used, "limit": limit, "remaining": remaining, + "resetsOn": reset.isoformat(), "paused": paused, "message": message} + + +def _short_message(remaining): + drafts = "1 draft" if remaining == 1 else f"{remaining} drafts" + channels = "1 channel" if remaining == 1 else f"up to {remaining} channels" + return f"This workspace has {drafts} left this month. Choose {channels}." + + +def _refusal(state, wanted): + """The 429 a draft of `wanted` drafts earns against `state`, or None when it fits.""" + if state["remaining"] <= 0: + return err(429, "drafting_allowance_used", state["message"]) + if state["remaining"] < wanted: + return err(429, "drafting_allowance_short", _short_message(state["remaining"])) + return None + + +def _read_allowance(st): + doc = st.get(ALLOWANCE_KEY) + return doc if isinstance(doc, dict) else {} + + +def _count(st, month, delta): + """Add `delta` drafts to `month` in one read-modify-write; keeps the last 12 months.""" + def _up(doc): + doc = dict(doc) if isinstance(doc, dict) else {} + months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {} + months[month] = max(0, int(months.get(month) or 0) + int(delta)) + for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]: + months.pop(stale, None) + doc["months"] = months + return doc + st.update(ALLOWANCE_KEY, _up, flush="sync") + + +def reserve_drafts(st, wanted, now=None): + """Check AND reserve `wanted` drafts in ONE store update, so two concurrent drafts cannot both + take the last slot. Raises the 429 (no write) when it does not fit; returns the month key the + reservation was booked to, for `release_drafts` on a failure.""" + now = now or _now() + try: + current = _read_allowance(st) + except Exception: # noqa: BLE001 + raise err(503, "allowance_unavailable", + "The drafting allowance could not be checked, so nothing was drafted. Try again.") + refusal = _refusal(allowance_state(current, now), wanted) + if refusal is not None: # fast path: a paused workspace writes nothing + raise refusal + month = _month_key(now) + verdict = {} + + def _up(doc): + doc = dict(doc) if isinstance(doc, dict) else {} + refused = _refusal(allowance_state(doc, now), wanted) + if refused is not None: + verdict["refusal"] = refused + return doc + months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {} + months[month] = max(0, int(months.get(month) or 0)) + int(wanted) + for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]: + months.pop(stale, None) + doc["months"] = months + return doc + + try: + st.update(ALLOWANCE_KEY, _up, flush="sync") + except Exception: # noqa: BLE001 + raise err(503, "allowance_unavailable", + "The drafting allowance could not be checked, so nothing was drafted. Try again.") + if "refusal" in verdict: + raise verdict["refusal"] + return month + + +def release_drafts(st, month, n): + """Hand back a reservation whose drafts were never written (AI failure, copy refusal, store).""" + try: + _count(st, month, -int(n)) + except Exception: # noqa: BLE001 + pass + + +@router.put("/allowance") +def set_allowance(body: dict = Body(default=None), session: Session = Depends(require_session)): + """The owner raises (or resets) one tenant's monthly drafting allowance. Platform admin only.""" + import core.platform_admin as platform_admin + from harness import runtime + if not platform_admin.is_platform_admin(session.user): + raise err(403, "forbidden", "your account does not have access to this surface") + body = body if isinstance(body, dict) else {} + slug = str(body.get("tenant") or "").strip().lower() + monthly = body.get("monthly") + if monthly is not None and (isinstance(monthly, bool) or not isinstance(monthly, int) + or monthly < 0): + raise err(400, "bad_allowance", + "The monthly allowance must be a whole number of drafts, 0 or more.") + if not slug or slug not in runtime.known_tenants(): + raise err(404, "tenant_not_found", "That workspace does not exist.") + try: + st = runtime.get_runtime(slug) + except KeyError: + raise err(404, "tenant_not_found", "That workspace does not exist.") + + def _up(doc): + doc = dict(doc) if isinstance(doc, dict) else {} + if monthly is None: + doc.pop("allowance", None) + else: + doc["allowance"] = int(monthly) + return doc + + st.update(ALLOWANCE_KEY, _up, flush="sync") + return {"tenant": slug, "allowance": allowance_state(_read_allowance(st))} + + +@router.get("/references") +def list_references(app: str = "", session: Session = Depends(require_session)): + """The References a Draft can use: id, title, author, network, and whether its structure is in. + Also this workspace's drafting allowance, so the panel reports the limit before it bites.""" + _app, _posts, references = _studio(session, app) + rows = _reference_rows(session, references) + try: + allowance = allowance_state(_read_allowance(session.runtime)) + except Exception: # noqa: BLE001 + allowance = None # unreadable: the draft route still enforces it + return {"references": [ + {"id": rid, "title": str(row.get("title") or row.get("url") or "Untitled reference")[:160], + "author": str(row.get("author") or "")[:120], "network": str(row.get("network") or ""), + "extracted": bool(str(row.get("structure") or "").strip())} + for rid, row in rows.items()], + "channels": list(CHANNELS), "advice": dict(CHANNEL_ADVICE), "allowance": allowance} + + +# --------------------------------------------------------------------------------------------- +# S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an +# App ... it should act like Catalog / Route where it just uses the data module as what its +# building on top." So the App opens in the App module on its OWN surface (`#/app?studio=`, +# `apps/contentStudio/ContentStudioApp.tsx`): a calendar and a board over Posts, the Draft door, +# and the two databases underneath. Data still holds References and Posts; the App never sends +# anyone there to use it. +# ⚠ The shared `/applications` doors (`routes_assets.py`) admit only topic-backed databases, so +# they refuse an App bound to a user table; the read, star and delete this App needs live here, +# on the registry's own writers, behind the same `_studio` + `may_read` / `may_edit` walls. +# --------------------------------------------------------------------------------------------- +def app_link(app_id): + """The one address a Content Studio App opens at (App module, its own surface).""" + return f"/app?studio={app_id}" + + +#: The Posts fields the surface draws, in the grid's own Field shape so `viewModes.CalendarView` +#: (by Scheduled) and `KanbanView` (by Status) render them unchanged. +SURFACE_FIELDS = ("caption", "channel", "scheduled", "status", "attribution", "campaign") + + +def _surface_field(defn): + kind = {"date": "date", "select": "select"}.get(defn.get("type"), "text") + field = {"key": defn["key"], "label": str(defn.get("label") or defn["key"]), "type": kind, + "source": "overlay"} + if kind == "select": + field["options"] = [str(o) for o in (defn.get("options") or ())] + return field + + +def _studio_or_missing(session, app_id): + """`_studio`, answering an absent App with the code the App home already reads as 'gone'.""" + try: + return _studio(session, app_id) + except Exception as exc: # noqa: BLE001 + if getattr(exc, "status_code", None) == 404: + raise err(404, "application_not_found", "That Content Studio no longer exists.") + raise + + +@router.get("/studio") +def studio(app: str = "", session: Session = Depends(require_session)): + """Everything the App surface draws: the App, Posts (surface fields + rows), References' + count, the statuses in board order, and the drafting allowance.""" + from core import user_tables + application, posts, references = _studio(session, app) + if not user_tables.may_open(posts, session.uname, session.admin, st=session.runtime): + raise err(404, "content_studio_not_found", "That Content Studio does not exist.") + posts_def = user_tables.get(posts, session.runtime) or {} + refs_def = user_tables.get(references, session.runtime) or {} + by_key = {f.get("key"): f for f in posts_def.get("fields") or () if isinstance(f, dict)} + fields = [_surface_field(by_key[key]) for key in SURFACE_FIELDS if key in by_key] + rows = [] + for rid, row in (posts_def.get("rows") or {}).items(): + try: + pid = int(rid) + except (TypeError, ValueError): + continue + row = row if isinstance(row, dict) else {} + rows.append({"pid": pid, **{key: row.get(key) for key in SURFACE_FIELDS}}) + rows.sort(key=lambda r: r["pid"]) + try: + allowance = allowance_state(_read_allowance(session.runtime)) + except Exception: # noqa: BLE001 + allowance = None + return {"app": {"id": application["id"], "name": application.get("name") or LABEL}, + "posts": {"database": posts, "label": str(posts_def.get("label") or "Posts"), + "fields": fields, "rows": rows}, + "references": {"database": references, + "label": str(refs_def.get("label") or "References"), + "count": len(refs_def.get("rows") or {})}, + "statuses": list(STATUSES), "allowance": allowance} + + +@router.patch("/apps/{app_id}") +def star_app(app_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Star or unstar a Content Studio on the App home (the registry's own star writer).""" + import application_registry + _app, posts, _references = _studio_or_missing(session, app_id) + body = body if isinstance(body, dict) else {} + if set(body) != {"starred"} or not isinstance(body.get("starred"), bool): + raise err(400, "bad_patch", "Only the star of a Content Studio changes here.") + try: + landed = application_registry.mutate(session.runtime, app_id, session.uname, session.admin, + {"starred": body["starred"]}, allowed_databases=[posts]) + except PermissionError: + raise err(403, "forbidden", "You cannot change this App.") + except application_registry.ApplicationError: + raise err(404, "application_not_found", "That Content Studio no longer exists.") + return {"application": landed} + + +@router.delete("/apps/{app_id}") +def delete_app(app_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Delete the App and nothing else (rule 13): References and Posts, every row in them, and + every View on them outlive it. The name is confirmed inside the registry's own update.""" + import application_registry + application, posts, _references = _studio_or_missing(session, app_id) + confirm = (body or {}).get("confirmName") if isinstance(body, dict) else None + if not isinstance(confirm, str) or confirm != application.get("name"): + raise err(400, "confirm_name", "Enter the exact App name to confirm.") + + class _NameConfirmed: + def update(self, key, delete): + def confirmed(raw): + deleted = delete(raw) + if (raw.get("applications") or {}).get(app_id, {}).get("name") != confirm: + raise application_registry.ApplicationError("the App name changed") + return deleted + return session.runtime.update(key, confirmed) + + try: + return application_registry.delete(_NameConfirmed(), app_id, session.uname, session.admin, + allowed_databases=[posts]) + except PermissionError: + raise err(403, "forbidden", "You no longer have permission to delete this App.") + except application_registry.ApplicationError: + raise err(400, "confirm_name", "The App name may have changed. Close this dialog and reload to retry.") + + +def _words(text): + return re.findall(r"[a-z0-9']+", str(text or "").lower()) + + +def copied_sentence(draft, sources): + """The first run of VERBATIM_WORDS consecutive words a draft shares with any source, or ''.""" + grams = set() + for source in sources: + words = _words(source) + grams.update(" ".join(words[i:i + VERBATIM_WORDS]) + for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1))) + words = _words(draft) + for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)): + gram = " ".join(words[i:i + VERBATIM_WORDS]) + if gram in grams: + return gram + return "" + + +def attribution(rows): + """Every selected reference's author, credited in plain words (R14 traceability).""" + names = list(dict.fromkeys( + str(row.get("author") or row.get("title") or "an unnamed source").strip() for row in rows)) + joined = names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1] + return f"Structure drawn from posts by {joined}." + + +def draft_prompt(topic, channel, rows): + parts = [f"Channel: {channel}. {CHANNEL_ADVICE.get(channel, '')}", f"Topic: {topic}", ""] + for n, row in enumerate(rows, 1): + parts.append(f"Reference {n} ({row.get('network') or 'post'}):") + for key, label in (("hook", "Hook technique"), ("structure", "Structure"), ("angle", "Angle")): + value = " ".join(str(row.get(key) or "").split())[:900] + if value: + parts.append(f" {label}: {value}") + parts.append("") + parts.append("Write the post now.") + return "\n".join(parts) + + +def draft_posts(session, app_id, reference_ids, topic, channels, *, ask=None): + """Generate EVERY channel first; write the Posts rows only if all succeeded (no half row).""" + topic = " ".join(str(topic or "").split())[:200] + ids = [str(x) for x in dict.fromkeys(reference_ids or ())][:50] + wanted = [c for c in dict.fromkeys(channels or ()) if c in CHANNELS] + if not ids: + raise err(400, "no_references", "Select at least one reference to draft from.") + if not topic: + raise err(400, "no_topic", "Type a one-line topic for the draft.") + if not wanted: + raise err(400, "no_channels", "Choose at least one channel.") + _app, posts, references = _studio(session, app_id) + rows_by_id = _reference_rows(session, references) + missing = [rid for rid in ids if rid not in rows_by_id] + if missing: + raise err(409, "references_missing", "A selected reference no longer exists. Reload and try again.") + chosen = [rows_by_id[rid] for rid in ids] + if not any(str(row.get("structure") or "").strip() for row in chosen): + raise err(409, "references_not_extracted", + "The selected references have no extracted structure yet. Paste their text first.") + # R35: the allowance is checked and reserved BEFORE any provider is resolved or called. One + # draft per channel; the reservation is handed back unless every row is written. + month = reserve_drafts(session.runtime, len(wanted)) + try: + return _generate_and_save(session, posts, ids, topic, wanted, chosen, ask=ask) + except BaseException: + release_drafts(session.runtime, month, len(wanted)) + raise + + +def _generate_and_save(session, posts, ids, topic, wanted, chosen, *, ask=None): + import ai_enrich + import ai_review + import usage_ledger + from core import user_tables + + live = ai_review.ladder() + if not live and ask is None: + raise err(503, "ai_unavailable", "No AI provider is configured here, so nothing was drafted.") + provider = (live or [{"name": "injected", "model": "injected"}])[0] + model = provider["model"] + caller = ask or ai_enrich._ask + sources = [row.get("pasted_text") or "" for row in chosen] + drafts, tokens, calls = [], 0, 0 + try: + for channel in wanted: + prompt = draft_prompt(topic, channel, chosen) + text = "" + for attempt in range(2): + answer, used, problem = caller(provider, model, prompt, DRAFT_MAX_TOKENS, + DRAFT_TIMEOUT_S, system=DRAFT_SYSTEM) + calls += 1 + tokens += int(used or 0) + if problem or not str(answer or "").strip(): + raise err(502, "draft_failed", f"The {channel} draft could not be written " + f"({problem or 'the model returned nothing'}). Nothing was saved.") + text = ai_enrich.no_dashes(str(answer).strip()) + copied = copied_sentence(text, sources) + if not copied: + break + prompt += f"\n\nYour last draft reused this wording from a reference: \"{copied}\". Rewrite it in new words." + text = "" + if not text: + raise err(502, "draft_copied", f"The {channel} draft kept repeating a reference's " + "wording, so it was not saved. Nothing was saved.") + drafts.append({"channel": channel, "caption": text}) + finally: + if calls: + # `usage_ledger.SURFACES` names four surfaces and refuses any other, so drafts are + # booked as `field_agent` (AI-written cells of a user table) until a `content_studio` + # surface exists there (a finding for the ledger's owner; this module cannot add it). + usage_ledger.record(LEDGER_SURFACE, provider["name"], model, total=tokens, + calls=calls, st=session.runtime, user=session.uname) + credit = attribution(chosen) + new_rows = [{"caption": d["caption"], "channel": d["channel"], "status": "Draft", + "references": ",".join(ids), "attribution": credit} for d in drafts] + created = user_tables.add_rows(posts, new_rows, username=session.uname, st=session.runtime) + if not created: + raise err(503, "store_unavailable", "The drafts could not be saved. Nothing was saved.") + return {"created": list(created) if isinstance(created, (list, tuple)) else created, + "drafts": drafts, "attribution": credit, "tokens": tokens, "calls": calls, + "provider": provider["name"], "model": model} + + +@router.post("/draft") +def draft(body: dict = Body(default=None), session: Session = Depends(require_session)): + body = body if isinstance(body, dict) else {} + return draft_posts(session, body.get("app"), body.get("references"), body.get("topic"), + body.get("channels")) diff --git a/api/deps.py b/api/deps.py index c1c42a56dcff4efb1625e6c85c3fae99166ddb0a..bcb27545f37c62d376975e77f2fffbae144d2844 100644 --- a/api/deps.py +++ b/api/deps.py @@ -232,6 +232,28 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields= else ("product_pool", team_id)) cache = getattr(session.runtime, "pool_cache", None) entry = cache.get(cache_key) if isinstance(cache, dict) else None + if not isinstance(entry, tuple) and isinstance(cache, dict): + # ⭐⭐ S67-T33 / D-778: A DATABASE NEVER OPENED ANSWERS. The snapshot used to exist only + # once somebody opened the grid in this process, so after every restart the assistant + # refused a database its caller demonstrably holds. The miss is now filled by the GRID'S + # OWN pool builder, for the SAME scope key the grid would fill (`derive_pool_scope` + # above), so this is the grid's first open, not a second reader. It is read-only on Odoo + # (rule 4) and honours the connector pause exactly as the grid does (503 when paused + # with nothing cached). Permission and field checks above have already run. + try: + if database == "customer_data": + import routes_customers + routes_customers._pool_for(session.runtime, team_id, agent) + else: + import routes_products + routes_products._pool_for(session.runtime, team_id) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + _assistant_refuse(409, "assistant_snapshot_unavailable", + f"this database's snapshot could not be built for your current " + f"scope ({type(exc).__name__}); try again shortly") + entry = cache.get(cache_key) if (not isinstance(entry, tuple) or len(entry) < 2 or not isinstance(entry[1], (list, tuple)) or any(not isinstance(row, dict) for row in entry[1])): @@ -247,6 +269,44 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields= _ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"}) +def _assistant_nav_opens(session: Session, database: str, lent, definition=None) -> bool: + """Does the NAVIGATION offer this `ut_*` database to this caller? S67-T33 / R33. + + ⭐⭐ THE AI CONNECTOR'S LIST IS THE NAV'S LIST, BY CONSTRUCTION RATHER THAN BY AGREEMENT. + `routes_nav.nav` admits a user table when `user_tables.nav_entries` lists it (which is + `may_open` plus the route-system exclusion), it is not a retired key, and + `perm_scope.nav_may_open` agrees (an administrator's explicit untick denies). The assistant + used to ask `may_open` alone, so a database the member's navigation did not show could still + be listed and read through the connector. This asks the nav's own predicates, in the nav's + order; it is not a second permission rule. + ⚠ `session.admin` short-circuits leg 1 of `nav_may_open` exactly as the nav's own admin + session does; `nav_may_open` itself returns True for an administrator. + """ + import core.perm_scope as perm_scope + from core import route_system, user_tables + + key = str(database or "") + try: + from odoo_relational import RETIRED_KEYS as retired + except Exception: # noqa: BLE001 + retired = () + if key in retired: + return False + if not user_tables.may_open(key, session.uname, session.admin, st=lent): + return False + if isinstance(definition, dict) and (route_system.is_managed(definition) + or route_system.is_managed(key)): + return False + if session.admin: + return True + user = getattr(session, "user", None) + if not isinstance(user, dict): + # No account record means no explicit per-key deny can exist, and `nav_may_open` leg 3 + # then defers to `may_open`, which has already answered above. + return True + return bool(perm_scope.nav_may_open(user, key, st=lent)) + + def _assistant_table_defn(session: Session, database: str): """The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`. @@ -276,7 +336,9 @@ def _assistant_table_defn(session: Session, database: str): if not isinstance(definition, dict): _assistant_refuse(404, "assistant_unknown_database", "that database is not available") definition_runtime = user_tables.lend(session.runtime, user_tables=definitions) - if not user_tables.may_open(database, session.uname, session.admin, st=definition_runtime): + # S67-T33: the NAV's predicate, not `may_open` alone, so a database the caller's navigation + # does not show is refused here too (403), not merely left off the list. + if not _assistant_nav_opens(session, database, definition_runtime, definition): # Deliberately no label/owner in this response: knowing a key must not become an oracle # for somebody else's database. _assistant_refuse(403, "assistant_forbidden", @@ -346,7 +408,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None, - *, defn=None, lent=None): + *, defn=None, lent=None, mode=None, extra=None): """Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7). Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the @@ -382,11 +444,24 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No # THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all? # A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror # about it would open a cursor to learn nothing. - if assistant_mirror.declared_fields(database) is None: + if database.startswith("ut_amzn_"): + # ⭐ S67-T33: THE GRID'S OWN ADMISSION FIRST. A dynamically sized Amazon snapshot enters + # the source registry only through `require_amazon_mirror`, so without it a database + # never opened in this process had no binding here (measured on staging for + # `ut_amzn_fba_inventory`). It refuses by name (503) while the Amazon mirror is unloaded. + import routes_odoo_tables + routes_odoo_tables.require_amazon_mirror(database, session, definition=definition) + # ⭐ S67-T33: the stored definition rides along, so an Amazon grid or a generated + # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid. + declared = assistant_mirror.declared_fields(database, definition) + if declared is None: _assistant_refuse(409, "assistant_snapshot_unavailable", "this database does not store its rows and it is not one of the " "connected databases the assistant can read from the mirror") - cur, refusal = assistant_mirror.open_cursor(session) + # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's + # first sync. Passed only for an Amazon key, which is the one family it changes. + cur, refusal = (assistant_mirror.open_cursor(session, database) + if database.startswith("ut_amzn_") else assistant_mirror.open_cursor(session)) if refusal is not None: _assistant_refuse(refusal.status, refusal.code, refusal.message) spec = assistant_mirror.binding(cur, database) @@ -394,7 +469,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No _assistant_refuse(409, "assistant_snapshot_unavailable", "this database is served live from its connector and this deployment " "has no read-through binding for it yet, so it holds no rows to read") - fields = assistant_mirror.answerable_fields(database, spec) + fields = assistant_mirror.answerable_fields(database, spec, declared) if not fields: _assistant_refuse(409, "assistant_snapshot_unavailable", "this database's connector mirror carries none of its declared columns " @@ -422,12 +497,36 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No if grant is None: wall_tree = None + filter_keys = [field["key"] for field in visible + if isinstance(field, dict) and field.get("key")] + # ⭐⭐ S67-T34: THREE READS BEHIND THE ONE PREFLIGHT ABOVE, chosen by `mode`, all compiling the + # SAME wall and filter (`assistant_mirror._predicate`). "schema" reads and counts no row + # (D-2630: describe answers the field list of a million-row table without counting it); + # ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631); + # ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1). try: - rows, stamp, total = assistant_mirror.read_rows( - session, database, cur, spec, fields, - wall_tree=wall_tree, filter_tree=filter_tree, - filter_keys=[field["key"] for field in visible - if isinstance(field, dict) and field.get("key")]) + if mode == "schema": + return [], fields, {"read": "connector mirror"}, definition.get("label") or database + if isinstance(mode, tuple) and mode and mode[0] == "aggregate": + values, total = assistant_mirror.aggregate( + session, database, cur, spec, fields, mode[1], + wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys) + if isinstance(extra, dict): + extra["aggregates"], extra["total"] = values, total + return ([], fields, {"read": "connector mirror", "rows": total}, + definition.get("label") or database) + if isinstance(mode, tuple) and mode and mode[0] == "page": + rows, stamp, total = assistant_mirror.read_page( + session, database, cur, spec, fields, offset=mode[1], limit=mode[2], + wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys, + defn=definition) + if isinstance(extra, dict): + extra["total"] = total + else: + rows, stamp, total = assistant_mirror.read_rows( + session, database, cur, spec, fields, + wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys, + defn=definition) except assistant_mirror.MirrorRefusal as exc: _assistant_refuse(exc.status, exc.code, exc.message) raise # unreachable; see above @@ -459,7 +558,33 @@ def _assistant_requested_fields(fields, visible_fields): return out -def assistant_read_scope(session: Session, database, fields=None, filters=None): +def _assistant_oversize_reason(database: str, definition, lent) -> str: + """The named cause an oversize stored table carries, or "". S67-T33 / rule 1. + + ⛔ AN EVICTED TABLE KEEPS ITS DEFINITION AND LOSES ITS ROWS, so the materialised reader used + to answer it as a complete table of 0 rows. `user_tables.limit_report` is the one evaluator + of that stamp (the grid serves the same sentence), so its cause and recommendation are read + from there rather than re-worded here. + """ + from core import user_tables + + if not user_tables.oversize_of(database, defn=definition if isinstance(definition, dict) + else {}): + return "" + report = user_tables.limit_report(database, st=lent) or {} + label = str((definition or {}).get("label") or database) + return (f"{label} is not stored: {report.get('cause') or 'its rows were not stored'}. " + f"{str(report.get('recommendation') or '').rstrip('.')}. The assistant refuses " + f"rather than answering 0 rows").replace("\u2014", ",").replace("\u2013", "-") + + +def _assistant_refuse_oversize(database: str, definition, lent) -> None: + reason = _assistant_oversize_reason(database, definition, lent) + if reason: + _assistant_refuse(409, "assistant_source_too_large", reason) + + +def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None): """Return a detached, permissioned snapshot for one Assistant tool call. Signature (the C8 contract for E): @@ -468,6 +593,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None): permission_scope_applied, fields, filters, records } + ⭐ S67-T34: `mode` changes only HOW a mirror-served database is read, never who may read it + (every wall below runs unchanged): "schema" returns the field contract with no row read, + ("aggregate", specs) adds `server_aggregates` and `total_rows` computed in SQL over every + permitted row, ("page", offset, limit) returns one SQL page plus `total_rows`. Every other + source kind ignores it and answers as before. + `database` is exactly one source. `fields` is an explicit list of visible field keys (or ``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by `core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and @@ -478,6 +609,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None): from harness import filter_eval key = str(database or "").strip() + extra = {} if key in _ASSISTANT_BUILT_INS: rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters) label = _ASSISTANT_BUILT_INS[key]["label"] @@ -494,12 +626,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None): from core import user_tables _defn, _lent = _assistant_table_defn(session, key) if user_tables.materialises(key, st=_lent, defn=_defn): + # S67-T33: a stored table whose rows were evicted refuses by name, never answers 0. + # Only on the STORED reader: a read-through table's rows come from the mirror. + _assistant_refuse_oversize(key, _defn, _lent) rows, canonical_fields, source_version, label = _assistant_user_table( session, key, fields, filters, defn=_defn, lent=_lent) source_kind = "materialised_user_table" else: rows, canonical_fields, source_version, label = _assistant_mirror_table( - session, key, fields, filters, defn=_defn, lent=_lent) + session, key, fields, filters, defn=_defn, lent=_lent, mode=mode, extra=extra) source_kind = "mirror_user_table" else: _assistant_refuse(404, "assistant_unknown_database", "that database is not available") @@ -556,7 +691,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None): records.append({"pid": copy.deepcopy(row["pid"]), **{field: copy.deepcopy(row.get(field)) for field in selected_keys}}) - return { + out = { "database": key, "label": str(label), "source_kind": source_kind, @@ -568,6 +703,11 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None): "filters": copy.deepcopy(requested_filter), "records": records, } + if "aggregates" in extra: + out["server_aggregates"] = copy.deepcopy(extra["aggregates"]) + if "total" in extra: + out["total_rows"] = int(extra["total"]) + return out def assistant_permitted_sources(session: Session, databases): @@ -672,6 +812,19 @@ def assistant_source_status(session: Session, databases=None): mirror_cur = None return mirror_cur + amazon_cur = {} + + def _amazon_mirror(key): + """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key.""" + if "cur" not in amazon_cur: + try: + import assistant_mirror + cur, refusal = assistant_mirror.open_cursor(session, key) + amazon_cur["cur"] = None if refusal is not None else cur + except Exception: # noqa: BLE001 + amazon_cur["cur"] = None + return amazon_cur["cur"] + def _definitions(): nonlocal definitions, read_definitions if not read_definitions: @@ -752,6 +905,10 @@ def assistant_source_status(session: Session, databases=None): entry = cache.get(cache_key) if isinstance(cache, dict) else None if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)): row["answerable"] = True + elif isinstance(cache, dict): + # ⭐ S67-T33 / D-778: a miss is no longer a refusal; the read builds the snapshot + # through the grid's own pool builder, so this source is answerable now. + row["answerable"] = True else: row["reason"] = "no snapshot loaded yet. Open this database once, then ask" continue @@ -762,10 +919,16 @@ def assistant_source_status(session: Session, databases=None): if not isinstance(definition, dict): continue lent = user_tables.lend(session.runtime, user_tables=defs) - if not user_tables.may_open(key, session.uname, session.admin, st=lent): + # ⭐ S67-T33 / R33: the NAV's predicate, so this list and the navigation are one set. + if not _assistant_nav_opens(session, key, lent, definition): continue row["permitted"] = True if user_tables.materialises(key, st=lent, defn=definition): + oversize = _assistant_oversize_reason(key, definition, lent) + if oversize: + # Rule 1's second sentence: listed, with its cause, never answered as 0 rows. + row["reason"] = oversize + continue row["answerable"] = True continue # ⭐⭐ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat @@ -778,7 +941,9 @@ def assistant_source_status(session: Session, databases=None): # is transient and retrying works, while a grid this deployment has no read-through # binding for will not fix itself by waiting. One sentence for both would make the first # look permanent, which is the failure mode the ruling exists to prevent. - cur = _mirror() + # ⭐ S67-T33: an Amazon grid asks ITS OWN mirror (`_table_cursor`), never the Odoo + # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon. + cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror() if cur is None: row["reason"] = ("this database is read live from its connector mirror, and that " "mirror is not open on this deployment right now; try again shortly") @@ -801,6 +966,49 @@ def assistant_source_status(session: Session, databases=None): return out +#: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every +#: authenticated request, uncached; on a saturated cluster that read queued ahead of every page. +#: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28 +#: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only +#: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres +#: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next +#: request. ⛔ FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still +#: answers 503 when the store is unreadable); a cached registry is never served in its place. +#: The key carries the identity of the reader and the store it reads, so a swapped store or a +#: patched `users.registry` / `store.get` misses instead of replaying another registry. +_SESSION_REGISTRY = {"key": None, "reg": None} +_SESSION_REGISTRY_LOCK = __import__("threading").Lock() + + +def _session_registry(): + """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed. + + ⛔ POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`), + and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a + fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the + memo on HF), and its `get()` already holds the bucket for the life of the process, so the + re-read this memo removes does not exist there: it reads straight through. + """ + key = None + try: + backend = store.backend() + token = (store.revision("users") or {}).get("token") if backend == "pg" else None + if token not in (None, ""): + where = os.environ.get("AIOS_TENANT") or "royal-imports" + key = (backend, where, id(users.registry), id(store.get), str(token)) + except Exception: # noqa: BLE001 -- fail-closed below + key = None + if key is not None: + with _SESSION_REGISTRY_LOCK: + if _SESSION_REGISTRY["key"] == key: + return _SESSION_REGISTRY["reg"] + reg = users.registry() or {} + if key is not None: + with _SESSION_REGISTRY_LOCK: + _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg) + return reg + + def _user_for(claims): """The account named by a verified cookie, or None — with the epoch check that makes a stateless cookie revocable. @@ -833,12 +1041,12 @@ def _user_for(claims): # `Exception` into `{}`, so a store `PoolTimeout` on staging answered a valid cookie with 401 # `invalid_session` and the client signed the owner out. Still fail-closed: nobody is admitted. try: - reg = users.registry() or {} + reg = _session_registry() except Exception as exc: print(f"[aios-api] session account store unavailable: {type(exc).__name__}: {exc}") raise err(503, "session_store_unavailable", "your account could not be checked just now. Try again in a moment") - rec = reg.get(uname) + rec = copy.deepcopy(reg.get(uname)) # the registry may be the held copy: never mutate it if (rec and rec.get("active", True) and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)): pub = users._public(uname, rec) diff --git a/api/formula_runtime.py b/api/formula_runtime.py index 65c76eb99f8fc3ec8fe305392c07db0c98049733..7f12e7df4885a3d442cb6a74dfc47c66d9a3b4d6 100644 --- a/api/formula_runtime.py +++ b/api/formula_runtime.py @@ -83,6 +83,8 @@ FN_ARITY = { "ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1), "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0), "FILTER": (2, 3), "SORT": (1, 4), + # S67-T24: the reference workbooks' `IF(ISNUMBER(SEARCH(...)))` and `ISFORMULA(...)`. + "SEARCH": (2, 3), "SEARCHB": (2, 3), "ISFORMULA": (1, 1), } # Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text # untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's @@ -457,6 +459,15 @@ FN_CATALOGUE = { "description": "Returns TRUE when a value is a number.", "contexts": _BOTH}, "ISTEXT": {"category": "Information", "signature": "ISTEXT(value)", "description": "Returns TRUE when a value is text.", "contexts": _BOTH}, + "ISFORMULA": {"category": "Information", "signature": "ISFORMULA(reference)", + "description": "Returns TRUE when the cell it names holds a formula.", + "contexts": _SHEET}, + "SEARCH": {"category": "Text", "signature": "SEARCH(find_text, within_text, [start_num])", + "description": "Returns the position of one text value inside another, ignoring " + "case; ? matches one character and * any run.", "contexts": _BOTH}, + "SEARCHB": {"category": "Text", "signature": "SEARCHB(find_text, within_text, [start_num])", + "description": "Counts like SEARCH; each character is one byte in this workbook.", + "contexts": _BOTH}, "ISERROR": {"category": "Information", "signature": "ISERROR(value)", "description": "Returns TRUE when a formula would refuse to produce a value.", "contexts": _BOTH}, @@ -625,6 +636,9 @@ PARITY_FIXTURES = { "EXACT": ("EXACT(\"A\", \"a\")", "FALSE"), "ISNUMBER": ("ISNUMBER(1)", "TRUE"), "ISTEXT": ("ISTEXT(\"a\")", "TRUE"), + "SEARCH": ("SEARCH(\"B\", \"abc\")", 2), + "SEARCHB": ("SEARCHB(\"c?\", \"abcd\")", 3), + "ISFORMULA": ("ISFORMULA(B2)", "FALSE"), "ISERROR": ("ISERROR(1 / 0)", "TRUE"), } @@ -917,7 +931,7 @@ RANGE_ARGS = { #: ⭐ THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read #: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`. -ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,)} +ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,), "ISFORMULA": (0,)} def _is_range_position(fn, index): @@ -1863,6 +1877,17 @@ def _eval_value(node, row, today, diagnostics=None): for code in local: _diagnose(diagnostics, code) return "" if first is None and isinstance(row, _SheetInputs) and not local else first + if fn in ("ISNUMBER", "ISTEXT") and len(node.get("args") or ()) == 1: + # ⭐ S67-T24: AN IS-FUNCTION NEVER PASSES AN ERROR ON. Excel answers FALSE for + # `ISNUMBER(#VALUE!)`, which is what makes `IF(ISNUMBER(SEARCH(x, y)), ...)` (2,512 cells + # in the Inventory System's Catalog) take its FALSE branch instead of failing whole. + local = [] + try: + value = _eval(node["args"][0], row, today, local) + except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, + FormulaError, _ReferenceFault): + return False + return False if local else _call(fn, [value], today, diagnostics) if fn == "ISERROR": # The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL. local = [] @@ -1893,6 +1918,11 @@ def _eval_value(node, row, today, diagnostics=None): _diagnose(diagnostics, "bad_value") return None args.append(here) + if fn == "ISFORMULA": + # ⭐ S67-T24: ASKED OF THE CELL, NOT OF ITS VALUE. Only a Sheet document knows which + # cells hold formulas (`SheetCalculation.specs`); a grid row has no formula cells. + spec = row.engine.specs.get(args[0]) if isinstance(row, _SheetInputs) and args[0] else None + return bool(isinstance(spec, dict) and spec.get("formula")) return _call(fn, args, today, diagnostics) # ⭐ RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`, # `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list, @@ -1928,6 +1958,27 @@ def _eval_value(node, row, today, diagnostics=None): raise FormulaError("formula AST node is not supported") +def _display_text(value): + """A calculated value as the grid shows it: Excel's TRUE / FALSE for a boolean (S67-T24; a + Python `str(True)` painted "True"), blank for nothing, else the value's own text.""" + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +def _search_pattern(text): + """SEARCH's wildcard text as a regex: `?` one character, `*` any run, `~x` a literal x.""" + out, index = [], 0 + while index < len(text): + ch = text[index] + if ch == "~" and index + 1 < len(text) and text[index + 1] in "?*~": + out.append(re.escape(text[index + 1])); index += 2; continue + out.append(".*?" if ch == "*" else "." if ch == "?" else re.escape(ch)); index += 1 + return "".join(out) + + def _addr_arg(node, diagnostics): """The ADDRESS a1 node names, as a key. Anything else is not an address and refuses.""" if isinstance(node, dict) and node.get("t") == "a1": @@ -2324,6 +2375,19 @@ def _call(fn, a, today, diagnostics=None): if fn == "FIND": start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start) return pos+1 if pos >= 0 else None + if fn in ("SEARCH", "SEARCHB"): + # Excel's SEARCH: case-insensitive, `?` one character, `*` any run, `~` escapes the next; + # absent text or a start outside the text is #VALUE!. SEARCHB counts bytes only in a + # double-byte locale, so in this one it is SEARCH. + within = _text(a[1]) + start = _strict_num(a[2], diagnostics) if len(a) > 2 else 1 + if start is None or not 1 <= int(start) <= len(within) + 1: + _diagnose(diagnostics, "bad_value"); return None + found = re.compile(_search_pattern(_text(a[0])), re.IGNORECASE | re.DOTALL).search( + within, int(start) - 1) + if not found: + _diagnose(diagnostics, "bad_value"); return None + return found.start() + 1 if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"): pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject) if fn == "REGEX_EXTRACT": return m.group(0) if m else None @@ -3727,7 +3791,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None): excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code) if excel: cell["error"]["excelCode"] = excel else: - cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"])) + cell.update(state="fresh", display=_display_text(cell["value"])) cell.pop("refusal", None) cell.pop("error", None) cell.pop("spill", None) @@ -3770,7 +3834,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None): base = engine.specs.get(child, {}) style = {**(owner.get("style") or {}), **(base.get("style") or {})} record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh", - "display": "" if engine.values.get(child) is None else str(engine.values[child]), + "display": _display_text(engine.values.get(child)), "spillParent": origin, "style": style, "formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child, **({"base": base} if base else {})}}} diff --git a/api/main.py b/api/main.py index 672626ff911d7595404b6caaf81fbf9928b18ef2..be7592d9ef565fde1ca8b11d69dfd4f7edcbf103 100644 --- a/api/main.py +++ b/api/main.py @@ -92,6 +92,7 @@ from core import store # noqa: E402 # 409: a view write whose caller was looking at an older revision. See its handler below. from core import view_audit # noqa: E402 from deps import Session, module_gate # noqa: E402 +from deps import require_session as deps_require_session # noqa: E402 (S67-T20 freshness door) _WEB_DIST = _HERE.parents[1] / "web" / "dist" @@ -351,6 +352,21 @@ app.include_router(routes_uploads.router) # control is that the enumeration must NAME every router in this file.** app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory + + +@app.get("/api/v1/connectors/odoo/freshness") +def odoo_sync_freshness(session: Session = Depends(deps_require_session)): + """⭐⭐ S67-T20 / C4 — IS THIS WORKSPACE'S ODOO DATA STALE, AND DID THE LAST SYNC FAIL. + + `{stale, lastGoodSyncAt, schedule, staleAfterSeconds, lastRunAt, phase, frozen, failure}` + from `odoo_relational.sync_freshness`. The Connectors card and Connectors > Odoo word it. + ⚠ A SIBLING OF `GET /connectors/odoo/status`, not a field on it, because that route is another + lane's file this wave; the integrator may fold these fields into it (COMBINE-AMEND, S67-T20). + Same administrator door as that route. + """ + routes_connectors._odoo_admin(session) + import odoo_relational as _rel_fresh + return _rel_fresh.sync_freshness(session.runtime) # ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its # two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so # it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token @@ -1359,11 +1375,17 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out cid = str(cid or default_cid) report = report if isinstance(report, dict) else {} failed = report.get("failed") + # S67-T19: a read-through `limits` notice is the design working, not a failed + # database; health reads only the entries that withheld rows (`_rel.health_view`). database_failures, unmatched_failures = ( - routes_connectors._odoo_result_database_failures(_rt, cid, report)) + routes_connectors._odoo_result_database_failures( + _rt, cid, _rel.health_view(report))) failure = (routes_connectors._odoo_failure(failed, "materialization") if failed else None) - if not failure and (database_failures or unmatched_failures): + # S67-T19: count drift within tolerance is recorded per database (below) and does not + # fail the connection; a real failure or a stale mirror's gap still does. + if not failure and (_rel.blocking_failures(database_failures, report) + or unmatched_failures): failure = routes_connectors._odoo_failure( "one or more databases failed", "materialization") tables = report.get("tables") if isinstance(report.get("tables"), dict) else {} @@ -1440,6 +1462,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out _ut.record_odoo_sync_status( phase="error" if safe_errors else "ok", errors=safe_errors, st=_rt, connection_id=cid) + # S67-T20 / C4: the tenant-wide freshness stamp follows the DEFAULT connection, the + # one this mirror serves. Best effort: a stamp must never fail the rebuild it records. + if cid == str(default_cid): + try: + _rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at) + except Exception: # noqa: BLE001 + pass meta_state = _rebuild_meta_relational( why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True) if isinstance(meta_state_out, dict): @@ -1469,6 +1498,9 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out _ut.record_odoo_sync_status( phase="error", errors=[f"{failure['code']}: {failure['message']}"], st=_rt, connection_id=cid) + # S67-T20 / C4: a failed rebuild is a sync that did NOT end well. + import odoo_relational as _rel_failed + _rel_failed.record_sync_outcome(_rt, ok=False) except Exception: pass return False diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 7df59114292c98d38c2e7f3201ec10d444b2e614..de4fc7159627e489e0a2c628da3cd6822d6ab9af 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -3619,6 +3619,112 @@ def sync_seconds(rt): return max(int(secs), SYNC_FLOOR_SECONDS) +#: ⭐⭐ S67-T20 / C4 — how far past its cadence a sync may run before its data is called stale. +#: Two hours of grace on top of the interval: a daily sync is stale after 26 hours (C4's number), +#: long enough that one slow pass does not raise a warning and short enough that a dead sync is +#: named on the next day, not three weeks later as staging's was. +STALE_GRACE_SECONDS = 2 * 3600 + + +def _utc_now(): + return _dt.datetime.now(_dt.timezone.utc) + + +def _parse_iso(value): + """An ISO timestamp as an aware UTC datetime, or None when it is not one.""" + if not value: + return None + try: + at = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + return at if at.tzinfo else at.replace(tzinfo=_dt.timezone.utc) + + +def record_sync_outcome(rt, ok, at=None): + """Stamp this tenant's last sync, and its last GOOD sync when `ok`, on the connector's config. + + ⛔ ON THE CONFIG DOCUMENT, NOT ON `odoo_sync_status`, because that document is REPLACED whole + on every phase write (`core.user_tables.record_odoo_sync_status`), so a stamp kept there would + be erased by the next `syncing`. Every writer of this config document updates it in place. + """ + stamp = at or _utc_now().isoformat() + + def _up(cur): + cur = dict(cur) if isinstance(cur, dict) else {} + cur["lastSyncAt"] = stamp + if ok: + cur["lastGoodSyncAt"] = stamp + return cur + + rt.update(CONFIG_KEY, _up, flush="sync") + return stamp + + +def _effective_sync_seconds(preset): + """The interval a sync can actually keep: the tenant's preset, floored, and never faster than + the deployment's own resync floor (`AIOS_STORE_RESYNC_SECONDS`), which `main.py` applies with + `max()`. None for `manual`: a person who asked not to be synced has no schedule to fall + behind.""" + secs = SYNC_PRESETS.get(preset) + if secs is None: + return None + secs = max(int(secs), SYNC_FLOOR_SECONDS) + import os + try: + env = int(str(os.environ.get("AIOS_STORE_RESYNC_SECONDS") or "").strip() or 0) + except ValueError: + env = 0 + return max(secs, env) + + +def sync_freshness(rt, now=None): + """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run + fail. PURE over the store reads. + + `stale` is true when the last good sync is older than the effective cadence plus + `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected + (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is + the last run's own reason, its code prefix removed, when that run ended in error. + """ + now = now or _utc_now() + try: + raw = rt.get(CONFIG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + raw = None + raw = raw if isinstance(raw, dict) else {} + cfg = read_config(rt) + try: + status = rt.get(_ut().ODOO_SYNC_STATUS_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + status = None + status = status if isinstance(status, dict) else {} + phase = str(status.get("phase") or "unknown") + last_good = raw.get("lastGoodSyncAt") or None + # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync. + if not last_good and phase == "ok": + last_good = status.get("lastRun") or None + every = _effective_sync_seconds(cfg["syncEvery"]) + allowed = None if every is None else every + STALE_GRACE_SECONDS + stale = False + if allowed is not None and not cfg["frozen"]: + good_at = _parse_iso(last_good) + stale = good_at is None or (now - good_at).total_seconds() > allowed + failure = None + errors = status.get("errors") if isinstance(status.get("errors"), list) else [] + if phase == "error" and errors: + text = str(errors[0]) + code, _sep, message = text.partition(": ") + if not _sep: + code, message = "sync_failed", text + failure = {"code": code.strip()[:80], "message": message.strip()[:300], + "at": status.get("lastRun") or None} + return {"stale": bool(stale), "lastGoodSyncAt": last_good, + "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed, + "lastRunAt": status.get("lastRun") or None, "phase": phase, + "frozen": cfg["frozen"], "failure": failure} + + def frozen(rt): """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" @@ -4558,6 +4664,93 @@ def available(rt=None, tenant_key=None, connection_id=None): return False +#: ⭐ S67-T19 — the `limits` effects that REPORT a serving choice rather than withheld rows. +#: `read_through` says a database is served THROUGH the mirror with no ceiling at all +#: (`core.user_tables.limit_report`), which is the design working, not a failure. +INFORMATIONAL_LIMIT_EFFECTS = frozenset({"read_through"}) + + +def failing_limits(limits): + """Only the `limits` entries that describe rows a pass did NOT serve, PURE. + + ⛔⛔ STAGING'S `materialization_failed` WAS THIS, NAMED 2026-09-28 FROM ITS OWN RECEIPTS. The + rebuild handed its whole `limits` map to the health classifier, which reads EVERY entry as a + materialization failure, so the two read-through grains (`ut_odoo_order_lines`, + `ut_odoo_gl_lines`, "no ceiling applies to them") failed the connection on every single sync + and `odoo_sync_status` could never read `ok`. The report keeps them; only health drops them. + """ + if not isinstance(limits, dict): + return limits + return {key: entry for key, entry in limits.items() + if not (isinstance(entry, dict) + and entry.get("effect") in INFORMATIONAL_LIMIT_EFFECTS)} + + +#: ⭐ S67-T19 — the reconciliation DRIFT a good sync may carry: Odoo keeps taking invoices while a +#: rebuild runs, and a hard delete leaves a mirror row until `reconcile_deletes` (every fourth +#: pass). Measured on a mirror pulled 25 minutes before its rebuild: invoices 32,980 vs 32,978, +#: orders 34,059 vs 34,058, agents 17 vs 18, GL lines 1,024,102 vs 1,024,178, customers 3,715 vs +#: 3,716. A stale mirror is nothing like that: staging's unpulled seed was 626 orders and 732 +#: invoices short. The receipt keeps its exact counts either way; this only decides whether the +#: CONNECTION's sync counts as good. +RECONCILE_DRIFT_ROWS = 10 +RECONCILE_DRIFT_RATIO = 0.002 + + +def reconciliation_drift(receipt): + """`(difference, tolerance)` for a count-reconciliation receipt, or None when it has no counts.""" + if not isinstance(receipt, dict): + return None + source = receipt.get("sourceCount") + if isinstance(receipt.get("unexplainedCount"), int): + diff = int(receipt["unexplainedCount"]) + else: + mirror = receipt.get("mirrorCount") + if not isinstance(source, int) or not isinstance(mirror, int): + return None + diff = abs(source - mirror) + base = source if isinstance(source, int) and source > 0 else 0 + return diff, max(RECONCILE_DRIFT_ROWS, int(base * RECONCILE_DRIFT_RATIO)) + + +def blocking_failures(database_failures, report): + """The database failures that make a sync NOT good, PURE. + + A reconciliation failure whose count drift is within `reconciliation_drift`'s tolerance is + recorded against its database and does not fail the connection; every other failure does. + """ + receipts = {} + for name in ("databases", "canonicalDatabases"): + group = report.get(name) if isinstance(report, dict) else None + if isinstance(group, dict): + receipts.update(group) + out = {} + for key, failure in (database_failures or {}).items(): + if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete": + drift = reconciliation_drift(receipts.get(key)) + if drift is not None and drift[0] <= drift[1]: + continue + out[key] = failure + return out + + +def health_view(report): + """The refresh report as the sync-health classifier must read it, PURE. + + Identical to `report` except that `limits` carries only `failing_limits`; the returned report + itself (the log line, the Sync report) still names every read-through database. + """ + if not isinstance(report, dict) or not isinstance(report.get("limits"), dict): + return report + view = dict(report) + remaining = failing_limits(report["limits"]) + if remaining: + view["limits"] = remaining + else: + view.pop("limits", None) + return view + + def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation", today=None, retire=True, on_apply=None, source_client=None): """Refresh each enabled database independently; retain failed databases unchanged. @@ -4844,6 +5037,16 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_ return out +#: ⭐ S67-T19 — the store errors a connection refresh retries ONCE. Staging's rebuild at +#: 2026-09-28 16:03Z failed whole with `ProtocolViolation` (SQLSTATE 08P01), which is how the +#: DigitalOcean pool (PgBouncer) reports its own timeouts while every lane container shares it; +#: nothing was written and the next scheduled pass was a day away. `refresh` is idempotent, so a +#: second attempt after a pause is safe; a refusal or any other exception is never retried. +TRANSIENT_STORE_ERRORS = frozenset({"ProtocolViolation", "OperationalError", "PoolTimeout", + "ConnectionTimeout", "AdminShutdown"}) +REFRESH_RETRY_SECONDS = 20 + + def refresh_all(rt, tenant, username="automation", cur=None, today=None): """Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4). @@ -4880,12 +5083,20 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None): primary = None for entry_cid in order: named = None if (entry_cid is None or entry_cid == default) else entry_cid - try: - one = refresh(rt, tenant, username=username, - cur=(cur if named is None else None), - today=today, connection_id=named) - except Exception as exc: # noqa: BLE001 - one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"} + for attempt in (1, 2): + try: + one = refresh(rt, tenant, username=username, + cur=(cur if named is None else None), + today=today, connection_id=named) + break + except Exception as exc: # noqa: BLE001 + # S67-T19: one retry for a TRANSIENT store error only; every other failure is + # recorded exactly as before. + if attempt == 1 and type(exc).__name__ in TRANSIENT_STORE_ERRORS: + _time.sleep(REFRESH_RETRY_SECONDS) + continue + one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"} + break reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one if named is None and primary is None: primary = one diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 7937e644e71765cb0aae0b66314b127668fc1810..832f240d6239efc43f1fb7502106b1035ffb5213 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -31,7 +31,7 @@ from datetime import datetime, timezone from collections import OrderedDict from typing import Annotated, Literal -from fastapi import APIRouter, Body, Depends, Query +from fastapi import APIRouter, Body, Depends, HTTPException, Query import oauth_connect from deps import Session, err, require_session, running_main @@ -326,9 +326,11 @@ TOKEN_CONNECTORS = [ "hint": "Create a Production app client in the Solution Provider Portal, self authorize " "it, then paste client_id, client_secret, refresh_token, seller_id and " "marketplace_id.", - "note": "The credential is verified live against your marketplace participations before " - "it is stored. Bulk reads use the Reports API, so a backfill is one document " - "rather than a page loop."}, + # S66-T27: storing is offline (`keychain._test_amazon_entry` probes only on an explicit + # Test), so this no longer claims the key was verified before it was stored. + "note": "Choose Test in the Keychain tab to verify the credential live against your " + "marketplace participations. Bulk reads use the Reports API, so a backfill is one " + "document rather than a page loop."}, ] #: ⭐⭐ WAVE 31 · R7 — THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many @@ -446,6 +448,11 @@ _register_probers() # W48C-T08 / C6 — the graph is a reader-scoped DTO over tables that exist in THIS workspace, # never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family; # `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter. +#: The directory states that mean "this workspace holds this connection": connected, stored but +#: asking to be reconnected, or stored past its rotation deadline (`ConnectorsPage`'s `held`). +_GRAPH_HELD_STATES = frozenset(("connected", "reconnect", "expired")) + + def _graph_refusal(connection_key, reason, status="refused"): return {"connectionKey": str(connection_key or ""), "status": status, "reason": str(reason or ""), "tables": [], "relationships": []} @@ -598,7 +605,12 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False): graphs = {} connected = set() for connection_key, row in row_by_key.items(): - if row.get("state") == "connected": + # ⭐ S67-T31 / D-718 — A HELD CONNECTION HAS A GRAPH. Staging's Amazon card reads + # `reconnect` (its stored key has no passing test on record) while 38 `ut_amzn_*` tables + # are materialised, and the old `== "connected"` test refused the graph as "Connect this + # provider" over a connection that is plainly there. The nodes are still only tables that + # exist and that this reader may open, so admitting the held states draws no template. + if row.get("state") in _GRAPH_HELD_STATES: connected.add(connection_key) graphs[connection_key] = _graph_refusal( connection_key, @@ -978,9 +990,21 @@ def directory(rt, uname, is_admin=False, incomplete=None): else "keychain_personal"), # Said once, on the row, rather than in a paragraph somewhere (R13): storing the # key is real and useful, and it is not the same as the data being on screen. - "note": "Storing the key connects the account; reading its data lands in a later " - "release.", + # ⭐ S66-T27 / S67-T31 — EXCEPT AMAZON, whose data DOES land (38 `ut_amzn_*` + # tables on staging), so the generic sentence told an administrator the opposite + # of the truth. Amazon carries its own authored note. + # S67 lane D: so does Meta Ads, whose authored note this line used to overwrite + # with the generic one; every card now shows its own (Stripe's and Shopify's own + # note IS the generic sentence, so theirs is unchanged). + "note": t["note"], }) + # S66-T27 — WHAT `reconnect` MEANS, said on the card. On Amazon it is "a key is stored + # but no passing Test is on record for it" (`keychain.entry_verified`), measured on + # staging's admin view with the data still syncing. The bare word read as a broken + # connection with no way to tell what to do. + if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp": + rows[-1]["needs"] = ("Amazon access for the stored key is not confirmed yet. " + "Choose Test in the Keychain tab, or replace the key.") contribute("token", _tokens, "the token-connector registry could not be composed", @@ -3791,7 +3815,17 @@ def _amazon_databases(rt, is_admin, spec=None): enabled = enablement.get("databases") or {} cat = amazon_schema.catalogue() tables = cat.get("tables") or {} - states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None + # ⭐ S67-T31 — A STATUS THE GUARD REFUSES COSTS ITS OWN ROW, NEVER THE MODAL. `sync_state` + # trims an over-long cause itself; a credential shape still refuses, and the refusal carries + # the guarded rows that passed, so only the refused tables read "Sync status is unavailable + # for this database." (a missing status is exactly what `_amazon_database_sync_health` + # degrades). Any other failure reading status degrades every row, never the whole page. + try: + states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None + except cx.AmazonStatusSafetyError as refused: + states = dict(getattr(refused, "safe_status", None) or {}) + except Exception: # noqa: BLE001 + states = {} # Which OTHER declared tables are filled from the SAME report document. Derived from the # report map so a table added to `REPORT_TYPES` gets this disclosure for free. @@ -4289,6 +4323,8 @@ def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offs # question about a PERSON and a table, not about an administrator flag alone. databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""), page=page, fields=fields) + if key == "amazon_sp": + databases = _screen_amazon_database_rows(databases) budget.check() overview = _detail_overview(row, databases, graph) if page is not None and page.total: @@ -4325,6 +4361,36 @@ def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offs "relationships": graph}) +def _screen_amazon_database_rows(rows): + """S67-T31: each Amazon database row answers the status guard ON ITS OWN. + + `_safe_connector_status` still guards the whole detail below, and it used to be the first and + only judge, so one oversized scalar in one row aborted every row, the Overview and the + Keychain with a bare 500. Here an over-long string is trimmed, and a row that still fails (a + credential shape anywhere in it) keeps only its key, its shipped name and its switch, with + "Sync status is unavailable for this database." as its status. The refused value is dropped, + never echoed. + """ + import connectors_amazon as cx # noqa: PLC0415 - provider owns the guard + listed = list(rows or []) + safe, _refused = cx.screen_status_rows(dict(enumerate(listed))) + out = [] + for index, row in enumerate(listed): + if index in safe: + out.append(safe[index]) + continue + key = str((row or {}).get("key") or "") + enabled = (row or {}).get("enabled") is True + label = LABELS_AMAZON.get(key) or key + out.append({"key": key, "label": label, "defaultLabel": label, "native": "", + "enabled": enabled, + **({"sync": None} if "sync" in (row or {}) else {}), + "syncHealth": _amazon_database_sync_health(None, enabled), + "fields": [], "excluded": [], "sharesFieldsWith": [], + **({"fieldsDeferred": True} if (row or {}).get("fieldsDeferred") else {})}) + return out + + def _set_amazon_database_enablement(database_key, body, session): """The testable core of Amazon's tenant-scoped administrator enablement door.""" if not session.admin: @@ -4406,16 +4472,46 @@ def connectors_detail(connector_key: str, session: Session = Depends(require_ses workspace is connected to. What a member does NOT get is the tenant-wide credential door (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue. """ - detail = connector_detail( - session.runtime, session.user, session.uname, - bool(session.admin), connector_key, databases_offset=databases_offset, - databases_limit=databases_limit, fields=fields) + try: + detail = connector_detail( + session.runtime, session.user, session.uname, + bool(session.admin), connector_key, databases_offset=databases_offset, + databases_limit=databases_limit, fields=fields) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + raise _plain_read_failure(exc, "connector_detail_failed", + "The connector details could not be read.") if detail is None: raise err(404, "unknown_connector", "this workspace has no connector with that key") return detail +def _plain_read_failure(exc, code, sentence): + """S67-T31: an unexpected failure on a connector read answers the app's JSON error shape. + + The owner's modal read "The server answered 500." because a guard refusal escaped as a bare + `Internal Server Error` with no body. Every read the modal makes now answers + `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log, + and no exception text (which can echo a payload) reaches the wire. + """ + import logging # noqa: PLC0415 + logging.getLogger(__name__).exception("connector read failed: %s", code) + name = type(exc).__name__ + if name in ("StoreUnavailable", "PoolTimeout"): + return err(503, "store_unavailable", + "The workspace store did not answer in time, so nothing could be read.") + try: + import connectors_amazon as cx # noqa: PLC0415 + if isinstance(exc, cx.AmazonStatusSafetyError): + return err(500, "connector_status_withheld", + "This connector's status could not be shown safely, so it was withheld.") + except Exception: # noqa: BLE001 + pass + return err(500, code, sentence) + + def _detail_row(rt, uname, is_admin, connector_key): row = next((row for row in directory(rt, uname, is_admin) if row.get("key") == connector_key), None) @@ -4429,22 +4525,35 @@ def connector_database_fields(connector_key: str, database_key: str, session: Session = Depends(require_session)): """The same field disclosure and database access as detail, for one database only.""" rt = _ConnectorDetailReads(session.runtime) - _detail_row(rt, session.uname, bool(session.admin), connector_key) - rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname, - page=_DatabasePage(database_key=database_key)) - if not rows: - raise err(404, "unknown_database", "This connector has no visible database with that key.") - return _safe_connector_status(connector_key, { - "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]}) + try: + _detail_row(rt, session.uname, bool(session.admin), connector_key) + rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname, + page=_DatabasePage(database_key=database_key)) + if not rows: + raise err(404, "unknown_database", + "This connector has no visible database with that key.") + return _safe_connector_status(connector_key, { + "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]}) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + raise _plain_read_failure(exc, "connector_fields_failed", + "The fields of this database could not be read.") @router.get("/connectors/{connector_key}/relationships") def connector_relationships(connector_key: str, session: Session = Depends(require_session)): """Load the permitted graph when the summary client's Relationships tab opens.""" rt = _ConnectorDetailReads(session.runtime) - row = _detail_row(rt, session.uname, bool(session.admin), connector_key) - return _safe_connector_status(connector_key, relationship_graphs( - [row], session.user, rt, session.uname, bool(session.admin))[connector_key]) + try: + row = _detail_row(rt, session.uname, bool(session.admin), connector_key) + return _safe_connector_status(connector_key, relationship_graphs( + [row], session.user, rt, session.uname, bool(session.admin))[connector_key]) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + raise _plain_read_failure(exc, "connector_relationships_failed", + "The relationship graph could not be read.") class _ReplacementConflict(Exception): diff --git a/api/routes_external.py b/api/routes_external.py index 6eb5bea833832b925031d7aea92fc7e2756e8f1a..f479438ef786347f27b90e520fba1c7084bdaefa 100644 --- a/api/routes_external.py +++ b/api/routes_external.py @@ -728,8 +728,12 @@ def _resolve_external_scope( if include_schema: if row["answerable"] or strict_schema: try: + # ⭐ S67-T34 (D-2630): THE FIELD CONTRACT, NOT THE ROWS. Describing a + # database used to read (and count) every row first, so the largest + # connected table refused its own schema. "schema" runs every wall and + # reads no row; sources that are not mirror-served ignore it. scoped = deps.assistant_read_scope( - session, database, fields=None, filters=None, + session, database, fields=None, filters=None, mode="schema", ) except HTTPException as exc: # HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror @@ -1821,8 +1825,11 @@ def _query_database(session: Session, database: str, payload: dict): # This is the same permissioned record reader the in-app Assistant uses. It resolves the # database, field and row walls on every call before this adapter sorts, caps, or aggregates. + # ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so + # its aggregates run in SQL over every permitted row and its rows are read only as needed. scoped = deps.assistant_read_scope( - session, database, fields=None, filters=payload.get("filters")) + session, database, fields=None, filters=payload.get("filters"), mode="schema") + mirror = scoped.get("source_kind") == "mirror_user_table" visible_fields = list(scoped.get("fields") or []) visible_keys = { field.get("key") for field in visible_fields @@ -1851,6 +1858,104 @@ def _query_database(session: Session, database: str, payload: dict): sorts.append({"field": rule["field"], "direction": rule["direction"]}) aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys) + server_aggregates = None + paged_total = None + if mirror: + # ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates + # are computed by the mirror in SQL over exactly the rows this caller may read, never + # over a fetched slice, and never by pulling a million rows into this process. + if aggregates: + summary = deps.assistant_read_scope( + session, database, fields=None, filters=payload.get("filters"), + mode=("aggregate", aggregates)) + server_aggregates = summary.get("server_aggregates") + import assistant_mirror + # The summary already counted the permitted population, so a table past the one-pass + # ceiling goes straight to SQL paging instead of pulling a first 5,000-row page only to + # refuse it. + known = (summary.get("total_rows") if aggregates else None) + try: + if known is not None and int(known) > assistant_mirror.ASSISTANT_READ_MAX: + raise err(409, "assistant_source_too_large", + f"this database matches {int(known):,} rows for that question, " + f"past the {assistant_mirror.ASSISTANT_READ_MAX:,} rows one answer " + f"holds in memory") + scoped = deps.assistant_read_scope( + session, database, fields=None, filters=payload.get("filters")) + except HTTPException as exc: + info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {} + if info.get("code") != "assistant_source_too_large": + raise + # ⭐ RULE 1: PAST THE ONE-PASS CEILING THE ROWS ARE PAGED IN SQL, NOT REFUSED. Only + # this page reaches the process. A sort cannot be honoured at this size without + # holding every row, so it is refused by name with the fix, never silently dropped. + if sorts: + raise err(409, "query_sort_too_large", + f"{info.get('message') or 'This database is too large to sort here.'} " + f"Sorting is refused at this size because it would need every row in " + f"memory. Ask without sorts to page it in record order, or narrow it " + f"with a filter and sort again.") from exc + paged_total = True + if paged_total: + offset = (0 if payload.get("cursor") is None else None) + page_digest = _json_digest({ + "database": database, "fields": selected_keys, "filters": payload.get("filters"), + "sorts": [], "aggregates": aggregates, "limit": limit, "paged": True, + }) + first = deps.assistant_read_scope( + session, database, fields=None, filters=payload.get("filters"), mode=("page", 0, 1)) + snap = _json_digest({ + "source_version": first.get("source_version"), + "visible_fields": [field["key"] for field in visible_fields + if isinstance(field, dict) and field.get("key") in visible_keys], + "total_rows": first.get("total_rows"), + }) + if offset is None: + offset = _decode_query_cursor(session, payload.get("cursor"), page_digest, snap) + total_rows = int(first.get("total_rows") or 0) + if offset > total_rows: + raise err(409, "query_cursor_stale", + "The permitted result set changed. Start again without a cursor.") + page = deps.assistant_read_scope( + session, database, fields=None, filters=payload.get("filters"), + mode=("page", offset, limit)) + page_rows = list(page.get("records") or []) + for row in page_rows: + _assert_finite_query_value({"pid": row.get("pid"), + **{key: row.get(key) for key in selected_keys}}) + records = [{"pid": copy.deepcopy(row.get("pid")), + **{key: copy.deepcopy(row.get(key)) for key in selected_keys}} + for row in page_rows] + next_offset = offset + limit + next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset) + if next_offset < total_rows else None) + result = { + "database": page.get("database"), "label": page.get("label"), + "source_kind": page.get("source_kind"), + "source_version": copy.deepcopy(page.get("source_version")), + "retrieved_at": page.get("retrieved_at"), "permission_scope_applied": True, + "fields": [copy.deepcopy(field) for field in visible_fields + if field.get("key") in selected_keys], + "filters": copy.deepcopy(page.get("filters")), "sorts": [], + "records": records, "aggregates": copy.deepcopy(server_aggregates or []), + "page": {"limit": limit, "returned": len(records), "offset": offset, + "has_more": next_cursor is not None, "next_cursor": next_cursor}, + "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS, + "max_aggregates": _QUERY_MAX_AGGREGATES, + "max_filter_depth": _QUERY_MAX_FILTER_DEPTH, + "max_filter_nodes": _QUERY_MAX_FILTER_NODES, + "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS}, + } + _assert_finite_query_value(result) + audit.record("external.query", session.tenant, session.uname, { + "surface": "ai_api", + "authentication": session.claims.get("external_auth_type", "api_key"), + "field_count": len(selected_keys), "filter_applied": page.get("filters") is not None, + "sort_count": 0, "aggregate_count": len(aggregates), "page_size": limit, + "returned_count": len(records), "cursor_used": offset > 0, + }) + return result + rows = list(scoped.get("records") or []) inspected_keys = (set(selected_keys) | {rule["field"] for rule in sorts} @@ -1901,7 +2006,8 @@ def _query_database(session: Session, database: str, payload: dict): "filters": copy.deepcopy(scoped.get("filters")), "sorts": copy.deepcopy(sorts), "records": records, - "aggregates": _aggregate_records(rows, aggregates), + "aggregates": (copy.deepcopy(server_aggregates) if server_aggregates is not None + else _aggregate_records(rows, aggregates)), "page": {"limit": limit, "returned": len(records), "offset": offset, "has_more": next_cursor is not None, "next_cursor": next_cursor}, "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS, diff --git a/api/routes_module_index.py b/api/routes_module_index.py index d776470f9b5f717ab9123e8df539d38543a0492f..b9d246e393da257a09298d657a1e2c06566172dc 100644 --- a/api/routes_module_index.py +++ b/api/routes_module_index.py @@ -5,8 +5,10 @@ from fastapi import APIRouter, Body, Depends, HTTPException from core.sheets import SheetFolderRefusal, validate_folder_tree from deps import Session, err, module_gate, require_session +import application_registry import routes_assets import routes_automation +from routes_applications import _may_open from routes_nav import _placeable_top_keys import voice_control_plane as voice @@ -18,10 +20,16 @@ def _sentence(value): return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split()) +# D-3282 (standing rule 1): a limit the server cannot lift is reported with its cause and the fix. +_REFUSAL_SENTENCES = { + "too_many_folders": "Folders hold at most 200 per rail. Delete or merge a folder, then try again.", +} + + def _refuse(exc): status = {"module_index_unknown": 404, "module_index_user_forbidden": 403, "module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400) - return err(status, exc.code, _sentence(exc.message)) + return err(status, exc.code, _sentence(_REFUSAL_SENTENCES.get(exc.code, exc.message))) def _access(module: str, session: Session = Depends(require_session)): @@ -57,7 +65,19 @@ def _readable_ids(module, session): # only listed items, so an unverifiable placement renders no tile. Use local voice # records, never audience-option assembly, just to verify placement ids. if module == "app": - return {row["id"] for row in routes_assets.application_index(session=session)["applications"]} + # S67-T38 (O24): the App home also renders the seeded Finance projections, so the wall + # admits their ids through the SAME grant filter /nav/applications uses. Never ungated: + # a Finance card the session cannot open is not placeable. + stored = {row["id"] for row in routes_assets.application_index(session=session)["applications"]} + finance = application_registry.finance_application_records( + may_open=lambda permission: _may_open(session, permission)) + # S67-T41 owner correction: a Content Studio App is bound to a USER table, which the + # topic-only `/applications` index refuses, so its tile arrives from /nav/applications. + # Its id is admitted through the registry's own `may_read` (the same read that lists it). + studios = {row["id"] for row in application_registry.list_apps( + session.runtime, session.uname, session.admin) + if row.get("templateKind") == application_registry.CONTENT_STUDIO_TEMPLATE_KIND} + return stored | {row["id"] for row in finance} | studios # ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read # *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until # it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index af8e30f2acd99543dfd536b1b39f7692a3c25b32..8a0457587d279e52a80df39d2b5be4b84911b74e 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -23,6 +23,7 @@ import hashlib import hmac import json import math +import threading import time from contextlib import contextmanager @@ -622,10 +623,27 @@ GRID_SOURCES = {} _SOURCE_PROVIDERS = [] _SOURCE_ANALYSES = {} +#: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every +#: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's +#: keys to the shared dict as it went, and returned the dict to any caller that found it +#: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered +#: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read +#: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under +#: this lock and publishes it in one `dict.update`; a reader waits on the lock until +#: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the +#: Amazon fallback) happens only after a build has been published, so it can never stand in +#: for one either. +_SOURCES_LOCK = threading.RLock() +_SOURCES_BUILT = False +_SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse + def invalidate_sources(): """Forget measured bindings after a connector publishes a new mirror schema.""" - GRID_SOURCES.clear() + global _SOURCES_BUILT + with _SOURCES_LOCK: + _SOURCES_BUILT = False + GRID_SOURCES.clear() def register_source_provider(fn): @@ -672,20 +690,60 @@ def _sources(): ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and importing that module at file-import time would drag the Odoo layer into every process that mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. + + ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the + lock's comment). An empty registry still means "build" (gates clear it to force a rebuild), + and so does one that holds only admissions made since the last invalidation. """ - if GRID_SOURCES: + global _SOURCES_BUILT, _SOURCES_BUILDER + if _SOURCES_BUILT and GRID_SOURCES: return GRID_SOURCES - for build in list(_SOURCE_PROVIDERS): + with _SOURCES_LOCK: + if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident(): + return GRID_SOURCES + _SOURCES_BUILDER = threading.get_ident() try: - extra = build() or {} - except Exception: # noqa: BLE001 - # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must - # not go dark because a newer connector's module is unhappy — and vice versa. - continue - if extra: + built = {} + for build in list(_SOURCE_PROVIDERS): + try: + extra = build() or {} + except Exception: # noqa: BLE001 + # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo + # grids must not go dark because a newer connector's module is unhappy, and + # vice versa. + continue + if extra: + built.update(extra) + _ut().register_connected(*extra) + # A key admitted since the last invalidation was measured at least as recently as + # this build, so it stands; everything else comes from the build. ONE update, so a + # lock-free reader of the dict sees the old entry or the new one, never a gap. + GRID_SOURCES.update({**built, **GRID_SOURCES}) + _SOURCES_BUILT = True + finally: + _SOURCES_BUILDER = None + return GRID_SOURCES + + +def _learn_amazon_sources(): + """Publish the Amazon provider's keys into a registry that was built without any of them. + + HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot + BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates + it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through + Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more + than one window. A registry that already knows one Amazon key is not re-asked. + """ + _sources() + if any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)): + return + extra = _amazon_sources() + if not extra: + return + with _SOURCES_LOCK: + if not any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)): GRID_SOURCES.update(extra) _ut().register_connected(*extra) - return GRID_SOURCES def _odoo_sources(): @@ -1055,11 +1113,13 @@ def _amazon_sources(): return {} -def require_amazon_mirror(table_key, session, definition=None): +def require_amazon_mirror(table_key, session, definition=None, st=None): """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry. The provider contributes no keys on an Odoo-only mirror. A persisted read-through definition still must refuse there instead of falling through to its empty stored rows. + S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the + window route stops paying a second definitions projection for this one check. """ if not str(table_key).startswith("ut_amzn_"): return @@ -1069,7 +1129,8 @@ def require_amazon_mirror(table_key, session, definition=None): if table_key not in amazon.SOURCE: return if definition is None: - definition = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + definition = _defn_or_refuse(session, table_key, st=st, defs_only=True, + scope_applied=True) try: cur, mismatch = _mirror_cur(session, table_key) except RuntimeError as exc: @@ -1086,8 +1147,17 @@ def require_amazon_mirror(table_key, session, definition=None): raise err(503, "amazon_mirror_not_loaded", current["missing"]) # A dynamically oversized snapshot may have a durable stub before this process # has measured it. Admit its binding without changing another source's registry. + # ⛔ S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon + # key. Admitted into an empty or Amazon-less registry, this one key used to make + # `_sources()` skip its build, so the grid resolved while its link targets + # (orders, read-through) were dispatched as stored tables: a first-page 500. if spec is None or spec.get("missing"): - GRID_SOURCES[table_key] = current + _learn_amazon_sources() + with _SOURCES_LOCK: + _sources() + held = GRID_SOURCES.get(table_key) + if held is None or held.get("missing"): + GRID_SOURCES[table_key] = current finally: cur.close() @@ -1322,18 +1392,14 @@ def _source_for(cur, table_key): if module_cache_source(key) is not None: return None spec = _sources().get(key) - if not spec and key.startswith("ut_amzn_") and not any( - k.startswith("ut_amzn_") for k in GRID_SOURCES): + if not spec and key.startswith("ut_amzn_"): # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every # read-through Amazon grid answered "no read-through binding". Ask the Amazon provider # alone, once it can answer; a registry that already knows one Amazon key is not re-asked. - extra = _amazon_sources() - if extra: - GRID_SOURCES.update(extra) - _ut().register_connected(*extra) - spec = GRID_SOURCES.get(key) + _learn_amazon_sources() + spec = GRID_SOURCES.get(key) if not spec: return None if spec.get("missing"): @@ -1721,11 +1787,133 @@ def _related_source_value(session, entry, record): def _materialised_table(table_key): - """Storage dispatch, independent of the browser's fallible status roster.""" - return module_cache_source(table_key) is not None or table_key not in _sources() + """Storage dispatch, independent of the browser's fallible status roster. + + ⛔ S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first + (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before + the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one. + """ + if module_cache_source(table_key) is not None: + return True + if table_key in _sources(): + return False + if str(table_key).startswith("ut_amzn_"): + _learn_amazon_sources() + return table_key not in GRID_SOURCES + return True + + +#: ⭐ S67-T28 (R38, D-3480 point 1) — the tenant's table DEFINITIONS and a stored link target's +#: ROWS, each held against the durable `user_tables` revision it was read at. A grouped Amazon page +#: used to re-read the definitions projection once for its own wall and once more for EVERY link +#: target, plus the stored listings entry, on every page; on staging each of those reads is one +#: large JSONB projection (measured 33 to 186 s a page under load, 2026-09-28). Every `put` and +#: `update` bumps the revision column in the same statement, so an unchanged token proves the +#: bytes unchanged; the token is read BEFORE the value, so a write racing the read can only cache +#: newer bytes under the older token (a miss next time, never a stale hit). ⚠ Nothing cached here +#: is a permission: the wall (`may_read`, `apply_row_scope`, the compiled predicate) is evaluated +#: per request against the session's own user, only the bytes it evaluates over are reused. +_REV_MEMO: dict = {} +_REV_MEMO_MAX = 64 +# Route handlers run on a thread pool: the lock makes the evict (`next(iter(...))`) and the +# insert one step, so a concurrent insert can never raise "dictionary changed size". +_REV_MEMO_LOCK = threading.Lock() +#: "No token was handed in, read one." Distinct from None, which means "unreadable, hold nothing". +#: It is the `...` literal (a singleton) so a signature can spell its default `...` and still +#: define when `verify_grid_ux.py` execs `_relation_rows` on its own, without this module name. +_UNREAD = ... + + +def _user_tables_token(rt): + """The durable change token of this tenant's `user_tables` bucket, or None when unreadable.""" + try: + return rt.revision(_ut().STORE_KEY)["token"] + except Exception: # noqa: BLE001 + return None + + +def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD): + """`read()`, reused while this tenant's `user_tables` revision is unchanged. + + ⛔ `clone` runs on the way in AND on every hit, so a caller that edits what it was handed + cannot edit the next request's answer. `keep(value)` decides what may be held: an empty + answer never is (it is the shape a failed read takes, and caching it would make one outage + outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure. + `token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round + trip, not one per memo. It was read before any value this call reads, which is the only + ordering the staleness argument above needs. + """ + if token is _UNREAD: + token = _user_tables_token(rt) + tenant = getattr(rt, "key", None) + slot = (kind, tenant, name) + if token is not None and tenant: + with _REV_MEMO_LOCK: + hit = _REV_MEMO.get(slot) + if hit is not None and hit[0] == token: + return clone(hit[1]) + value = read() + if token is not None and tenant and keep(value): + held = (token, clone(value)) + with _REV_MEMO_LOCK: + if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX: + _REV_MEMO.pop(next(iter(_REV_MEMO)), None) + _REV_MEMO[slot] = held + return value + + +def _page_defs(session): + """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's. + + Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked + against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link + target does not pay a second revision round trip. + The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per + `user_tables` revision instead of once per question. Kept only when every definition really is + a projection (`store.is_projected`), because `all_defs` silently falls back to the whole + 28 MB document on a failed read and that must never be held, let alone copied per request. + ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it + lends lazily live for this one request and are never carried to the next. + """ + import copy + import core.store as store + + ut = _ut() + rt = session.runtime + token = _user_tables_token(rt) + + def keep(v): + tables = [d for d in (v or {}).values() if isinstance(d, dict)] + return bool(tables) and all(store.is_projected(d) for d in tables) + + defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt), + keep=keep, clone=copy.deepcopy, token=token) + return ut.lend(rt, **{ut.STORE_KEY: defs}), token + + +def _stored_link_rows(session, table, token=_UNREAD): + """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision. + + ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module + cache, an app reader) or a read-through binding answers from somewhere this token does not + watch, so those read exactly as before. The wall is NOT applied here; the caller applies + `apply_row_scope` for its own user on every request. Holding it across users is sound only + because `perm_scope._read_user_table` never consults `user` for a `ut_*` key: every + user-dependent step (hidden fields, row wall, BU scope) sits above it, in the caller. + """ + import core.perm_scope as perm_scope + + rt = session.runtime + if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES + or module_cache_source(table) is not None or not _materialised_table(table)): + return perm_scope._read(table, session.user, rt) + return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt), + keep=lambda v: bool(v[1]), + clone=lambda v: ([dict(f) for f in v[0]], [dict(r) for r in v[1]]), + token=token) -def _traversal_defn_or_refuse(session, table_key): +def _traversal_defn_or_refuse(session, table_key, st=None): """R50 / C4: admit an intermediate after the caller has established both end grants. The session may already open both ends of a declared two-hop link. Its intermediate @@ -1733,7 +1921,7 @@ def _traversal_defn_or_refuse(session, table_key): read still applies the row wall, hidden-field closure, BU scope and mirror guard. """ ut = _ut() - st = ut.lend_defs(session.runtime) + st = st if st is not None else ut.lend_defs(session.runtime) defn = ut.get(table_key, st=st) if not defn: raise err(404, "unknown_table", "that database does not exist") @@ -1741,21 +1929,26 @@ def _traversal_defn_or_refuse(session, table_key): def _relation_rows(session, table, columns, *, ids=None, match=None, where=None, - traversal_of=None): + traversal_of=None, st=None, rev=...): # `...` IS `_UNREAD` """Internal join values, after the ordinary row wall, with no row-count ceiling. Join columns may be hidden on the wire. Only identities leave the resolver; the native payload builders still apply their complete hidden-field closure before returning cells. With traversal_of, the caller must already have established both ends of the declared edge named by that key. Only join values leave this intermediate read, never a cell. + S67-T28: `st` is a lent DEFINITIONS snapshot the caller already holds (`_page_defs`); the wall + is asked of it instead of re-reading every definition per target. A stored target's rows come + through `_stored_link_rows`, reused per `user_tables` revision; its wall still runs here. + `rev` is the revision token `_page_defs` already read for this request; absent, the stored + target is read directly, as before S67-T28. """ import core.perm_scope as perm_scope import core.relations as relations columns = set(columns) | {relations.IDENTITY_COL} if not _materialised_table(table): - read = (_compiled_read(table, session, traversal_intermediate=True) - if traversal_of is not None else _compiled_read(table, session)) + read = (_compiled_read(table, session, traversal_intermediate=True, st=st) + if traversal_of is not None else _compiled_read(table, session, st=st)) spec = read["spec"] bindings = dict(spec["cols"], pid=(spec["id"], int)) if columns - set(bindings): @@ -1778,11 +1971,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None, # This is an internal join, never a cell response. Apply the same permanent wall on the # complete contract before taking any join value, including a join on a hidden column. - if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime): + if traversal_of is None and not perm_scope.may_read( + session.user, table, st=st if st is not None else session.runtime): raise err(403, "forbidden_table", "This account cannot open the linked database.") if module_cache_source(table) is not None: session.require(table) - fields, rows = perm_scope._read(table, session.user, session.runtime) + # The revision memo serves a caller that handed this request's token (the page's link fills, + # S67-T28); any other caller reads exactly as before, which is also what lets + # `verify_grid_ux.py` exec this function alone with a stubbed `perm_scope`. + fields, rows = (perm_scope._read(table, session.user, session.runtime) if rev is ... + else _stored_link_rows(session, table, token=rev)) rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime) if where: raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.") @@ -2570,7 +2768,8 @@ def _may_see_view(session, vid, store): def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None, - search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False): + search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False, + st=None): """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once. Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits, @@ -2630,10 +2829,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts= # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows # used to be. So the wall is APPLIED here, in SQL, three paragraphs down. + # S67-T28: `st` lends a definitions snapshot the caller already holds (`_page_defs`), the + # same `st=` `_defn_or_refuse` has always accepted. None keeps its own `lend_defs` read. if traversal_intermediate: - defn = _traversal_defn_or_refuse(session, table_key) + defn = _traversal_defn_or_refuse(session, table_key, st=st) else: - defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) fields = None for _bucket, key, _label, mk_fields in rel.TABLES: if key == table_key: @@ -3105,7 +3306,12 @@ def odoo_table_rows(table_key: str, "limit, filters, sorts and search in the query string.") ids = _record_ids(body) - require_amazon_mirror(table_key, session) + # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check + # asks it instead of reading its own definitions projection (one store round trip fewer on + # every Amazon page). Not reordered past it: the check may register this grid's binding, + # which is what `_materialised_table` below then reads. + page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None + require_amazon_mirror(table_key, session, st=page[0] if page else None) if _materialised_table(table_key): return _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related, groupBy, groupKey, ids=ids) @@ -3116,8 +3322,11 @@ def odoo_table_rows(table_key: str, # is a page of* - and that is provable only while there is one of it. What stays here is what # a WINDOW alone needs: the total order, the projection, the overlay merge and the two # sentences a page earns (`window_clamped`, the deep-page walk). + # S67-T28 (D-3480 point 1): ONE definitions snapshot answers this page's own wall and every + # link target's below, re-read only when `user_tables` has moved (`_page_defs`). + page_st, page_rev = page if page is not None else _page_defs(session) read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj, - sorts=sorts, search=search, related=related) + sorts=sorts, search=search, related=related, st=page_st) read = _restrict_row_ids(read, ids) defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"] cols, limits, frm = read["cols"], read["limits"], read["frm"] @@ -3184,14 +3393,16 @@ def odoo_table_rows(table_key: str, def read_amazon_targets(target, column, values): if target not in _opened: - _opened[target] = _may_open(session, target) + _opened[target] = _may_open(session, target, st=page_st) if not _opened[target]: return [] if not _materialised_table(target): - return _relation_rows(session, target, [column], match=(column, values)) + return _relation_rows(session, target, [column], match=(column, values), + st=page_st, rev=page_rev) if target not in _stored: _stored[target] = _relation_rows( - session, target, sorted(_stored_cols.get(target, set()) | {column})) + session, target, sorted(_stored_cols.get(target, set()) | {column}), + st=page_st, rev=page_rev) wanted = {str(value) for value in values} # The same string comparison `_relation_rows`' own `match=` applies to a stored table. return [row for row in _stored[target] if str(row.get(column)) in wanted] @@ -3214,11 +3425,14 @@ def odoo_table_rows(table_key: str, # the wall and the predicate, so the argument it demands is the list we just fetched. # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already # answered "may this session open this surface". - shared_defs = _so().fields(table_key, st=session.runtime) + # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request + # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two. + # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row). + shared_defs = _so().fields(table_key, st=page_st) if shared_defs: fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()] for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], - st=session.runtime).items(): + st=page_st).items(): overlays.setdefault(pid, {}).update(cells) # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula @@ -3549,9 +3763,11 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None), # who assumed both were bodies would have wired the rows and left the summaries answering over # the entire database under a tab that names one record. `_compiled_read` accepts either shape, # so the two doors cannot pin differently without this argument being deleted. + # S67-T28: the same page snapshot `/rows` takes, so opening a grouped view (its group counts + # come from here) no longer re-reads every definition while `user_tables` is unchanged. read = _compiled_read(table_key, session, tree=query["filters"], filterConj=query["filterConj"], search=query["search"], - related=payload.get("related")) + related=payload.get("related"), st=_page_defs(session)[0]) read = _restrict_row_ids(read, ids) out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"], read["params"], read["frm"], aggs, group_by=group_by, diff --git a/api/routes_sheet.py b/api/routes_sheet.py index ff9cd53376982d3c2af708194321571c510b6b67..fc5e75933931ad8188feb9605f663a8d31c72ad2 100644 --- a/api/routes_sheet.py +++ b/api/routes_sheet.py @@ -2565,8 +2565,13 @@ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=() def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(), - code="ending_cell_unreadable"): - """Shared cell read for Ending and Filter, including scoped bound and formula inputs.""" + code="ending_cell_unreadable", as_date=False): + """Shared cell read for Ending and Filter, including scoped bound and formula inputs. + + S67-T10: with ``as_date`` a date-formatted number (typed, or a formula's result such as + 46293) is returned as its ISO day, so a Filter condition compares it as that date for every + operator; a plain number with no date format is returned unchanged. + """ from modules import sheet as sheet_mod import formula_runtime as fr @@ -2648,6 +2653,13 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=() if cell.get("state") == "stale": refuse("the cell needs to be refreshed.") value = cell.get("value", cell.get("text")) + if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool) + and sheet_mod.date_formatted_cell(cell)): + from harness import windows as W + day = W.ending_cell_date(value, book.get("dateSystem", "1900")) + if not day: + refuse("its value is not a real date.") + return day return value @@ -2655,7 +2667,8 @@ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen from modules import sheet as sheet_mod try: return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell( - session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable")) + session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable", + as_date=True)) except sheet_mod.SheetRefusal as exc: raise err(400, exc.code, exc.message) diff --git a/api/routes_templates.py b/api/routes_templates.py index 6b499da97ce889b05feb8192ed10bf612a23ba7c..0c7ada55c0c120ee79cbf30734cade841497e0a4 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -137,6 +137,16 @@ def apply_template(key: str, body: dict = Body(default=None), import core.table_store as table_store ops = table_store.make(ws_key, st=session.runtime) applied = [] + # S67-T39 (R12, D-2233): a template View bound to an App that no longer exists instantiates as + # a plain view of its database; the live App ids are read once, and an unreadable registry + # keeps the reference (the read projection in `grid_events.table_workspace` still masks it). + try: + import application_registry + raw_registry = session.runtime.get(application_registry._KEY) + live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict) + and isinstance(raw_registry.get("applications"), dict) else None) + except Exception: + live_apps = None for view in tpl.get('views') or (): # ⛔ A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level # — mutating one would write this caller's username into the template every other tenant @@ -145,6 +155,10 @@ def apply_template(key: str, body: dict = Body(default=None), # side. payload = dict(view) payload['config'] = dict(view.get('config') or {}) + dead_ref = payload['config'].get('application') + if (live_apps is not None and isinstance(dead_ref, dict) + and str(dead_ref.get('id') or '') not in live_apps): + payload['config'].pop('application') payload['createdBy'] = session.uname saved = ops.save_view(session.uname, payload) applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']}) @@ -316,8 +330,17 @@ def _agent_templates(): ] +def _content_studio(): + """The Content Studio App's own module, or None when it is absent (rule 13: delete-safe).""" + try: + import content_studio + except ImportError: + return None + return content_studio + + def _module_templates(session): - from core import sheet_templates, user_tables + from core import registry, sheet_templates, user_tables from routes_connectors import _entry_types from routes_keychain import env_odoo_available from routes_nav import application_templates @@ -332,7 +355,9 @@ def _module_templates(session): saved = _saved_sheet_templates(session.runtime) data_templates = [(row, _data_template_database(row, definitions)) for row in _templates().TEMPLATES] - agent_templates = _agent_templates() if _may_open(session, "automation") else [] + # S67-T36 (R25): an archived Agents module offers no agent template; the create route stays. + agent_templates = (_agent_templates() if _may_open(session, "automation") + and not registry.is_archived("automation") else []) databases = {row["database_hint"] for row in sheet_templates.TEMPLATES} databases.update(db for row in saved for db in row["databases"]) databases.update(database for _row, database in data_templates if database) @@ -361,6 +386,16 @@ def _module_templates(session): sections = [{"module": "sheet", "templates": sheet}, {"module": "data", "templates": data}, {"module": "app", "templates": app}] + # S67-T40 (R18): every tenant's App catalogue offers Content Studio, which creates its own + # databases. Guarded (rule 13): without the module the catalogue simply omits the row. + # ⚠ Appended AFTER `sections` (it is the App section's own list, so the order stays Finance, + # then Studio): `verify_shell_rail.py` scopes its S65-T10 needles to the span from the Finance + # offer to `sections = [`, and its finance-offer-wizard control mutates the FIRST + # `creation_mode="post"` in this file. A Studio row inside that span, or above it, masks it. + studio = _content_studio() + if studio is not None: + app.append(_catalog_row(studio.TEMPLATE_KEY, studio.LABEL, studio.DESCRIPTION, [], + creation_mode="post")) if agent_templates: sections.append({"module": "agents", "templates": [ _catalog_row(row["key"], row["name"], row["description"], @@ -509,6 +544,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None raise err(404, "unknown_template", "That template does not exist.") if not template["creatable"]: raise err(409, "template_unavailable", "Connect the required source and switch on its databases.") + studio = _content_studio() if module == "app" else None + if studio is not None and key == studio.TEMPLATE_KEY: + if not session.runtime.available(): + raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was created.") + try: + return studio.create(session, body.get("name") or "") + except studio.ContentStudioError as exc: + raise err(409, "content_studio_refused", str(exc)) + except PermissionError as exc: + raise err(403, "forbidden", str(exc)) if module == "sheet": database = template["requires"][0]["database"] if template["requires"] else "" result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session) @@ -529,3 +574,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None app = result["application"] return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""), **result} + +# S67-T41: Content Studio's own routes (`/api/v1/content-studio/*`) ride this router because the +# App wires no mount of its own into main.py; GUARDED, so deleting content_studio.py drops the +# routes and nothing else (rule 13). +# ⚠ An inline guarded import, NOT a call to `_content_studio()`: this tail belongs to the +# dispatcher block, which `verify_home.py`'s W61-T23 declaration-order control moves ABOVE that +# helper's definition, where a call is a NameError that crashes the whole gate. +try: + import content_studio as _studio_module +except ImportError: + _studio_module = None +if _studio_module is not None and getattr(_studio_module, "router", None) is not None: + router.include_router(_studio_module.router) diff --git a/platform/core/daily_sales_repair.py b/platform/core/daily_sales_repair.py index d57257b99ace950452b7130711e8bcf22512d8b4..c069974c44d057c66e32b6bf98bd7fd7d8fdaa56 100644 --- a/platform/core/daily_sales_repair.py +++ b/platform/core/daily_sales_repair.py @@ -498,7 +498,8 @@ def _checked_projection(projected, edits, formula_check): != {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}: raise RepairRefusal( "repair_unplanned_formula_change", - "Formula validation would change a cell outside the supplied source diff.") + "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0]) + + "!" + slot[1] + ", a cell outside the supplied source diff.") desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1]) if row is None: sheet = sheet_names[slot[0]] @@ -515,7 +516,8 @@ def _checked_projection(projected, edits, formula_check): or row["after"].get("kind") != "formula": raise RepairRefusal( "repair_unplanned_formula_change", - "Formula validation would change a cell outside the supplied source diff.") + "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0]) + + "!" + slot[1] + ", a cell outside the supplied source diff.") recalculated = after.get(slot) if not isinstance(recalculated, dict) \ or (row["kind"] == "source" @@ -1333,6 +1335,404 @@ def _build_daily_sales_bindings_plan(current, request, resolve_binding, formula_ "helper": helper, "ending_row": ending, "observed": observed} +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ S67-T21 / R34 — EVERY METRIC COMPUTED; ONLY THE OWNER'S OWN INPUTS STAY TYPED +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner's weekly scorecard read pasted numbers: the weekly block and the middle of a dozen +# rows were values copied once and never recomputed. This mode applies an explicit, fully named +# cell plan (each cell a formula, a Cell Database binding or a blank), resolved and recalculated +# BEFORE a byte moves, and REFUSES unless the projected workbook keeps no hardcoded number but the +# owner's inputs. The plan itself is built by the operator from the Excel reference; this module is +# what makes it safe to apply: owner inputs and labels are untouchable, every binding resolves to +# a fresh value with no limits, the formula census stays empty, and the backup lands first. + +SR_TAB = "Summary (SR)" + +#: R34 (owner, 2026-09-28), verbatim cells. Summary (All)'s six are the owner's inputs; Summary +#: (SR)'s are the only numbers the Excel file itself types on that tab (its targets and goals). +OWNER_INPUTS = { + SUMMARY_TAB: frozenset({"Q25", "T20", "T37", "T38", "T51", "T55"}), + SR_TAB: frozenset({"T24", "T25", "T34", "T67", "T71", "T105", "T112"}), +} +_PLANNED_KINDS = ("formula", "bound", "empty") +#: The tab-level visual keys a plan may DECLARE (O42: match the Excel file's freeze). Anything else +#: visual still refuses, exactly as every content-only repair above does. +_PLANNED_TAB_VISUAL_KEYS = ("freeze",) + + +def hardcoded_numbers(book): + """`{tab name: {address: number}}` for every literal numeric cell, PURE. R34's census.""" + out = {} + for tab in book.get("sheets") or (): + found = {} + for address, cell in (tab.get("cells") or {}).items(): + value = (cell or {}).get("value") + if (cell or {}).get("kind") == "literal" and type(value) in (int, float): + found[address] = value + out[str(tab.get("name") or "")] = found + return out + + +def _bound_value(value, fn): + """A resolved binding's value as the document stores it, or a refusal. + + ⭐ A `min`/`max` over a date column answers a DATE ("Latest data" is `max(invoice_date)`), which + is stored as its ISO text, the shape every formula date in this workbook already has. + """ + import datetime as _dtm + if isinstance(value, (_dtm.date, _dtm.datetime)): + if fn not in ("min", "max"): + raise RepairRefusal("repair_binding_unreadable", "Only min or max may answer a date.") + return value.isoformat()[:10] + if isinstance(value, str) and fn in ("min", "max") and re.fullmatch(r"\d{4}-\d{2}-\d{2}.*", value): + return value[:10] + if value is None and fn in ("avg", "min", "max"): + return None + if type(value) not in (int, float) or not math.isfinite(value): + raise RepairRefusal("repair_binding_unreadable", "A binding must resolve a finite number.") + return value + + +def _visual_without(book, planned_notes, keys=("note",)): + """`visual_metadata` with each planned cell's declared visual `keys` removed (a note for + T21, a sparkline for T23), so a declared change is the only visual change a plan may make.""" + shaped = copy.deepcopy(book) + for tab in shaped.get("sheets") or (): + name = str(tab.get("name") or "").casefold() + for address, cell in (tab.get("cells") or {}).items(): + if (name, address) in planned_notes and isinstance(cell, dict): + for key in keys: + cell.pop(key, None) + return visual_metadata(shaped) + + +def _build_computed_metrics_plan(current, request, resolve_binding, formula_check): + """R34: validate and resolve an explicit per-cell plan; refuse unless it leaves no hardcode.""" + spec = request["computed_metrics"] + if not isinstance(spec, dict) or any(key in request for key in ( + "formula_repair", "related_migration", "summary_block", "source_cells", + "binding_cells", "daily_sales_bindings")): + raise RepairRefusal("repair_metrics_plan_conflict", + "Run the computed metrics plan as its own repair.") + if not callable(formula_check) or not callable(resolve_binding): + raise RepairRefusal("repair_binding_resolver_required", + "Computed metrics require the resolver and the formula engine.") + refreshed_at = str(request.get("refreshed_at") or "").strip() + if not refreshed_at: + raise RepairRefusal("repair_timestamp_required", "Computed metrics require a refresh timestamp.") + rows = spec.get("cells") + if not isinstance(rows, list) or not rows: + raise RepairRefusal("repair_metrics_plan_empty", "The computed metrics plan names no cell.") + edits, observed, seen, planned_notes = [], [], set(), set() + # ⭐ A binding may read a cell this same plan writes (a month's helper start/end dates, a + # year header), so bindings resolve against the workbook WITH the planned formulas and blanks + # applied and recalculated, never against the stale current one. + staged = [] + for row in rows: + want = row.get("cell") if isinstance(row, dict) else None + if isinstance(want, dict) and want.get("kind") in ("formula", "empty"): + tab = _tab(current, str(row.get("sheet") or "")) + address = _addr(row.get("address")) + before = copy.deepcopy((tab.get("cells") or {}).get(address)) + cell = ({"kind": "formula", "state": "stale", "formula": str(want.get("formula") or "")} + if want["kind"] == "formula" else {"kind": "empty", "state": "typed"}) + staged.append({"sheet": str(tab.get("name")), "address": address, "before": before, + "after": sheets.clean_cell(_preserve_visual(before, cell) or cell, address)}) + def recalculated(extra): + projected = _project(current, staged + extra) + checked = formula_check(copy.deepcopy(projected)) + return checked["book"] if isinstance(checked, dict) and isinstance( + checked.get("book"), dict) else projected + + def resolve_row(row, book): + sheet = str(_tab(current, str(row.get("sheet") or "")).get("name")) + address = _addr(row.get("address")) + binding = sheets.clean_cell({"kind": "bound", "binding": row["cell"].get("binding")}, + address)["binding"] + try: + result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(book), + copy.deepcopy(_tab(book, sheet))) + except Exception as exc: # noqa: BLE001 + detail = getattr(exc, "detail", None) + info = (detail.get("error") or {}) if isinstance(detail, dict) else {} + code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable" + return None, RepairRefusal(code, sheet + "!" + address + " binding resolution refused.") + return (binding, result), None + + # ⭐ ROUNDS, because a binding may read a date another binding produces through formulas + # (Summary (SR)'s week axis runs E12 `max(invoice_date)` -> E15 -> R8 -> F8 -> F31's window). + # Each round resolves what it can read, the next projection carries those values, and only a + # cell whose filter input was not yet readable waits; no progress in a round refuses by name. + resolved, pending = {}, [row for row in rows if isinstance(row, dict) + and isinstance(row.get("cell"), dict) + and row["cell"].get("kind") == "bound"] + for _round in range(6): + if not pending: + break + extra = [] + for (sheet, address), (binding, result) in resolved.items(): + value = result.get("value") if isinstance(result, dict) else None + try: + value = _bound_value(value, (binding.get("aggregate") or {}).get("fn")) + except RepairRefusal: + value = None + extra.append({"sheet": sheet, "address": address, "before": None, + "after": sheets.clean_cell({"kind": "bound", "state": "fresh", + "binding": binding, "value": value, + "display": "" if value is None else str(value)}, + address)}) + book = recalculated(extra) + waiting, first_error = [], None + for row in pending: + got, error = resolve_row(row, book) + if got is not None: + resolved[(str(_tab(current, str(row["sheet"])).get("name")), + _addr(row.get("address")))] = got + elif error.code == "filter_cell_unreadable": + waiting.append(row) + first_error = first_error or error + else: + raise error + if len(waiting) == len(pending): + raise first_error + pending = waiting + if pending: + raise RepairRefusal("filter_cell_unreadable", "A binding's filter cell never became readable.") + for row in rows: + if not isinstance(row, dict): + raise RepairRefusal("repair_metrics_cell_invalid", "Every planned cell must be an object.") + sheet_name = str(row.get("sheet") or "").strip() + address = _addr(row.get("address")) + tab = _tab(current, sheet_name) + sheet_name = str(tab.get("name") or sheet_name) + slot = (sheet_name.casefold(), address) + if slot in seen: + raise RepairRefusal("repair_edit_duplicate", "A planned cell may be named only once.") + seen.add(slot) + if address in OWNER_INPUTS.get(sheet_name, ()): + raise RepairRefusal("repair_owner_input", sheet_name + "!" + address + + " is one of the owner's typed inputs and stays typed.") + cells = tab.get("cells") or {} + before = copy.deepcopy(cells.get(address)) + prior_value = (before or {}).get("value") + want = row.get("cell") + if not isinstance(want, dict) or want.get("kind") not in (*_PLANNED_KINDS, "note_only"): + raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address + + " must be planned as a formula, a binding, a blank or a note.") + kind = want["kind"] + # A typed ISO date is a value (Summary (SR)'s frozen E13/E15), never a label. + if (before or {}).get("kind") == "literal" and isinstance(prior_value, str) \ + and prior_value.strip() and kind != "note_only" \ + and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", prior_value.strip()): + raise RepairRefusal("repair_label_protected", sheet_name + "!" + address + + " holds a label, which a metrics plan never replaces.") + if kind == "note_only": + desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"} + elif kind == "formula": + text = str(want.get("formula") or "") + if not text.startswith("="): + raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address + + " needs a formula that starts with =.") + desired = _preserve_visual(before, {"kind": "formula", "state": "stale", "formula": text}) + elif kind == "empty": + desired = _preserve_visual(before, {"kind": "empty", "state": "typed"}) \ + or {"kind": "empty", "state": "typed"} + else: + binding, result = resolved[(sheet_name, address)] + fn = (binding.get("aggregate") or {}).get("fn") + if not isinstance(result, dict) or "value" not in result or result.get("refusal") \ + or result.get("error") or result.get("state") not in (None, "fresh"): + raise RepairRefusal("repair_binding_unreadable", sheet_name + "!" + address + + " did not resolve a fresh bound value.") + if result.get("limits"): + raise RepairRefusal("repair_binding_limits", sheet_name + "!" + address + + " has unapplied or limited binding conditions.") + value = _bound_value(result["value"], fn) + if (before or {}).get("kind") == "bound" and before.get("binding") == binding \ + and type(prior_value) in (int, float) and type(value) in (int, float) \ + and math.isclose(prior_value, value, rel_tol=1e-9, abs_tol=1e-9): + value = prior_value # the same number; a rerun changes nothing + desired = _preserve_visual(before, { + "kind": "bound", "state": "fresh", "binding": binding, "value": value, + "display": "" if value is None else str(value), "refreshed_at": refreshed_at}) + observed.append({"sheet": sheet_name, "address": address, "value": value}) + if "note" in row: + note = str(row.get("note") or "").strip() + if note: + desired["note"] = note + else: + desired.pop("note", None) + planned_notes.add(slot) + after = sheets.clean_cell(desired, address) + # A planned formula is `derived`: the recalculation below is what fills its value, and + # `_checked_projection` accepts a recalculated planned cell only under that kind. + edit_kind = {"formula": "derived", "bound": "binding"}.get(kind, "metric") + edits.append({"kind": edit_kind, "name": sheet_name + "!" + address, "sheet": sheet_name, + "address": address, "before": before, "after": after}) + tab_visual = spec.get("tab_visual") or {} + if not isinstance(tab_visual, dict): + raise RepairRefusal("repair_metrics_visual_invalid", "Tab visual changes must be an object.") + for sheet_name, keys in tab_visual.items(): + _tab(current, sheet_name) + if not isinstance(keys, dict) or not keys or set(keys) - set(_PLANNED_TAB_VISUAL_KEYS): + raise RepairRefusal("repair_metrics_visual_invalid", + "A metrics plan may declare only a tab's freeze.") + projected, _errors = _checked_projection(_project(current, edits), edits, formula_check) + _daily_sales_invariants(projected, formula_check) + remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset())) + for name, found in hardcoded_numbers(projected).items() + if name in OWNER_INPUTS} + if any(remaining.values()): + raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers would remain: " + + json.dumps(remaining, sort_keys=True)) + return [edit for edit in edits if edit["before"] != edit["after"]], { + "observed": observed, "planned_notes": sorted(f"{s}!{a}" for s, a in planned_notes), + "tab_visual": copy.deepcopy(tab_visual), + "hardcoded_before": hardcoded_numbers(current), "hardcoded_after": hardcoded_numbers(projected)} + + +def _apply_tab_visual(st, workbook_id, tab_visual, *, actor="", actor_name=""): + """Write the DECLARED tab-level visual keys through the ordinary document door, then prove + nothing else moved. Returns the readback snapshot.""" + doc = sheets.workbook_snapshot(st, workbook_id) + before_cells = {str(t.get("name")): copy.deepcopy(t.get("cells") or {}) for t in doc["sheets"]} + before_visual = visual_metadata(doc) + for sheet_name, keys in tab_visual.items(): + tab = _tab(doc, sheet_name) + for key, value in keys.items(): + tab[key] = copy.deepcopy(value) + sheets.put_workbook(st, workbook_id, doc, expect_rev=int(doc.get("rev") or 0), + actor=actor, actor_name=actor_name, lossless=True) + readback = sheets.workbook_snapshot(st, workbook_id) + for sheet_name, keys in tab_visual.items(): + tab = _tab(readback, sheet_name) + for key, value in keys.items(): + if _canonical(tab.get(key)) != _canonical(value): + raise RepairRefusal("repair_visual_readback", sheet_name + " " + key + + " did not read back as declared.") + if {str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} != before_cells: + raise RepairRefusal("repair_visual_readback", "A declared visual change moved cell content.") + expected = copy.deepcopy(before_visual) + for item in expected: + for key, value in (tab_visual.get(item.get("name")) or {}).items(): + item[key] = copy.deepcopy(value) + if visual_metadata(readback) != _canonical(expected): + raise RepairRefusal("repair_visual_readback", "An undeclared visual property changed.") + return readback + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ S67-T22 / S67-T23 — DECLARED VISUAL REPAIRS: Excel's rule colours and its 3M sparklines +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# Every repair above REFUSES a visual change, on purpose. These two tickets ARE visual changes, so +# they get their own mode that names exactly what may move: a tab's `rules` (T22, O31) and a cell's +# `sparkline` (T23, O32). Cell content, every other visual key and every other cell must read back +# unchanged; the backup lands before anything is written. + +_VISUAL_REPAIR_TAB_KEYS = ("rules",) + + +def _build_visual_repair_plan(current, request): + spec = request["visual_repair"] + if not isinstance(spec, dict) or any(key in request for key in ( + "formula_repair", "related_migration", "summary_block", "source_cells", + "binding_cells", "daily_sales_bindings", "computed_metrics")): + raise RepairRefusal("repair_visual_plan_conflict", "Run the visual repair as its own repair.") + tab_visual = copy.deepcopy(spec.get("tab_visual") or {}) + for sheet_name, keys in tab_visual.items(): + _tab(current, sheet_name) + if not isinstance(keys, dict) or not keys or set(keys) - set(_VISUAL_REPAIR_TAB_KEYS): + raise RepairRefusal("repair_visual_invalid", "A visual repair may declare only a tab's rules.") + keys["rules"] = sheets._clean_rules(keys["rules"], lossless=True) + edits, seen = [], set() + for row in spec.get("cells") or (): + if not isinstance(row, dict) or "sparkline" not in row: + raise RepairRefusal("repair_visual_invalid", "A visual repair cell names only its sparkline.") + tab = _tab(current, str(row.get("sheet") or "")) + sheet_name, address = str(tab.get("name")), _addr(row.get("address")) + if (sheet_name.casefold(), address) in seen: + raise RepairRefusal("repair_edit_duplicate", "A repaired cell may be named only once.") + seen.add((sheet_name.casefold(), address)) + before = copy.deepcopy((tab.get("cells") or {}).get(address)) + desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"} + if row["sparkline"] is None: + desired.pop("sparkline", None) + else: + clean = sheets._clean_sparkline(row["sparkline"]) + if clean is None: + raise RepairRefusal("repair_visual_invalid", sheet_name + "!" + address + + " names a sparkline that cannot be drawn.") + desired["sparkline"] = clean + after = sheets.clean_cell(desired, address) + content = ("kind", "state", "value", "display", "formula", "binding", "text") + if any((before or {}).get(key) != after.get(key) for key in content if key != "state") \ + and not (before is None and after.get("kind") == "empty"): + raise RepairRefusal("repair_visual_content_changed", sheet_name + "!" + address + + " would change content, which a visual repair never does.") + edits.append({"kind": "visual", "name": sheet_name + "!" + address, "sheet": sheet_name, + "address": address, "before": before, "after": after}) + return [e for e in edits if e["before"] != e["after"]], { + "tab_visual": tab_visual, "sparkline_cells": sorted(e["name"] for e in edits)} + +def _execute_visual_repair(st, backup_store, request, current, revision, digest, workbook_id, + *, actor="", actor_name=""): + """Back up, then write ONLY the declared sparklines (exact cell edits) and tab rules.""" + edits, evidence = _build_visual_repair_plan(current, request) + if not edits and not evidence["tab_visual"]: + return {"workbook_id": workbook_id, "revision_before": revision, "revision_after": revision, + "applied": 0, "noop": True, "visual_repair": evidence} + planned = {(row["sheet"].casefold(), row["address"]) for row in edits} + projected = _project(current, edits) + if _visual_without(projected, planned, ("sparkline",)) != \ + _visual_without(current, planned, ("sparkline",)): + raise RepairRefusal("repair_visual_metadata_changed", + "The projected repair changes a visual property it did not declare.") + exact = [{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")} + for row in edits] + diff_digest = sheets.workbook_digest({"edits": exact, "tab_visual": evidence["tab_visual"]}) + backup_key = _write_backup(backup_store, current, digest, diff_digest, workbook_id) + revision_after = revision + if exact: + sheets.apply_exact_cell_edits(st, workbook_id, exact, revision, digest, + actor=actor, actor_name=actor_name) + readback = sheets.workbook_snapshot(st, workbook_id) + if int(readback.get("rev") or 0) != revision + 1: + raise RepairRefusal("repair_readback_revision", + "The repaired workbook revision did not read back as expected.") + for row in edits: + actual = (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]) + if actual != sheets.clean_cell(row["after"], row["address"]): + raise RepairRefusal("repair_readback_cell", + row["sheet"] + "!" + row["address"] + " did not read back exactly.") + revision_after = int(readback["rev"]) + if evidence["tab_visual"]: + revision_after = int(_apply_tab_visual(st, workbook_id, evidence["tab_visual"], + actor=actor, actor_name=actor_name)["rev"]) + return {"workbook_id": workbook_id, "revision_before": revision, + "revision_after": revision_after, "applied": len(exact), "backup_key": backup_key, + "backup_digest": digest, "diff_digest": diff_digest, "visual_repair": evidence} + + +def restore_backup(st, backup_store, backup_key, *, workbook_id, actor="", actor_name=""): + """NC for every mode here: put a backup's workbook back and prove it round-trips.""" + record = backup_store.get(backup_key) + if not isinstance(record, dict) or record.get("workbook_id") != workbook_id: + raise RepairRefusal("repair_backup_missing", "That backup does not exist for this workbook.") + saved = copy.deepcopy(record["workbook"]) + current = sheets.workbook_snapshot(st, workbook_id) + sheets.put_workbook(st, workbook_id, saved, expect_rev=int(current.get("rev") or 0), + actor=actor, actor_name=actor_name, lossless=True) + readback = sheets.workbook_snapshot(st, workbook_id) + same = ({str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} + == {str(t.get("name")): t.get("cells") or {} for t in saved["sheets"]} + and visual_metadata(readback) == visual_metadata(saved)) + if not same: + raise RepairRefusal("repair_restore_unverified", "The restored workbook did not round-trip.") + return readback + + def _build_plan(current, request, resolve_binding): if not isinstance(request.get("source_cells"), list) or not request["source_cells"]: raise RepairRefusal( @@ -1525,12 +1925,24 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID, raise RepairRefusal("repair_revision_mismatch", "The named workbook is not at the expected repair revision.") digest = sheets.workbook_digest(current) + if "visual_repair" in request: + return _execute_visual_repair(st, backup_store, request, current, revision, digest, + target_workbook_id, actor=actor, actor_name=actor_name) summary_evidence = None summary_errors_before = None migrated = None formula_evidence = None daily_evidence = None - if "daily_sales_bindings" in request: + metrics_evidence = None + if "computed_metrics" in request: + formula_check = _daily_sales_formula_checker(formula_check) + edits, metrics_evidence = _build_computed_metrics_plan( + current, request, resolve_binding, formula_check) + if not edits and not metrics_evidence["tab_visual"]: + return {"workbook_id": target_workbook_id, "revision_before": revision, + "revision_after": revision, "applied": 0, "noop": True, + "computed_metrics": metrics_evidence} + elif "daily_sales_bindings" in request: formula_check = _daily_sales_formula_checker(formula_check) edits, daily_evidence = _build_daily_sales_bindings_plan( current, request, resolve_binding, formula_check) @@ -1579,7 +1991,11 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID, "The projected repair introduces a formula error or leaves one in its repair scope, " "so no repair was applied.") visual_before = visual_metadata(current) - if visual_metadata(projected) != visual_before: + planned_notes = set() + if metrics_evidence is not None: + planned_notes = {tuple(item.split("!", 1)) for item in metrics_evidence["planned_notes"]} + planned_notes = {(sheet.casefold(), address) for sheet, address in planned_notes} + if _visual_without(projected, planned_notes) != _visual_without(current, planned_notes): raise RepairRefusal("repair_visual_metadata_changed", "The projected repair changes Daily Sales visual metadata.") @@ -1611,7 +2027,7 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID, or _cell_map(checked_readback) != _cell_map(readback): raise RepairRefusal("repair_readback_formula_state", "The repaired workbook did not read back calculation-current.") - if visual_metadata(readback) != visual_before: + if _visual_without(readback, planned_notes) != _visual_without(current, planned_notes): raise RepairRefusal("repair_readback_visual_metadata", "Daily Sales visual metadata changed during repair.") if formula_evidence is not None: @@ -1667,4 +2083,19 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID, result["daily_sales_bindings"] = daily_evidence result["formula_error_census"] = { "before": _formula_repair_census(current), "after": _formula_repair_census(readback)} + if metrics_evidence is not None: + _daily_sales_invariants(readback, formula_check) + remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset())) + for name, found in hardcoded_numbers(readback).items() if name in OWNER_INPUTS} + if any(remaining.values()): + raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers remain after the write: " + + json.dumps(remaining, sort_keys=True)) + if metrics_evidence["tab_visual"]: + visual_readback = _apply_tab_visual(st, target_workbook_id, metrics_evidence["tab_visual"], + actor=actor, actor_name=actor_name) + result["revision_after"] = visual_readback["rev"] + metrics_evidence["hardcoded_readback"] = hardcoded_numbers(readback) + result["computed_metrics"] = metrics_evidence + result["formula_error_census"] = { + "before": _formula_repair_census(current), "after": _formula_repair_census(readback)} return result diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 2ecd0e110759998d3811c07e4c79f7e7368afbc6..ac5c6cb060582da8833d5e98c6df5d987724ae52 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -42,12 +42,19 @@ _STRING = {"type": "string"} _ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."} _FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"}, "currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",)) +# S67-T14 / contract C2: a border side is the legacy `true` (read as a thin black line) or +# `{style, color}` with one of Excel's seven line styles and a six digit hex colour; `false` or an +# absent side is no side. `sheets.py::_clean_border_side` refuses anything else by code, and this +# publishes the same shape so an AI client can draw a styled border at all. +_BORDER_SIDE = {"oneOf": [{"type": "boolean"}, _object({ + "style": {"enum": list(sheets.CELL_BORDER_STYLES)}, + "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"}}, ("style", "color"))]} _STYLE = _object({ **{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")}, **{key: _STRING for key in ("fill", "color", "font", "align", "valign")}, "size": {"type": ["number", "string"]}, "indent": {"type": "integer"}, "textRotation": {"type": "number"}, "format": _FORMAT, - "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}), + "border": _object({side: _BORDER_SIDE for side in ("top", "right", "bottom", "left")}), }) _VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",)) _FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({ @@ -62,15 +69,25 @@ _BINDING = _object({ "period": _object({"unit": _STRING, "start": _STRING}), "ending": {"type": "object"}, }, ("database", "aggregate", "scope")) -_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]}, - "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING, - "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",)) _RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0} for key in ("col0", "row0", "col1", "row1")}, ("col0", "row0", "col1", "row1"))]} +# S67-T18: an in-cell sparkline, `{kind, range}` exactly as `sheets.py::_clean_sparkline` stores +# it; the range is a rectangle on the SAME tab (an A1 range such as "B2:M2", or the dict). +_SPARKLINE = _object({"kind": {"enum": list(sheets.SPARKLINE_KINDS)}, "range": _RANGE}, + ("kind", "range")) +_CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]}, + "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING, + "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING, + "sparkline": _SPARKLINE}, ("kind",)) +# S67-T13: a colour scale's two ends and what each end means. `sheets.py::_clean_rules` refuses +# a type outside `SHEET_RULE_SCALE_TYPES`; the enum publishes the same list to the caller. +_SCALE_TYPE = {"enum": list(sheets.SHEET_RULE_SCALE_TYPES)} _RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING, "value": {}, "value2": {}, "formula": _STRING, "style": _STYLE, - "scale": {"type": "object"}}, ("range", "op")) + "scale": _object({"low": _STRING, "high": _STRING, + "minType": _SCALE_TYPE, "maxType": _SCALE_TYPE})}, + ("range", "op")) _CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING, "stacking": _STRING, "source": {"type": "object"}, "series": {"type": "array", "items": {"type": "object"}}, diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index f2c308477c631434959a219e36f24acfdf963bda..496b8c568257451ec0014edbbae423f1b7a163ae 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -632,6 +632,57 @@ def _grant_marker(ctx, ws): return _has_grants +#: S67-T39 / R36: the Application registry's store key. READ here, never imported: +#: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE). +_APPLICATIONS_KEY = 'applications_v1' + + +def _live_application_ids(ctx): + """The set of App ids the tenant registry holds, or None when that is not KNOWN. + + None (read failed, or no registry document with an `applications` map) means "strip + nothing": an unknown answer must never turn a live App binding into a plain view. + """ + try: + doc = _store_of(ctx).get(_APPLICATIONS_KEY) + except Exception as exc: + _tel.error('applications:live', exc) + return None + apps = doc.get('applications') if isinstance(doc, dict) else None + return {str(key) for key in apps} if isinstance(apps, dict) else None + + +def _application_ref_id(view): + config = view.get('config') if isinstance(view, dict) else None + ref = config.get('application') if isinstance(config, dict) else None + return str(ref.get('id') or '') if isinstance(ref, dict) else None + + +def _drop_dead_application_refs(ctx, views): + """S67-T39 / R36 / D-2233: a View bound to a DELETED App is served as a plain view of its + database (the `application` member dropped, everything else kept), so it opens with no + error. Projection only, and it never raises: any failure serves the views unchanged. + The registry is read only when some View actually carries a reference. + """ + try: + if not any(_application_ref_id(v) is not None for v in (views or {}).values()): + return views + live = _live_application_ids(ctx) + if live is None: + return views + out = {} + for vid, view in views.items(): + ref_id = _application_ref_id(view) + if ref_id is not None and ref_id not in live: + config = {k: v for k, v in view['config'].items() if k != 'application'} + view = {**view, 'config': config} + out[vid] = view + return out + except Exception as exc: + _tel.error('applications:dead_refs', exc) + return views + + def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False): """Durable Airtable-style view/schema/overlay state, with an honest session fallback. @@ -707,6 +758,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do merged.update(ws.get('views') or {}) # own views win the id collision ws['views'] = {vid: scope_view(v, uname, allowed_pids) for vid, v in merged.items()} + if ws.get('views'): + ws['views'] = _drop_dead_application_refs(ctx, ws['views']) return ws # Store down. Streamlit degrades to its captioned session workspace exactly as before; an # API caller has no durable dict and must be told, not quietly served memory. @@ -1584,7 +1637,13 @@ def handle_one(event, ctx): _refuse(ctx, 'invalid_application_ref', 'that Application reference is not a valid typed identity', view_id) return False - config['application'] = application + # S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View + # saves as a plain view of its database. Unknown registry state keeps the ref. + _live = _live_application_ids(ctx) + if _live is not None and application['id'] not in _live: + application = None + else: + config['application'] = application # Wave-6 item 10: a direct non-grid display mode rides the view config, validated # structurally (mode whitelist, refs must be this table's fields). A bound Application's # config remains in the registry and is resolved on read, so an offered clone is dropped. diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index efeb99085ec3818f71338190ddd7819a040f4b7e..bb82e8790330ec3d9fa76a02f3ef0b4770d0ac51 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -221,12 +221,39 @@ def assistant_entry(user, module): that reader must be able to name the migrated grant that admitted a database. In particular, a store-outage admin identity with no ``perms`` document is not an unresolved permission that may be widened into data access. + + ⭐ S67-T34 / D-276: ONE NAMED EXCEPTION, the grant the NAVIGATION uses. For the two registry + topics in ``ASSISTANT_NAV_TOPICS`` an administrator account is admitted unfiltered (unless + explicitly unticked), because ``nav_may_open`` leg 1 opens them to it; the AI list must equal + the nav (R33). See the comment at that leg for why the outage identity no longer reaches it. """ e = entry(user, module) - if (not is_migrated(user) or not isinstance(e, dict) - or not bool(e.get('access', True)) or not may_access(user, module)): - return None - return e + if (is_migrated(user) and isinstance(e, dict) + and bool(e.get('access', True)) and may_access(user, module)): + return e + # ⭐⭐ S67-T34 / D-276 / R26: THE ADMINISTRATOR THE NAVIGATION ADMITS. `nav_may_open` leg 1 + # opens every registry topic to an administrator, and `_clean_perms` writes no per-topic + # entry for one, so an administrator saw `customer_data` and `product_data` (Odoo products, + # R26's Inventory source) in the nav and had them refused here. MEASURED on staging + # 2026-09-28: aaron (admin, no `perms_v`) had both in the nav and neither in the AI list. The + # answer is the grant the nav already uses, admitted for exactly these topics: an account + # whose ROLE is admin, minus an explicit `access: false` (narrower than the nav, never wider). + # Unfiltered, as the nav's break-glass is. + # ⚠ THE STORE-OUTAGE IDENTITY the docstring names is no longer built by `deps._user_for` + # (it admits a stored record or the QA runner, nothing else), so the role is read off a + # stored account here. If a synthetic admin identity is ever reintroduced, it must be kept + # out of this leg. + # ⛔ NOT A MEMBER. A non-administrator still needs the explicit migrated grant above, so a + # user without it does not see the database (the NC). + if (module in ASSISTANT_NAV_TOPICS and perms.is_admin(user) + and not (isinstance(e, dict) and e.get('access') is False)): + return dict(e) if isinstance(e, dict) else {'access': True, 'filter': None} + return None + + +#: S67-T34: the registry topics the navigation opens to an administrator by break-glass, which +#: `assistant_entry` therefore admits for a migrated administrator without a per-topic entry. +ASSISTANT_NAV_TOPICS = frozenset({'customer_data', 'product_data'}) # ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── diff --git a/platform/core/registry.py b/platform/core/registry.py index ae3c90ed139055dca5823c8d1b6c4b4b95af3310..ad4901aeb538a3ddb2752d5f474c8cf52e5bc8e6 100644 --- a/platform/core/registry.py +++ b/platform/core/registry.py @@ -203,7 +203,12 @@ REGISTRY = [ 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete ' 'them") made that literal: `validate_only` takes it out of the account menu too, ' 'which is the last place it was still visible. The proof survives; the door does not.'}, + # S67-T36 (owner instr 24, R25, 2026-09-28): Agents is TABLED. `archived` hides the UI only: + # no rail row, no Settings module toggle, no Home section, no agent templates. The + # `/automations` APIs, stored automations and their schedules are untouched and keep running. + # ⭐ RESTORING IS THIS ONE LINE: delete `'archived': True` below and every door comes back. {'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False, + 'archived': True, 'nav': True, 'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases ' '(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS ' @@ -280,6 +285,11 @@ def brand_filterable(key) -> bool: return bool(BY_KEY.get(key, {}).get('brand')) +def is_archived(key) -> bool: + """True when the registry row `key` is archived (tabled for everyone). Unknown keys: False.""" + return bool(BY_KEY.get(key, {}).get('archived')) + + # ═════════════════════════════════════════════════════════════════════════════════════════════ # ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE # ═════════════════════════════════════════════════════════════════════════════════════════════ diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py index 1bc4d6ce5726f9cf67e7db286b78602b3944eeb1..ec057f106cf47a08b0f589b77aee9b29ce004ce6 100644 --- a/platform/core/sheet_import.py +++ b/platform/core/sheet_import.py @@ -43,9 +43,11 @@ from __future__ import annotations import re import zipfile import hashlib +import io import json import math import os +import posixpath import tempfile import xml.etree.ElementTree as ET from datetime import date, datetime, time, timezone @@ -392,12 +394,11 @@ def style_of(cell, notes): if border is not None: sides = {} for side in sheets._BORDER_SIDES: - edge = getattr(border, side, None) - if edge is not None and getattr(edge, "style", None): - sides[side] = True + line = border_line(getattr(border, side, None), notes) + if line is not None: + sides[side] = line if sides: out["border"] = sides - notes.add("a border keeps its side but not its weight, style or colour") fmt, note = classify_format(getattr(cell, "number_format", "")) if fmt: @@ -407,6 +408,47 @@ def style_of(cell, notes): return out +#: S67-T14: the OOXML line styles that are not one of C2's seven, each drawn as its nearest. +#: The dash and dot variants keep their PATTERN over their weight, because a patterned line drawn +#: solid reads as a different border while a medium dash drawn at 1px still reads as dashed. +_BORDER_NEAREST = { + "mediumDashed": "dashed", "dashDot": "dashed", "dashDotDot": "dashed", + "mediumDashDot": "dashed", "mediumDashDotDot": "dashed", "slantDashDot": "dashed", +} + + +def border_line(edge, notes): + """One openpyxl border side as C2's `{style, color}`, or None when the side draws nothing. + + The seven Excel names C2 shares pass through unchanged; the dash dot variants become their + nearest with a note naming the approximation. The colour is the explicit RGB when there is one; + an automatic colour is black, which is what Excel draws; a theme or indexed colour is drawn in + black too, with a note, because a colour nobody chose is never guessed (see `argb_of`). + """ + style = str(getattr(edge, "style", None) or "") if edge is not None else "" + if not style or style == "none": + return None + if style not in sheets.CELL_BORDER_STYLES: + nearest = _BORDER_NEAREST.get(style, "thin") + notes.add("a " + style + " border is drawn as " + nearest) + style = nearest + colour = getattr(edge, "color", None) + argb = argb_of(colour) + if argb: + hex_colour = "#" + argb.lower() + else: + hex_colour = "#000000" + kind = getattr(colour, "type", "") if colour is not None else "" + # ⚠ `type`, never `.auto`: openpyxl answers an unset descriptor with a (truthy) error + # STRING, so `colour.auto` reads as set on every theme colour. + automatic = (colour is None or kind == "auto" + or (kind == "indexed" and getattr(colour, "indexed", None) == 64) + or (kind == "rgb" and str(getattr(colour, "rgb", "")).upper() == "00000000")) + if not automatic: + notes.add("a theme or indexed border colour was drawn in black") + return {"style": style, "color": hex_colour} + + # ── values ─────────────────────────────────────────────────────────────────────────────────── def json_value(value): @@ -1037,6 +1079,134 @@ def guard_import_archive(path): "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc +# ── sparklines (S67-T18) ───────────────────────────────────────────────────────────────────── + +#: ⭐ S67-T18: Excel keeps sparkline groups in the worksheet's x14 extension, which openpyxl drops, +#: so they are read from the worksheet PART itself. A group with no `type` draws lines, `column` +#: draws columns and `stacked` draws win/loss bars (Excel's name for `sheets.SPARKLINE_KINDS`'s +#: `winloss`). The part was already admitted by `guard_import_archive`, so this second read adds +#: no new exposure. +_X14 = "{http://schemas.microsoft.com/office/spreadsheetml/2009/9/main}" +_XM = "{http://schemas.microsoft.com/office/excel/2006/main}" +_MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" +_DOC_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}" +_PKG_REL_NS = "{http://schemas.openxmlformats.org/package/2006/relationships}" +_SPARKLINE_KIND_OF = {"line": "line", "column": "column", "stacked": "winloss"} + + +def _worksheet_parts(archive): + """`{tab name: worksheet part}` through `xl/workbook.xml` and its relationships, never by index.""" + names = set(archive.namelist()) + if "xl/workbook.xml" not in names or "xl/_rels/workbook.xml.rels" not in names: + return {} + targets = {} + for rel in ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")).iter(_PKG_REL_NS + "Relationship"): + target = str(rel.get("Target") or "") + targets[rel.get("Id")] = (target.lstrip("/") if target.startswith("/") + else posixpath.normpath("xl/" + target)) + out = {} + for sheet in ET.fromstring(archive.read("xl/workbook.xml")).iter(_MAIN_NS + "sheet"): + part = targets.get(sheet.get(_DOC_REL_NS + "id")) + if part in names: + out[str(sheet.get("name") or "")] = part + return out + + +def _sparkline_groups(stream): + """`[(type, [(formula, sqref), ...]), ...]` for one worksheet part's x14 sparkline groups.""" + groups = [] + inside = 0 # a group's subtree is kept until the group itself ends + for event, element in ET.iterparse(stream, events=("start", "end")): + if element.tag == _X14 + "sparklineGroup": + if event == "start": + inside += 1 + continue + inside -= 1 + entries = [((spark.findtext(_XM + "f") or "").strip(), + (spark.findtext(_XM + "sqref") or "").strip()) + for spark in element.iter(_X14 + "sparkline")] + groups.append((str(element.get("type") or "line"), entries)) + element.clear() + elif event == "end" and not inside: + element.clear() + return groups + + +def _sparklines_of(path, notes): + """`{tab name: groups}` for every worksheet that carries x14 sparklines; `{}` for the rest. + + A part whose bytes never mention a sparkline group is not parsed at all, so a workbook with + no sparklines imports exactly as it did before S67-T18. + """ + out = {} + try: + with zipfile.ZipFile(path) as archive: + for name, part in _worksheet_parts(archive).items(): + raw = archive.read(part) + if b"sparklineGroup" not in raw: + continue + groups = _sparkline_groups(io.BytesIO(raw)) + if groups: + out[name] = groups + except (KeyError, OSError, ET.ParseError, zipfile.BadZipFile): + notes.add("the workbook's sparklines could not be read, so none were imported") + return {} + return out + + +def _count_phrase(count, one, many): + return (one if count == 1 else str(count) + " " + many) + + +def _apply_sparklines(tab_name, cells, groups, blank_styles, notes): + """Write `cell.sparkline {kind, range}` on each location cell of this tab's groups. + + `sheets._clean_sparkline` stores a rectangle LOCAL to its tab, so a sparkline drawn from + another tab's data cannot be carried: it is left out and NAMED, never dropped in silence + (the module note). A location that is not one cell, or data that is not a range, is named too. + A blank location becomes an `empty` cell holding only the sparkline, which is exactly what + `clean_cell`'s empty branch keeps; its compact blank-style reference moves onto it. + """ + other_tab = unreadable = 0 + for kind_name, entries in groups: + kind = _SPARKLINE_KIND_OF.get(kind_name) + if kind is None: + unreadable += len(entries) + continue + for formula, sqref in entries: + source = sheet_of_ref(formula) + if source and source.casefold() != tab_name.casefold(): + other_tab += 1 + continue + rect = rect_of(formula) + places = sqref.replace("$", "").upper().split() + if (rect is None or len(places) != 1 or ":" in places[0] + or not _address_in_document(places[0])): + unreadable += 1 + continue + addr = places[0] + spark = {"kind": kind, "range": rect} + cell = cells.get(addr) + if cell is None: + if len(cells) >= sheets._MAX_CELLS_PER_SHEET: + unreadable += 1 + continue + cell = {"kind": "empty", "sparkline": spark} + if addr in blank_styles: + cell["excel"] = {"styleId": blank_styles.pop(addr)} + cells[addr] = cell + else: + cell["sparkline"] = spark + if other_tab: + notes.add(_count_phrase(other_tab, "a sparkline", "sparklines") + + " drawn from another tab's data " + ("was" if other_tab == 1 else "were") + + " left out") + if unreadable: + notes.add(_count_phrase(unreadable, "a sparkline", "sparklines") + + " with a location or data range that could not be read " + + ("was" if unreadable == 1 else "were") + " left out") + + def _print_of(ws): """Store print setup and inclusive print areas; export executes these settings later.""" setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__ @@ -1148,6 +1318,7 @@ def import_workbook(path, *, source="", progress=None): counts = {"cells": 0, "formulas": 0} book_notes = set() + sparklines = _sparklines_of(path, book_notes) # S67-T18 authored = _authored_chart_document(path, book_notes) chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {} if authored and len(chart_tabs) != len(authored["sheets"]): @@ -1209,6 +1380,8 @@ def import_workbook(path, *, source="", progress=None): cells = _cells_of(ws, cached, notes, counts, excel) _say("formats", 0.7 + 0.25 * (index / total)) _validations_of(ws, cells, notes) + if name in sparklines: # S67-T18, before usedExtent + _apply_sparklines(name, cells, sparklines[name], excel["blankStyles"], notes) cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes) tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows, "cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]), diff --git a/platform/core/sheets.py b/platform/core/sheets.py index 8d09b6673d7255f99a61de847113b3a3deeae803..7c8c11cfd433fcae1387ad729085acff30d4134f 100644 --- a/platform/core/sheets.py +++ b/platform/core/sheets.py @@ -152,6 +152,14 @@ MAX_FONT_PT = 96 _STYLE_MARKS = ("bold", "italic", "underline") _BORDER_SIDES = ("top", "right", "bottom", "left") +#: ⭐⭐ S67-T14 / CONTRACT C2: Excel's seven border line styles, name for name. Mirrored in +#: `aios-web/web/src/sheet/types.ts::CELL_BORDER_STYLES`, and `verify_sheet.py` compares the two +#: lists. A side is either the legacy `True` (read as a thin black line and stored unchanged, so a +#: document written before S67 reads back byte for byte) or `{"style": , +#: "color": "#rrggbb"}`. +CELL_BORDER_STYLES = ("thin", "medium", "thick", "dashed", "dotted", "double", "hair") +_BORDER_COLOUR = re.compile(r"^#[0-9a-f]{6}$") + #: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that #: outlives every layout change, exactly as a stored hex outlives a palette. _MAX_INDENT = 8 @@ -204,6 +212,12 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains", #: every rule written before this ticket already is, so writing the word onto them would make a #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`. SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula") +#: ⭐ S67-T13: Excel's colour scale point types, one list for both ends. `lowest` / `highest` are +#: measured from the range at paint, so the scale re-spreads as the data changes; the other three +#: read their number from the rule's `value` (low end) or `value2` (high end). A type outside this +#: list is REFUSED by name, never dropped: a dropped type reads back as a different scale. Mirrored +#: in `web/src/sheet/types.ts::SheetRuleScaleType`. +SHEET_RULE_SCALE_TYPES = ("lowest", "highest", "number", "percent", "percentile") # The client compiler accepts 500 characters per condition. MAX_RULE_FORMULA_LENGTH = 500 @@ -652,13 +666,44 @@ def _clean_style(raw): border = raw.get("border") if isinstance(border, dict): - sides = {side: True for side in _BORDER_SIDES if border.get(side) is True} + sides = {} + for side in _BORDER_SIDES: + line = _clean_border_side(border.get(side)) + if line is not None: + sides[side] = line if sides: out["border"] = sides return out or None +def _clean_border_side(raw): + """One C2 border side: `True`, a `{style, color}` dict, or None when the side is absent. + + ⛔⛔ THE SECOND DOCUMENTED EXCEPTION TO "UNKNOWN KEYS ARE DROPPED IN SILENCE". A border side + that names a line style outside the seven, or a colour that is not a six digit hex, is + REFUSED by name. Dropping it would store a cell with no line where the person drew one, and + the caller could not learn its request went nowhere: `_clean_format`'s reason, one key over. + A side that is `False`, None or absent is simply no side. A legacy `True` stays `True`. + """ + if raw is None or raw is False: + return None + if raw is True: + return True + if not isinstance(raw, dict): + return None + style = raw.get("style") + colour = raw.get("color") + if isinstance(colour, str): + colour = colour.strip().lower() + if (not isinstance(style, str) or style not in CELL_BORDER_STYLES + or not isinstance(colour, str) or not _BORDER_COLOUR.match(colour)): + raise SheetRefusal( + "cell_border_invalid", + "A border needs a line style such as thin and a six digit hex colour such as #000000.") + return {"style": style, "color": colour} + + # ── the cell ───────────────────────────────────────────────────────────────────────────────── def _clean_validation(raw): @@ -1131,15 +1176,31 @@ def _clean_rule_scale(raw): ⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which `_clean_rules` already persists for every op that takes operands, so a colour scale needed no new numeric slot at all. + + ⭐ S67-T13: `minType` / `maxType` say what each end MEANS (`SHEET_RULE_SCALE_TYPES`). Each is + kept only when present and never injected, so a stored boundless scale reads back byte for + byte, and input with no types returns exactly the `{low, high}` it always did. An unknown type + is refused by name. """ if not isinstance(raw, dict): return None + types = {} + for key in ("minType", "maxType"): + kind = raw.get(key) + if kind is None or kind == "": + continue + if kind not in SHEET_RULE_SCALE_TYPES: + raise SheetRefusal( + "sheet_rule_scale_type_unknown", + "A colour scale end must be one of " + ", ".join(SHEET_RULE_SCALE_TYPES) + ".") + types[key] = kind out = {} for key in ("low", "high"): token = _text(raw.get(key), 40) if not token or not _COLOUR.match(token): return None out[key] = token + out.update(types) return out diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index 77ab07c24ac0521a36ddfd604fbc8b463d676293..39707a197c8e0955cf7783c7dd2b3c2b833033de 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -1457,7 +1457,10 @@ def group_a_records(fields, operation, *, start=None, end=None, asins=(), params.update(includedData="summaries,issues,offers,fulfillmentAvailability,attributes", pageSize=20) root, payload_key, next_key = "items", None, "pageToken" else: - params.update(identifiersType="ASIN", includedData="summaries,salesRanks,dimensions,attributes", + # `relationships` (S67-T29, O23): same Catalog Items endpoint and Product Listing role; + # it carries VARIATION parentAsins so the mirror can map every own ASIN to its parent. + params.update(identifiersType="ASIN", + includedData="summaries,salesRanks,dimensions,attributes,relationships", pageSize=20) root, payload_key, next_key = "items", None, "pageToken" batches = [None] diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 8785dbdcaf2370fb320547eddae97961c4bb3c92..2bc94de4e1557e3a35c099519802bfad0b349c80 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -853,6 +853,206 @@ def _sqp_catalogue(con): return [] +_ASIN_RE = re.compile(r"[A-Z0-9]{10}") + + +def _columns_of(con, table): + return {r[0] for r in con.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()} + + +_NEWEST_ORDER = ("start_date", "_period_start", "_snapshot_date", "_report_row_number") + + +def _newest_child_parent_sql(columns, table, only=None): + """SELECT child_asin, parent_asin: one row per child, the newest period's parent; None if absent. + + `only` is an optional SQL SELECT of the child ASINs wanted. It filters BEFORE the window, which + keeps or drops whole `child_asin` partitions, so every kept child's answer is unchanged. Once + the catalog carries parents (its first sync after S67-T29), `asin_parents_sql` passes only the + few own ASINs the catalog leaves without one, so S67-T30's per-page lookup stops ranking all of + economics (572,320 rows on staging, about 100 ms a query) to answer them. + """ + if not {"child_asin", "parent_asin"} <= columns: + return None + order = ", ".join(f'"{c}" DESC NULLS LAST' for c in _NEWEST_ORDER if c in columns) or "1" + wanted = f" AND TRIM(CAST(child_asin AS VARCHAR)) IN ({only})" if only else "" + return (f'SELECT child_asin, parent_asin FROM (SELECT TRIM(CAST(child_asin AS VARCHAR)) AS child_asin, ' + f'TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, ROW_NUMBER() OVER (' + f'PARTITION BY TRIM(CAST(child_asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM "{table}" ' + f"WHERE child_asin IS NOT NULL AND parent_asin IS NOT NULL " + f"AND TRIM(CAST(parent_asin AS VARCHAR)) <> ''{wanted}) WHERE _rn = 1") + + +def _child_parent_map(con, table): + """child ASIN -> parent ASIN from the newest rows of `table`; {} when it has no child_asin.""" + sql = _newest_child_parent_sql(_columns_of(con, table), table) + if sql is None: + return {} + return {str(c): str(p) for c, p in con.execute(sql).fetchall() + if _ASIN_RE.fullmatch(str(c)) and _ASIN_RE.fullmatch(str(p))} + + +def _pricing_parent_map(con): + """O23 fallback. Used only when pricing HAS child_asin: the live Sales and Traffic mirror is + stored at parent granularity (parent_asin only), so this is {} there and economics carries on.""" + return _child_parent_map(con, "amzn_pricing") + + +def _economics_parent_map(con): + """Data Kiosk economics child_asin -> parent_asin, newest start_date wins.""" + return _child_parent_map(con, "amzn_economics") + + +def _catalog_parent(item, asin, pricing_map, marketplace, economics_map=None): + """One item's (parent_asin, source); never raises on a missing or malformed `relationships`.""" + parents, is_parent = [], False + groups = item.get("relationships") + for group in groups if isinstance(groups, list) else (): + if not isinstance(group, dict): + continue + if marketplace and group.get("marketplaceId") not in (None, marketplace): + continue + entries = group.get("relationships") + for rel in entries if isinstance(entries, list) else (): + # Only VARIATION defines a parent; PACKAGE_HIERARCHY is a pack, not a family. + if not isinstance(rel, dict) or rel.get("type") != "VARIATION": + continue + found = rel.get("parentAsins") + parents += [str(a).strip() for a in (found if isinstance(found, list) else ()) + if _ASIN_RE.fullmatch(str(a).strip())] + if isinstance(rel.get("childAsins"), list) and rel["childAsins"]: + is_parent = True + if parents: + return parents[0], "relationships" + if is_parent: + return asin, "relationships" + for source, mapping in (("pricing", pricing_map), ("economics", economics_map)): + fallback = str((mapping or {}).get(asin) or "").strip() + if _ASIN_RE.fullmatch(fallback): + return fallback, source + return asin, "self" + + +def catalog_parent_map(items, pricing_map=None, marketplace=None, economics_map=None): + """S67-T29 (O23): own ASIN -> (parent ASIN, source) for every catalog item. + + Source order: the catalog VARIATION `parentAsins` for the requested marketplace; an item that + is itself a variation parent maps to itself; else pricing's `parent_asin` for that ASIN as + `child_asin` (O23); else economics child_asin -> parent_asin; else itself. A standalone ASIN + therefore maps to itself, never an error. + """ + out = {} + for item in items or (): + if not isinstance(item, dict): + continue + asin = str(item.get("asin") or "").strip() + if asin: + out[asin] = _catalog_parent(item, asin, pricing_map, marketplace, economics_map) + return out + + +#: S67 lane D (S67-T29 finding): Sales and Traffic defaults `asinGranularity` to PARENT, so the +#: saved `amzn_pricing` mirror carried `parent_asin` with no `child_asin` and no `sku`, and +#: `connector_spec`'s pricing identity `(child_asin, parent_asin)` and its two listings links never +#: formed. SKU grain returns parentAsin, childAsin and sku per row: the SAME report type and role +#: (the PRD's Amazon compliance line: no new endpoint or role). +REPORT_OPTIONS = {"pricing": {"asinGranularity": "SKU"}} + + +def _pricing_regrain(con, mirror, rows, log=print): + """Clear a PARENT-grain pricing mirror before the first SKU-grain rows land; True if cleared. + + `amzn_pricing` keeps history across incremental passes, so without this the old parent-grain + days would sit beside SKU-grain days and any sum across days would count twice. A mirror that + already holds any `child_asin`, an empty or absent mirror, or rows without `child_asin` are + left alone. + """ + if not any((row or {}).get("child_asin") for row in rows): + return False + have = _columns_of(con, mirror) # empty for an absent table (first load), never raises + if not have: + return False + if "child_asin" in have and con.execute( + f'SELECT COUNT(*) FROM {mirror} WHERE "child_asin" IS NOT NULL').fetchone()[0]: + return False + stale = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0] + if not stale: + return False + con.execute(f"DELETE FROM {mirror} WHERE 1=1") + log(f" pricing: replaced {stale} parent-grain rows with SKU-grain rows (one grain per table)") + return True + + +def asin_parents_sql(con): + """S67-T29: ONE read-time resolver, one row per own ASIN: `(asin, parent_asin, parent_asin_source)`. + + Own ASINs = DISTINCT listings `asin1` UNION the LATEST catalog `_snapshot_date`'s `asin` (the + snapshot tables keep one copy per sync day, D-3031, so only the newest day counts). Parent = + that latest catalog row's `parent_asin` (source as stamped at sync), else pricing child_asin -> + parent_asin when the pricing mirror carries child_asin (source `pricing`, O23), else economics + child_asin -> parent_asin (newest start_date, source `economics`), else the ASIN itself + (`self`). Covers own ASINs Amazon's catalog search returns no item for, without stub catalog + rows. Degrades with absent tables or columns; None only when no own-ASIN source exists. + Reusable verbatim by a SQL join (S67-T30): wrap it as a subquery. + """ + listings, catalog = _columns_of(con, "amzn_listings"), _columns_of(con, "amzn_catalog_items") + latest = ('(SELECT MAX("_snapshot_date") FROM amzn_catalog_items)') + own = [] + if "asin1" in listings: + own.append("SELECT TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM amzn_listings WHERE asin1 IS NOT NULL") + has_catalog = {"asin", "_snapshot_date"} <= catalog + if has_catalog: + own.append("SELECT TRIM(CAST(asin AS VARCHAR)) AS asin FROM amzn_catalog_items " + f'WHERE asin IS NOT NULL AND "_snapshot_date" = {latest}') + if not own: + return None + ctes = ["own AS (SELECT DISTINCT asin FROM (" + " UNION ".join(own) + ") WHERE asin <> '')"] + joins, parents, sources = [], [], [] + if has_catalog and "parent_asin" in catalog: + source = ("CAST(parent_asin_source AS VARCHAR)" if "parent_asin_source" in catalog + else "'relationships'") + order = '"_report_row_number"' if "_report_row_number" in catalog else "1" + ctes.append( + "cat AS (SELECT asin, parent_asin, parent_asin_source FROM (SELECT TRIM(CAST(asin AS VARCHAR)) AS asin, " + "TRIM(CAST(parent_asin AS VARCHAR)) AS parent_asin, " + f"COALESCE({source}, 'relationships') AS parent_asin_source, ROW_NUMBER() OVER (" + f"PARTITION BY TRIM(CAST(asin AS VARCHAR)) ORDER BY {order}) AS _rn FROM amzn_catalog_items " + f"WHERE \"_snapshot_date\" = {latest} AND parent_asin IS NOT NULL " + "AND TRIM(CAST(parent_asin AS VARCHAR)) <> '') WHERE _rn = 1)") + joins.append("LEFT JOIN cat ON cat.asin = own.asin") + parents.append("cat.parent_asin") + sources.append("WHEN cat.parent_asin IS NOT NULL THEN cat.parent_asin_source") + # Economics is only the fallback, so rank only the own ASINs the catalog left without a parent. + only = "SELECT asin FROM own" + (" WHERE asin NOT IN (SELECT asin FROM cat)" if joins else "") + # O23: pricing is the fallback BEFORE economics. None (no step) while the pricing mirror has no + # `child_asin`, which is the live Sales and Traffic mirror today (S67-T29 finding). + pricing = _newest_child_parent_sql(_columns_of(con, "amzn_pricing"), "amzn_pricing", only=only) + if pricing: + ctes.append(f"pri AS ({pricing})") + joins.append("LEFT JOIN pri ON pri.child_asin = own.asin") + parents.append("pri.parent_asin") + sources.append("WHEN pri.parent_asin IS NOT NULL THEN 'pricing'") + economics = _newest_child_parent_sql(_columns_of(con, "amzn_economics"), "amzn_economics", + only=only) + if economics: + ctes.append(f"eco AS ({economics})") + joins.append("LEFT JOIN eco ON eco.child_asin = own.asin") + parents.append("eco.parent_asin") + sources.append("WHEN eco.parent_asin IS NOT NULL THEN 'economics'") + return ("WITH " + ", ".join(ctes) + " SELECT own.asin AS asin, " + f"COALESCE({', '.join([*parents, 'own.asin'])}) AS parent_asin, " + + (f"CASE {' '.join(sources)} ELSE 'self' END" if sources else "'self'") + + " AS parent_asin_source " + "FROM own " + " ".join(joins) + " ORDER BY own.asin") + + +def asin_parents(con): + """`asin_parents_sql` as `{asin: (parent_asin, source)}`; {} when no own-ASIN source exists.""" + sql = asin_parents_sql(con) + return {} if sql is None else {a: (p, s) for a, p, s in con.execute(sql).fetchall()} + + def sqp_limit_sentence(marker): """Pure projection of saved sync facts; never asks Amazon for a catalogue.""" numbers = [marker[k] for k in ("maxChars", "maxPerBatch", "asinCount", "batchCount")] @@ -1470,6 +1670,21 @@ def _blob_metadata(data): return {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} +#: S67-T32 (D-3481): v1 names in-place keys (`
.parquet`, `sync_state.json`) and still +#: restores; v2 names each blob's CONTENT key, so a manifest only ever names complete blobs and a +#: key it names is never overwritten. `keys.json` is the write-ahead ledger of every key ever +#: uploaded: the store has no list-by-prefix, so without it an orphan is unpurgeable. +MANIFEST_VERSIONS = (1, 2) +MANIFEST_VERSION = 2 +_MANIFEST_KEY = "manifest.json" +_KEY_LEDGER = "keys.json" +_TABLE_BLOB_RE = re.compile(r"(amzn_[a-z0-9_]+)(?:\.([0-9a-f]{16}))?\.parquet") +_STATE_BLOB_RE = re.compile(r"sync_state(?:\.([0-9a-f]{16}))?\.json") +#: A ledger key no manifest names is a killed checkpoint's orphan once it is this old. A live +#: checkpoint publishes within minutes, so the grace never races an in-flight writer. +ORPHAN_GRACE = timedelta(hours=24) + + def _checked_mirror_blob(rt, path, metadata): data = rt.download_bytes(MIRROR_PREFIX + path) if data is None or _blob_metadata(data) != {k: metadata.get(k) for k in ("bytes", "sha256")}: @@ -1477,11 +1692,195 @@ def _checked_mirror_blob(rt, path, metadata): return data +def _content_key(stem, suffix, metadata): + return f"{stem}.{metadata['sha256'][:16]}{suffix}" + + +def _table_blob_key(name, metadata, *, strict=True): + """The key a manifest entry names: its content key, or the legacy in-place key.""" + key = str(metadata.get("key") or name + ".parquet") + match = _TABLE_BLOB_RE.fullmatch(key) + if not match or match.group(1) != name or ( + strict and match.group(2) and not str(metadata.get("sha256") or "").startswith(match.group(2))): + raise ValueError("Amazon mirror manifest names an invalid table blob key") + return key + + +def _state_blob_key(metadata, *, strict=True): + key = str(metadata.get("key") or "sync_state.json") + match = _STATE_BLOB_RE.fullmatch(key) + if not match or (strict and match.group(1) + and not str(metadata.get("sha256") or "").startswith(match.group(1))): + raise ValueError("Amazon mirror manifest names an invalid state blob key") + return key + + +def _is_mirror_blob_key(key): + return bool(_TABLE_BLOB_RE.fullmatch(str(key)) or _STATE_BLOB_RE.fullmatch(str(key))) + + +def _manifest_keys(manifest, *, strict=True): + """{key: sha256} for every blob a manifest names; malformed entries are skipped, never raised.""" + out = {} + if not isinstance(manifest, dict): + return out + tables = manifest.get("tables") + for name, metadata in (tables.items() if isinstance(tables, dict) else ()): + try: + out[_table_blob_key(str(name), metadata, strict=strict)] = metadata.get("sha256") + except (ValueError, AttributeError, TypeError): + continue + state = manifest.get("sync_state") + if isinstance(state, dict): + try: + out[_state_blob_key(state, strict=strict)] = state.get("sha256") + except (ValueError, AttributeError, TypeError): + pass + return out + + +def _read_key_ledger(rt, tenant_key): + """The write-ahead key ledger. Unreadable is an error: a purge must not guess.""" + raw = rt.download_bytes(MIRROR_PREFIX + _KEY_LEDGER) + key = _tenant_key(tenant_key) + if raw is None: + return {"version": 1, "tenant": key, "keys": {}} + doc = json.loads(raw) + if (not isinstance(doc, dict) or doc.get("tenant") != key or not isinstance(doc.get("keys"), dict)): + raise ValueError("Amazon mirror key ledger is unreadable or belongs to another tenant") + doc["keys"] = {str(k): v for k, v in doc["keys"].items() if _is_mirror_blob_key(k)} + return doc + + +def _ledger_add(rt, tenant_key, keys): + """Record keys BEFORE their upload, so a killed checkpoint's blob is always enumerable.""" + ledger = _read_key_ledger(rt, tenant_key) + stamp = _iso(datetime.now(timezone.utc)) + fresh = [key for key in keys if key not in ledger["keys"]] + if not fresh: + return + for key in fresh: + if not _is_mirror_blob_key(key): + raise ValueError("Amazon mirror key is outside the naming contract") + ledger["keys"][key] = stamp + rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger)) + + +def _validated_snapshot(tenant_key, rt, con, raw, tmp): + """Every check restore makes, without writing. -> (manifest, state rows, parquet paths, rows). + + Raises ValueError for anything the stored snapshot itself gets wrong; a store-access failure + propagates as its own type, so a caller can tell "unrestorable" from "unreachable". + """ + try: + manifest = json.loads(raw) + except ValueError as exc: + raise ValueError("Amazon mirror manifest is not JSON") from exc + if (not isinstance(manifest, dict) or manifest.get("version") not in MANIFEST_VERSIONS + or manifest.get("tenant") != _tenant_key(tenant_key) + or not isinstance(manifest.get("tables"), dict) or not isinstance(manifest.get("sync_state"), dict)): + raise ValueError("Amazon mirror manifest has an invalid version, tenant or table map") + state_meta = manifest["sync_state"] + state_bytes = _checked_mirror_blob(rt, _state_blob_key(state_meta), state_meta) + rows = json.loads(state_bytes) + if (not isinstance(rows, list) or len(rows) != state_meta.get("rows") + or not rows or any(not isinstance(row, list) or len(row) != 6 + or not isinstance(row[0], str) or not row[0].startswith(PREFIX) for row in rows) + or len({row[0] for row in rows}) != len(rows)): + raise ValueError("Amazon mirror state is incomplete or contains unrelated entities") + paths, total = {}, 0 + for name, metadata in manifest["tables"].items(): + if not re.fullmatch(r"amzn_[a-z0-9_]+", name) or not isinstance(metadata, dict): + raise ValueError("Amazon mirror manifest contains an invalid table name") + data = _checked_mirror_blob(rt, _table_blob_key(name, metadata), metadata) + path = Path(tmp) / (name + ".parquet") + path.write_bytes(data) + try: + count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0] + except Exception as exc: # noqa: BLE001 + raise ValueError("Amazon mirror parquet is unreadable") from exc + if type(metadata.get("rows")) is not int or count != metadata["rows"]: + raise ValueError("Amazon mirror parquet row count mismatch") + paths[name] = str(path) + total += count + return manifest, rows, paths, total + + +def _snapshot_tables(): + """Current-state tables: every report or API the client declares `snapshot`.""" + cx = _client() + return tuple(t for t in TABLES if (cx.REPORT_TYPES.get(t) or {}).get("snapshot")) + + +def _collapse_snapshots(con, log=None): + """D-3031 heal: a snapshot table holding more than one pull keeps only the latest. + + -> {mirror: (rows before, rows after, copies)}. The ledger keeps its latest document's + dates (a DAILY document spans several), every other table its newest `_snapshot_date`; + AWD rows are another leg and never touched. Idempotent: one copy is left alone. + """ + existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"} + collapsed = {} + for table in _snapshot_tables(): + mirror = table_name(table) + if mirror not in existing: + continue + cols = _columns_of(con, mirror) + if "_snapshot_date" not in cols: + continue + scope = ("COALESCE(inventory_source, 'FBA') <> 'AWD'" + if table == "inventory" and "inventory_source" in cols else "TRUE") + dates = [row[0] for row in con.execute( + f"SELECT DISTINCT CAST(_snapshot_date AS VARCHAR) FROM {mirror} WHERE {scope}").fetchall()] + present = sorted(d for d in dates if d is not None) + if len(dates) <= 1 or not present: + continue + keep = [] + if table == "inventory": + saved = _read_state(con, entity_key("reconcile_" + table)) + try: + marked = json.loads(saved["detail"])["dates"] if saved and saved.get("detail") else {} + except (ValueError, KeyError, TypeError): + marked = {} + keep = [d for d in present if d in marked] + keep = keep or present[-1:] + stale = (f"{scope} AND (_snapshot_date IS NULL OR " + f"CAST(_snapshot_date AS VARCHAR) NOT IN ({','.join('?' for _ in keep)}))") + if not con.execute(f"SELECT count(*) FROM {mirror} WHERE {stale}", keep).fetchone()[0]: + continue + before = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] + con.execute("BEGIN TRANSACTION") + try: + con.execute(f"DELETE FROM {mirror} WHERE {stale}", keep) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise + after = con.execute(f"SELECT count(*) FROM {mirror}").fetchone()[0] + collapsed[mirror] = (before, after, len(dates)) + if log: + log(f"Amazon snapshot {mirror}: {len(dates)} copies collapsed to the latest, " + f"{before:,} rows to {after:,}") + return collapsed + + +def _heal_snapshots(con, result, log): + """Collapse after a restore; the healed tables are dirty so the next checkpoint saves one copy.""" + try: + result["dirty"].update(_collapse_snapshots(con, log)) + except Exception as exc: # noqa: BLE001 + message = (f"Amazon snapshot collapse failed ({type(exc).__name__}); the next resync of each " + "snapshot table replaces its earlier copies.") + result["problems"].append(message) + log(message) + + def restore_mirror(tenant_key, rt, log=None): """Restore once per process and tenant, with all blobs validated before one atomic commit. The tenant runtime is the only store door. A bad snapshot never partially replaces tables; the caller gets a problem and the unchanged first-load decision can rebuild it from Amazon. + Both manifest versions restore: v1's in-place keys exactly as saved, v2's content keys. """ log = print if log is None else log cache_key = _mirror_cache_key(tenant_key, rt) @@ -1496,46 +1895,25 @@ def restore_mirror(tenant_key, rt, log=None): # a process whose mirror survived re-uploaded EVERY table on its first checkpoint # (39 tables, 19.5 MB). The small stored manifest lets unchanged bytes be skipped. try: - raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json") + raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) saved = json.loads(raw) if raw else {} - if (saved.get("version") == 1 and saved.get("tenant") == _tenant_key(tenant_key) + if (saved.get("version") in MANIFEST_VERSIONS and saved.get("tenant") == _tenant_key(tenant_key) and isinstance(saved.get("tables"), dict)): result["manifest"] = saved except Exception: # noqa: BLE001 pass + _heal_snapshots(con, result, log) _MIRROR_CACHE[cache_key] = result return result - raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json") + raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) if raw is None: result["state"] = "absent" else: obligation = retention_check(tenant_key, rt=rt, raise_overdue=True) if obligation["earliestTriggerAt"]: raise ValueError("Amazon deletion is outstanding; saved data cannot be restored") - manifest = json.loads(raw) - if (manifest.get("version") != 1 or manifest.get("tenant") != _tenant_key(tenant_key) - or not isinstance(manifest.get("tables"), dict)): - raise ValueError("Amazon mirror manifest has an invalid version, tenant or table map") - state_bytes = _checked_mirror_blob(rt, "sync_state.json", manifest["sync_state"]) - rows = json.loads(state_bytes) - if (not isinstance(rows, list) or len(rows) != manifest["sync_state"]["rows"] - or not rows or any(not isinstance(row, list) or len(row) != 6 - or not isinstance(row[0], str) or not row[0].startswith(PREFIX) for row in rows) - or len({row[0] for row in rows}) != len(rows)): - raise ValueError("Amazon mirror state is incomplete or contains unrelated entities") with tempfile.TemporaryDirectory(prefix="amazon-restore-") as tmp: - paths, total = {}, 0 - for name, metadata in manifest["tables"].items(): - if not re.fullmatch(r"amzn_[a-z0-9_]+", name): - raise ValueError("Amazon mirror manifest contains an invalid table name") - data = _checked_mirror_blob(rt, name + ".parquet", metadata) - path = Path(tmp) / (name + ".parquet") - path.write_bytes(data) - count = con.execute("SELECT count(*) FROM read_parquet(?)", [str(path)]).fetchone()[0] - if type(metadata.get("rows")) is not int or count != metadata["rows"]: - raise ValueError("Amazon mirror parquet row count mismatch") - paths[name] = str(path) - total += count + manifest, rows, paths, total = _validated_snapshot(tenant_key, rt, con, raw, tmp) con.execute("BEGIN TRANSACTION") try: for name, path in paths.items(): @@ -1559,6 +1937,7 @@ def restore_mirror(tenant_key, rt, log=None): line = (f"Amazon mirror restored from the store: {len(paths)} tables, {total:,} rows; " f"resuming {resume}; settlements after document {document}") log(line.replace("\u2014", "-").replace("\u2013", "-").replace("\n", " ").replace("\r", " ")) + _heal_snapshots(con, result, log) except Exception as exc: result["state"] = "refused" message = (f"Amazon mirror restore refused ({type(exc).__name__}); saved manifest, checksums, " @@ -1572,26 +1951,110 @@ def restore_mirror(tenant_key, rt, log=None): return result +def _refused_may_publish(tenant_key, rt, con, cache): + """D-3481: a process that could not restore must not publish over a snapshot that is valid. + + It publishes once its own first load is complete (its rebuild is then whole), or once a + fresh re-validation proves the stored snapshot unrestorable (nothing valid is lost). A + store-access failure during that re-check proves nothing, so it holds and asks again later. + """ + if cache.get("publish_ok"): + return True + load = _read_state(con, entity_key(FIRST_LOAD_END_KEY)) + if load and load["state"] == "done": + cache["publish_ok"] = True + return True + if cache.get("stored_valid") is None: + try: + raw = rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) + if raw is None: + cache["stored_valid"] = False + else: + with tempfile.TemporaryDirectory(prefix="amazon-recheck-") as tmp: + _validated_snapshot(tenant_key, rt, con, raw, tmp) + cache["stored_valid"] = True + except (ValueError, KeyError, TypeError, AttributeError): + cache["stored_valid"] = False + except Exception: # noqa: BLE001 + return False + if cache["stored_valid"] is False: + cache["publish_ok"] = True + return True + return False + + +def _retire_blobs(tenant_key, rt, published, previous, log, *, orphans=True): + """After a publish only: delete what the previous manifest named and the new one does not, + plus ledger orphans older than ORPHAN_GRACE. Best effort; a failure is retried next time.""" + try: + if rt.download_bytes(MIRROR_PREFIX + _MANIFEST_KEY) != published: + log("Amazon mirror cleanup skipped: the stored manifest changed after this checkpoint") + return 0 + current = _manifest_keys(json.loads(published), strict=False) + ledger = _read_key_ledger(rt, tenant_key) + doomed = set(previous) - set(current) + if orphans: + cutoff = datetime.now(timezone.utc) - ORPHAN_GRACE + for key, stamp in ledger["keys"].items(): + try: + aged = _utc(stamp) < cutoff + except Exception: # noqa: BLE001 + aged = True + if key not in current and aged: + doomed.add(key) + removed = [] + for key in sorted(doomed): + if key in current or not _is_mirror_blob_key(key): + continue + rt.delete_path(MIRROR_PREFIX + key) + removed.append(key) + if any(key in ledger["keys"] for key in removed): + ledger = _read_key_ledger(rt, tenant_key) + ledger["keys"] = {k: v for k, v in ledger["keys"].items() if k not in removed} + rt.upload_bytes(MIRROR_PREFIX + _KEY_LEDGER, _mirror_json(ledger)) + return len(removed) + except Exception as exc: # noqa: BLE001 + log(f"Amazon mirror cleanup deferred ({type(exc).__name__}); superseded blobs stay in the " + "key ledger for the next checkpoint or purge.") + return 0 + + def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print): """Checkpoint changed tables and all Amazon markers, publishing the manifest last. Candidates include partially written tables. Hashing their parquet locally avoids reuploading identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain dirty for retry on the next pass, while landed local rows and cursors stay usable. + + S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a + CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists + every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row + upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4) + leaves the previous manifest naming only complete, untouched blobs. """ uploaded = 0 with _MIRROR_LOCK: cache = restore_mirror(tenant_key, rt, log=log) cache["dirty"].update(changed_tables) try: + if cache.get("state") == "refused" and not _refused_may_publish(tenant_key, rt, con, cache): + message = ("Amazon mirror checkpoint held: this process could not restore the saved snapshot " + "at start, and that snapshot is still valid or unreachable. It is kept; restart to " + "restore it. This process saves only after its own first load completes.") + report["problems"].append(message) + log(message) + return old = cache["manifest"] + named = _manifest_keys(old) tables = {name for name, kind in _amazon_objects(con) if name.startswith(PREFIX) and kind == "BASE TABLE"} - manifest = {"version": 1, "tenant": _tenant_key(tenant_key), + manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key), "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}} - changed = [] + changed, pending = [], [] with tempfile.TemporaryDirectory(prefix="amazon-persist-") as tmp: - for name in sorted((cache["dirty"] | (tables - set(manifest["tables"]))) & tables): + # A legacy entry (no content key) is a candidate too: the first checkpoint migrates. + legacy = {name for name, meta in manifest["tables"].items() if not meta.get("key")} + for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables): if not re.fullmatch(r"amzn_[a-z0-9_]+", name): raise ValueError("Amazon mirror table name is invalid") path = Path(tmp) / (name + ".parquet") @@ -1599,28 +2062,49 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print): "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) data = path.read_bytes() metadata = _blob_metadata(data) - if metadata["sha256"] == manifest["tables"].get(name, {}).get("sha256"): + prior = manifest["tables"].get(name, {}) + if metadata["sha256"] == prior.get("sha256") and prior.get("key"): continue + key = _content_key(name, ".parquet", metadata) metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], - written_at=_iso(datetime.now(timezone.utc))) - rt.upload_bytes(MIRROR_PREFIX + name + ".parquet", data) - uploaded += len(data) + written_at=_iso(datetime.now(timezone.utc)), key=key) manifest["tables"][name] = metadata changed.append(name) - rows = [row for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)] - state = _mirror_json(rows) - state_meta = {**_blob_metadata(state), "rows": len(rows)} - if rows and (changed or state_meta != old.get("sync_state") - or manifest["tables"] != old.get("tables", {})): - rt.upload_bytes(MIRROR_PREFIX + "sync_state.json", state) - uploaded += len(state) - manifest.update(sync_state=state_meta, written_at=_iso(datetime.now(timezone.utc))) - data = _mirror_json(manifest) - rt.upload_bytes(MIRROR_PREFIX + "manifest.json", data) - uploaded += len(data) - cache["manifest"] = manifest - cache["problems"].clear() - cache["dirty"].clear() + if key in named: + # The same content under its own name is already stored and published. + if named[key] != metadata["sha256"]: + raise ValueError("Amazon mirror content key collides with a published blob") + continue + pending.append((key, path)) + rows = [row for row in _sync_state_rows(con) if str(row[0]).startswith(PREFIX)] + state = _mirror_json(rows) + state_meta = {**_blob_metadata(state), "rows": len(rows)} + state_meta["key"] = _content_key("sync_state", ".json", state_meta) + published = None + if rows and (changed or state_meta != old.get("sync_state") + or manifest["tables"] != old.get("tables", {}) + or old.get("version") != MANIFEST_VERSION): + state_path = None + if state_meta["key"] not in named: + state_path = Path(tmp) / "sync_state.json" + state_path.write_bytes(state) + pending.append((state_meta["key"], state_path)) + if pending: + _ledger_add(rt, tenant_key, [key for key, _path in pending]) + for key, path in pending: + data = path.read_bytes() + rt.upload_bytes(MIRROR_PREFIX + key, data) + uploaded += len(data) + manifest.update(sync_state=state_meta, written_at=_iso(datetime.now(timezone.utc))) + published = _mirror_json(manifest) + rt.upload_bytes(MIRROR_PREFIX + _MANIFEST_KEY, published) + uploaded += len(published) + cache["manifest"] = manifest + cache["problems"].clear() + if published: + _retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused") + if published or not changed: + cache["dirty"].clear() log(f"Amazon mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded") except Exception as exc: message = (f"Amazon mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. " @@ -1629,20 +2113,41 @@ def persist_mirror(tenant_key, rt, con, changed_tables, *, report, log=print): log(message) +def _purge_json(rt, name): + """A saved mirror JSON document for a purge: the parsed dict, or {} when absent or unreadable.""" + try: + raw = rt.download_bytes(MIRROR_PREFIX + name) + doc = json.loads(raw) if raw else {} + except (TypeError, ValueError, UnicodeError, RecursionError): + return {} + return doc if isinstance(doc, dict) else {} + + def _purge_saved_mirror(tenant_key, rt, local_tables): - """Delete data before its inventory so a failed deletion remains retryable after restart.""" + """Delete data before its inventory so a failed deletion remains retryable after restart. + + Every version goes: each legacy in-place key, each key any manifest names, and each key the + write-ahead ledger lists, which includes blobs a killed checkpoint uploaded but never + published. Then the manifest, then the ledger itself, each read back. + """ with _MIRROR_LOCK: key = _mirror_cache_key(tenant_key, rt) cached = _MIRROR_CACHE.get(key, {}) - raw = rt.download_bytes(MIRROR_PREFIX + "manifest.json") - manifest = json.loads(raw) if raw else {} + # Deletion never refuses on unreadable inventory: a corrupt or foreign manifest or ledger + # still yields every blob name it holds, and every path below goes through THIS tenant's + # runtime, so a ledger labelled with another tenant cannot point a delete outside it. + manifest = _purge_json(rt, _MANIFEST_KEY) + ledger_keys = {str(k) for k in (_purge_json(rt, _KEY_LEDGER).get("keys") or {}) + if _is_mirror_blob_key(str(k))} names = ({table_name(t) for t in TABLES} | set(local_tables) | set(manifest.get("tables", {})) | set(cached.get("manifest", {}).get("tables", {})) | set(cached.get("dirty", ()))) - paths = [] - for name in sorted(names): - if re.fullmatch(r"amzn_[a-z0-9_]+", name): - paths.append(MIRROR_PREFIX + name + ".parquet") - paths.extend((MIRROR_PREFIX + "sync_state.json", MIRROR_PREFIX + "manifest.json")) + blobs = {name + ".parquet" for name in names if re.fullmatch(r"amzn_[a-z0-9_]+", name)} + blobs.add("sync_state.json") + blobs |= set(_manifest_keys(manifest, strict=False)) + blobs |= set(_manifest_keys(cached.get("manifest", {}), strict=False)) + blobs |= ledger_keys + paths = [MIRROR_PREFIX + blob for blob in sorted(blobs)] + paths.extend((MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER)) for path in paths: if rt.delete_path(path) is False or rt.download_bytes(path) is not None: raise RuntimeError("Amazon saved mirror deletion failed or left residue") @@ -2117,8 +2622,14 @@ def _record_reconciliation(con, table, marker): _record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":"))) -def _replace_period(con, table, rows, columns, column, start, end=None, marker=None): - """Atomic range/date replacement and proof; an empty restatement removes old rows too.""" +def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False): + """Atomic range/date replacement and proof; an empty restatement removes old rows too. + + `snapshot=True` (S67-T32, D-3031): a current-state report is ONE copy. A successful, parsed + pull replaces every earlier `_snapshot_date` of the table, not only its own day, because the + grid pid hashes the date and a kept day doubles every row downstream. A failed pull raises + before this runs, so it never empties the table. + """ mirror = table_name(table) columns = list(dict.fromkeys([*columns, column, *_ORIGIN_TYPES])) # All JSON money columns remain fractional even when today's amounts are integers. @@ -2129,6 +2640,10 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N predicate = ('CAST(posted_date AS TIMESTAMPTZ) >= CAST(? AS TIMESTAMPTZ) AND ' 'CAST(posted_date AS TIMESTAMPTZ) < CAST(? AS TIMESTAMPTZ)') params = [start, end] if end else [start] + if snapshot: + if column != "_snapshot_date" or end: + raise ValueError("a snapshot replacement is keyed by one _snapshot_date") + predicate, params = "TRUE", [] con.execute("BEGIN TRANSACTION") try: con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params) @@ -2177,11 +2692,13 @@ def _snapshot_identity(table, row): def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count): - """Replace complete returned data dates, preserving history and the separate AWD leg. + """Replace the whole FBA snapshot with one parsed document, preserving the separate AWD leg. Listings is a catalogue as of the vendor's data/creation date. The ledger is a DAILY summary: its row date wins over the report creation date, including multi-date documents. Duplicate source identities refuse the whole document rather than silently dropping rows. + S67-T32 (D-3031): one copy. The document's dates replace every earlier date, so the table + holds exactly one pull, the row count a fresh first load would store. """ stamp = meta.get("dataEndTime") or meta.get("dataStartTime") or meta.get("createdTime") if not stamp and (table == "listings" or not rows): @@ -2210,16 +2727,11 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count): dates = sorted(counts) marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date", "count": source_count, "dates": counts, "document": meta.get("reportDocumentId")} - placeholders = ",".join("?" for _ in dates) - predicate = f"_snapshot_date IN ({placeholders})" - # Undated pre-migration copies cannot be attributed to a day. Replace them once with - # dated source evidence; never fabricate a date from the time we happened to fetch. - predicate = f"({predicate} OR _snapshot_date IS NULL)" - if table == "inventory": - predicate += " AND COALESCE(inventory_source, 'FBA') <> 'AWD'" + # Every earlier date and any undated pre-migration copy goes; AWD rows are another leg. + predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE" con.execute("BEGIN TRANSACTION") try: - con.execute(f"DELETE FROM {mirror} WHERE {predicate}", dates) + con.execute(f"DELETE FROM {mirror} WHERE {predicate}") count = _replace(con, mirror, columns, types, rows) _record_reconciliation(con, table, marker) con.execute("COMMIT") @@ -2306,6 +2818,13 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log): document = "operation:" + uuid.uuid4().hex rows = [_origin(_flatten_row(_without_buyer_data(row)), document, i, spec["api"]) for i, row in enumerate(records, 1)] + if table == "catalog_items": + parents = catalog_parent_map(records, _pricing_parent_map(con), + fields.get("marketplace_id") or cx.US_MARKETPLACE_ID, + _economics_parent_map(con)) + for row in rows: + asin = str(row.get("asin") or "").strip() + row["parent_asin"], row["parent_asin_source"] = parents.get(asin, (asin or None, "self")) column = "_snapshot_date" if spec.get("snapshot") else "_period_start" period = datetime.now(timezone.utc).date().isoformat() if spec.get("snapshot") else start for row in rows: @@ -2322,7 +2841,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log): raise ValueError("Amazon order metrics returned an interval outside the requested period") row[column] = _iso(day) return _replace_period(con, table, rows, sorted({k for r in rows for k in r}), - column, period, end if table == "order_metrics" else None, marker=marker) + column, period, end if table == "order_metrics" else None, marker=marker, + snapshot=bool(spec.get("snapshot"))) kwargs = {} if spec.get("snapshot") else { "start": _iso(w0), "end": _iso(w1 - timedelta(seconds=1))} options = dict(spec.get("options") or {}) @@ -2370,7 +2890,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log): marker = {"start": period, "end": _iso(w1) if column == "_period_start" else period, "column": column, "count": document_row_count( text, (spec["json_root"], *spec.get("json_extra_roots", ())) if spec.get("json_root") else ())} - return _replace_period(con, table, rows, columns, column, period, marker=marker) + return _replace_period(con, table, rows, columns, column, period, marker=marker, + snapshot=bool(spec.get("snapshot"))) def _order_metric_totals(records, *, flattened=False): @@ -2605,7 +3126,13 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items() if head in want and sibling in want] + want)) handled, noted = set(), set() + # D-3031: a mirror that already holds two pulls of a snapshot heals here too, idempotently. changed_tables = set() + try: + changed_tables.update(_collapse_snapshots(con, log)) + except Exception as exc: # noqa: BLE001 + report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next " + "resync of each snapshot table replaces its earlier copies.") def write_window(table, cols, types, rows): count, note = replace_window(con, table_name(table), cols, types, rows, @@ -2875,7 +3402,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None {"state": "access", "operation": "report_document"}, rt, report, log) text, meta = cx.report_document( fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, - log=lambda message: log(f"{table}: {message}")) + log=lambda message: log(f"{table}: {message}"), + **({"options": dict(REPORT_OPTIONS[table])} if table in REPORT_OPTIONS else {})) if root: parsed_cols, rows = parse_json_report( text, root, document_id=meta.get("reportDocumentId"), report_type=rtype) @@ -2937,6 +3465,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None cols.append("inventory_source") if table == "pricing": pricing_asins.extend(_pricing_asins(rows)) + if not cleared and _pricing_regrain(con, mirror, rows, log): + cleared = True types = _ensure(con, mirror, cols, rows, declared=declared, preserve_rows=True) if cols else {} # Explicit windows and first loads replace the mirror; incremental passes keep history. @@ -3364,16 +3894,16 @@ def snapshot_selftest(log=print, check=None): "FROM amzn_listings").fetchone() == (2, 2)) meta["createdTime"] = "2026-09-26T11:00:00Z" sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) - ck("NC two listing dates keep two copies", lambda: + # S67-T32 (D-3031, reversed): a second day REPLACES the first, one load's rows. + ck("a second listing day replaces the first, leaving one copy", lambda: con.execute("SELECT _snapshot_date, COUNT(*) FROM amzn_listings " - "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-25", 2), ("2026-09-26", 2)] + "GROUP BY 1 ORDER BY 1").fetchall() == [("2026-09-26", 2)] and len(calls) == 2 and all("start" not in c and "end" not in c for c in calls)) payload[0] = "seller-sku\tquantity\nA\t9\n" sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None) - ck("same listing date replaces removed SKUs and retains the previous date", lambda: + ck("same listing date replaces removed SKUs", lambda: con.execute("SELECT _snapshot_date, seller_sku, quantity FROM amzn_listings " - "ORDER BY 1, 2").fetchall() - == [("2026-09-25", "A", 2), ("2026-09-25", "B", 3), ("2026-09-26", "A", 9)] + "ORDER BY 1, 2").fetchall() == [("2026-09-26", "A", 9)] and valid()["state"] == "ok") else: ck("ledger keeps one row per data date, SKU, disposition and location", lambda: @@ -3387,10 +3917,11 @@ def snapshot_selftest(log=print, check=None): before_awd = con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() payload[0] = ledger_header + "09/24/2026\tA\tFA\tAA\tSELLABLE\tUS\t9\n" sync(rt.key, tables=[table], since=start, rt=rt, log=lambda *_a: None) - ck("ledger same data date replaces while older dates survive", lambda: + # S67-T32 (D-3031, reversed): the next document replaces every earlier FBA date. + ck("ledger second document replaces every earlier data date", lambda: con.execute("SELECT _snapshot_date, ending_warehouse_balance FROM amzn_inventory " "WHERE inventory_source='FBA' ORDER BY 1").fetchall() - == [("2026-09-23", 2), ("2026-09-24", 9)]) + == [("2026-09-24", 9)]) ck("FBA replacement preserves AWD rows exactly", lambda: con.execute("SELECT * FROM amzn_inventory WHERE inventory_source='AWD'").fetchall() == before_awd) # The AWD operation's existing reconciliation still checks its own rows. @@ -3419,8 +3950,8 @@ def snapshot_selftest(log=print, check=None): if table == "listings": payload[0] = "seller-sku\tquantity\n" sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) - ck("empty listing restatement clears only that date and stays unproven", lambda: - count() == 2 and valid()["state"] == "unproven") + ck("empty listing restatement replaces the copy and stays unproven", lambda: + count() == 0 and valid()["state"] == "unproven") meta["dataEndTime"] = "2026-09-27T23:00:00Z" payload[0] = "seller-sku\tquantity\n" + "".join(f"S{i}\t1\n" for i in range(1205)) sync(rt.key, tables=[table], rt=rt, log=lambda *_a: None) @@ -3436,6 +3967,308 @@ def snapshot_selftest(log=print, check=None): "SELECT COUNT(*) FROM amzn_inventory WHERE _snapshot_date='2026-09-25' " "AND fnsku='F-fallback'").fetchone()[0] == 1) con.close() + ck("S67-T32 saved mirror: one copy, content keys, kill-safe checkpoint, full purge", + lambda: mirror_selftest(log, check) == 0) + return len(failures) + + +def mirror_selftest(log=print, check=None): + """S67-T32 (D-3031 + D-3481): the saved snapshot is one copy, whole, and restorable. + + Offline: a DuckDB :memory: mirror and a dictionary blob store. Covers the legacy (v1) + manifest restoring exactly, the doubled-mirror collapse, content keys never overwriting a + named key, a checkpoint dying after one table upload, purge reaching the orphan, a normal + checkpoint round trip (NC) and a refused process never publishing over a valid snapshot. + """ + from unittest.mock import patch + import duckdb + failures = [] + + def ck(name, probe): + try: + ok = bool(probe()) + except Exception as exc: # noqa: BLE001 + ok = False + log(f" {name}: {type(exc).__name__}: {exc}") + if check: + check("[mirror] " + name, ok) + else: + log(f" {'ok' if ok else 'FAIL'} {name}") + if not ok: + failures.append(name) + + class Store: + key = "qa-mirror" + + def __init__(self): + self.blobs, self.uploads, self.hook = {}, [], None + + def get(self, *_a, **_k): + return None + + def upload_bytes(self, path, data, message=None): + if self.hook: + self.hook("upload", path) + self.blobs[path] = bytes(data) + self.uploads.append(path) + + def download_bytes(self, path): + if self.hook: + self.hook("download", path) + return self.blobs.get(path) + + def delete_path(self, path): + self.blobs.pop(path, None) + return True + + rt = Store() + manifest_path, ledger_path = MIRROR_PREFIX + _MANIFEST_KEY, MIRROR_PREFIX + _KEY_LEDGER + con = duckdb.connect(":memory:") + con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, " + "last_id BIGINT, cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") + + def forget(): + _MIRROR_CACHE.pop(_mirror_cache_key(rt.key, rt), None) + + def wipe(): + """A new process whose local disk was lost.""" + for name, _kind in _amazon_objects(con): + con.execute(f"DROP TABLE {_quoted_identifier(name)}") + con.execute("DELETE FROM _sync_state WHERE starts_with(entity, 'amzn_')") + forget() + + def dump(): + out = {name: sorted(con.execute(f"SELECT * FROM {_quoted_identifier(name)}").fetchall(), key=repr) + for name, _kind in _amazon_objects(con)} + out["_sync_state"] = sorted(r for r in _sync_state_rows(con) if r[0].startswith(PREFIX)) + return out + + def ledger_doc(): + try: + return json.loads(rt.blobs[ledger_path]) + except (KeyError, ValueError): + return {"version": 1, "tenant": rt.key, "keys": {}} + + def mirror_blobs(): + return {k for k in rt.blobs if k.startswith(MIRROR_PREFIX)} + + def persist(tables=()): + report = {"problems": []} + persist_mirror(rt.key, rt, con, set(tables), report=report, log=lambda *_a: None) + return report + + def restore(): + return restore_mirror(rt.key, rt, log=lambda *_a: None) + + def legacy_save(): + """Today's staging layout, byte for byte: v1, in-place keys, no ledger.""" + manifest = {"version": 1, "tenant": rt.key, "tables": {}} + import tempfile as _tmp + with _tmp.TemporaryDirectory(prefix="amazon-legacy-") as tmp: + for name, _kind in _amazon_objects(con): + path = Path(tmp) / (name + ".parquet") + con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " + "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) + data = path.read_bytes() + rt.blobs[MIRROR_PREFIX + name + ".parquet"] = data + manifest["tables"][name] = {**_blob_metadata(data), "written_at": "2026-09-27T00:00:00Z", + "rows": con.execute(f"SELECT count(*) FROM {name}").fetchone()[0]} + rows = [list(r) for r in _sync_state_rows(con) if r[0].startswith(PREFIX)] + state = _mirror_json(rows) + rt.blobs[MIRROR_PREFIX + "sync_state.json"] = state + manifest.update(sync_state={**_blob_metadata(state), "rows": len(rows)}, written_at="2026-09-27T00:00:00Z") + rt.blobs[manifest_path] = _mirror_json(manifest) + + def seed(): + con.execute("CREATE TABLE amzn_restock_recommendations AS SELECT * FROM (VALUES " + "('2026-09-25', 'A', 1), ('2026-09-25', 'B', 2), ('2026-09-25', 'C', 3), " + "('2026-09-26', 'A', 4), ('2026-09-26', 'B', 5), ('2026-09-26', 'C', 6)) " + "AS t(_snapshot_date, sku, qty)") + con.execute("CREATE TABLE amzn_inventory AS SELECT * FROM (VALUES " + "('2026-09-22', 'A', 'FBA', 1), ('2026-09-23', 'A', 'FBA', 2), " + "('2026-09-24', 'A', 'FBA', 3), (NULL, 'AWD-1', 'AWD', 9)) " + "AS t(_snapshot_date, msku, inventory_source, qty)") + con.execute("CREATE TABLE amzn_orders AS SELECT * FROM (VALUES " + "('2026-09-01', 'O1', 10.5), ('2026-09-02', 'O2', 11.5)) AS t(_period_start, id, amount)") + con.execute("CREATE TABLE amzn_listings AS SELECT * FROM (VALUES ('2026-09-26', 'S1', 1)) " + "AS t(_snapshot_date, seller_sku, quantity)") + _record_state(con, entity_key("reconcile_inventory"), "done", detail=json.dumps( + {"start": "2026-09-23", "end": "2026-09-24", "column": "_snapshot_date", "count": 2, + "dates": {"2026-09-23": 1, "2026-09-24": 1}})) + record_watermark(con, "orders", datetime(2026, 9, 26, tzinfo=timezone.utc), 2) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z") + + try: + with patch.object(sys.modules[__name__], "_connect_tenant_mirror", + side_effect=lambda *_a: (con.cursor(), None)): + # D-3031 at the write: an API snapshot pulled on a new day replaces the old day. + for day, qty in (("2026-09-25", 1), ("2026-09-26", 2)): + rows = [_origin({"sku": s, "qty": qty, "_snapshot_date": day}, "doc-" + day, i, "fixture") + for i, s in enumerate(("A", "B"), 1)] + _replace_period(con, "fba_inventory", rows, ["sku", "qty", "_snapshot_date"], + "_snapshot_date", day, snapshot=True) + ck("two snapshot days leave one copy of one load's rows", lambda: con.execute( + "SELECT _snapshot_date, count(*) FROM amzn_fba_inventory GROUP BY 1").fetchall() + == [("2026-09-26", 2)]) + ck("a period table keeps its history", lambda: _replace_period( + con, "storage_fees", [_origin({"month": "2026-08", "fee": 1}, "d", 1, "f")], ["month", "fee"], + "month", "2026-08") == 1 and _replace_period( + con, "storage_fees", [_origin({"month": "2026-09", "fee": 1}, "d", 1, "f")], ["month", "fee"], + "month", "2026-09") == 1 and con.execute( + "SELECT count(DISTINCT month) FROM amzn_storage_fees").fetchone()[0] == 2) + con.execute("DROP TABLE amzn_fba_inventory") + con.execute("DROP TABLE amzn_storage_fees") + + # The legacy manifest staging boots from, holding a doubled snapshot. + seed() + original = dump() + legacy_save() + legacy_blobs = dict(rt.blobs) + wipe() + result = restore() + ck("a legacy v1 manifest with in-place keys restores every table and marker", lambda: + result["state"] == "restored" and not result["problems"] + and dump()["amzn_orders"] == original["amzn_orders"] + and dump()["amzn_listings"] == original["amzn_listings"] + and dump()["_sync_state"] == original["_sync_state"]) + ck("a doubled mirror collapses on restore to the latest pull and is marked dirty", lambda: + con.execute("SELECT _snapshot_date, count(*) FROM amzn_restock_recommendations " + "GROUP BY 1").fetchall() == [("2026-09-26", 3)] + and con.execute("SELECT _snapshot_date FROM amzn_inventory WHERE inventory_source='FBA' " + "ORDER BY 1").fetchall() == [("2026-09-23",), ("2026-09-24",)] + and con.execute("SELECT count(*) FROM amzn_inventory WHERE inventory_source='AWD'").fetchone()[0] == 1 + and {"amzn_restock_recommendations", "amzn_inventory"} <= result["dirty"] + and "amzn_orders" not in result["dirty"]) + ck("the collapse is idempotent", lambda: _collapse_snapshots(con) == {}) + + # First checkpoint after the legacy boot: every blob moves to a content key. + legacy_keys = set(_manifest_keys(json.loads(legacy_blobs[manifest_path]))) + rt.uploads.clear() + report = persist() + saved = json.loads(rt.blobs[manifest_path]) + named = set(_manifest_keys(saved)) + ck("the first checkpoint migrates to content keys and retires the in-place blobs", lambda: + not report["problems"] and saved["version"] == MANIFEST_VERSION + and all(_TABLE_BLOB_RE.fullmatch(k).group(2) for k in named if k.endswith(".parquet")) + and not any(MIRROR_PREFIX + k in rt.blobs for k in legacy_keys) + and not any(MIRROR_PREFIX + k in rt.uploads for k in legacy_keys) + and set(ledger_doc()["keys"]) == named + and rt.uploads.index(ledger_path) < min(rt.uploads.index(MIRROR_PREFIX + k) for k in named) + and rt.uploads[-2 if rt.uploads[-1] == ledger_path else -1] == manifest_path) + healed = dump() + wipe() + ck("NC a normal checkpoint restores exactly, one copy each", lambda: + restore()["state"] == "restored" and dump() == healed + and con.execute("SELECT count(*) FROM amzn_restock_recommendations").fetchone()[0] == 3) + rt.uploads.clear() + before = dict(rt.blobs) + persist(set(original)) + ck("an unchanged checkpoint uploads nothing", lambda: rt.uploads == [] and rt.blobs == before) + + # A changed table never overwrites a key the published manifest names. + con.execute("UPDATE amzn_orders SET amount = amount + 1") + rt.uploads.clear() + persist({"amzn_orders"}) + ck("content keys never overwrite a named key", lambda: + not any(path[len(MIRROR_PREFIX):] in named for path in rt.uploads) + and _manifest_keys(json.loads(rt.blobs[manifest_path])) + and not (MIRROR_PREFIX + saved["tables"]["amzn_orders"]["key"]) in rt.blobs) + + # A checkpoint that dies after its first table blob lands. + published = rt.blobs[manifest_path] + good = dump() + con.execute("UPDATE amzn_orders SET amount = amount + 1") + con.execute("UPDATE amzn_listings SET quantity = quantity + 1") + parquet = [] + + def die_on_second(kind, path): + if kind == "upload" and path.endswith(".parquet"): + parquet.append(path) + if len(parquet) == 2: + raise OSError("fixture: the process died mid-checkpoint") + + rt.hook = die_on_second + report = persist({"amzn_orders", "amzn_listings"}) + rt.hook = None + orphan = parquet[0][len(MIRROR_PREFIX):] if parquet else "absent" + ck("a checkpoint that dies after one table upload leaves the manifest untouched", lambda: + any("persistence failed" in p for p in report["problems"]) + and rt.blobs[manifest_path] == published and MIRROR_PREFIX + orphan in rt.blobs) + ck("the key ledger lists the unpublished orphan", lambda: + orphan in ledger_doc()["keys"] + and orphan not in _manifest_keys(json.loads(published))) + wipe() + ck("the next process restores the previous snapshot whole", lambda: + restore()["state"] == "restored" and dump() == good) + + # An aged orphan is retired by the next successful publish; a fresh one is not. + ledger = ledger_doc() + ledger["keys"][orphan] = "2026-01-01T00:00:00Z" + rt.blobs[ledger_path] = _mirror_json(ledger) + con.execute("UPDATE amzn_orders SET amount = amount + 5") + persist({"amzn_orders"}) + ck("an aged ledger orphan is retired after the next publish", lambda: + MIRROR_PREFIX + orphan not in rt.blobs + and orphan not in ledger_doc()["keys"] + and set(ledger_doc()["keys"]) >= set( + _manifest_keys(json.loads(rt.blobs[manifest_path])))) + + # A refused restore (store unreachable at boot) must not publish its rebuild. + valid_manifest = rt.blobs[manifest_path] + wipe() + + def unreachable(kind, path): + if kind == "download" and path == manifest_path: + raise OSError("fixture: store unreachable at boot") + + rt.hook = unreachable + refused = restore() + rt.hook = None + con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-02' AS _period_start, 'O2' AS id, 1.0 AS amount") + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z") + rt.uploads.clear() + report = persist({"amzn_orders"}) + ck("a refused restore does not publish a partial rebuild over the valid manifest", lambda: + refused["state"] == "refused" and rt.blobs[manifest_path] == valid_manifest + and rt.uploads == [] and any("checkpoint held" in p for p in report["problems"])) + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "done", detail="2026-09-26T00:00:00Z") + report = persist({"amzn_orders"}) + ck("the refused process publishes once its own first load completes", lambda: + not report["problems"] and rt.blobs[manifest_path] != valid_manifest + and set(json.loads(rt.blobs[manifest_path])["tables"]) == {"amzn_orders"}) + + # A stored snapshot that is itself unrestorable names nothing valid; the rebuild may publish. + wipe() + corrupt = json.loads(rt.blobs[manifest_path]) + corrupt["tables"]["amzn_orders"]["rows"] += 1 + rt.blobs[manifest_path] = _mirror_json(corrupt) + refused = restore() + con.execute("CREATE TABLE amzn_orders AS SELECT '2026-09-03' AS _period_start, 'O3' AS id, 2.0 AS amount") + _record_state(con, entity_key(FIRST_LOAD_END_KEY), "partial", detail="2026-09-26T00:00:00Z") + report = persist({"amzn_orders"}) + ck("a corrupt stored snapshot refuses whole and the rebuild may publish over it", lambda: + refused["state"] == "refused" and not report["problems"] + and json.loads(rt.blobs[manifest_path])["tables"]["amzn_orders"]["rows"] == 1) + + # Purge reaches every version: legacy keys, named keys and a killed checkpoint's orphan. + for path, data in legacy_blobs.items(): + if path != manifest_path: + rt.blobs[path] = data + con.execute("UPDATE amzn_orders SET amount = 7") + parquet.clear() + rt.hook = lambda kind, path: (_ for _ in ()).throw(OSError("fixture death")) if ( + kind == "upload" and path == manifest_path) else None + persist({"amzn_orders"}) + rt.hook = None + rt.blobs["unrelated/keep.bin"] = b"keep" + orphans = set(ledger_doc()["keys"]) - set( + _manifest_keys(json.loads(rt.blobs[manifest_path]))) + _purge_saved_mirror(rt.key, rt, [name for name, _kind in _amazon_objects(con)]) + ck("purge deletes every stored version including the orphan", lambda: + orphans and not mirror_blobs() and rt.blobs == {"unrelated/keep.bin": b"keep"}) + finally: + forget() + con.close() return len(failures) @@ -3666,8 +4499,19 @@ def margin_selftest(log=print, check=None): "sku\tasin\nS1\tA1\n", {"reportDocumentId": "next-snapshot", "dataEndTime": "2026-09-03T00:00:00Z"})): _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) - ck("margin two snapshot dates survive while same-date replay stays idempotent", - con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date) FROM amzn_referral_fees").fetchone() == (2, 2)) + # S67-T32 (D-3031, reversed): a new day REPLACES the earlier one; one copy, one load's rows. + ck("margin a second snapshot day replaces the first and same-date replay stays idempotent", + con.execute("SELECT COUNT(*), COUNT(DISTINCT _snapshot_date), MAX(_snapshot_date) " + "FROM amzn_referral_fees").fetchone() == (1, 1, "2026-09-03") + and _validate_margin(con, "referral_fees")["state"] == "ok") + kept = con.execute("SELECT * FROM amzn_referral_fees").fetchall() + with patch.object(cx, "report_document", side_effect=RuntimeError("fixture vendor failure")): + try: + _sync_margin_window(con, "referral_fees", {}, None, None, max_wait=1, log=log) + except RuntimeError: + pass + ck("margin a failed snapshot pull keeps the stored copy", + con.execute("SELECT * FROM amzn_referral_fees").fetchall() == kept and len(kept) == 1) _replace_period(con, "economics", [], [c[1] for c in con.execute("PRAGMA table_info('amzn_economics')").fetchall()], "start_date", "2026-09-01", "2026-09-03") ck("margin empty economics restatement deletes stale detail and stays unproven", @@ -4188,6 +5032,9 @@ def group_a_selftest(log=print, check=None): sent = [a for _, params in requests[::3] for a in params["identifiers"].split(",")] ck("catalog search covers all forty-one own ASINs in batches of twenty", sent == args["asins"] and all(len(p["identifiers"].split(",")) <= 20 for _, p in requests)) + ck("catalog search asks for relationships and nothing beyond the approved includes", + all(p["includedData"].split(",") == ["summaries", "salesRanks", "dimensions", "attributes", + "relationships"] for _, p in requests)) if operation == "getInventorySummaries": ck("FBA inventory requests detailed inventory on every page", all(p["details"] == "true" for _, p in requests)) @@ -4204,6 +5051,205 @@ def group_a_selftest(log=print, check=None): return len(requests) == 3 and len(rows) == 2 and requests[1]["createdBefore"] == requests[2]["createdAfter"] ck("Listings search splits date ranges at the real thousand-SKU ceiling", not partition_probe(False) and partition_probe(True)) + ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0) + return 1 if failures else 0 + + +def catalog_parent_selftest(log=print, check=None): + """S67-T29 (O23) offline fixtures: the catalog Parent ASIN map covers every own ASIN. + + `python -c "import sys; sys.path[:0]=['platform','platform/harness']; import amazon_store as s; + sys.exit(s.catalog_parent_selftest())"` from the repo root; no credentials, no network. + """ + from unittest.mock import patch + import duckdb + cx = _client() + failures = [] + + def ck(name, ok): + if check: + check("[catalog parent] " + name, bool(ok)) + else: + log(f" {'ok' if ok else 'FAIL'} catalog parent {name}") + if not ok: + failures.append(name) + + mkt = cx.US_MARKETPLACE_ID + child, parent, standalone, lonely, broken = ("B0CHILD001", "B0PARENT01", "B0STAND001", + "B0LONELY01", "B0BROKEN01") + items = [ + {"asin": child, "relationships": [ + {"marketplaceId": "A1F83G8C2ARO7P", "relationships": [ + {"type": "VARIATION", "parentAsins": ["B0WRONGMKT"]}]}, + {"marketplaceId": mkt, "relationships": [ + {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}, + {"type": "VARIATION", "parentAsins": [parent], + "variationTheme": {"attributes": ["color"], "theme": "COLOR"}}]}]}, + {"asin": parent, "relationships": [{"marketplaceId": mkt, "relationships": [ + {"type": "VARIATION", "childAsins": [child, "B0SIBLING1"]}]}]}, + {"asin": standalone, "relationships": [{"marketplaceId": mkt, "relationships": []}]}, + {"asin": lonely}, + {"asin": broken, "relationships": "not-a-list"}, + {"asin": "B0BROKEN02", "relationships": [None, {"marketplaceId": mkt, "relationships": {"x": 1}}, + {"marketplaceId": mkt, "relationships": [ + {"type": "VARIATION", "parentAsins": "B0NOTALIST"}, + {"type": "VARIATION", "parentAsins": [None, "bad"]}]}]}, + ] + eco_only = "B0ECONLY01" + items.append({"asin": eco_only, "relationships": [{"marketplaceId": mkt, "relationships": []}]}) + pricing = {standalone: "B0PRICEPAR", broken: "not an asin"} + economics = {eco_only: "B0ECOPARNT", standalone: "B0ECOLOSER", child: "B0ECOWRONG", broken: "junk"} + got = catalog_parent_map(items, pricing, mkt, economics) + ck("variation child maps to its parent from relationships", got.get(child) == (parent, "relationships")) + ck("variation parent maps to itself", got.get(parent) == (parent, "relationships")) + ck("standalone falls back to pricing parent", got.get(standalone) == ("B0PRICEPAR", "pricing")) + ck("no relationships and no pricing maps to itself", got.get(lonely) == (lonely, "self")) + ck("malformed relationships degrade to itself, never an error", + got.get(broken) == (broken, "self") and got.get("B0BROKEN02") == ("B0BROKEN02", "self")) + ck("economics child to parent is the next fallback after pricing", + got.get(eco_only) == ("B0ECOPARNT", "economics")) + ck("every item gets a parent", set(got) == {i["asin"] for i in items}) + # Red-first: economics never beats relationships or pricing, and is absent without its map. + ck("economics never outranks relationships or pricing, and needs its map", + got.get(child) == (parent, "relationships") and got.get(standalone) == ("B0PRICEPAR", "pricing") + and catalog_parent_map(items, pricing, mkt).get(eco_only) == (eco_only, "self")) + # Red-first: another marketplace or a PACKAGE_HIERARCHY parent must never win. + solo_other = catalog_parent_map([dict(items[0], relationships=items[0]["relationships"][:1])], {}, mkt) + pack_only = catalog_parent_map([{"asin": child, "relationships": [{"marketplaceId": mkt, "relationships": [ + {"type": "PACKAGE_HIERARCHY", "parentAsins": ["B0PACKAGE1"]}]}]}], {}, mkt) + ck("another marketplace and a package parent are ignored", + solo_other.get(child) == (child, "self") and pack_only.get(child) == (child, "self")) + ck("empty and junk inputs never raise", catalog_parent_map(None) == {} + and catalog_parent_map([None, "x", {"asin": ""}]) == {}) + + con = duckdb.connect(":memory:") + try: + con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " + "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") + # Resolver before any table exists: no own-ASIN source means None, never an error. + ck("resolver with no listings or catalog is None and an empty map", + asin_parents_sql(con) is None and asin_parents(con) == {}) + own = [child, parent, standalone, lonely, broken, eco_only] + # Listed ASINs Amazon's catalog returns no item for (the live 5 of 1164). + eco_listed, nothing = "B0ECOLIST1", "B0NOTHING1" + listed = own + [eco_listed, nothing, child] + con.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") + con.executemany("INSERT INTO amzn_listings VALUES (?)", [[a] for a in listed]) + only_listings = asin_parents(con) + ck("resolver with listings only maps every listed ASIN to itself", + set(only_listings) == set(listed) and all(v == (k, "self") for k, v in only_listings.items())) + con.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR, start_date VARCHAR)") + con.executemany("INSERT INTO amzn_economics VALUES (?, ?, ?)", [ + [eco_only, "B0ECOOLDPR", "2026-08-01"], [eco_only, "B0ECOPARNT", "2026-09-01"], + [eco_listed, "B0ECOLDPAR", "2026-07-01"], [eco_listed, "B0ECOLSTPR", "2026-09-10"], + [eco_listed, None, "2026-09-20"]]) + con.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR, _period_start VARCHAR)") + con.executemany("INSERT INTO amzn_pricing VALUES (?, ?, ?)", [ + [standalone, "B0OLDPARNT", "2026-08-01T00:00:00Z"], + [standalone, "B0PRICEPAR", "2026-09-01T00:00:00Z"]]) + asked = [] + + def records(_fields, operation, **kwargs): + asked.append(list(kwargs.get("asins") or ())) + return [i for i in items if i["asin"] in kwargs.get("asins", ())] + + # An older snapshot day with a different parent: the readback must use the LATEST day. + with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \ + patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in fixtures")), \ + patch.object(cx, "group_a_records", side_effect=records): + count = _sync_margin_window(con, "catalog_items", {}, None, None, max_wait=1, log=lambda *_: None) + con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " + "VALUES (?, 'B0STALEPAR', 'relationships', '2000-01-01')", [child]) + latest = con.execute("SELECT MAX(_snapshot_date) FROM amzn_catalog_items").fetchone()[0] + mapped = dict((a, (p, s)) for a, p, s in con.execute( + "SELECT asin, parent_asin, parent_asin_source FROM amzn_catalog_items WHERE _snapshot_date=?", + [latest]).fetchall()) + columns = {r[1] for r in con.execute("PRAGMA table_info('amzn_catalog_items')").fetchall()} + state = _read_state(con, entity_key("reconcile_catalog_items")) or {} + marker = json.loads(state.get("detail") or "{}") + ck("store readback shows the parent of a known child ASIN on the latest snapshot", + mapped.get(child) == (parent, "relationships")) + ck("catalog asks for every listed ASIN and maps every item it returns", + asked == [sorted(set(listed))] and set(mapped) == set(own) + and all(p for p, _ in mapped.values())) + ck("catalog sync stamps the economics fallback", mapped.get(eco_only) == ("B0ECOPARNT", "economics")) + # Resolver: two catalog snapshot days, an old-day-only ASIN, a latest-day catalog-only ASIN. + con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " + "VALUES ('B0OLDONLY1', 'B0OLDONLY1', 'self', '2000-01-01')") + con.execute("INSERT INTO amzn_catalog_items (asin, parent_asin, parent_asin_source, _snapshot_date) " + "VALUES ('B0CATONLY1', 'B0CATPARNT', 'relationships', ?)", [latest]) + resolved = asin_parents(con) + rows = con.execute(asin_parents_sql(con)).fetchall() + ck("resolver covers every own ASIN exactly once, latest catalog day only", + set(resolved) == set(listed) | {"B0CATONLY1"} and len(rows) == len(resolved) + and "B0OLDONLY1" not in resolved and all(p for p, _ in resolved.values())) + ck("resolver readback shows the latest parent of a known child", + resolved.get(child) == (parent, "relationships") and resolved.get("B0CATONLY1") + == ("B0CATPARNT", "relationships")) + ck("resolver falls back to the newest economics parent, then to itself", + resolved.get(eco_listed) == ("B0ECOLSTPR", "economics") and resolved.get(nothing) == (nothing, "self")) + # A catalog mirror synced before S67-T29 has no parent_asin column: economics and self carry it. + old = duckdb.connect(":memory:") + try: + old.execute("CREATE TABLE amzn_catalog_items (asin VARCHAR, _snapshot_date VARCHAR)") + old.execute(f"INSERT INTO amzn_catalog_items VALUES ('{child}', '2026-09-01'), ('{eco_listed}', '2026-09-01')") + old.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)") + old.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR')") + ck("resolver degrades on a pre-T29 catalog with no listings table", + asin_parents(old) == {child: (child, "self"), eco_listed: ("B0ECOLSTPR", "economics")}) + finally: + old.close() + # O23 order in the READ-TIME resolver: catalog, then pricing, then economics, then self. + pri = duckdb.connect(":memory:") + try: + pri.execute("CREATE TABLE amzn_listings (asin1 VARCHAR)") + pri.execute(f"INSERT INTO amzn_listings VALUES ('{eco_listed}'), ('{nothing}')") + pri.execute("CREATE TABLE amzn_economics (child_asin VARCHAR, parent_asin VARCHAR)") + pri.execute(f"INSERT INTO amzn_economics VALUES ('{eco_listed}', 'B0ECOLSTPR'), " + f"('{nothing}', 'B0ECONOTHG')") + pri.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)") + pri.execute(f"INSERT INTO amzn_pricing VALUES ('{eco_listed}', 'B0PRICEPA2')") + got = asin_parents(pri) + ck("resolver: a pricing parent wins over economics, economics still answers the rest", + got.get(eco_listed) == ("B0PRICEPA2", "pricing") + and got.get(nothing) == ("B0ECONOTHG", "economics")) + pri.execute("CREATE OR REPLACE TABLE amzn_pricing (parent_asin VARCHAR)") + ck("NC: a pricing mirror with no child_asin (the live mirror today) adds no pricing step", + asin_parents(pri).get(eco_listed) == ("B0ECOLSTPR", "economics")) + finally: + pri.close() + # SKU grain (S67-T29 finding): the request asks for it, and the first SKU-grain pull + # replaces a parent-grain mirror instead of sitting beside it. + grain = duckdb.connect(":memory:") + try: + quiet = lambda *_a, **_k: None # noqa: E731 + sku_rows = [{"parent_asin": "B0PARENT01", "child_asin": "B0CHILD001", "sku": "RI-1"}] + ck("regrain: an absent mirror is left alone", _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False) + grain.execute("CREATE TABLE amzn_pricing (parent_asin VARCHAR, report_date VARCHAR)") + grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-01'), ('B0PARENT02', '2026-09-02')") + ck("regrain: parent-grain rows WITHOUT child_asin never clear the mirror", + _pricing_regrain(grain, "amzn_pricing", [{"parent_asin": "B0PARENT03"}], quiet) is False + and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 2) + ck("regrain: the first SKU-grain pull clears the parent-grain mirror (one grain per table)", + _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is True + and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 0) + grain.execute("ALTER TABLE amzn_pricing ADD COLUMN child_asin VARCHAR") + grain.execute("INSERT INTO amzn_pricing VALUES ('B0PARENT01', '2026-09-03', 'B0CHILD001')") + ck("regrain: an already SKU-grain mirror keeps its history", + _pricing_regrain(grain, "amzn_pricing", sku_rows, quiet) is False + and grain.execute("SELECT COUNT(*) FROM amzn_pricing").fetchone()[0] == 1) + finally: + grain.close() + with open(__file__, encoding="utf-8") as _src: + passes_options = _src.read().count('REPORT_OPTIONS[table]') >= 2 + ck("the Sales and Traffic request asks for SKU grain, the same report and role", + REPORT_OPTIONS.get("pricing") == {"asinGranularity": "SKU"} and passes_options) + ck("pricing fallback reads the newest pricing period", mapped.get(standalone) == ("B0PRICEPAR", "pricing")) + ck("raw relationships JSON is kept beside the derived columns", + {"relationships", "parent_asin", "parent_asin_source"} <= columns) + ck("row count stays Amazon's record count", count == len(own) and marker.get("count") == len(own)) + finally: + con.close() return 1 if failures else 0 diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py index 6b6280534c8874f3f78e28a3e1603d4f2cddcee3..4a0a55616efb7e6cb1da86dc6446f57c3730108e 100644 --- a/platform/modules/sheet.py +++ b/platform/modules/sheet.py @@ -682,6 +682,24 @@ def clean_binding(raw): return out +def date_formatted_cell(cell): + """S67-T10 (QI4, D-767): a cell is a date when its number format says so. + + A Date format set by hand, the format the engine infers for DATE() / TODAY() / NOW() (written + into ``style.format``), or an imported Excel code carrying day or year tokens. Mirrors + ``aios-web/web/src/sheet/bindingRefs.ts::isDateFormattedCell``. + """ + import re as _re + fmt = ((cell or {}).get("style") or {}).get("format") if isinstance(cell, dict) else None + if not isinstance(fmt, dict): + return False + if fmt.get("kind") == "date": + return True + code = fmt.get("code") + return (fmt.get("kind") == "code" and isinstance(code, str) + and _re.search(r"[dy]", _re.sub(r'"[^"]*"|\\.', "", code), _re.I) is not None) + + def filter_cell_nodes(nodes): """Cell value slots live on leaves at any depth, never on a fixed object path.""" for node in nodes or (): diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index 4f2f8cf804bc0cc2970bbc2881886994a31203ad..4b9f44b361da6a2d3e4419803daeb6a33e374c18 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -4bddbdac0a66f19f85828e1077bbbc16266db0d5f1e63fa1448e0a2b41a57313 +eb084c3f9d52c9da87adb2168567d2ecf4bb936d483cfea8bcdcfe15160dedde diff --git a/web/src/apps/contentStudio/ContentStudioApp.tsx b/web/src/apps/contentStudio/ContentStudioApp.tsx new file mode 100644 index 0000000000000000000000000000000000000000..5281bbfcdcbc6e72564751b49413a78df60b61a5 --- /dev/null +++ b/web/src/apps/contentStudio/ContentStudioApp.tsx @@ -0,0 +1,271 @@ +import { Suspense, lazy, useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; +import { dataPathHash } from "../../customer-grid/dataPath"; +import type { Field, Row } from "../../customer-grid/types"; +import { CalendarView, KanbanView } from "../../customer-grid/viewModes"; +import "../../styles/wave50/grid.css"; + +/** Drafting is its own drawer (S67-T41/T42); it loads only when the owner asks to draft. */ +const DraftPanel = lazy(() => import("./DraftPanel")); + +interface StudioPost { pid: number; caption?: string | null; channel?: string | null; scheduled?: string | null; + status?: string | null; attribution?: string | null; campaign?: string | null } +interface Studio { + app: { id: string; name: string }; + posts: { database: string; label: string; fields: Field[]; rows: StudioPost[] }; + references: { database: string; label: string; count: number }; + statuses: string[]; +} +type Load = { phase: "loading" } | { phase: "error"; message: string } | { phase: "ready"; studio: Studio }; +type Mode = "calendar" | "board"; + +/** The server's own sentence first (top-level `error.message`), then the legacy nesting. */ +const sentence = (body: unknown, fallback: string): string => { + const row = body as { error?: { message?: string }; detail?: { error?: { message?: string } } } | null; + return row?.error?.message || row?.detail?.error?.message || fallback; +}; + +const plural = (n: number, one: string, many: string) => `${n} ${n === 1 ? one : many}`; + +/** A post's one-line name on a calendar chip or a board card: its channel, then its opening words. */ +const titleOf = (post: StudioPost): string => { + const words = String(post.caption ?? "").replace(/\s+/g, " ").trim(); + const head = words.length > 70 ? `${words.slice(0, 70).trimEnd()}…` : words || "Untitled post"; + return post.channel ? `${post.channel}: ${head}` : head; +}; + +const TITLE_FIELD: Field = { key: "title", label: "Post", type: "text", source: "overlay" }; + +/** + * S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an + * App ... it should act like Catalog / Route where it just uses the data module as what its + * building on top." So this is the App: it opens in the App module on its own surface and works + * the Posts and References databases underneath, which stay in Data. A calendar of posts by + * Scheduled beside the posts not yet scheduled, a board of posts by Status (a drag moves one), + * a post drawer to schedule it, and the Draft door. Plan-only (round 1 R5): nothing publishes. + * The calendar and the board are the grid's own `CalendarView` and `KanbanView`, unchanged. + */ +export default function ContentStudioApp({ appId }: { appId: string }) { + const [load, setLoad] = useState({ phase: "loading" }); + const [epoch, setEpoch] = useState(0); + const [mode, setMode] = useState("calendar"); + const [draftOpen, setDraftOpen] = useState(false); + const [openPid, setOpenPid] = useState(null); + const [notice, setNotice] = useState(""); + const [moved, setMoved] = useState>({}); + + useEffect(() => { + const controller = new AbortController(); + void fetch(`${API_V1}/content-studio/studio?app=${encodeURIComponent(appId)}`, + { credentials: CREDENTIALS, signal: controller.signal }) + .then(async (response) => { + const body = await response.json().catch(() => null); + if (!response.ok) throw new Error(sentence(body, "This Content Studio could not be loaded. Reload to retry.")); + setLoad({ phase: "ready", studio: body as Studio }); + setMoved({}); + }) + .catch((failure: unknown) => { + if (controller.signal.aborted) return; + setLoad({ phase: "error", + message: failure instanceof Error ? failure.message : "This Content Studio could not be loaded. Reload to retry." }); + }); + return () => controller.abort(); + }, [appId, epoch]); + + // A draft written from the drawer lands in Posts; read the App again so it appears here. + useEffect(() => { + const reload = () => setEpoch((value) => value + 1); + window.addEventListener("aios:grid-refresh", reload); + return () => window.removeEventListener("aios:grid-refresh", reload); + }, []); + + const studio = load.phase === "ready" ? load.studio : null; + const fieldByKey = useMemo(() => { + const map = new Map((studio?.posts.fields ?? []).map((field) => [field.key, field])); + map.set(TITLE_FIELD.key, TITLE_FIELD); + return map; + }, [studio]); + const rows: Row[] = useMemo(() => (studio?.posts.rows ?? []).map((post) => ({ + ...post, status: moved[post.pid] ?? post.status ?? null, title: titleOf(post), + }) as Row), [studio, moved]); + const scheduledField = fieldByKey.get("scheduled"); + const statusField = fieldByKey.get("status"); + const unscheduled = rows.filter((row) => !String(row.scheduled ?? "").trim()); + const openPost = studio?.posts.rows.find((post) => post.pid === openPid) ?? null; + + const postsDatabase = studio?.posts.database ?? ""; + // KanbanView is memo'd and keeps its props stable on purpose, so both handlers are too. + const openCard = useCallback((pid: number) => setOpenPid(pid), []); + const move = useCallback((pid: number, value: string) => { + if (!postsDatabase) return; + setNotice(""); + setMoved((current) => ({ ...current, [pid]: value })); + void fetch(`${API_V1}/tables/${encodeURIComponent(postsDatabase)}/rows/${pid}`, { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ status: value }), + }).then(async (response) => { + if (!response.ok) throw new Error(sentence(await response.json().catch(() => null), "The post could not be moved. Try again.")); + setEpoch((current) => current + 1); + }).catch((failure: unknown) => { + setMoved((current) => { const next = { ...current }; delete next[pid]; return next; }); + setNotice(failure instanceof Error ? failure.message : "The post could not be moved. Try again."); + }); + }, [postsDatabase]); + + if (load.phase === "loading") { + return
+

Content Studio

+
+ {[0, 1, 2].map((row) => )} +
+
; + } + if (load.phase === "error" || !studio) { + return
+

Content Studio

+

{load.phase === "error" ? load.message : ""}

+
; + } + + return
+
+
+

{studio.app.name}

+

+ {plural(studio.posts.rows.length, "post", "posts")} · {plural(studio.references.count, "reference", "references")} +

+
+
+ {studio.references.label} + {studio.posts.label} + +
+
+ + {/* The same worded toggle the Templates picker uses (`home-layout-btn`, sized to its words). */} +
+ {(["calendar", "board"] as const).map((value) => )} +
+ {notice ?

{notice}

: null} + + {mode === "calendar" ?
+
+ {!scheduledField ?

Posts has no Scheduled date field, so there is no calendar.

+ : rows.length === 0 ?

+ No posts yet. Use Draft posts to write some from your references.

+ : rows.length === unscheduled.length ?

+ No post is scheduled yet. Open one under Not scheduled to give it a date.

+ : } +
+
+

Not scheduled

+ {unscheduled.length === 0 ?

+ {rows.length === 0 ? "Nothing is waiting for a date." : "Every post has a date."}

+ :
    + {unscheduled.map((row) =>
  • + +
  • )} +
} +
+
: statusField ?
+ +
:

Posts has no Status field, so there is no board.

} + + {openPost ? setOpenPid(null)} onSaved={() => { setOpenPid(null); setEpoch((value) => value + 1); }} /> : null} + {draftOpen ? + setDraftOpen(false)} /> + : null} +
; +} + +/** One post: read its caption and credit, set its date and status. Saved through Posts' own row door. */ +function PostDrawer({ post, database, statuses, onClose, onSaved }: { + post: StudioPost; database: string; statuses: string[]; onClose: () => void; onSaved: () => void; +}) { + const panelRef = useRef(null); + const titleId = useId(); + const dateId = useId(); + const statusId = useId(); + const [scheduled, setScheduled] = useState(String(post.scheduled ?? "").slice(0, 10)); + const [status, setStatus] = useState(String(post.status ?? "") || statuses[0] || ""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const dismiss = () => { if (!busy) onClose(); }; + useOverlayLayer({ panelRef, onDismiss: dismiss, dismissOnOutside: false, + initialFocus: "[data-overlay-autofocus]", restoreFocus: true, trapFocus: true }); + + const save = async () => { + setBusy(true); + setError(""); + try { + const response = await fetch(`${API_V1}/tables/${encodeURIComponent(database)}/rows/${post.pid}`, { + method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ scheduled, status }), + }); + if (!response.ok) throw new Error(sentence(await response.json().catch(() => null), "The post could not be saved. Try again.")); + onSaved(); + } catch (failure) { + setError(failure instanceof Error ? failure.message : "The post could not be saved. Try again."); + setBusy(false); + } + }; + + return +
+
+

{post.channel ? `${post.channel} post` : "Post"}

+ +
+
+

{post.caption || "No caption yet."}

+ {post.attribution ?

{post.attribution}

: null} + + setScheduled(event.target.value)} /> + + + {error ?

{error}

: null} +
+
+ + +
+
+
; +} diff --git a/web/src/apps/contentStudio/ContentStudioSetup.tsx b/web/src/apps/contentStudio/ContentStudioSetup.tsx new file mode 100644 index 0000000000000000000000000000000000000000..11856c1a51dbf9ca33fd83db1bf8e2fbef024acf --- /dev/null +++ b/web/src/apps/contentStudio/ContentStudioSetup.tsx @@ -0,0 +1,83 @@ +import { useId, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; + +/** The Templates catalogue key this dialog answers for (`content_studio.TEMPLATE_KEY`). */ +export const CONTENT_STUDIO_TEMPLATE_KEY = "content_studio"; + +/** + * S67-T40 (R21, O2): Content Studio's create-for-me step, IN PLACE of "Choose a database". The + * template makes its own tenant-owned References and Posts databases, so there is nothing to + * choose; this dialog names what will be created and hands back the App's address. + */ +export default function ContentStudioSetup({ onCancel, onCreated }: { + onCancel: () => void; + onCreated: (address: string) => void; +}) { + const panelRef = useRef(null); + const titleId = useId(); + const nameId = useId(); + const [name, setName] = useState("Content Studio"); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const dismiss = () => { if (!busy) onCancel(); }; + useOverlayLayer({ panelRef, onDismiss: dismiss, dismissOnOutside: false, + initialFocus: "[data-overlay-autofocus]", restoreFocus: true, trapFocus: true }); + + const create = async () => { + if (busy) return; + setBusy(true); + setError(""); + try { + const response = await fetch(`${API_V1}/templates/app/${CONTENT_STUDIO_TEMPLATE_KEY}`, { + method: "POST", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name }), + }); + const body = await response.json().catch(() => null); + if (!response.ok || typeof body?.address !== "string") { + throw new Error(body?.error?.message || body?.detail?.error?.message + || "Content Studio could not be created. Nothing was changed. Try again."); + } + window.dispatchEvent(new Event("aios:applications-changed")); + onCreated(body.address); + } catch (failure) { + setError(failure instanceof Error ? failure.message : "Content Studio could not be created. Try again."); + setBusy(false); + } + }; + + return +
+
+
+

Content Studio

+ +
+
+

This creates two databases in this workspace, then opens a calendar of your posts.

+
    +
  • References: URL, network, author, pasted text, tags, hook, structure and angle.
  • +
  • Posts: caption, channel, scheduled date, status, media, campaign, post URL, references and attribution.
  • +
+ + setName(event.target.value)} /> + {error ?

{error}

: null} +
+
+ + +
+
+
+
; +} diff --git a/web/src/apps/contentStudio/DraftPanel.tsx b/web/src/apps/contentStudio/DraftPanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e9d93ee7f7486516efc6af154a53c567a757485c --- /dev/null +++ b/web/src/apps/contentStudio/DraftPanel.tsx @@ -0,0 +1,179 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { API_V1, CREDENTIALS } from "../../apiContract"; +import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface"; + +interface StudioReference { id: string; title: string; author: string; network: string; extracted: boolean } +interface Drafted { channel: string; caption: string } +/** S67-T42 (R35): this workspace's monthly drafting allowance, as the server counts it. */ +interface Allowance { used: number; limit: number; remaining: number; resetsOn: string; paused: boolean; message: string } + +/** The server's own sentence first (top-level `error.message`), then the legacy nesting. */ +const sentence = (body: unknown, fallback: string): string => { + const row = body as { error?: { message?: string }; detail?: { error?: { message?: string } } } | null; + return row?.error?.message || row?.detail?.error?.message || fallback; +}; + +const listed = (names: string[]): string => + names.length <= 1 ? names.join("") : `${names.slice(0, -1).join(", ")} and ${names[names.length - 1]}`; + +/** + * S67-T41 (R13-R15, R19, R20, R5, O4): Content Studio's drafting drawer. Several References plus + * a one-line topic become one Draft row per chosen channel in Posts, written from the references' + * extracted structure and crediting their authors. Plan-only: nothing is published. + */ +export default function DraftPanel({ appId, onClose }: { appId: string; onClose: () => void }) { + const panelRef = useRef(null); + const titleId = useId(); + const topicId = useId(); + const [phase, setPhase] = useState<"loading" | "ready" | "error">("loading"); + const [loadError, setLoadError] = useState(""); + const [references, setReferences] = useState([]); + const [channels, setChannels] = useState([]); + const [advice, setAdvice] = useState>({}); + const [selected, setSelected] = useState([]); + const [chosen, setChosen] = useState([]); + const [topic, setTopic] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [drafted, setDrafted] = useState([]); + const [allowance, setAllowance] = useState(null); + const dismiss = () => { if (!busy) onClose(); }; + useOverlayLayer({ panelRef, onDismiss: dismiss, dismissOnOutside: false, + initialFocus: "[data-overlay-autofocus]", restoreFocus: true, trapFocus: true }); + + useEffect(() => { + const controller = new AbortController(); + void fetch(`${API_V1}/content-studio/references?app=${encodeURIComponent(appId)}`, + { credentials: CREDENTIALS, signal: controller.signal }) + .then(async (response) => { + const body = await response.json().catch(() => null); + if (!response.ok) throw new Error(sentence(body, "References could not be loaded. Try again.")); + setReferences(Array.isArray(body?.references) ? body.references : []); + setChannels(Array.isArray(body?.channels) ? body.channels : []); + setAdvice(body?.advice && typeof body.advice === "object" ? body.advice : {}); + setAllowance(body?.allowance && typeof body.allowance === "object" ? body.allowance : null); + setPhase("ready"); + }) + .catch((failure: unknown) => { + if (controller.signal.aborted) return; + setLoadError(failure instanceof Error ? failure.message : "References could not be loaded. Try again."); + setPhase("error"); + }); + return () => controller.abort(); + }, [appId]); + + const toggle = (list: string[], value: string) => + list.includes(value) ? list.filter((item) => item !== value) : [...list, value]; + // NC: Draft stays disabled, WITH the reason, until every input it needs is there. + // R35: a used allowance pauses Draft and says when it resets, ahead of every other reason. + const reason = allowance?.paused ? allowance.message + : selected.length === 0 ? "Select at least one reference to draft from." + : !topic.trim() ? "Type a one-line topic for the draft." + : chosen.length === 0 ? "Choose at least one channel." : ""; + + const draft = async () => { + if (busy || reason) return; + setBusy(true); + setError(""); + setDrafted([]); + try { + const response = await fetch(`${API_V1}/content-studio/draft`, { + method: "POST", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ app: appId, references: selected, topic, channels: chosen }), + }); + const body = await response.json().catch(() => null); + if (response.status === 429 && body?.error?.code === "drafting_allowance_used") { + const message = sentence(body, "This workspace has used this month's drafting allowance."); + setAllowance((current) => current ? { ...current, remaining: 0, paused: true, message } + : { used: 0, limit: 0, remaining: 0, resetsOn: "", paused: true, message }); + } + if (!response.ok) throw new Error(sentence(body, "The drafts could not be written. Nothing was saved.")); + const made: Drafted[] = Array.isArray(body?.drafts) ? body.drafts : []; + setDrafted(made); + // The server owns the count and its sentence: re-read it rather than guess it here. + void fetch(`${API_V1}/content-studio/references?app=${encodeURIComponent(appId)}`, { credentials: CREDENTIALS }) + .then((reply) => reply.ok ? reply.json() : null) + .then((fresh) => { if (fresh?.allowance && typeof fresh.allowance === "object") setAllowance(fresh.allowance); }) + .catch(() => undefined); + window.dispatchEvent(new Event("aios:grid-refresh")); + } catch (failure) { + setError(failure instanceof Error ? failure.message : "The drafts could not be written. Nothing was saved."); + } finally { + setBusy(false); + } + }; + + return +
+
+

Draft posts

+ +
+
+

References

+ {phase === "loading" ?
+ {[0, 1, 2].map((row) => )} +
: phase === "error" ?

{loadError}

+ : references.length === 0 ?

No references yet. Add one to References, then paste its text.

+ :
    + {references.map((row) =>
  • + +
  • )} +
} + + setTopic(event.target.value)} /> +
+ Channels +
+ {channels.map((channel) => )} +
+
+ {chosen.length ?
    + {chosen.map((channel) =>
  • {advice[channel] || channel}
  • )} +
: null} + {reason ?

{reason}

: null} + {busy ?

Drafting {listed(chosen)}…

: null} + {error ?

{error}

: null} + {drafted.length ?
+

{drafted.length === 1 ? "1 draft" : `${drafted.length} drafts`} added to Posts as Draft.

+
    + {drafted.map((row) =>
  • {row.channel}: {row.caption.slice(0, 160)}{row.caption.length > 160 ? "…" : ""}
  • )} +
+
: null} +
+
+ + +
+ {allowance && !allowance.paused ?

+ {allowance.used} of {allowance.limit} drafts used this month +

: null} +
+
; +} diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx index d56705aad476a3516984e864d5af98e8373f3b21..bd1090083c7851cccab804f03e8483d7f29c0173 100644 --- a/web/src/connectors/ConnectorModal.tsx +++ b/web/src/connectors/ConnectorModal.tsx @@ -890,6 +890,17 @@ const CONNECTOR_DETAIL_DEADLINE_MS = 20_000; // Closing the modal unmounts it; keep the last successful read for the next open. const connectorDetailCache = new Map(); +// S67-T31: a refused read shows the server's own plain cause, or this component's sentence when +// the server sent none (a bare 500 has no JSON body). A status code never reaches the screen: +// "The server answered 500." was the owner's whole report, and it told him nothing he could act on. +function plainReadFailure( + body: { error?: { message?: string }; detail?: { error?: { message?: string } } } | null, + fallback: string, +): string { + const said = (body?.error?.message || body?.detail?.error?.message || "").trim(); + return said || fallback; +} + // Shell.tsx::signOut emits SIGNED_OUT_EVENT; an unloaded module has no cache to clear. export function clearConnectorDetailCache(): void { connectorDetailCache.clear(); @@ -1120,9 +1131,7 @@ export default function ConnectorModal({ if (dead) throw new Error("closed"); if (timedOut) throw new Error(timeoutMessage); if (!res.ok) { - throw new Error( - body?.error?.message || body?.detail?.error?.message || `The server answered ${res.status}.` - ); + throw new Error(plainReadFailure(body, `The details for ${row.label} could not be read.`)); } const page = parseConnectorDetail(body, connectorKey); if (!page) { @@ -1230,9 +1239,7 @@ export default function ConnectorModal({ | null; if (controller.signal.aborted) return; if (!res.ok) { - throw new Error( - body?.error?.message || body?.detail?.error?.message || `The server answered ${res.status}.` - ); + throw new Error(plainReadFailure(body, "The relationships for this connection could not be read.")); } // The same reader, and the same selected-key guard, as the inline graph on a full read. setGraphRead({ reads: forReads, graph: parseRelationshipGraph(body, connectorKey) }); diff --git a/web/src/connectors/ConnectorsPage.tsx b/web/src/connectors/ConnectorsPage.tsx index b4ec1307b6004662d51ff0b40851ff7ef2dc2cd6..108d578277f6e883906c961ba8c37bd89da4a232 100644 --- a/web/src/connectors/ConnectorsPage.tsx +++ b/web/src/connectors/ConnectorsPage.tsx @@ -67,6 +67,15 @@ import { type OdooSchema, type OdooStatus, } from "../automation/automationApi"; +// ⭐ S67-T20 / C4: the freshness READER and its words, never the settings panel. The tombstone +// above still holds: `OdooConfigPanel` stays out of this file. +import { + freshnessWarnings, + getOdooFreshness, + OdooFreshnessNote, + syncDate, + type OdooFreshness, +} from "./OdooConfig"; import "../styles/wave45/agent-operator.css"; // ⭐ W49-T12 — the card's own open control and the relationship canvas rules live here now. The // `CONNECTOR_GRAPH_CSS` string this file used to inject through `